🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { createHash } from "crypto";
import pLimit from "p-limit";
import { prisma } from "@langfuse/shared/src/db";
import {
commandClickhouse,
convertDateToClickhouseDateTime,
logger,
parseClickhouseUTCDateTimeFormat,
queryClickhouse,
queryClickhouseStream,
recordGauge,
recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";
import { env } from "../../env";
import { getRetentionCutoffDate } from "../utils/utilities";
import {
PeriodicExclusiveRunner,
PeriodicExclusiveRunnerLeaseLostError,
} from "../../utils/PeriodicExclusiveRunner";
// Tables for batch data retention cleaning (ClickHouse only; also no dataset_run_items)
export const BATCH_DATA_RETENTION_TABLES = [
"traces",
"observations",
"scores",
"events_full",
"events_core",
] as const;
export type BatchDataRetentionTable =
(typeof BATCH_DATA_RETENTION_TABLES)[number];
const RUNNER_NAME = "batch_data_retention_cleaner";
const METRIC_PREFIX = `langfuse.${RUNNER_NAME}`;
const BATCH_DATA_RETENTION_CLEANER_LOCK_PREFIX =
"langfuse:batch-data-retention-cleaner";
export const TIMESTAMP_COLUMN_MAP: Record<BatchDataRetentionTable, string> = {
traces: "timestamp",
observations: "start_time",
scores: "timestamp",
events_full: "start_time",
events_core: "start_time",
};
interface ProjectRetention {
projectId: string;
retentionDays: number;
cutoffDate: Date;
}
interface ProjectWorkload extends ProjectRetention {
rowCount: number | null;
secondsPastCutoff: number | null;
}
interface ProjectWorkloadSelection {
observedWorkloads: ProjectWorkload[];
selectedWorkloads: ProjectWorkload[];
projectCountMeasurementComplete: boolean;
lagMeasurementComplete: boolean;
}
interface CandidateDiscoveryResult {
candidates: ProjectRetention[];
complete: boolean;
}
interface EnrichmentResult {
workloads: ProjectWorkload[];
complete: boolean;
}
function summarizeBacklog(workloads: ProjectWorkload[]) {
return {
pendingProjects: workloads.length,
secondsPastCutoff: workloads.reduce(
(maximum, workload) =>
workload.secondsPastCutoff === null
? maximum
: Math.max(maximum, workload.secondsPastCutoff),
0,
),
};
}
/**
* Hash projectId to a short key for ClickHouse parameter names.
* sha256 rather than md5: an OpenSSL FIPS provider refuses md5. The key only
* names query parameters within one query, so it is never persisted.
*/
function toParamKey(projectId: string): string {
return createHash("sha256").update(projectId).digest("hex").slice(0, 8);
}
/**
* Build OR conditions for project-specific cutoffs.
* Used by batch deletion.
* Uses hashed projectId keys to prevent index mismatch bugs.
*/
function buildRetentionConditions(
timestampColumn: string,
projects: ProjectRetention[],
): { conditions: string; params: Record<string, unknown> } {
// Compute hashes once and check for collisions
const projectToKey = new Map<string, string>();
const keyToProject = new Map<string, string>();
for (const p of projects) {
const key = toParamKey(p.projectId);
const existing = keyToProject.get(key);
if (existing && existing !== p.projectId) {
throw new Error(
`Hash collision detected: projectIds "${existing}" and "${p.projectId}" both hash to "${key}"`,
);
}
projectToKey.set(p.projectId, key);
keyToProject.set(key, p.projectId);
}
const conditions = projects
.map((p) => {
const key = projectToKey.get(p.projectId)!;
return `(project_id = {pid_${key}: String} AND ${timestampColumn} < {cutoff_${key}: DateTime64(3)})`;
})
.join(" OR ");
const params: Record<string, unknown> = {};
for (const p of projects) {
const key = projectToKey.get(p.projectId)!;
params[`pid_${key}`] = p.projectId;
params[`cutoff_${key}`] = convertDateToClickhouseDateTime(p.cutoffDate);
}
return { conditions, params };
}
/**
* BatchDataRetentionCleaner handles bulk deletion of ClickHouse data based on
* project retention settings.
*
* Each instance processes one table (traces, observations, scores, events_full, events_core).
* Multiple workers coordinate via Redis distributed locking to ensure only one
* worker deletes from a given table at a time.
*
* Flow:
* 1. Query PG for all projects with retentionDays > 0
* 2. Calculate retention cutoff dates for each project
* 3. Stream projects with expired rows from bounded project chunks
* 4. Best-effort enrich candidates with total row count and oldest timestamp
* 5. Sort by lag past cutoff DESC, using total row count as a tie-breaker
* 6. Execute a single batch DELETE with OR conditions
*/
export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
private readonly tableName: BatchDataRetentionTable;
private readonly candidateQueryHttpTimeoutSeconds: number;
private runOutcome: "success" | "failed" | "skipped" = "success";
protected get defaultIntervalMs(): number {
return env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_INTERVAL_MS;
}
constructor(tableName: BatchDataRetentionTable) {
const candidateQueryTimeoutMs =
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS;
// TTL = longest bounded operation + 5 minutes buffer
const lockTtlSeconds =
Math.ceil(
Math.max(
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
candidateQueryTimeoutMs,
) / 1000,
) + 300;
super({
name: `BatchDataRetentionCleaner(${tableName})`,
metricName: RUNNER_NAME,
metricScope: tableName,
lockKey: `${BATCH_DATA_RETENTION_CLEANER_LOCK_PREFIX}:${tableName}`,
lockTtlSeconds,
onUnavailable: "fail",
});
this.tableName = tableName;
this.candidateQueryHttpTimeoutSeconds = Math.ceil(
candidateQueryTimeoutMs / 1000,
);
}
/**
* Start the batch cleaner service
*/
public override start(): void {
logger.info(`Starting ${this.instanceName}`, {
intervalMs: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_INTERVAL_MS,
projectLimit: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_PROJECT_LIMIT,
queryConcurrency:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_QUERY_CONCURRENCY,
candidateQueryTimeoutMs:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
deleteTimeoutMs:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
});
super.start();
}
/**
* Process a batch for data retention for this table.
* Preflight and deletion are both under lock to avoid redundant expensive queries.
*/
protected async execute(): Promise<void> {
this.runOutcome = "success";
const startedAt = Date.now();
try {
await this.executeRetentionBatch();
} catch (error) {
this.runOutcome = "failed";
throw error;
} finally {
this.logCompletion(Date.now() - startedAt);
}
}
protected override markRunFailed(error: unknown): void {
this.runOutcome = "failed";
super.markRunFailed(error);
}
protected override markRunSkipped(): void {
if (this.runOutcome === "success") {
this.runOutcome = "skipped";
}
super.markRunSkipped();
}
private async executeRetentionBatch(): Promise<void> {
// Data retention is an enterprise feature. Without a license the setting
// cannot be viewed or changed, so stored policies must not be enforced
// either: a deployment that loses its license would otherwise keep
// deleting data under a policy its operators can no longer reach.
if (!isEnterpriseLicenseAvailable()) {
return;
}
const timestampColumn = TIMESTAMP_COLUMN_MAP[this.tableName];
let observedBacklog: ReturnType<typeof summarizeBacklog> | undefined;
let observedLagMeasurementComplete = false;
const recordBacklog = (
{
pendingProjects,
secondsPastCutoff,
}: ReturnType<typeof summarizeBacklog>,
includeLag = true,
) => {
recordGauge(`${METRIC_PREFIX}.pending_projects`, pendingProjects, {
table: this.tableName,
});
if (includeLag) {
recordGauge(`${METRIC_PREFIX}.seconds_past_cutoff`, secondsPastCutoff, {
table: this.tableName,
});
}
};
await this.withLock(
async () => {
// Step 1: Get project workloads (streamed CH candidates + PG config)
const {
observedWorkloads,
selectedWorkloads,
projectCountMeasurementComplete,
lagMeasurementComplete,
} = await this.getProjectWorkloads();
const currentBacklog = summarizeBacklog(observedWorkloads);
if (projectCountMeasurementComplete) {
observedBacklog = currentBacklog;
observedLagMeasurementComplete = lagMeasurementComplete;
}
// Step 2: Execute DELETE
if (selectedWorkloads.length > 0) {
logger.info(
`${this.instanceName}: Processing ${selectedWorkloads.length} projects`,
{
projectIds: selectedWorkloads.map(
(workload) => workload.projectId,
),
pendingProjectsSeen: observedWorkloads.length,
observedMaxSecondsPastCutoff: currentBacklog.secondsPastCutoff,
lagMeasurementComplete,
},
);
await this.executeBatchDelete(timestampColumn, selectedWorkloads);
logger.info(`${this.instanceName}: Batch deletion completed`, {
table: this.tableName,
projectsProcessed: selectedWorkloads.length,
});
} else {
logger.info(
`${this.instanceName}: No projects with retention and data to delete`,
);
}
if (projectCountMeasurementComplete) {
// selectedWorkloads is the leading PROJECT_LIMIT slice.
recordBacklog(
summarizeBacklog(observedWorkloads.slice(selectedWorkloads.length)),
lagMeasurementComplete,
);
}
// Record successful deletion metrics
recordIncrement(`${METRIC_PREFIX}.delete_successes`, 1, {
table: this.tableName,
});
recordIncrement(
`${METRIC_PREFIX}.projects_processed`,
selectedWorkloads.length,
{
table: this.tableName,
},
);
},
() => {
if (observedBacklog) {
recordBacklog(observedBacklog, observedLagMeasurementComplete);
}
recordIncrement(`${METRIC_PREFIX}.delete_failures`, 1, {
table: this.tableName,
});
},
() => recordBacklog({ pendingProjects: 0, secondsPastCutoff: 0 }),
);
}
private logCompletion(durationMs: number): void {
try {
logger.info(`${this.instanceName}: Retention run completed`, {
event: "periodic_runner.completed",
runner: RUNNER_NAME,
scope: this.tableName,
outcome: this.runOutcome,
durationMs,
});
} catch {
/** Best-effort logging preserves the cleanup outcome. */
}
}
/**
* Get project workloads using bounded candidate discovery:
* 1. PostgreSQL: Get all projects with retention enabled
* 2. Calculate cutoffs for all projects
* 3. Stream projects with expired data from each CHUNK_SIZE project chunk
* 4. Best-effort enrich each chunk's candidates with total count/min
* 5. Sort by lag, then total count, and select top PROJECT_LIMIT
*/
private async getProjectWorkloads(): Promise<ProjectWorkloadSelection> {
const timestampColumn = TIMESTAMP_COLUMN_MAP[this.tableName];
// Step 1: Get all projects with retention from PostgreSQL
const projectsWithRetention = await prisma.project.findMany({
select: { id: true, retentionDays: true },
where: {
retentionDays: { gt: 0 },
deletedAt: null,
},
});
if (projectsWithRetention.length === 0) {
return {
observedWorkloads: [],
selectedWorkloads: [],
projectCountMeasurementComplete: true,
lagMeasurementComplete: true,
};
}
// Step 2: Calculate cutoffs for all projects upfront
const now = new Date();
const allProjectRetentions: ProjectRetention[] = projectsWithRetention.map(
(project) => ({
projectId: project.id,
retentionDays: project.retentionDays!,
cutoffDate: getRetentionCutoffDate(project.retentionDays!, now),
}),
);
// Step 3: Keep each bounded discovery chunk within one exact cutoff so
// its predicate uses a single project IN set and timestamp comparison.
const projectChunks = chunkProjectsByCutoff(
allProjectRetentions,
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,
);
// Step 4: Oldest timestamp provides the main ordering signal and total
// count breaks ties, but deletion must not depend on this grouped query
// succeeding. Keep enrichment per input chunk so its parameters remain
// bounded too.
const limitRetentionQuery = pLimit(
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_QUERY_CONCURRENCY,
);
let leaseLost = false;
// Wait for every limited pipeline so no queued query outlives the lock.
const settledChunkResults = await Promise.allSettled(
projectChunks.map((projectChunk) =>
limitRetentionQuery(async () => {
if (leaseLost) {
return null;
}
try {
const discovery = await this.findExpiredProjectCandidates(
timestampColumn,
projectChunk,
);
if (leaseLost) {
return null;
}
const enrichment = await this.enrichProjectCandidates(
timestampColumn,
discovery.candidates,
);
return {
workloads: enrichment.workloads,
discoveryComplete: discovery.complete,
complete: discovery.complete && enrichment.complete,
};
} catch (error) {
if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
leaseLost = true;
}
throw error;
}
}),
),
);
const failedChunk = settledChunkResults.find(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
if (failedChunk) {
throw failedChunk.reason;
}
const chunkResults = settledChunkResults.flatMap((result) =>
result.status === "fulfilled" && result.value !== null
? [result.value]
: [],
);
const workloads = chunkResults.flatMap((result) => result.workloads);
// Prioritize the projects furthest behind their cutoff. A failed lag or
// count estimate can indicate a particularly large workload, so keep
// unknown values ahead of known ones to avoid starvation.
workloads.sort((left, right) => {
if (left.secondsPastCutoff === null) {
if (right.secondsPastCutoff !== null) {
return -1;
}
} else if (right.secondsPastCutoff === null) {
return 1;
} else {
const lagDifference = right.secondsPastCutoff - left.secondsPastCutoff;
if (lagDifference !== 0) {
return lagDifference;
}
}
if (left.rowCount === null) {
return right.rowCount === null ? 0 : -1;
}
if (right.rowCount === null) {
return 1;
}
return right.rowCount - left.rowCount;
});
return {
observedWorkloads: workloads,
selectedWorkloads: workloads.slice(
0,
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_PROJECT_LIMIT,
),
projectCountMeasurementComplete: chunkResults.every(
(result) => result.discoveryComplete,
),
lagMeasurementComplete: chunkResults.every((result) => result.complete),
};
}
/**
* Stream projects with expired rows from one bounded project chunk. Rows
* yielded before a query failure remain eligible for deletion.
*/
private async findExpiredProjectCandidates(
timestampColumn: string,
{ projects, cutoff }: ProjectRetentionChunk,
): Promise<CandidateDiscoveryResult> {
if (projects.length === 0) {
return { candidates: [], complete: true };
}
const query = `
SELECT DISTINCT project_id
FROM ${this.tableName}
PREWHERE project_id IN ({discoveryProjectIds: Array(String)})
AND ${timestampColumn} < {discoveryCutoff: DateTime64(3)}
LIMIT {candidateLimit: UInt32}
`;
const candidatesById = new Map<string, ProjectRetention>();
let complete = true;
const projectsById = new Map(
projects.map((project) => [project.projectId, project]),
);
try {
const stream = queryClickhouseStream<{ project_id: string }>({
query,
params: {
discoveryProjectIds: projects.map((project) => project.projectId),
discoveryCutoff: cutoff,
candidateLimit: projects.length,
},
useMultipartParamsAuto: true,
clickhouseConfigs: {
// The shared client derives max_execution_time five seconds after
// this request timeout and sends progress headers.
request_timeout:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
clickhouse_settings: {
http_send_timeout: this.candidateQueryHttpTimeoutSeconds,
http_receive_timeout: this.candidateQueryHttpTimeoutSeconds,
},
},
});
for await (const row of stream) {
const project = projectsById.get(row.project_id);
if (project) {
candidatesById.set(row.project_id, project);
}
await this.extendLockOnProgress();
}
} catch (error) {
if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
throw error;
}
complete = false;
recordIncrement(`${METRIC_PREFIX}.candidate_query_failures`, 1, {
table: this.tableName,
});
this.markRunFailed(error);
logger.warn(`${this.instanceName}: Candidate query did not complete`, {
error,
candidatesFound: candidatesById.size,
});
}
// A completed query is progress even when it did not yield a candidate.
await this.extendLockOnProgress();
return {
candidates: Array.from(candidatesById.values()),
complete,
};
}
/**
* Best-effort total count and oldest retention timestamp for discovered
* candidates. The query deliberately filters only on project_id. Retention
* cutoffs come from Postgres and are frozen once per run, so they are applied
* after the result is returned. A failed primary query is retried once on a
* read-only pool. If both attempts fail, candidates remain eligible for
* deletion without publishing a partial lag maximum.
*/
private async enrichProjectCandidates(
timestampColumn: string,
candidates: ProjectRetention[],
): Promise<EnrichmentResult> {
if (candidates.length === 0) {
return { workloads: [], complete: true };
}
const query = `
SELECT
project_id,
count() AS row_count,
min(${timestampColumn}) AS oldest_timestamp
FROM ${this.tableName}
PREWHERE project_id IN ({candidateProjectIds: Array(String)})
GROUP BY project_id
`;
const queryExactWorkloads = async (
preferredClickhouseService?: "ReadOnly" | "EventsReadOnly",
): Promise<ProjectWorkload[]> => {
const result = await queryClickhouse<{
project_id: string;
row_count: number;
oldest_timestamp: string;
}>({
query,
params: {
candidateProjectIds: candidates.map(
(candidate) => candidate.projectId,
),
},
useMultipartParamsAuto: true,
preferredClickhouseService,
clickhouseConfigs: {
request_timeout:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
clickhouse_settings: {
http_send_timeout: this.candidateQueryHttpTimeoutSeconds,
http_receive_timeout: this.candidateQueryHttpTimeoutSeconds,
},
},
});
const resultByProjectId = new Map(
result.map((row) => {
const oldestTimestamp = parseClickhouseUTCDateTimeFormat(
row.oldest_timestamp,
);
if (Number.isNaN(oldestTimestamp.getTime())) {
throw new Error(
`Invalid oldest timestamp for project ${row.project_id}`,
);
}
return [
row.project_id,
{
rowCount: Number(row.row_count),
oldestTimestamp,
},
] as const;
}),
);
return candidates.flatMap((candidate) => {
const resultForProject = resultByProjectId.get(candidate.projectId);
if (
!resultForProject ||
resultForProject.oldestTimestamp.getTime() >=
candidate.cutoffDate.getTime()
) {
return [];
}
return [
{
...candidate,
rowCount: resultForProject.rowCount,
secondsPastCutoff:
(candidate.cutoffDate.getTime() -
resultForProject.oldestTimestamp.getTime()) /
1000,
},
];
});
};
try {
const workloads = await queryExactWorkloads();
await this.extendLockOnProgress();
return { workloads, complete: true };
} catch (error) {
if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
throw error;
}
// Preserve the APM error, but let a successful retry keep the runner outcome healthy.
traceException(error);
logger.warn(
`${this.instanceName}: Candidate enrichment failed; retrying on read-only`,
{
error,
candidatesFound: candidates.length,
},
);
// Reset the lease before another potentially long exact query.
await this.extendLockOnProgress(true);
const preferredClickhouseService =
this.tableName === "events_full" || this.tableName === "events_core"
? "EventsReadOnly"
: "ReadOnly";
try {
const workloads = await queryExactWorkloads(preferredClickhouseService);
await this.extendLockOnProgress();
return { workloads, complete: true };
} catch (retryError) {
if (retryError instanceof PeriodicExclusiveRunnerLeaseLostError) {
throw retryError;
}
this.markRunFailed(retryError);
recordIncrement(`${METRIC_PREFIX}.enrichment_query_failures`, 1, {
table: this.tableName,
});
logger.warn(`${this.instanceName}: Candidate enrichment failed`, {
error: retryError,
primaryError: error,
candidatesFound: candidates.length,
});
await this.extendLockOnProgress();
return {
workloads: candidates.map((candidate) => ({
...candidate,
rowCount: null,
secondsPastCutoff: null,
})),
complete: false,
};
}
}
}
/**
* Execute batch DELETE with OR conditions for selected projects.
* Single query deletes data for all selected projects at once.
*/
private async executeBatchDelete(
timestampColumn: string,
workloads: ProjectWorkload[],
): Promise<void> {
if (workloads.length === 0) {
return;
}
const { conditions, params } = buildRetentionConditions(
timestampColumn,
workloads,
);
const query = `DELETE FROM ${this.tableName} WHERE ${conditions}`;
// Reset the full lease immediately before the only destructive query. Its
// TTL includes the configured DELETE timeout plus the existing buffer.
await this.extendLockOnProgress(true);
await commandClickhouse({
query,
params,
clickhouseConfigs: {
request_timeout:
env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
},
});
}
}
/** Keep discovery chunks bounded and tied to one exact serialized cutoff. */
function chunkProjectsByCutoff(
projects: ProjectRetention[],
chunkSize: number,
): ProjectRetentionChunk[] {
const projectsByCutoff = new Map<string, ProjectRetention[]>();
for (const project of projects) {
const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);
const group = projectsByCutoff.get(cutoff);
if (group) {
group.push(project);
} else {
projectsByCutoff.set(cutoff, [project]);
}
}
const chunks: ProjectRetentionChunk[] = [];
for (const [cutoff, group] of projectsByCutoff) {
for (let offset = 0; offset < group.length; offset += chunkSize) {
chunks.push({
cutoff,
projects: group.slice(offset, offset + chunkSize),
});
}
}
return chunks;
}
interface ProjectRetentionChunk {
cutoff: string;
projects: ProjectRetention[];
}