Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

24.7 KB
Raw Blame
import { createHash } from "crypto";
import pLimit from "p-limit";

import { prisma } from "@langfuse/shared/src/db";
import {
  commandClickhouse,
  convertDateToClickhouseDateTime,
  logger,
  parseClickhouseUTCDateTimeFormat,
  queryClickhouse,
  queryClickhouseStream,
  recordGauge,
  recordIncrement,
  traceException,
} from "@langfuse/shared/src/server";
import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";
import { env } from "../../env";
import { getRetentionCutoffDate } from "../utils/utilities";
import {
  PeriodicExclusiveRunner,
  PeriodicExclusiveRunnerLeaseLostError,
} from "../../utils/PeriodicExclusiveRunner";

// Tables for batch data retention cleaning (ClickHouse only; also no dataset_run_items)
export const BATCH_DATA_RETENTION_TABLES = [
  "traces",
  "observations",
  "scores",
  "events_full",
  "events_core",
] as const;

export type BatchDataRetentionTable =
  (typeof BATCH_DATA_RETENTION_TABLES)[number];

const RUNNER_NAME = "batch_data_retention_cleaner";
const METRIC_PREFIX = `langfuse.${RUNNER_NAME}`;

const BATCH_DATA_RETENTION_CLEANER_LOCK_PREFIX =
  "langfuse:batch-data-retention-cleaner";

export const TIMESTAMP_COLUMN_MAP: Record<BatchDataRetentionTable, string> = {
  traces: "timestamp",
  observations: "start_time",
  scores: "timestamp",
  events_full: "start_time",
  events_core: "start_time",
};

interface ProjectRetention {
  projectId: string;
  retentionDays: number;
  cutoffDate: Date;
}

interface ProjectWorkload extends ProjectRetention {
  rowCount: number | null;
  secondsPastCutoff: number | null;
}

interface ProjectWorkloadSelection {
  observedWorkloads: ProjectWorkload[];
  selectedWorkloads: ProjectWorkload[];
  projectCountMeasurementComplete: boolean;
  lagMeasurementComplete: boolean;
}

interface CandidateDiscoveryResult {
  candidates: ProjectRetention[];
  complete: boolean;
}

interface EnrichmentResult {
  workloads: ProjectWorkload[];
  complete: boolean;
}

function summarizeBacklog(workloads: ProjectWorkload[]) {
  return {
    pendingProjects: workloads.length,
    secondsPastCutoff: workloads.reduce(
      (maximum, workload) =>
        workload.secondsPastCutoff === null
          ? maximum
          : Math.max(maximum, workload.secondsPastCutoff),
      0,
    ),
  };
}

/**
 * Hash projectId to a short key for ClickHouse parameter names.
 * sha256 rather than md5: an OpenSSL FIPS provider refuses md5. The key only
 * names query parameters within one query, so it is never persisted.
 */
function toParamKey(projectId: string): string {
  return createHash("sha256").update(projectId).digest("hex").slice(0, 8);
}

/**
 * Build OR conditions for project-specific cutoffs.
 * Used by batch deletion.
 * Uses hashed projectId keys to prevent index mismatch bugs.
 */
function buildRetentionConditions(
  timestampColumn: string,
  projects: ProjectRetention[],
): { conditions: string; params: Record<string, unknown> } {
  // Compute hashes once and check for collisions
  const projectToKey = new Map<string, string>();
  const keyToProject = new Map<string, string>();
  for (const p of projects) {
    const key = toParamKey(p.projectId);
    const existing = keyToProject.get(key);
    if (existing && existing !== p.projectId) {
      throw new Error(
        `Hash collision detected: projectIds "${existing}" and "${p.projectId}" both hash to "${key}"`,
      );
    }
    projectToKey.set(p.projectId, key);
    keyToProject.set(key, p.projectId);
  }

  const conditions = projects
    .map((p) => {
      const key = projectToKey.get(p.projectId)!;
      return `(project_id = {pid_${key}: String} AND ${timestampColumn} < {cutoff_${key}: DateTime64(3)})`;
    })
    .join(" OR ");

  const params: Record<string, unknown> = {};
  for (const p of projects) {
    const key = projectToKey.get(p.projectId)!;
    params[`pid_${key}`] = p.projectId;
    params[`cutoff_${key}`] = convertDateToClickhouseDateTime(p.cutoffDate);
  }

  return { conditions, params };
}

/**
 * BatchDataRetentionCleaner handles bulk deletion of ClickHouse data based on
 * project retention settings.
 *
 * Each instance processes one table (traces, observations, scores, events_full, events_core).
 * Multiple workers coordinate via Redis distributed locking to ensure only one
 * worker deletes from a given table at a time.
 *
 * Flow:
 * 1. Query PG for all projects with retentionDays > 0
 * 2. Calculate retention cutoff dates for each project
 * 3. Stream projects with expired rows from bounded project chunks
 * 4. Best-effort enrich candidates with total row count and oldest timestamp
 * 5. Sort by lag past cutoff DESC, using total row count as a tie-breaker
 * 6. Execute a single batch DELETE with OR conditions
 */
export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {
  private readonly tableName: BatchDataRetentionTable;
  private readonly candidateQueryHttpTimeoutSeconds: number;
  private runOutcome: "success" | "failed" | "skipped" = "success";

  protected get defaultIntervalMs(): number {
    return env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_INTERVAL_MS;
  }

  constructor(tableName: BatchDataRetentionTable) {
    const candidateQueryTimeoutMs =
      env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS;

    // TTL = longest bounded operation + 5 minutes buffer
    const lockTtlSeconds =
      Math.ceil(
        Math.max(
          env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
          candidateQueryTimeoutMs,
        ) / 1000,
      ) + 300;

    super({
      name: `BatchDataRetentionCleaner(${tableName})`,
      metricName: RUNNER_NAME,
      metricScope: tableName,
      lockKey: `${BATCH_DATA_RETENTION_CLEANER_LOCK_PREFIX}:${tableName}`,
      lockTtlSeconds,
      onUnavailable: "fail",
    });
    this.tableName = tableName;
    this.candidateQueryHttpTimeoutSeconds = Math.ceil(
      candidateQueryTimeoutMs / 1000,
    );
  }

  /**
   * Start the batch cleaner service
   */
  public override start(): void {
    logger.info(`Starting ${this.instanceName}`, {
      intervalMs: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_INTERVAL_MS,
      projectLimit: env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_PROJECT_LIMIT,
      queryConcurrency:
        env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_QUERY_CONCURRENCY,
      candidateQueryTimeoutMs:
        env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
      deleteTimeoutMs:
        env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
    });
    super.start();
  }

  /**
   * Process a batch for data retention for this table.
   * Preflight and deletion are both under lock to avoid redundant expensive queries.
   */
  protected async execute(): Promise<void> {
    this.runOutcome = "success";
    const startedAt = Date.now();
    try {
      await this.executeRetentionBatch();
    } catch (error) {
      this.runOutcome = "failed";
      throw error;
    } finally {
      this.logCompletion(Date.now() - startedAt);
    }
  }

  protected override markRunFailed(error: unknown): void {
    this.runOutcome = "failed";
    super.markRunFailed(error);
  }

  protected override markRunSkipped(): void {
    if (this.runOutcome === "success") {
      this.runOutcome = "skipped";
    }
    super.markRunSkipped();
  }

  private async executeRetentionBatch(): Promise<void> {
    // Data retention is an enterprise feature. Without a license the setting
    // cannot be viewed or changed, so stored policies must not be enforced
    // either: a deployment that loses its license would otherwise keep
    // deleting data under a policy its operators can no longer reach.
    if (!isEnterpriseLicenseAvailable()) {
      return;
    }

    const timestampColumn = TIMESTAMP_COLUMN_MAP[this.tableName];
    let observedBacklog: ReturnType<typeof summarizeBacklog> | undefined;
    let observedLagMeasurementComplete = false;
    const recordBacklog = (
      {
        pendingProjects,
        secondsPastCutoff,
      }: ReturnType<typeof summarizeBacklog>,
      includeLag = true,
    ) => {
      recordGauge(`${METRIC_PREFIX}.pending_projects`, pendingProjects, {
        table: this.tableName,
      });
      if (includeLag) {
        recordGauge(`${METRIC_PREFIX}.seconds_past_cutoff`, secondsPastCutoff, {
          table: this.tableName,
        });
      }
    };

    await this.withLock(
      async () => {
        // Step 1: Get project workloads (streamed CH candidates + PG config)
        const {
          observedWorkloads,
          selectedWorkloads,
          projectCountMeasurementComplete,
          lagMeasurementComplete,
        } = await this.getProjectWorkloads();

        const currentBacklog = summarizeBacklog(observedWorkloads);

        if (projectCountMeasurementComplete) {
          observedBacklog = currentBacklog;
          observedLagMeasurementComplete = lagMeasurementComplete;
        }

        // Step 2: Execute DELETE
        if (selectedWorkloads.length > 0) {
          logger.info(
            `${this.instanceName}: Processing ${selectedWorkloads.length} projects`,
            {
              projectIds: selectedWorkloads.map(
                (workload) => workload.projectId,
              ),
              pendingProjectsSeen: observedWorkloads.length,
              observedMaxSecondsPastCutoff: currentBacklog.secondsPastCutoff,
              lagMeasurementComplete,
            },
          );

          await this.executeBatchDelete(timestampColumn, selectedWorkloads);

          logger.info(`${this.instanceName}: Batch deletion completed`, {
            table: this.tableName,
            projectsProcessed: selectedWorkloads.length,
          });
        } else {
          logger.info(
            `${this.instanceName}: No projects with retention and data to delete`,
          );
        }

        if (projectCountMeasurementComplete) {
          // selectedWorkloads is the leading PROJECT_LIMIT slice.
          recordBacklog(
            summarizeBacklog(observedWorkloads.slice(selectedWorkloads.length)),
            lagMeasurementComplete,
          );
        }

        // Record successful deletion metrics
        recordIncrement(`${METRIC_PREFIX}.delete_successes`, 1, {
          table: this.tableName,
        });
        recordIncrement(
          `${METRIC_PREFIX}.projects_processed`,
          selectedWorkloads.length,
          {
            table: this.tableName,
          },
        );
      },
      () => {
        if (observedBacklog) {
          recordBacklog(observedBacklog, observedLagMeasurementComplete);
        }
        recordIncrement(`${METRIC_PREFIX}.delete_failures`, 1, {
          table: this.tableName,
        });
      },
      () => recordBacklog({ pendingProjects: 0, secondsPastCutoff: 0 }),
    );
  }

  private logCompletion(durationMs: number): void {
    try {
      logger.info(`${this.instanceName}: Retention run completed`, {
        event: "periodic_runner.completed",
        runner: RUNNER_NAME,
        scope: this.tableName,
        outcome: this.runOutcome,
        durationMs,
      });
    } catch {
      /** Best-effort logging preserves the cleanup outcome. */
    }
  }

  /**
   * Get project workloads using bounded candidate discovery:
   * 1. PostgreSQL: Get all projects with retention enabled
   * 2. Calculate cutoffs for all projects
   * 3. Stream projects with expired data from each CHUNK_SIZE project chunk
   * 4. Best-effort enrich each chunk's candidates with total count/min
   * 5. Sort by lag, then total count, and select top PROJECT_LIMIT
   */
  private async getProjectWorkloads(): Promise<ProjectWorkloadSelection> {
    const timestampColumn = TIMESTAMP_COLUMN_MAP[this.tableName];

    // Step 1: Get all projects with retention from PostgreSQL
    const projectsWithRetention = await prisma.project.findMany({
      select: { id: true, retentionDays: true },
      where: {
        retentionDays: { gt: 0 },
        deletedAt: null,
      },
    });

    if (projectsWithRetention.length === 0) {
      return {
        observedWorkloads: [],
        selectedWorkloads: [],
        projectCountMeasurementComplete: true,
        lagMeasurementComplete: true,
      };
    }

    // Step 2: Calculate cutoffs for all projects upfront
    const now = new Date();
    const allProjectRetentions: ProjectRetention[] = projectsWithRetention.map(
      (project) => ({
        projectId: project.id,
        retentionDays: project.retentionDays!,
        cutoffDate: getRetentionCutoffDate(project.retentionDays!, now),
      }),
    );

    // Step 3: Keep each bounded discovery chunk within one exact cutoff so
    // its predicate uses a single project IN set and timestamp comparison.
    const projectChunks = chunkProjectsByCutoff(
      allProjectRetentions,
      env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CHUNK_SIZE,
    );

    // Step 4: Oldest timestamp provides the main ordering signal and total
    // count breaks ties, but deletion must not depend on this grouped query
    // succeeding. Keep enrichment per input chunk so its parameters remain
    // bounded too.
    const limitRetentionQuery = pLimit(
      env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_QUERY_CONCURRENCY,
    );
    let leaseLost = false;
    // Wait for every limited pipeline so no queued query outlives the lock.
    const settledChunkResults = await Promise.allSettled(
      projectChunks.map((projectChunk) =>
        limitRetentionQuery(async () => {
          if (leaseLost) {
            return null;
          }

          try {
            const discovery = await this.findExpiredProjectCandidates(
              timestampColumn,
              projectChunk,
            );
            if (leaseLost) {
              return null;
            }

            const enrichment = await this.enrichProjectCandidates(
              timestampColumn,
              discovery.candidates,
            );
            return {
              workloads: enrichment.workloads,
              discoveryComplete: discovery.complete,
              complete: discovery.complete && enrichment.complete,
            };
          } catch (error) {
            if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
              leaseLost = true;
            }
            throw error;
          }
        }),
      ),
    );

    const failedChunk = settledChunkResults.find(
      (result): result is PromiseRejectedResult => result.status === "rejected",
    );
    if (failedChunk) {
      throw failedChunk.reason;
    }

    const chunkResults = settledChunkResults.flatMap((result) =>
      result.status === "fulfilled" && result.value !== null
        ? [result.value]
        : [],
    );
    const workloads = chunkResults.flatMap((result) => result.workloads);

    // Prioritize the projects furthest behind their cutoff. A failed lag or
    // count estimate can indicate a particularly large workload, so keep
    // unknown values ahead of known ones to avoid starvation.
    workloads.sort((left, right) => {
      if (left.secondsPastCutoff === null) {
        if (right.secondsPastCutoff !== null) {
          return -1;
        }
      } else if (right.secondsPastCutoff === null) {
        return 1;
      } else {
        const lagDifference = right.secondsPastCutoff - left.secondsPastCutoff;
        if (lagDifference !== 0) {
          return lagDifference;
        }
      }

      if (left.rowCount === null) {
        return right.rowCount === null ? 0 : -1;
      }
      if (right.rowCount === null) {
        return 1;
      }
      return right.rowCount - left.rowCount;
    });

    return {
      observedWorkloads: workloads,
      selectedWorkloads: workloads.slice(
        0,
        env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_PROJECT_LIMIT,
      ),
      projectCountMeasurementComplete: chunkResults.every(
        (result) => result.discoveryComplete,
      ),
      lagMeasurementComplete: chunkResults.every((result) => result.complete),
    };
  }

  /**
   * Stream projects with expired rows from one bounded project chunk. Rows
   * yielded before a query failure remain eligible for deletion.
   */
  private async findExpiredProjectCandidates(
    timestampColumn: string,
    { projects, cutoff }: ProjectRetentionChunk,
  ): Promise<CandidateDiscoveryResult> {
    if (projects.length === 0) {
      return { candidates: [], complete: true };
    }

    const query = `
      SELECT DISTINCT project_id
      FROM ${this.tableName}
      PREWHERE project_id IN ({discoveryProjectIds: Array(String)})
        AND ${timestampColumn} < {discoveryCutoff: DateTime64(3)}
      LIMIT {candidateLimit: UInt32}
    `;

    const candidatesById = new Map<string, ProjectRetention>();
    let complete = true;
    const projectsById = new Map(
      projects.map((project) => [project.projectId, project]),
    );

    try {
      const stream = queryClickhouseStream<{ project_id: string }>({
        query,
        params: {
          discoveryProjectIds: projects.map((project) => project.projectId),
          discoveryCutoff: cutoff,
          candidateLimit: projects.length,
        },
        useMultipartParamsAuto: true,
        clickhouseConfigs: {
          // The shared client derives max_execution_time five seconds after
          // this request timeout and sends progress headers.
          request_timeout:
            env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
          clickhouse_settings: {
            http_send_timeout: this.candidateQueryHttpTimeoutSeconds,
            http_receive_timeout: this.candidateQueryHttpTimeoutSeconds,
          },
        },
      });

      for await (const row of stream) {
        const project = projectsById.get(row.project_id);
        if (project) {
          candidatesById.set(row.project_id, project);
        }
        await this.extendLockOnProgress();
      }
    } catch (error) {
      if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
        throw error;
      }
      complete = false;
      recordIncrement(`${METRIC_PREFIX}.candidate_query_failures`, 1, {
        table: this.tableName,
      });
      this.markRunFailed(error);
      logger.warn(`${this.instanceName}: Candidate query did not complete`, {
        error,
        candidatesFound: candidatesById.size,
      });
    }

    // A completed query is progress even when it did not yield a candidate.
    await this.extendLockOnProgress();

    return {
      candidates: Array.from(candidatesById.values()),
      complete,
    };
  }

  /**
   * Best-effort total count and oldest retention timestamp for discovered
   * candidates. The query deliberately filters only on project_id. Retention
   * cutoffs come from Postgres and are frozen once per run, so they are applied
   * after the result is returned. A failed primary query is retried once on a
   * read-only pool. If both attempts fail, candidates remain eligible for
   * deletion without publishing a partial lag maximum.
   */
  private async enrichProjectCandidates(
    timestampColumn: string,
    candidates: ProjectRetention[],
  ): Promise<EnrichmentResult> {
    if (candidates.length === 0) {
      return { workloads: [], complete: true };
    }

    const query = `
      SELECT
        project_id,
        count() AS row_count,
        min(${timestampColumn}) AS oldest_timestamp
      FROM ${this.tableName}
      PREWHERE project_id IN ({candidateProjectIds: Array(String)})
      GROUP BY project_id
    `;

    const queryExactWorkloads = async (
      preferredClickhouseService?: "ReadOnly" | "EventsReadOnly",
    ): Promise<ProjectWorkload[]> => {
      const result = await queryClickhouse<{
        project_id: string;
        row_count: number;
        oldest_timestamp: string;
      }>({
        query,
        params: {
          candidateProjectIds: candidates.map(
            (candidate) => candidate.projectId,
          ),
        },
        useMultipartParamsAuto: true,
        preferredClickhouseService,
        clickhouseConfigs: {
          request_timeout:
            env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_CANDIDATE_QUERY_TIMEOUT_MS,
          clickhouse_settings: {
            http_send_timeout: this.candidateQueryHttpTimeoutSeconds,
            http_receive_timeout: this.candidateQueryHttpTimeoutSeconds,
          },
        },
      });

      const resultByProjectId = new Map(
        result.map((row) => {
          const oldestTimestamp = parseClickhouseUTCDateTimeFormat(
            row.oldest_timestamp,
          );
          if (Number.isNaN(oldestTimestamp.getTime())) {
            throw new Error(
              `Invalid oldest timestamp for project ${row.project_id}`,
            );
          }

          return [
            row.project_id,
            {
              rowCount: Number(row.row_count),
              oldestTimestamp,
            },
          ] as const;
        }),
      );

      return candidates.flatMap((candidate) => {
        const resultForProject = resultByProjectId.get(candidate.projectId);
        if (
          !resultForProject ||
          resultForProject.oldestTimestamp.getTime() >=
            candidate.cutoffDate.getTime()
        ) {
          return [];
        }

        return [
          {
            ...candidate,
            rowCount: resultForProject.rowCount,
            secondsPastCutoff:
              (candidate.cutoffDate.getTime() -
                resultForProject.oldestTimestamp.getTime()) /
              1000,
          },
        ];
      });
    };

    try {
      const workloads = await queryExactWorkloads();
      await this.extendLockOnProgress();
      return { workloads, complete: true };
    } catch (error) {
      if (error instanceof PeriodicExclusiveRunnerLeaseLostError) {
        throw error;
      }
      // Preserve the APM error, but let a successful retry keep the runner outcome healthy.
      traceException(error);
      logger.warn(
        `${this.instanceName}: Candidate enrichment failed; retrying on read-only`,
        {
          error,
          candidatesFound: candidates.length,
        },
      );

      // Reset the lease before another potentially long exact query.
      await this.extendLockOnProgress(true);

      const preferredClickhouseService =
        this.tableName === "events_full" || this.tableName === "events_core"
          ? "EventsReadOnly"
          : "ReadOnly";

      try {
        const workloads = await queryExactWorkloads(preferredClickhouseService);
        await this.extendLockOnProgress();
        return { workloads, complete: true };
      } catch (retryError) {
        if (retryError instanceof PeriodicExclusiveRunnerLeaseLostError) {
          throw retryError;
        }

        this.markRunFailed(retryError);
        recordIncrement(`${METRIC_PREFIX}.enrichment_query_failures`, 1, {
          table: this.tableName,
        });
        logger.warn(`${this.instanceName}: Candidate enrichment failed`, {
          error: retryError,
          primaryError: error,
          candidatesFound: candidates.length,
        });

        await this.extendLockOnProgress();
        return {
          workloads: candidates.map((candidate) => ({
            ...candidate,
            rowCount: null,
            secondsPastCutoff: null,
          })),
          complete: false,
        };
      }
    }
  }

  /**
   * Execute batch DELETE with OR conditions for selected projects.
   * Single query deletes data for all selected projects at once.
   */
  private async executeBatchDelete(
    timestampColumn: string,
    workloads: ProjectWorkload[],
  ): Promise<void> {
    if (workloads.length === 0) {
      return;
    }

    const { conditions, params } = buildRetentionConditions(
      timestampColumn,
      workloads,
    );

    const query = `DELETE FROM ${this.tableName} WHERE ${conditions}`;

    // Reset the full lease immediately before the only destructive query. Its
    // TTL includes the configured DELETE timeout plus the existing buffer.
    await this.extendLockOnProgress(true);

    await commandClickhouse({
      query,
      params,
      clickhouseConfigs: {
        request_timeout:
          env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_DELETE_TIMEOUT_MS,
      },
    });
  }
}

/** Keep discovery chunks bounded and tied to one exact serialized cutoff. */
function chunkProjectsByCutoff(
  projects: ProjectRetention[],
  chunkSize: number,
): ProjectRetentionChunk[] {
  const projectsByCutoff = new Map<string, ProjectRetention[]>();
  for (const project of projects) {
    const cutoff = convertDateToClickhouseDateTime(project.cutoffDate);
    const group = projectsByCutoff.get(cutoff);
    if (group) {
      group.push(project);
    } else {
      projectsByCutoff.set(cutoff, [project]);
    }
  }

  const chunks: ProjectRetentionChunk[] = [];
  for (const [cutoff, group] of projectsByCutoff) {
    for (let offset = 0; offset < group.length; offset += chunkSize) {
      chunks.push({
        cutoff,
        projects: group.slice(offset, offset + chunkSize),
      });
    }
  }
  return chunks;
}

interface ProjectRetentionChunk {
  cutoff: string;
  projects: ProjectRetention[];
}