🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(worker): use full job_executions index when deleting traces (#19451)
Niklas Semmlerb1067f3a8c7d
diff --git a/worker/src/__tests__/traceDeletion.test.ts b/worker/src/__tests__/traceDeletion.test.tsindex 8828cd0144..d8c5797599 100644--- a/worker/src/__tests__/traceDeletion.test.ts+++ b/worker/src/__tests__/traceDeletion.test.ts@@ -24,8 +24,9 @@ import {import { randomUUID } from "crypto";import { createRequire } from "node:module";import { processClickhouseTraceDelete } from "../features/traces/processClickhouseTraceDelete";+import { processPostgresTraceDelete } from "../features/traces/processPostgresTraceDelete";import { env } from "../env";-import { prisma } from "@langfuse/shared/src/db";+import { prisma, type JobConfigState } from "@langfuse/shared/src/db";import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";vi.mock("@langfuse/shared/src/server", async (importOriginal) => {@@ -76,6 +77,83 @@ describe("trace deletion", () => {vi.restoreAllMocks();});+ it("deletes job executions of deleted traces across evaluation rules, rule-less evaluators and active and inactive configurations", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();+ const deletedTraceId = randomUUID();+ const keptTraceId = randomUUID();++ const createConfig = (configProjectId: string, status: JobConfigState) =>+ prisma.jobConfiguration.create({+ data: {+ projectId: configProjectId,+ status,+ jobType: "EVAL",+ scoreName: "score",+ filter: [],+ targetObject: "trace",+ variableMapping: [],+ sampling: 1,+ delay: 0,+ },+ });+ const activeConfig = await createConfig(projectId, "ACTIVE");+ const inactiveConfig = await createConfig(projectId, "INACTIVE");+ const otherProjectConfig = await createConfig(otherProjectId, "ACTIVE");+ const evaluationRule = await prisma.evaluationRule.create({+ data: {+ projectId,+ name: "score",+ filter: [],+ sampling: 1,+ delay: 0,+ targetObject: "trace",+ },+ });+ const ruleLessEvaluator = await prisma.evaluator.create({+ data: {+ projectId,+ name: "standalone",+ type: "LLM_AS_JUDGE",+ versions: { create: { version: 1 } },+ },+ });++ const createExecution = (+ executionProjectId: string,+ jobConfigurationId: string,+ jobInputTraceId: string,+ ) =>+ prisma.jobExecution.create({+ data: {+ projectId: executionProjectId,+ jobConfigurationId,+ jobInputTraceId,+ status: "PENDING",+ },+ });+ await createExecution(projectId, activeConfig.id, deletedTraceId);+ await createExecution(projectId, inactiveConfig.id, deletedTraceId);+ await createExecution(projectId, evaluationRule.id, deletedTraceId);+ await createExecution(projectId, ruleLessEvaluator.id, deletedTraceId);+ const kept = await createExecution(projectId, activeConfig.id, keptTraceId);+ const otherProject = await createExecution(+ otherProjectId,+ otherProjectConfig.id,+ deletedTraceId,+ );++ await processPostgresTraceDelete(projectId, [deletedTraceId]);++ const remaining = await prisma.jobExecution.findMany({+ where: { projectId: { in: [projectId, otherProjectId] } },+ select: { id: true },+ });+ expect(remaining.map((r) => r.id).sort()).toEqual(+ [kept.id, otherProject.id].sort(),+ );+ });+it("should delete all traces, observations, and scores from Clickhouse", async () => {// Setupconst { projectId } = await createOrgProjectAndApiKey();diff --git a/worker/src/features/traces/processPostgresTraceDelete.ts b/worker/src/features/traces/processPostgresTraceDelete.tsindex caec5f1313..938a5e33de 100644--- a/worker/src/features/traces/processPostgresTraceDelete.ts+++ b/worker/src/features/traces/processPostgresTraceDelete.ts@@ -1,6 +1,35 @@import { prisma } from "@langfuse/shared/src/db";import { logger, traceException } from "@langfuse/shared/src/server";+// Filtering on these ids lets Postgres use the+// (project_id, job_configuration_id, job_input_trace_id) index instead of+// scanning every job execution of the project. job_configuration_id+// references evaluation_rules, legacy job_configurations, or, for batch runs+// of evaluators without a rule, evaluators.+const getJobConfigurationIds = async (projectId: string) => {+ const [evaluationRules, jobConfigurations, evaluators] = await Promise.all([+ prisma.evaluationRule.findMany({+ where: { projectId },+ select: { id: true },+ }),+ prisma.jobConfiguration.findMany({+ where: { projectId },+ select: { id: true },+ }),+ prisma.evaluator.findMany({+ where: { projectId },+ select: { id: true },+ }),+ ]);+ return [+ ...new Set(+ [...evaluationRules, ...jobConfigurations, ...evaluators].map(+ (r) => r.id,+ ),+ ),+ ];+};+export const processPostgresTraceDelete = async (projectId: string,traceIds: string[],@@ -9,12 +38,18 @@ export const processPostgresTraceDelete = async (`Deleting traces ${JSON.stringify(traceIds)} in project ${projectId} from Postgres`,);try {+ const jobConfigurationIds = await getJobConfigurationIds(projectId);+ if (jobConfigurationIds.length === 0) return;+await prisma.jobExecution.deleteMany({where: {+ projectId: projectId,+ jobConfigurationId: {+ in: jobConfigurationIds,+ },jobInputTraceId: {in: traceIds,},- projectId: projectId,},});} catch (e) {