Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(worker): use full job_executions index when deleting traces (#19451)

Niklas Semmlerb1067f3a8c7d
diff --git a/worker/src/__tests__/traceDeletion.test.ts b/worker/src/__tests__/traceDeletion.test.ts
index 8828cd0144..d8c5797599 100644
--- a/worker/src/__tests__/traceDeletion.test.ts
+++ b/worker/src/__tests__/traceDeletion.test.ts
@@ -24,8 +24,9 @@ import {
import { randomUUID } from "crypto";
import { createRequire } from "node:module";
import { processClickhouseTraceDelete } from "../features/traces/processClickhouseTraceDelete";
+import { processPostgresTraceDelete } from "../features/traces/processPostgresTraceDelete";
import { env } from "../env";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma, type JobConfigState } from "@langfuse/shared/src/db";
import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
@@ -76,6 +77,83 @@ describe("trace deletion", () => {
vi.restoreAllMocks();
});
+ it("deletes job executions of deleted traces across evaluation rules, rule-less evaluators and active and inactive configurations", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();
+ const deletedTraceId = randomUUID();
+ const keptTraceId = randomUUID();
+
+ const createConfig = (configProjectId: string, status: JobConfigState) =>
+ prisma.jobConfiguration.create({
+ data: {
+ projectId: configProjectId,
+ status,
+ jobType: "EVAL",
+ scoreName: "score",
+ filter: [],
+ targetObject: "trace",
+ variableMapping: [],
+ sampling: 1,
+ delay: 0,
+ },
+ });
+ const activeConfig = await createConfig(projectId, "ACTIVE");
+ const inactiveConfig = await createConfig(projectId, "INACTIVE");
+ const otherProjectConfig = await createConfig(otherProjectId, "ACTIVE");
+ const evaluationRule = await prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name: "score",
+ filter: [],
+ sampling: 1,
+ delay: 0,
+ targetObject: "trace",
+ },
+ });
+ const ruleLessEvaluator = await prisma.evaluator.create({
+ data: {
+ projectId,
+ name: "standalone",
+ type: "LLM_AS_JUDGE",
+ versions: { create: { version: 1 } },
+ },
+ });
+
+ const createExecution = (
+ executionProjectId: string,
+ jobConfigurationId: string,
+ jobInputTraceId: string,
+ ) =>
+ prisma.jobExecution.create({
+ data: {
+ projectId: executionProjectId,
+ jobConfigurationId,
+ jobInputTraceId,
+ status: "PENDING",
+ },
+ });
+ await createExecution(projectId, activeConfig.id, deletedTraceId);
+ await createExecution(projectId, inactiveConfig.id, deletedTraceId);
+ await createExecution(projectId, evaluationRule.id, deletedTraceId);
+ await createExecution(projectId, ruleLessEvaluator.id, deletedTraceId);
+ const kept = await createExecution(projectId, activeConfig.id, keptTraceId);
+ const otherProject = await createExecution(
+ otherProjectId,
+ otherProjectConfig.id,
+ deletedTraceId,
+ );
+
+ await processPostgresTraceDelete(projectId, [deletedTraceId]);
+
+ const remaining = await prisma.jobExecution.findMany({
+ where: { projectId: { in: [projectId, otherProjectId] } },
+ select: { id: true },
+ });
+ expect(remaining.map((r) => r.id).sort()).toEqual(
+ [kept.id, otherProject.id].sort(),
+ );
+ });
+
it("should delete all traces, observations, and scores from Clickhouse", async () => {
// Setup
const { projectId } = await createOrgProjectAndApiKey();
diff --git a/worker/src/features/traces/processPostgresTraceDelete.ts b/worker/src/features/traces/processPostgresTraceDelete.ts
index caec5f1313..938a5e33de 100644
--- a/worker/src/features/traces/processPostgresTraceDelete.ts
+++ b/worker/src/features/traces/processPostgresTraceDelete.ts
@@ -1,6 +1,35 @@
import { prisma } from "@langfuse/shared/src/db";
import { logger, traceException } from "@langfuse/shared/src/server";
+// Filtering on these ids lets Postgres use the
+// (project_id, job_configuration_id, job_input_trace_id) index instead of
+// scanning every job execution of the project. job_configuration_id
+// references evaluation_rules, legacy job_configurations, or, for batch runs
+// of evaluators without a rule, evaluators.
+const getJobConfigurationIds = async (projectId: string) => {
+ const [evaluationRules, jobConfigurations, evaluators] = await Promise.all([
+ prisma.evaluationRule.findMany({
+ where: { projectId },
+ select: { id: true },
+ }),
+ prisma.jobConfiguration.findMany({
+ where: { projectId },
+ select: { id: true },
+ }),
+ prisma.evaluator.findMany({
+ where: { projectId },
+ select: { id: true },
+ }),
+ ]);
+ return [
+ ...new Set(
+ [...evaluationRules, ...jobConfigurations, ...evaluators].map(
+ (r) => r.id,
+ ),
+ ),
+ ];
+};
+
export const processPostgresTraceDelete = async (
projectId: string,
traceIds: string[],
@@ -9,12 +38,18 @@ export const processPostgresTraceDelete = async (
`Deleting traces ${JSON.stringify(traceIds)} in project ${projectId} from Postgres`,
);
try {
+ const jobConfigurationIds = await getJobConfigurationIds(projectId);
+ if (jobConfigurationIds.length === 0) return;
+
await prisma.jobExecution.deleteMany({
where: {
+ projectId: projectId,
+ jobConfigurationId: {
+ in: jobConfigurationIds,
+ },
jobInputTraceId: {
in: traceIds,
},
- projectId: projectId,
},
});
} catch (e) {