🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(assistant): Respect data-retention settings (#17850)
Ben Bachem0fe908209d23
diff --git a/packages/shared/prisma/migrations/20260923120000_in_app_agent_conversation_retention_index/migration.sql b/packages/shared/prisma/migrations/20260923120000_in_app_agent_conversation_retention_index/migration.sqlnew file mode 100644index 0000000000..ae8afb108a--- /dev/null+++ b/packages/shared/prisma/migrations/20260923120000_in_app_agent_conversation_retention_index/migration.sql@@ -0,0 +1,2 @@+CREATE INDEX CONCURRENTLY "in_app_agent_conversations_project_updated_at_idx"+ ON "in_app_agent_conversations"("project_id", "updated_at");diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex a50420f0df..0abf3fe4fb 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -318,6 +318,7 @@ model InAppAgentConversation {@@id([id, projectId])@@index([projectId, createdByUserId, deletedAt, updatedAt, id], map: "in_app_agent_conversations_project_user_list_idx")+ @@index([projectId, updatedAt], map: "in_app_agent_conversations_project_updated_at_idx")@@map("in_app_agent_conversations")}diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.tsindex 3d822a2fbd..76b2a97715 100644--- a/worker/src/__tests__/dataRetentionProcessing.test.ts+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts@@ -20,6 +20,8 @@ import {import { prisma } from "@langfuse/shared/src/db";import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";import { Job } from "bullmq";+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";+import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";describe("DataRetentionProcessingJob", () => {let storageService: StorageService;@@ -53,6 +55,229 @@ describe("DataRetentionProcessingJob", () => {s3Prefix = null;});+ it("deletes expired conversations and their runs and events, but keeps recent and active conversations", async () => {+ const expiredAt = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);+ const recentAt = new Date();+ const expiredId = randomUUID();+ const recentId = randomUUID();+ const runningId = randomUUID();+ const staleId = randomUUID();+ const eventlessId = randomUUID();+ const legacyId = randomUUID();+ const recentLegacyId = randomUUID();+ const keyIds: string[] = [];+ const ids = [+ expiredId,+ recentId,+ runningId,+ staleId,+ eventlessId,+ legacyId,+ recentLegacyId,+ ];++ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: 7 },+ });++ try {+ const key = await createAndAddApiKeysToDb({+ prisma,+ entityId: projectId,+ scope: "PROJECT",+ isInAppAgentKey: true,+ });+ keyIds.push(key.id);+ await prisma.inAppAgentConversation.createMany({+ data: [+ {+ id: expiredId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ {+ id: recentId,+ projectId,+ createdAt: expiredAt,+ updatedAt: recentAt,+ },+ {+ id: runningId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ {+ id: staleId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ {+ id: eventlessId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ {+ id: legacyId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ {+ id: recentLegacyId,+ projectId,+ createdAt: expiredAt,+ updatedAt: expiredAt,+ },+ ],+ });+ const expiredRunIds = Array.from({ length: 101 }, () => randomUUID());+ const runningRunId = randomUUID();+ await prisma.inAppAgentRun.createMany({+ data: [+ ...expiredRunIds.map((id) => ({+ id,+ projectId,+ conversationId: expiredId,+ createdAt: expiredAt,+ finishedAt: expiredAt,+ mcpApiKeyId: key.id,+ })),+ { id: runningRunId, projectId, conversationId: runningId },+ {+ id: randomUUID(),+ projectId,+ conversationId: staleId,+ status: InAppAgentRunStatus.RUNNING,+ createdAt: expiredAt,+ claimedAt: expiredAt,+ heartbeatAt: expiredAt,+ },+ {+ id: randomUUID(),+ projectId,+ conversationId: legacyId,+ status: null,+ createdAt: expiredAt,+ request: { message: "old request" },+ },+ {+ id: randomUUID(),+ projectId,+ conversationId: recentLegacyId,+ status: null,+ createdAt: recentAt,+ },+ ],+ });+ await prisma.inAppAgentEvent.createMany({+ data: [+ {+ projectId,+ conversationId: expiredId,+ runId: expiredRunIds[0],+ sequenceNumber: 0,+ type: "test",+ event: {},+ createdAt: recentAt,+ },+ {+ projectId,+ conversationId: runningId,+ runId: runningRunId,+ sequenceNumber: 0,+ type: "test",+ event: {},+ createdAt: expiredAt,+ },+ ],+ });++ await handleDataRetentionProcessingJob({+ data: { payload: { projectId, retention: 7 } },+ } as Job);++ const conversations = await prisma.inAppAgentConversation.findMany({+ where: { projectId, id: { in: ids } },+ select: { id: true },+ });+ expect(conversations.map(({ id }) => id).sort()).toEqual(+ [recentId, runningId, staleId, legacyId, recentLegacyId].sort(),+ );+ expect(+ await prisma.inAppAgentRun.count({+ where: { projectId, conversationId: expiredId },+ }),+ ).toBe(0);+ expect(+ await prisma.inAppAgentEvent.count({+ where: { projectId, conversationId: expiredId },+ }),+ ).toBe(0);+ expect(+ await prisma.inAppAgentEvent.count({+ where: { projectId, conversationId: runningId },+ }),+ ).toBe(1);+ expect(await prisma.apiKey.count({ where: { id: key.id } })).toBe(0);+ expect(+ await prisma.inAppAgentRun.count({+ where: { projectId, conversationId: legacyId },+ }),+ ).toBe(1);+ expect(+ await prisma.inAppAgentRun.count({+ where: { projectId, conversationId: staleId },+ }),+ ).toBe(1);+ } finally {+ await prisma.inAppAgentConversation.deleteMany({+ where: { projectId, id: { in: ids } },+ });+ await prisma.apiKey.deleteMany({ where: { id: { in: keyIds } } });+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: null },+ });+ }+ });++ it("retains assistant conversations when retention is disabled after the job was queued", async () => {+ const id = randomUUID();+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: null },+ });++ try {+ await prisma.inAppAgentConversation.create({+ data: {+ id,+ projectId,+ createdAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),+ },+ });++ await handleDataRetentionProcessingJob({+ data: { payload: { projectId, retention: 7 } },+ } as Job);++ expect(+ await prisma.inAppAgentConversation.findUnique({+ where: { id_projectId: { id, projectId } },+ }),+ ).not.toBeNull();+ } finally {+ await prisma.inAppAgentConversation.deleteMany({+ where: { id, projectId },+ });+ }+ });+it("should NOT delete event files from cloud storage if after expiry cutoff", async () => {// Setup: Set retention in database to match job payloadawait prisma.project.update({diff --git a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.tsindex 900fed957a..aa6784f284 100644--- a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts+++ b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts@@ -9,9 +9,13 @@ import {logger,removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,getCurrentSpan,+ redis,} from "@langfuse/shared/src/server";import { Job } from "bullmq";import { prisma } from "@langfuse/shared/src/db";+import { Prisma } from "@prisma/client";+import { isMissingInAppAgentMcpApiKeyError } from "@langfuse/shared/in-app-agent/server/runLifecycle";+import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";import { env, v4WritesToEventsTable } from "../../env";export const handleDataRetentionProcessingJob = async (job: Job) => {@@ -59,6 +63,62 @@ export const handleDataRetentionProcessingJob = async (job: Job) => {Date.now() - currentRetention * 24 * 60 * 60 * 1000,);+ let processedRuns = 0;+ while (processedRuns < 10_000) {+ const keyRuns = await prisma.inAppAgentRun.findMany({+ where: {+ projectId,+ conversation: { updatedAt: { lt: cutoffDate } },+ finishedAt: { not: null },+ mcpApiKeyId: { not: null },+ },+ select: { id: true, mcpApiKeyId: true },+ take: Math.min(100, 10_000 - processedRuns),+ });+ if (keyRuns.length === 0) break;+ for (const run of keyRuns) {+ // Prisma does not narrow the nullable field type from the `not: null` query filter.+ if (!run.mcpApiKeyId) continue;+ await deleteInAppAgentMcpApiKeyFromDb({+ prisma,+ id: run.mcpApiKeyId,+ projectId,+ redis,+ }).catch((error: unknown) => {+ if (!isMissingInAppAgentMcpApiKeyError(error)) throw error;+ });+ }+ await prisma.$executeRaw`+ UPDATE in_app_agent_runs AS runs+ SET mcp_api_key_id = NULL, updated_at = NOW()+ FROM (VALUES ${Prisma.join(+ keyRuns+ .filter((run): run is { id: string; mcpApiKeyId: string } =>+ Boolean(run.mcpApiKeyId),+ )+ .map((run) => Prisma.sql`(${run.id}, ${run.mcpApiKeyId})`),+ )}) AS selected(id, key_id)+ WHERE runs.id = selected.id+ AND runs.project_id = ${projectId}+ AND runs.mcp_api_key_id = selected.key_id+ `;+ processedRuns += keyRuns.length;+ }+ // Conversations with unfinished runs are skipped until those runs are reconciled.+ const deleted = await prisma.inAppAgentConversation.deleteMany({+ where: {+ projectId,+ updatedAt: { lt: cutoffDate },+ AND: [+ { runs: { none: { finishedAt: null } } },+ { runs: { none: { mcpApiKeyId: { not: null } } } },+ ],+ },+ });+ logger.info(+ `[Data Retention] Deleted ${deleted.count} expired assistant conversations for project ${projectId}`,+ );+// Delete media files if bucket is configuredif (env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET) {logger.info(