🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(assistant): Migrate retention handling to cleaner class (#18719)
Ben Bachem3d4b9c6314da
diff --git a/.env.dev.example b/.env.dev.exampleindex bba3a84935..3f8b0c68de 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -379,6 +379,8 @@ LANGFUSE_AI_SMALL_MODEL="eu.anthropic.claude-haiku-5-5"# workers can opt out without turning the instance off:# QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false+# Retention runs even when the assistant or ingestion retention queue is disabled.+# LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS=900000diff --git a/.env.prod.example b/.env.prod.exampleindex 51f93207db..945e2a82fb 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -389,6 +389,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# instance off:# QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false+# Retention runs even when the assistant or ingestion retention queue is disabled.+# LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5# Admission ceilings for concurrent Assistant runs, web onlydiff --git a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts b/packages/shared/src/in-app-agent/server/runLifecycle.test.tsindex 7054b07035..7cbaa9483e 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts@@ -8,6 +8,7 @@ import {import { Prisma, type PrismaClient } from "../../db";import { logger } from "../../server";import {+ cleanupRunMcpApiKey,cleanupTerminalRunMcpApiKeys,createQueuedRun,reconcileConversationRuns,@@ -375,9 +376,55 @@ describe("cleanupTerminalRunMcpApiKeys", () => {}expect(updateMany).toHaveBeenCalledWith({- where: { id: "run-1", projectId: "project-1" },+ where: { id: "run-1", projectId: "project-1", mcpApiKeyId: "key-1" },data: { mcpApiKeyId: null },});expect(errorSpy).not.toHaveBeenCalled();});++ it("leaves the pointer intact when revocation fails", async () => {+ const updateMany = vi.fn();+ const prisma = {+ inAppAgentRun: { updateMany },+ } as unknown as PrismaClient;+ const error = new Error("Revocation failed");++ await expect(+ cleanupRunMcpApiKey({+ prisma,+ projectId: "project-1",+ runId: "run-1",+ apiKeyId: "key-1",+ deleteApiKey: async () => {+ throw error;+ },+ }),+ ).rejects.toBe(error);+ expect(updateMany).not.toHaveBeenCalled();+ });++ it("does not clear a replacement key installed during revocation", async () => {+ let keyId: string | null = "key-1";+ const updateMany = vi.fn(+ async ({ where }: { where: { mcpApiKeyId: string } }) => {+ if (keyId !== where.mcpApiKeyId) return { count: 0 };+ keyId = null;+ return { count: 1 };+ },+ );+ const prisma = {+ inAppAgentRun: { updateMany },+ } as unknown as PrismaClient;++ await cleanupRunMcpApiKey({+ prisma,+ projectId: "project-1",+ runId: "run-1",+ apiKeyId: "key-1",+ deleteApiKey: async () => {+ keyId = "key-2";+ },+ });+ expect(keyId).toBe("key-2");+ });});diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.tsindex a5d7ff90af..dc39ecb220 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts@@ -764,6 +764,29 @@ export function isMissingInAppAgentMcpApiKeyError(error: unknown): boolean {);}+/** Keep the pointer retryable until revocation succeeds, including concurrent cleanup. */+export async function cleanupRunMcpApiKey(params: {+ prisma: PrismaClient;+ projectId: string;+ runId: string;+ apiKeyId: string;+ deleteApiKey: (apiKeyId: string) => Promise<void>;+}): Promise<void> {+ try {+ await params.deleteApiKey(params.apiKeyId);+ } catch (error) {+ if (!isMissingInAppAgentMcpApiKeyError(error)) throw error;+ }+ await params.prisma.inAppAgentRun.updateMany({+ where: {+ id: params.runId,+ projectId: params.projectId,+ mcpApiKeyId: params.apiKeyId,+ },+ data: { mcpApiKeyId: null },+ });+}+/** Retry terminal-run MCP-key cleanup outside the lifecycle transaction. */export async function cleanupTerminalRunMcpApiKeys(params: {prisma: PrismaClient;@@ -787,18 +810,12 @@ export async function cleanupTerminalRunMcpApiKeys(params: {if (!run.mcpApiKeyId) continue;try {- try {- await params.deleteApiKey(run.mcpApiKeyId);- } catch (error) {- // Concurrent cleanup or a prior delete already removed the row.- if (!isMissingInAppAgentMcpApiKeyError(error)) {- throw error;- }- }- await clearRunMcpApiKeyPointer({+ await cleanupRunMcpApiKey({prisma: params.prisma,projectId: params.projectId,runId: run.id,+ apiKeyId: run.mcpApiKeyId,+ deleteApiKey: params.deleteApiKey,});cleaned += 1;} catch (error) {diff --git a/web/src/features/in-app-agent/README.md b/web/src/features/in-app-agent/README.mdindex 8f6bb0909d..e15545e239 100644--- a/web/src/features/in-app-agent/README.md+++ b/web/src/features/in-app-agent/README.md@@ -154,9 +154,10 @@ separate click event would answer.Rollback is a code revert; background execution does not require a schemarollback.-Workers with `LANGFUSE_IN_APP_AGENT_ENABLED` set consume the run queue and-reconcile stale runs; without it, runs commit as `QUEUED` and die at-`queue_timeout`. Cloud is on unless the flag is `"false"`. Split-role workers+Workers with `LANGFUSE_IN_APP_AGENT_ENABLED` set consume the run queue.+Stale-run reconciliation runs independently of assistant enablement so abandoned+runs cannot block retention after opt-out; it never executes agent work.+Cloud is on unless the flag is `"false"`. Split-role workerscan keep the instance on and set`QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false` (and`LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false`) to skip those@@ -167,6 +168,14 @@ its default of 20 equals full US capacity and exceeds JP and staging.Conversation switching, detached invalidations, retry, and conversation-list runstatuses remain separate project concerns.+Assistant conversation retention runs independently of ingestion retention mode+and assistant enablement, under an enterprise license. It is enabled by default;+`QUEUE_CONSUMER_DATA_RETENTION_QUEUE_IS_ENABLED=false` no longer pauses assistant+retention. Set `LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false` on every+worker to pause it. Conversations with unfinished runs remain excluded until+reconciliation settles them. Keep at least one integrity runner enabled for+stale-run reconciliation, including after assistant opt-out.+Environment ownership:- Worker: queue concurrency, sandbox provider and Lambda MicroVM values.diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 8969b7f36e..b8d1243660 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -163,6 +163,11 @@## In-App Agent Runtime+- `src/features/in-app-agent-retention-cleaner/` expires conversations under+ current licensed project retention policies. Its exclusive periodic runner+ starts independently of batch-retention and assistant enablement, processes+ bounded batches, revokes MCP keys before cascading deletion, and leaves+ unfinished runs to the integrity runner.- `src/features/in-app-agent/runtime/` owns Mastra adaptation, agent execution,instrumentation, prompt loading, continuation handling, tools, skills, andsandbox providers.diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.tsindex 67c6e8e626..effaf47990 100644--- a/worker/src/__tests__/dataRetentionProcessing.test.ts+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts@@ -25,7 +25,7 @@ import {StorageServiceFactory,toClickhouseDateTime,} from "@langfuse/shared/src/server";-import { prisma } from "@langfuse/shared/src/db";+import { prisma, Prisma, PrismaClient } from "@langfuse/shared/src/db";import { env as sharedEnv } from "@langfuse/shared/src/env";import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";@@ -33,6 +33,7 @@ import { Job } from "bullmq";import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";import { ProjectId, SystemRoleId, UserId } from "@langfuse/shared/rbac";import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";+import { InAppAgentRetentionCleaner } from "../features/in-app-agent-retention-cleaner";type LicenseCheckModule =typeof import("@langfuse/shared/src/server/ee/licenseCheck");@@ -114,11 +115,16 @@ describe("DataRetentionProcessingJob", () => {recentLegacyId,];- await prisma.project.update({- where: { id: projectId },- data: { retentionDays: 7 },+ const { id: projectId, orgId } = await prisma.project.create({+ data: {+ name: "Assistant retention test",+ retentionDays: 7,+ organization: { create: { name: "Assistant retention test" } },+ },});+ const queryRaw = prisma.$queryRaw.bind(prisma);+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");try {const creator = await prisma.user.create({data: {@@ -241,9 +247,49 @@ describe("DataRetentionProcessingJob", () => {],});- await handleDataRetentionProcessingJob({- data: { payload: { projectId, retention: 7 } },- } as Job);+ // API-key fixture setup also queries projects while assigning its role.+ candidateQuery.mockClear();+ candidateQuery.mockImplementation((query, ...values) =>+ queryRaw(Prisma.sql`+ WITH in_app_agent_conversations AS (+ SELECT * FROM public.in_app_agent_conversations+ WHERE project_id = ${projectId}+ )+ ${"sql" in query ? query : Prisma.sql(query, ...values)}+ `),+ );+ const cleaner = new InAppAgentRetentionCleaner();+ // A conversation with 101 keys remains eligible until its last key is cleaned.+ expect(await cleaner.processBatch()).toBe(0);+ expect(+ await prisma.inAppAgentRun.count({+ where: {+ projectId,+ conversationId: expiredId,+ mcpApiKeyId: { not: null },+ },+ }),+ ).toBe(1);+ expect(await cleaner.processBatch()).toBe(0);+ expect(await cleaner.processBatch()).toBe(600_000);+ const [backlog] = await candidateQuery.mock.results[3]!.value;+ // Thirty-day-old conversations with seven-day retention are 23 days overdue.+ expect(backlog.overdueSeconds).toBeGreaterThanOrEqual(23 * 24 * 60 * 60);+ expect(backlog.overdueSeconds).toBeLessThan(+ (Date.now() - expiredAt.getTime()) / 1000 - 7 * 24 * 60 * 60 + 1,+ );+ const discovered = (+ await Promise.all(candidateQuery.mock.results.map(({ value }) => value))+ )+ .flat()+ .filter((row) => "projectId" in row);+ expect(discovered).toHaveLength(3);+ expect(discovered).toEqual(+ expect.arrayContaining([+ { id: expiredId, projectId },+ { id: eventlessId, projectId },+ ]),+ );const conversations = await prisma.inAppAgentConversation.findMany({where: { projectId, id: { in: ids } },@@ -279,48 +325,178 @@ describe("DataRetentionProcessingJob", () => {}),).toBe(1);} finally {+ candidateQuery.mockRestore();await prisma.inAppAgentConversation.deleteMany({where: { projectId, id: { in: ids } },});await prisma.apiKey.deleteMany({ where: { id: { in: keyIds } } });- await prisma.project.update({- where: { id: projectId },- data: { retentionDays: null },- });+ await prisma.organization.delete({ where: { id: orgId } });}});- it("retains assistant conversations when retention is disabled after the job was queued", async () => {- const id = randomUUID();- await prisma.project.update({- where: { id: projectId },- data: { retentionDays: null },- });+ it.each([+ { retentionDays: null, deletedAt: null },+ { retentionDays: 0, deletedAt: null },+ { retentionDays: 7, deletedAt: new Date() },+ ])(+ "does not discover assistant conversations with policy %j",+ async ({ retentionDays, deletedAt }) => {+ const id = randomUUID();+ const { id: projectId, orgId } = await prisma.project.create({+ data: {+ name: "Assistant retention policy test",+ retentionDays,+ deletedAt,+ organization: { create: { name: "Assistant retention policy test" } },+ },+ });+ const queryRaw = prisma.$queryRaw.bind(prisma);+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");+ candidateQuery.mockImplementation((query, ...values) =>+ queryRaw(Prisma.sql`+ WITH in_app_agent_conversations AS (+ SELECT * FROM public.in_app_agent_conversations+ WHERE project_id = ${projectId}+ )+ ${"sql" in query ? query : Prisma.sql(query, ...values)}+ `),+ );+ try {+ const expiredAt = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);+ await prisma.inAppAgentConversation.create({+ data: { id, projectId, createdAt: expiredAt, updatedAt: expiredAt },+ });+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(+ 600_000,+ );+ expect(await candidateQuery.mock.results[0]!.value).toEqual([]);+ expect(+ await prisma.inAppAgentConversation.count({+ where: { id, projectId },+ }),+ ).toBe(1);+ } finally {+ candidateQuery.mockRestore();+ await prisma.inAppAgentConversation.deleteMany({+ where: { id, projectId },+ });+ await prisma.organization.delete({ where: { id: orgId } });+ }+ },+ );- try {- await prisma.inAppAgentConversation.create({+ it.each(["Asia/Tokyo", "America/Los_Angeles"])(+ "uses UTC retention boundaries in the %s session timezone",+ async (timezone) => {+ const { id: projectId, orgId } = await prisma.project.create({data: {- id,- projectId,- createdAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),+ name: "Assistant retention timezone test",+ retentionDays: 7,+ organization: {+ create: { name: "Assistant retention timezone test" },+ },},});+ const expiredId = randomUUID();+ const retainedId = randomUUID();+ const cutoff = Date.now() - 7 * 24 * 60 * 60 * 1000;+ const hour = 60 * 60 * 1000;+ const timezonePrisma = new PrismaClient();+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");+ const deletionQuery = vi.spyOn(prisma, "$executeRaw");- await handleDataRetentionProcessingJob({- data: { payload: { projectId, retention: 7 } },- } as Job);+ try {+ await prisma.inAppAgentConversation.createMany({+ data: [+ { id: expiredId, projectId, updatedAt: new Date(cutoff - hour) },+ { id: retainedId, projectId, updatedAt: new Date(cutoff + hour) },+ ],+ });+ // SET LOCAL stays on the transaction's connection and cannot leak+ // timezone changes to other tests using the pool.+ candidateQuery.mockImplementation((query, ...values) =>+ timezonePrisma.$transaction(async (tx) => {+ await tx.$queryRaw`SELECT set_config('TimeZone', ${timezone}, true)`;+ return tx.$queryRaw(Prisma.sql`+ WITH in_app_agent_conversations AS (+ SELECT * FROM public.in_app_agent_conversations+ WHERE project_id = ${projectId}+ )+ ${"sql" in query ? query : Prisma.sql(query, ...values)}+ `);+ }),+ );+ deletionQuery.mockImplementation((query, ...values) =>+ timezonePrisma.$transaction(async (tx) => {+ await tx.$queryRaw`SELECT set_config('TimeZone', ${timezone}, true)`;+ return tx.$executeRaw(query, ...values);+ }),+ );++ await new InAppAgentRetentionCleaner().processBatch();+ expect(await candidateQuery.mock.results[0]!.value).toEqual([+ { id: expiredId, projectId },+ ]);+ expect(+ await prisma.inAppAgentConversation.findMany({+ where: { projectId },+ select: { id: true },+ }),+ ).toEqual([{ id: retainedId }]);++ // Independently exercise the live deletion guard with a candidate+ // whose timestamp no longer qualifies for retention.+ candidateQuery.mockResolvedValue([{ id: retainedId, projectId }]);+ await new InAppAgentRetentionCleaner().processBatch();+ expect(+ await prisma.inAppAgentConversation.count({ where: { projectId } }),+ ).toBe(1);+ } finally {+ candidateQuery.mockRestore();+ deletionQuery.mockRestore();+ await timezonePrisma.$disconnect();+ await prisma.organization.delete({ where: { id: orgId } });+ }+ },+ );- expect(- await prisma.inAppAgentConversation.findUnique({- where: { id_projectId: { id, projectId } },- }),- ).not.toBeNull();- } finally {- await prisma.inAppAgentConversation.deleteMany({- where: { id, projectId },+ it.each([null, 0])(+ "retains assistant conversations when retention changes to %s after discovery",+ async (retentionDays) => {+ const id = randomUUID();+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays },});- }- });++ const candidateQuery = vi.spyOn(prisma, "$queryRaw");+ try {+ await prisma.inAppAgentConversation.create({+ data: {+ id,+ projectId,+ createdAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),+ updatedAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),+ },+ });++ // Simulate a policy change after discovery without sweeping other fixtures.+ candidateQuery.mockResolvedValue([{ id, projectId }]);+ await new InAppAgentRetentionCleaner().processBatch();++ expect(+ await prisma.inAppAgentConversation.findUnique({+ where: { id_projectId: { id, projectId } },+ }),+ ).not.toBeNull();+ } finally {+ candidateQuery.mockRestore();+ await prisma.inAppAgentConversation.deleteMany({+ where: { id, projectId },+ });+ }+ },+ );it("should NOT delete event files from cloud storage if after expiry cutoff", async () => {// Setup: Set retention in database to match job payloaddiff --git a/worker/src/app.ts b/worker/src/app.tsindex d6c2a52ce4..d4b4d98fb2 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -101,6 +101,7 @@ import {BATCH_DATA_RETENTION_TABLES,} from "./features/batch-data-retention-cleaner";import { MediaRetentionCleaner } from "./features/media-retention-cleaner";+import { InAppAgentRetentionCleaner } from "./features/in-app-agent-retention-cleaner";import { BatchTraceDeletionCleaner } from "./features/batch-trace-deletion-cleaner";import { BatchProjectMediaCleaner } from "./features/batch-project-media-cleaner";import { BatchProjectBlobCleaner } from "./features/batch-project-blob-cleaner";@@ -804,6 +805,14 @@ if (env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_ENABLED === "true") {}}+// Assistant retention runs independently of ingestion retention mode and+// assistant enablement. Conversations with unfinished runs still need reconciliation.+export let inAppAgentRetentionCleaner: InAppAgentRetentionCleaner | null = null;+if (env.LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED === "true") {+ inAppAgentRetentionCleaner = new InAppAgentRetentionCleaner();+ inAppAgentRetentionCleaner.start();+}+// Media retention cleaner for media files and blob storageexport let mediaRetentionCleaner: MediaRetentionCleaner | null = null;@@ -854,11 +863,9 @@ if (env.LANGFUSE_TRACE_DELETE_BATCH_ACTION_RUNNER_ENABLED === "true") {// Reconciles stale in-app agent runs that nobody reopened, then reports remainders.export let inAppAgentIntegrityRunner: InAppAgentIntegrityRunner | null = null;-if (- isInAppAgentWorkerSurfaceEnabled(- env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED,- )-) {+// Reconciliation must continue after assistant opt-out so unfinished runs+// cannot prevent retention forever. This runner never executes agent work.+if (env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED !== "false") {inAppAgentIntegrityRunner = new InAppAgentIntegrityRunner();inAppAgentIntegrityRunner.start();}diff --git a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.tsindex 8a97b190eb..b8c9885ae7 100644--- a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts+++ b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts@@ -9,13 +9,9 @@ import {logger,removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,getCurrentSpan,- redis,} from "@langfuse/shared/src/server";import { Job } from "bullmq";import { prisma } from "@langfuse/shared/src/db";-import { Prisma } from "@prisma/client";-import { isMissingInAppAgentMcpApiKeyError } from "@langfuse/shared/in-app-agent/server/runLifecycle";-import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { env, v4WritesToEventsTable } from "../../env";@@ -72,62 +68,6 @@ export const handleDataRetentionProcessingJob = async (job: Job) => {Date.now() - currentRetention * 24 * 60 * 60 * 1000,);- let processedRuns = 0;- while (processedRuns < 10_000) {- const keyRuns = await prisma.inAppAgentRun.findMany({- where: {- projectId,- conversation: { updatedAt: { lt: cutoffDate } },- finishedAt: { not: null },- mcpApiKeyId: { not: null },- },- select: { id: true, mcpApiKeyId: true },- take: Math.min(100, 10_000 - processedRuns),- });- if (keyRuns.length === 0) break;- for (const run of keyRuns) {- // Prisma does not narrow the nullable field type from the `not: null` query filter.- if (!run.mcpApiKeyId) continue;- await deleteInAppAgentMcpApiKeyFromDb({- prisma,- id: run.mcpApiKeyId,- projectId,- redis,- }).catch((error: unknown) => {- if (!isMissingInAppAgentMcpApiKeyError(error)) throw error;- });- }- await prisma.$executeRaw`- UPDATE in_app_agent_runs AS runs- SET mcp_api_key_id = NULL, updated_at = NOW()- FROM (VALUES ${Prisma.join(- keyRuns- .filter((run): run is { id: string; mcpApiKeyId: string } =>- Boolean(run.mcpApiKeyId),- )- .map((run) => Prisma.sql`(${run.id}, ${run.mcpApiKeyId})`),- )}) AS selected(id, key_id)- WHERE runs.id = selected.id- AND runs.project_id = ${projectId}- AND runs.mcp_api_key_id = selected.key_id- `;- processedRuns += keyRuns.length;- }- // Conversations with unfinished runs are skipped until those runs are reconciled.- const deleted = await prisma.inAppAgentConversation.deleteMany({- where: {- projectId,- updatedAt: { lt: cutoffDate },- AND: [- { runs: { none: { finishedAt: null } } },- { runs: { none: { mcpApiKeyId: { not: null } } } },- ],- },- });- logger.info(- `[Data Retention] Deleted ${deleted.count} expired assistant conversations for project ${projectId}`,- );-// Delete media files if bucket is configuredif (env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET) {logger.info(diff --git a/worker/src/env.ts b/worker/src/env.tsindex 17b74b1534..c329eb1d5a 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -388,14 +388,16 @@ const EnvSchema = z.object({QUEUE_CONSUMER_MONITOR_QUEUE_IS_ENABLED: z.enum(["true", "false"]).default("true"),- // Optional opt-outs for split-role workers. Unset follows- // LANGFUSE_IN_APP_AGENT_ENABLED; "false" skips that surface.+ // Run execution follows LANGFUSE_IN_APP_AGENT_ENABLED unless opted out.QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED: z.enum(["true", "false"]).optional(),LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED: z.enum(["true", "false"]).optional(),+ LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED: z+ .enum(["true", "false"])+ .default("true"),LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER: z.enum(["dangerous-docker", "lambda-microvm"]).optional(),diff --git a/worker/src/features/in-app-agent-retention-cleaner/index.test.ts b/worker/src/features/in-app-agent-retention-cleaner/index.test.tsnew file mode 100644index 0000000000..f26f5d680c--- /dev/null+++ b/worker/src/features/in-app-agent-retention-cleaner/index.test.ts@@ -0,0 +1,213 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import { InAppAgentRetentionCleaner } from "./index";+import { Prisma } from "@prisma/client";++const mocks = vi.hoisted(() => ({+ query: vi.fn(),+ execute: vi.fn(),+ findRuns: vi.fn(),+ updateRuns: vi.fn(),+ deleteKey: vi.fn(),+ licensed: vi.fn(),+ extend: vi.fn(),+ gauge: vi.fn(),+}));++vi.mock("@langfuse/shared/src/db", async () => {+ const { Prisma } = await import("@prisma/client");+ return {+ Prisma,+ prisma: {+ $queryRaw: mocks.query,+ $executeRaw: mocks.execute,+ inAppAgentRun: {+ findMany: mocks.findRuns,+ updateMany: mocks.updateRuns,+ },+ },+ };+});+vi.mock("@langfuse/shared/src/server", () => ({+ logger: { info: vi.fn(), error: vi.fn(), debug: vi.fn() },+ redis: null,+ recordIncrement: vi.fn(),+ recordGauge: mocks.gauge,+ traceException: vi.fn(),+ getCurrentSpan: vi.fn(),+}));+vi.mock("@langfuse/shared/src/server/ee/licenseCheck", () => ({+ isEnterpriseLicenseAvailable: mocks.licensed,+}));+vi.mock("@langfuse/shared/src/server/auth/apiKeys", () => ({+ deleteInAppAgentMcpApiKeyFromDb: mocks.deleteKey,+}));+vi.mock("../../utils/RedisLock", () => ({+ RedisLock: class {+ key = "test-lock";+ extend = mocks.extend;+ async withLock<T>(operation: () => Promise<T>) {+ return operation();+ }+ },+}));++describe("InAppAgentRetentionCleaner", () => {+ beforeEach(() => {+ vi.resetAllMocks();+ mocks.licensed.mockReturnValue(true);+ mocks.extend.mockResolvedValue(true);+ mocks.query.mockResolvedValue([+ { id: "conversation", projectId: "project" },+ ]);+ mocks.findRuns.mockResolvedValue([+ {+ id: "run",+ projectId: "project",+ conversationId: "conversation",+ mcpApiKeyId: "key",+ },+ ]);+ mocks.execute.mockResolvedValue(1);+ mocks.updateRuns.mockResolvedValue({ count: 1 });+ mocks.deleteKey.mockResolvedValue(undefined);+ });++ it("does not inspect or delete data without a license", async () => {+ mocks.licensed.mockReturnValue(false);+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(600_000);+ expect(mocks.query).not.toHaveBeenCalled();+ expect(mocks.deleteKey).not.toHaveBeenCalled();+ expect(mocks.execute).not.toHaveBeenCalled();+ });++ it("revokes keys before clearing pointers and deleting conversations", async () => {+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(0);+ expect(mocks.deleteKey).toHaveBeenCalledWith({+ prisma: expect.objectContaining({ $queryRaw: mocks.query }),+ id: "key",+ projectId: "project",+ redis: null,+ });+ expect(mocks.updateRuns).toHaveBeenCalledWith({+ where: { id: "run", projectId: "project", mcpApiKeyId: "key" },+ data: { mcpApiKeyId: null },+ });+ expect(mocks.deleteKey.mock.invocationCallOrder[0]).toBeLessThan(+ mocks.updateRuns.mock.invocationCallOrder[0]!,+ );+ expect(mocks.updateRuns.mock.invocationCallOrder[0]).toBeLessThan(+ mocks.execute.mock.invocationCallOrder[0]!,+ );+ });++ it("keeps key pointers and conversations retryable when revocation fails", async () => {+ mocks.deleteKey.mockRejectedValue(new Error("Redis unavailable"));+ mocks.execute.mockResolvedValue(0);+ mocks.query+ .mockResolvedValueOnce([{ id: "conversation", projectId: "project" }])+ .mockResolvedValueOnce([{ overdueSeconds: 864_000 }]);+ const cleaner = new InAppAgentRetentionCleaner();+ expect(await cleaner.processBatch()).toBe(600_000);+ expect(mocks.gauge).toHaveBeenCalledWith(+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",+ 864_000,+ { unit: "seconds" },+ );+ expect(mocks.updateRuns).not.toHaveBeenCalled();+ expect(mocks.execute).toHaveBeenCalledOnce();++ mocks.query.mockResolvedValueOnce([+ { id: "conversation", projectId: "project" },+ ]);+ mocks.deleteKey.mockResolvedValue(undefined);+ mocks.execute.mockResolvedValue(1);+ expect(await cleaner.processBatch()).toBe(0);+ expect(mocks.updateRuns).toHaveBeenCalledOnce();+ });++ it("continues revocation and deletion for another project after a key fails", async () => {+ mocks.query.mockResolvedValue([+ { id: "failed-conversation", projectId: "failed-project" },+ { id: "healthy-conversation", projectId: "healthy-project" },+ ]);+ mocks.findRuns.mockResolvedValue([+ {+ id: "failed-run",+ conversationId: "failed-conversation",+ projectId: "failed-project",+ mcpApiKeyId: "failed-key",+ },+ {+ id: "healthy-run",+ conversationId: "healthy-conversation",+ projectId: "healthy-project",+ mcpApiKeyId: "healthy-key",+ },+ ]);+ mocks.deleteKey.mockRejectedValueOnce(new Error("Revocation failed"));++ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(0);++ expect(mocks.deleteKey).toHaveBeenCalledTimes(2);+ expect(mocks.updateRuns).toHaveBeenCalledExactlyOnceWith({+ where: {+ id: "healthy-run",+ projectId: "healthy-project",+ mcpApiKeyId: "healthy-key",+ },+ data: { mcpApiKeyId: null },+ });+ expect(mocks.execute).toHaveBeenCalledOnce();+ });++ it.each([0, 864_000])(+ "reports maximum retention overdue seconds %s at the end of a sweep",+ async (overdueSeconds) => {+ mocks.query+ .mockResolvedValueOnce([])+ .mockResolvedValueOnce([{ overdueSeconds }]);++ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(+ 600_000,+ );+ expect(mocks.gauge).toHaveBeenCalledExactlyOnceWith(+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",+ overdueSeconds,+ { unit: "seconds" },+ );+ expect(mocks.execute).not.toHaveBeenCalled();+ },+ );++ it("does not collect retention overdue seconds for each active batch", async () => {+ await new InAppAgentRetentionCleaner().processBatch();+ expect(mocks.query).toHaveBeenCalledOnce();+ expect(mocks.gauge).not.toHaveBeenCalled();+ });++ it("continues cleanup when the key was already removed", async () => {+ mocks.deleteKey.mockRejectedValue(+ new Prisma.PrismaClientKnownRequestError("Missing key", {+ code: "P2025",+ clientVersion: "test",+ }),+ );+ await new InAppAgentRetentionCleaner().processBatch();+ expect(mocks.updateRuns).toHaveBeenCalledOnce();+ expect(mocks.execute).toHaveBeenCalledOnce();+ });++ it("stops destructive work when the lease is lost", async () => {+ mocks.extend.mockResolvedValue(false);+ await new InAppAgentRetentionCleaner().processBatch();+ expect(mocks.deleteKey).not.toHaveBeenCalled();+ expect(mocks.execute).not.toHaveBeenCalled();+ });++ it("rechecks the lease before deletion even after key cleanup made progress", async () => {+ mocks.extend.mockResolvedValueOnce(true).mockResolvedValueOnce(false);+ await new InAppAgentRetentionCleaner().processBatch();+ expect(mocks.updateRuns).toHaveBeenCalledOnce();+ expect(mocks.execute).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/features/in-app-agent-retention-cleaner/index.ts b/worker/src/features/in-app-agent-retention-cleaner/index.tsnew file mode 100644index 0000000000..1608ad6613--- /dev/null+++ b/worker/src/features/in-app-agent-retention-cleaner/index.ts@@ -0,0 +1,165 @@+import { prisma, Prisma } from "@langfuse/shared/src/db";+import {+ logger,+ recordGauge,+ recordIncrement,+ redis,+} from "@langfuse/shared/src/server";+import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";+import { cleanupRunMcpApiKey } from "@langfuse/shared/in-app-agent/server/runLifecycle";+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";++const BATCH_LIMIT = 100;+const INTERVAL_MS = 600_000;++export class InAppAgentRetentionCleaner extends PeriodicExclusiveRunner {+ protected get defaultIntervalMs(): number {+ return INTERVAL_MS;+ }++ constructor() {+ super({+ name: "InAppAgentRetentionCleaner",+ metricName: "in_app_agent_retention_cleaner",+ lockKey: "langfuse:in-app-agent-retention-cleaner",+ lockTtlSeconds: 300,+ onUnavailable: "fail",+ });+ }++ protected async execute(): Promise<number> {+ if (!isEnterpriseLicenseAvailable()) {+ return this.defaultIntervalMs;+ }++ const nextDelay = await this.withLock(async () => {+ // Unfinished runs belong to the integrity runner. Excluding them here+ // prevents an unrepairable conversation from blocking actionable work.+ const conversations = await prisma.$queryRaw<+ Array<{ id: string; projectId: string }>+ >`+ SELECT c.id, c.project_id AS "projectId"+ FROM in_app_agent_conversations c+ JOIN projects p ON p.id = c.project_id+ WHERE p.retention_days > 0+ AND p.deleted_at IS NULL+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'+ AND NOT EXISTS (+ SELECT 1 FROM in_app_agent_runs r+ WHERE r.project_id = c.project_id AND r.conversation_id = c.id+ AND r.finished_at IS NULL+ )+ ORDER BY c.id, c.project_id+ LIMIT ${BATCH_LIMIT}+ `;++ if (conversations.length > 0) {+ const keyRuns = await prisma.inAppAgentRun.findMany({+ where: {+ OR: conversations.map(({ id, projectId }) => ({+ projectId,+ conversationId: id,+ })),+ finishedAt: { not: null },+ mcpApiKeyId: { not: null },+ },+ select: {+ id: true,+ projectId: true,+ mcpApiKeyId: true,+ },+ take: BATCH_LIMIT,+ });++ let cleanedRuns = 0;+ for (const run of keyRuns) {+ if (!run.mcpApiKeyId) continue;+ await this.extendLockOnProgress();+ try {+ await cleanupRunMcpApiKey({+ prisma,+ projectId: run.projectId,+ runId: run.id,+ apiKeyId: run.mcpApiKeyId,+ deleteApiKey: async (id) => {+ await deleteInAppAgentMcpApiKeyFromDb({+ prisma,+ id,+ projectId: run.projectId,+ redis,+ });+ },+ });+ cleanedRuns++;+ } catch (error) {+ this.markRunFailed(error);+ logger.error(+ `${this.instanceName}: Failed to clean run credentials`,+ {+ error,+ id: run.id,+ projectId: run.projectId,+ },+ );+ }+ }++ await this.extendLockOnProgress(true);+ // Re-evaluate the live policy and run guards at deletion time, after+ // potentially slow credential cleanup. Runs and events cascade.+ const deleted = await prisma.$executeRaw`+ DELETE FROM in_app_agent_conversations c+ USING projects p, (VALUES ${Prisma.join(+ conversations.map(+ ({ id, projectId }) => Prisma.sql`(${id}, ${projectId})`,+ ),+ )}) AS selected(id, project_id)+ WHERE c.id = selected.id AND c.project_id = selected.project_id+ AND p.id = c.project_id+ AND p.retention_days > 0 AND p.deleted_at IS NULL+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'+ AND NOT EXISTS (+ SELECT 1 FROM in_app_agent_runs r+ WHERE r.project_id = c.project_id AND r.conversation_id = c.id+ AND (r.finished_at IS NULL OR r.mcp_api_key_id IS NOT NULL)+ )+ `;++ recordIncrement(+ "langfuse.in_app_agent_retention_cleaner.conversations_deleted",+ deleted,+ );+ logger.info(`${this.instanceName}: Deleted expired conversations`, {+ deleted,+ cleanedRuns,+ });+ if (cleanedRuns > 0 || deleted > 0) {+ return 0;+ }+ }++ // Include blocked conversations, and report even when failures prevent progress.+ const [backlog] = await prisma.$queryRaw<+ Array<{ overdueSeconds: number }>+ >`+ SELECT GREATEST(COALESCE(MAX(EXTRACT(EPOCH FROM (+ (NOW() AT TIME ZONE 'UTC') - c.updated_at - p.retention_days * INTERVAL '1 day'+ ))), 0), 0)::double precision AS "overdueSeconds"+ FROM in_app_agent_conversations c+ JOIN projects p ON p.id = c.project_id+ WHERE p.retention_days > 0+ AND p.deleted_at IS NULL+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'+ `;+ recordGauge(+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",+ backlog!.overdueSeconds,+ { unit: "seconds" },+ );+ return this.defaultIntervalMs;+ });++ return nextDelay ?? this.defaultIntervalMs;+ }+}diff --git a/worker/src/utils/PeriodicRunner.test.ts b/worker/src/utils/PeriodicRunner.test.tsnew file mode 100644index 0000000000..9894b5611c--- /dev/null+++ b/worker/src/utils/PeriodicRunner.test.ts@@ -0,0 +1,76 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { PeriodicRunner } from "./PeriodicRunner";++vi.mock("@langfuse/shared/src/server", () => ({+ logger: { error: vi.fn(), warn: vi.fn() },+ contextWithLangfuseProps: () => ({}),+ instrumentAsync: async (+ _options: unknown,+ operation: (span: { setAttribute: () => void }) => Promise<void>,+ ) => operation({ setAttribute: () => {} }),+ recordDistribution: vi.fn(),+ recordGauge: vi.fn(),+ recordIncrement: vi.fn(),+ traceException: vi.fn(),+}));+vi.mock("@opentelemetry/api", () => ({+ context: {+ with: (_context: unknown, operation: () => unknown) => operation(),+ },+ SpanKind: { INTERNAL: 0 },+}));++class TestRunner extends PeriodicRunner {+ protected name = "test";+ protected defaultIntervalMs = 10;+ constructor(private readonly operation: () => Promise<void>) {+ super("test");+ }+ protected execute() {+ return this.operation();+ }+}++afterEach(() => vi.useRealTimers());++describe("PeriodicRunner.drain", () => {+ it("waits for the active batch and does not schedule another", async () => {+ vi.useFakeTimers();+ let finish: () => void = () => {};+ const operation = vi.fn(+ () =>+ new Promise<void>((resolve) => {+ finish = resolve;+ }),+ );+ const runner = new TestRunner(operation);+ runner.start();+ let drained = false;+ const drain = runner.drain(5_000).then(() => {+ drained = true;+ });+ await vi.advanceTimersByTimeAsync(100);+ expect(drained).toBe(false);+ finish();+ await drain;+ await vi.advanceTimersByTimeAsync(100);+ expect(operation).toHaveBeenCalledOnce();+ expect(vi.getTimerCount()).toBe(0);+ });++ it("bounds shutdown when the active batch is stuck", async () => {+ vi.useFakeTimers();+ const runner = new TestRunner(() => new Promise<void>(() => {}));+ runner.start();+ const drain = runner.drain(50);+ await vi.advanceTimersByTimeAsync(50);+ await drain;+ expect(vi.getTimerCount()).toBe(0);+ });++ it("returns immediately when no batch started", async () => {+ vi.useFakeTimers();+ await new TestRunner(async () => {}).drain(5_000);+ expect(vi.getTimerCount()).toBe(0);+ });+});diff --git a/worker/src/utils/PeriodicRunner.ts b/worker/src/utils/PeriodicRunner.tsindex 0f716091cd..91d3754913 100644--- a/worker/src/utils/PeriodicRunner.ts+++ b/worker/src/utils/PeriodicRunner.ts@@ -22,6 +22,7 @@ export abstract class PeriodicRunner {private timeoutId: NodeJS.Timeout | null = null;private isRunning = false;private activeRun: { outcome: RunOutcome } | null = null;+ private activeRunPromise: Promise<void> | null = null;protected abstract get name(): string;protected abstract get defaultIntervalMs(): number;@@ -44,7 +45,7 @@ export abstract class PeriodicRunner {if (this.initialDelayMs > 0) {this.scheduleNext(this.initialDelayMs);} else {- this.runAndScheduleNext();+ this.activeRunPromise = this.runAndScheduleNext();}}}@@ -57,6 +58,28 @@ export abstract class PeriodicRunner {}}+ public async drain(timeoutMs: number): Promise<void> {+ this.stop();+ if (!this.activeRunPromise) return;++ let timeout: NodeJS.Timeout | undefined;+ try {+ await Promise.race([+ this.activeRunPromise,+ new Promise<void>((resolve) => {+ timeout = setTimeout(() => {+ logger.warn(`${this.name}: shutdown drain timed out`, {+ timeoutMs,+ });+ resolve();+ }, timeoutMs);+ }),+ ]);+ } finally {+ clearTimeout(timeout);+ }+ }+protected markRunFailed(error: unknown): void {if (this.activeRun) {this.activeRun.outcome = "failed";@@ -157,7 +180,7 @@ export abstract class PeriodicRunner {return;}this.timeoutId = setTimeout(() => {- this.runAndScheduleNext();+ this.activeRunPromise = this.runAndScheduleNext();}, delayMs);}}diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 150510c4bf..e9b3ab49b4 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -15,6 +15,7 @@ import {batchProjectCleaners,batchDataRetentionCleaners,mediaRetentionCleaner,+ inAppAgentRetentionCleaner,batchProjectMediaCleaner,batchProjectBlobCleaner,batchTraceDeletionCleaner,@@ -68,6 +69,7 @@ const runDrainAndClose = async () => {// Stop media retention cleanermediaRetentionCleaner?.stop();+ inAppAgentRetentionCleaner?.stop();// Stop batch project media cleanerbatchProjectMediaCleaner?.stop();@@ -117,6 +119,11 @@ const runDrainAndClose = async () => {await ClickhouseWriter.shutdownAll();logger.info("Clickhouse writer has been shut down.");+ await Promise.all([+ inAppAgentRetentionCleaner?.drain(5_000),+ inAppAgentIntegrityRunner?.drain(5_000),+ ]);+// Closes the shared client and every per-queue client in one pass. Each// queue holds its own client; without this they stay connected, retry// forever, and keep the event loop alive so the process never exits.