Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(assistant): Migrate retention handling to cleaner class (#18719)

Ben Bachem3d4b9c6314da
diff --git a/.env.dev.example b/.env.dev.example
index bba3a84935..3f8b0c68de 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -379,6 +379,8 @@ LANGFUSE_AI_SMALL_MODEL="eu.anthropic.claude-haiku-5-5"
# workers can opt out without turning the instance off:
# QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false
# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false
+# Retention runs even when the assistant or ingestion retention queue is disabled.
+# LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false
# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5
# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS=900000
diff --git a/.env.prod.example b/.env.prod.example
index 51f93207db..945e2a82fb 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -389,6 +389,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# instance off:
# QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false
# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false
+# Retention runs even when the assistant or ingestion retention queue is disabled.
+# LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false
# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5
# Admission ceilings for concurrent Assistant runs, web only
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
index 7054b07035..7cbaa9483e 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
@@ -8,6 +8,7 @@ import {
import { Prisma, type PrismaClient } from "../../db";
import { logger } from "../../server";
import {
+ cleanupRunMcpApiKey,
cleanupTerminalRunMcpApiKeys,
createQueuedRun,
reconcileConversationRuns,
@@ -375,9 +376,55 @@ describe("cleanupTerminalRunMcpApiKeys", () => {
}
expect(updateMany).toHaveBeenCalledWith({
- where: { id: "run-1", projectId: "project-1" },
+ where: { id: "run-1", projectId: "project-1", mcpApiKeyId: "key-1" },
data: { mcpApiKeyId: null },
});
expect(errorSpy).not.toHaveBeenCalled();
});
+
+ it("leaves the pointer intact when revocation fails", async () => {
+ const updateMany = vi.fn();
+ const prisma = {
+ inAppAgentRun: { updateMany },
+ } as unknown as PrismaClient;
+ const error = new Error("Revocation failed");
+
+ await expect(
+ cleanupRunMcpApiKey({
+ prisma,
+ projectId: "project-1",
+ runId: "run-1",
+ apiKeyId: "key-1",
+ deleteApiKey: async () => {
+ throw error;
+ },
+ }),
+ ).rejects.toBe(error);
+ expect(updateMany).not.toHaveBeenCalled();
+ });
+
+ it("does not clear a replacement key installed during revocation", async () => {
+ let keyId: string | null = "key-1";
+ const updateMany = vi.fn(
+ async ({ where }: { where: { mcpApiKeyId: string } }) => {
+ if (keyId !== where.mcpApiKeyId) return { count: 0 };
+ keyId = null;
+ return { count: 1 };
+ },
+ );
+ const prisma = {
+ inAppAgentRun: { updateMany },
+ } as unknown as PrismaClient;
+
+ await cleanupRunMcpApiKey({
+ prisma,
+ projectId: "project-1",
+ runId: "run-1",
+ apiKeyId: "key-1",
+ deleteApiKey: async () => {
+ keyId = "key-2";
+ },
+ });
+ expect(keyId).toBe("key-2");
+ });
});
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.ts
index a5d7ff90af..dc39ecb220 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts
@@ -764,6 +764,29 @@ export function isMissingInAppAgentMcpApiKeyError(error: unknown): boolean {
);
}
+/** Keep the pointer retryable until revocation succeeds, including concurrent cleanup. */
+export async function cleanupRunMcpApiKey(params: {
+ prisma: PrismaClient;
+ projectId: string;
+ runId: string;
+ apiKeyId: string;
+ deleteApiKey: (apiKeyId: string) => Promise<void>;
+}): Promise<void> {
+ try {
+ await params.deleteApiKey(params.apiKeyId);
+ } catch (error) {
+ if (!isMissingInAppAgentMcpApiKeyError(error)) throw error;
+ }
+ await params.prisma.inAppAgentRun.updateMany({
+ where: {
+ id: params.runId,
+ projectId: params.projectId,
+ mcpApiKeyId: params.apiKeyId,
+ },
+ data: { mcpApiKeyId: null },
+ });
+}
+
/** Retry terminal-run MCP-key cleanup outside the lifecycle transaction. */
export async function cleanupTerminalRunMcpApiKeys(params: {
prisma: PrismaClient;
@@ -787,18 +810,12 @@ export async function cleanupTerminalRunMcpApiKeys(params: {
if (!run.mcpApiKeyId) continue;
try {
- try {
- await params.deleteApiKey(run.mcpApiKeyId);
- } catch (error) {
- // Concurrent cleanup or a prior delete already removed the row.
- if (!isMissingInAppAgentMcpApiKeyError(error)) {
- throw error;
- }
- }
- await clearRunMcpApiKeyPointer({
+ await cleanupRunMcpApiKey({
prisma: params.prisma,
projectId: params.projectId,
runId: run.id,
+ apiKeyId: run.mcpApiKeyId,
+ deleteApiKey: params.deleteApiKey,
});
cleaned += 1;
} catch (error) {
diff --git a/web/src/features/in-app-agent/README.md b/web/src/features/in-app-agent/README.md
index 8f6bb0909d..e15545e239 100644
--- a/web/src/features/in-app-agent/README.md
+++ b/web/src/features/in-app-agent/README.md
@@ -154,9 +154,10 @@ separate click event would answer.
Rollback is a code revert; background execution does not require a schema
rollback.
-Workers with `LANGFUSE_IN_APP_AGENT_ENABLED` set consume the run queue and
-reconcile stale runs; without it, runs commit as `QUEUED` and die at
-`queue_timeout`. Cloud is on unless the flag is `"false"`. Split-role workers
+Workers with `LANGFUSE_IN_APP_AGENT_ENABLED` set consume the run queue.
+Stale-run reconciliation runs independently of assistant enablement so abandoned
+runs cannot block retention after opt-out; it never executes agent work.
+Cloud is on unless the flag is `"false"`. Split-role workers
can keep the instance on and set
`QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED=false` (and
`LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED=false`) to skip those
@@ -167,6 +168,14 @@ its default of 20 equals full US capacity and exceeds JP and staging.
Conversation switching, detached invalidations, retry, and conversation-list run
statuses remain separate project concerns.
+Assistant conversation retention runs independently of ingestion retention mode
+and assistant enablement, under an enterprise license. It is enabled by default;
+`QUEUE_CONSUMER_DATA_RETENTION_QUEUE_IS_ENABLED=false` no longer pauses assistant
+retention. Set `LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED=false` on every
+worker to pause it. Conversations with unfinished runs remain excluded until
+reconciliation settles them. Keep at least one integrity runner enabled for
+stale-run reconciliation, including after assistant opt-out.
+
Environment ownership:
- Worker: queue concurrency, sandbox provider and Lambda MicroVM values.
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 8969b7f36e..b8d1243660 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -163,6 +163,11 @@
## In-App Agent Runtime
+- `src/features/in-app-agent-retention-cleaner/` expires conversations under
+ current licensed project retention policies. Its exclusive periodic runner
+ starts independently of batch-retention and assistant enablement, processes
+ bounded batches, revokes MCP keys before cascading deletion, and leaves
+ unfinished runs to the integrity runner.
- `src/features/in-app-agent/runtime/` owns Mastra adaptation, agent execution,
instrumentation, prompt loading, continuation handling, tools, skills, and
sandbox providers.
diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.ts
index 67c6e8e626..effaf47990 100644
--- a/worker/src/__tests__/dataRetentionProcessing.test.ts
+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts
@@ -25,7 +25,7 @@ import {
StorageServiceFactory,
toClickhouseDateTime,
} from "@langfuse/shared/src/server";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma, Prisma, PrismaClient } from "@langfuse/shared/src/db";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";
import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";
@@ -33,6 +33,7 @@ import { Job } from "bullmq";
import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";
import { ProjectId, SystemRoleId, UserId } from "@langfuse/shared/rbac";
import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";
+import { InAppAgentRetentionCleaner } from "../features/in-app-agent-retention-cleaner";
type LicenseCheckModule =
typeof import("@langfuse/shared/src/server/ee/licenseCheck");
@@ -114,11 +115,16 @@ describe("DataRetentionProcessingJob", () => {
recentLegacyId,
];
- await prisma.project.update({
- where: { id: projectId },
- data: { retentionDays: 7 },
+ const { id: projectId, orgId } = await prisma.project.create({
+ data: {
+ name: "Assistant retention test",
+ retentionDays: 7,
+ organization: { create: { name: "Assistant retention test" } },
+ },
});
+ const queryRaw = prisma.$queryRaw.bind(prisma);
+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");
try {
const creator = await prisma.user.create({
data: {
@@ -241,9 +247,49 @@ describe("DataRetentionProcessingJob", () => {
],
});
- await handleDataRetentionProcessingJob({
- data: { payload: { projectId, retention: 7 } },
- } as Job);
+ // API-key fixture setup also queries projects while assigning its role.
+ candidateQuery.mockClear();
+ candidateQuery.mockImplementation((query, ...values) =>
+ queryRaw(Prisma.sql`
+ WITH in_app_agent_conversations AS (
+ SELECT * FROM public.in_app_agent_conversations
+ WHERE project_id = ${projectId}
+ )
+ ${"sql" in query ? query : Prisma.sql(query, ...values)}
+ `),
+ );
+ const cleaner = new InAppAgentRetentionCleaner();
+ // A conversation with 101 keys remains eligible until its last key is cleaned.
+ expect(await cleaner.processBatch()).toBe(0);
+ expect(
+ await prisma.inAppAgentRun.count({
+ where: {
+ projectId,
+ conversationId: expiredId,
+ mcpApiKeyId: { not: null },
+ },
+ }),
+ ).toBe(1);
+ expect(await cleaner.processBatch()).toBe(0);
+ expect(await cleaner.processBatch()).toBe(600_000);
+ const [backlog] = await candidateQuery.mock.results[3]!.value;
+ // Thirty-day-old conversations with seven-day retention are 23 days overdue.
+ expect(backlog.overdueSeconds).toBeGreaterThanOrEqual(23 * 24 * 60 * 60);
+ expect(backlog.overdueSeconds).toBeLessThan(
+ (Date.now() - expiredAt.getTime()) / 1000 - 7 * 24 * 60 * 60 + 1,
+ );
+ const discovered = (
+ await Promise.all(candidateQuery.mock.results.map(({ value }) => value))
+ )
+ .flat()
+ .filter((row) => "projectId" in row);
+ expect(discovered).toHaveLength(3);
+ expect(discovered).toEqual(
+ expect.arrayContaining([
+ { id: expiredId, projectId },
+ { id: eventlessId, projectId },
+ ]),
+ );
const conversations = await prisma.inAppAgentConversation.findMany({
where: { projectId, id: { in: ids } },
@@ -279,48 +325,178 @@ describe("DataRetentionProcessingJob", () => {
}),
).toBe(1);
} finally {
+ candidateQuery.mockRestore();
await prisma.inAppAgentConversation.deleteMany({
where: { projectId, id: { in: ids } },
});
await prisma.apiKey.deleteMany({ where: { id: { in: keyIds } } });
- await prisma.project.update({
- where: { id: projectId },
- data: { retentionDays: null },
- });
+ await prisma.organization.delete({ where: { id: orgId } });
}
});
- it("retains assistant conversations when retention is disabled after the job was queued", async () => {
- const id = randomUUID();
- await prisma.project.update({
- where: { id: projectId },
- data: { retentionDays: null },
- });
+ it.each([
+ { retentionDays: null, deletedAt: null },
+ { retentionDays: 0, deletedAt: null },
+ { retentionDays: 7, deletedAt: new Date() },
+ ])(
+ "does not discover assistant conversations with policy %j",
+ async ({ retentionDays, deletedAt }) => {
+ const id = randomUUID();
+ const { id: projectId, orgId } = await prisma.project.create({
+ data: {
+ name: "Assistant retention policy test",
+ retentionDays,
+ deletedAt,
+ organization: { create: { name: "Assistant retention policy test" } },
+ },
+ });
+ const queryRaw = prisma.$queryRaw.bind(prisma);
+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");
+ candidateQuery.mockImplementation((query, ...values) =>
+ queryRaw(Prisma.sql`
+ WITH in_app_agent_conversations AS (
+ SELECT * FROM public.in_app_agent_conversations
+ WHERE project_id = ${projectId}
+ )
+ ${"sql" in query ? query : Prisma.sql(query, ...values)}
+ `),
+ );
+ try {
+ const expiredAt = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
+ await prisma.inAppAgentConversation.create({
+ data: { id, projectId, createdAt: expiredAt, updatedAt: expiredAt },
+ });
+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(
+ 600_000,
+ );
+ expect(await candidateQuery.mock.results[0]!.value).toEqual([]);
+ expect(
+ await prisma.inAppAgentConversation.count({
+ where: { id, projectId },
+ }),
+ ).toBe(1);
+ } finally {
+ candidateQuery.mockRestore();
+ await prisma.inAppAgentConversation.deleteMany({
+ where: { id, projectId },
+ });
+ await prisma.organization.delete({ where: { id: orgId } });
+ }
+ },
+ );
- try {
- await prisma.inAppAgentConversation.create({
+ it.each(["Asia/Tokyo", "America/Los_Angeles"])(
+ "uses UTC retention boundaries in the %s session timezone",
+ async (timezone) => {
+ const { id: projectId, orgId } = await prisma.project.create({
data: {
- id,
- projectId,
- createdAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),
+ name: "Assistant retention timezone test",
+ retentionDays: 7,
+ organization: {
+ create: { name: "Assistant retention timezone test" },
+ },
},
});
+ const expiredId = randomUUID();
+ const retainedId = randomUUID();
+ const cutoff = Date.now() - 7 * 24 * 60 * 60 * 1000;
+ const hour = 60 * 60 * 1000;
+ const timezonePrisma = new PrismaClient();
+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");
+ const deletionQuery = vi.spyOn(prisma, "$executeRaw");
- await handleDataRetentionProcessingJob({
- data: { payload: { projectId, retention: 7 } },
- } as Job);
+ try {
+ await prisma.inAppAgentConversation.createMany({
+ data: [
+ { id: expiredId, projectId, updatedAt: new Date(cutoff - hour) },
+ { id: retainedId, projectId, updatedAt: new Date(cutoff + hour) },
+ ],
+ });
+ // SET LOCAL stays on the transaction's connection and cannot leak
+ // timezone changes to other tests using the pool.
+ candidateQuery.mockImplementation((query, ...values) =>
+ timezonePrisma.$transaction(async (tx) => {
+ await tx.$queryRaw`SELECT set_config('TimeZone', ${timezone}, true)`;
+ return tx.$queryRaw(Prisma.sql`
+ WITH in_app_agent_conversations AS (
+ SELECT * FROM public.in_app_agent_conversations
+ WHERE project_id = ${projectId}
+ )
+ ${"sql" in query ? query : Prisma.sql(query, ...values)}
+ `);
+ }),
+ );
+ deletionQuery.mockImplementation((query, ...values) =>
+ timezonePrisma.$transaction(async (tx) => {
+ await tx.$queryRaw`SELECT set_config('TimeZone', ${timezone}, true)`;
+ return tx.$executeRaw(query, ...values);
+ }),
+ );
+
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(await candidateQuery.mock.results[0]!.value).toEqual([
+ { id: expiredId, projectId },
+ ]);
+ expect(
+ await prisma.inAppAgentConversation.findMany({
+ where: { projectId },
+ select: { id: true },
+ }),
+ ).toEqual([{ id: retainedId }]);
+
+ // Independently exercise the live deletion guard with a candidate
+ // whose timestamp no longer qualifies for retention.
+ candidateQuery.mockResolvedValue([{ id: retainedId, projectId }]);
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(
+ await prisma.inAppAgentConversation.count({ where: { projectId } }),
+ ).toBe(1);
+ } finally {
+ candidateQuery.mockRestore();
+ deletionQuery.mockRestore();
+ await timezonePrisma.$disconnect();
+ await prisma.organization.delete({ where: { id: orgId } });
+ }
+ },
+ );
- expect(
- await prisma.inAppAgentConversation.findUnique({
- where: { id_projectId: { id, projectId } },
- }),
- ).not.toBeNull();
- } finally {
- await prisma.inAppAgentConversation.deleteMany({
- where: { id, projectId },
+ it.each([null, 0])(
+ "retains assistant conversations when retention changes to %s after discovery",
+ async (retentionDays) => {
+ const id = randomUUID();
+ await prisma.project.update({
+ where: { id: projectId },
+ data: { retentionDays },
});
- }
- });
+
+ const candidateQuery = vi.spyOn(prisma, "$queryRaw");
+ try {
+ await prisma.inAppAgentConversation.create({
+ data: {
+ id,
+ projectId,
+ createdAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),
+ updatedAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000),
+ },
+ });
+
+ // Simulate a policy change after discovery without sweeping other fixtures.
+ candidateQuery.mockResolvedValue([{ id, projectId }]);
+ await new InAppAgentRetentionCleaner().processBatch();
+
+ expect(
+ await prisma.inAppAgentConversation.findUnique({
+ where: { id_projectId: { id, projectId } },
+ }),
+ ).not.toBeNull();
+ } finally {
+ candidateQuery.mockRestore();
+ await prisma.inAppAgentConversation.deleteMany({
+ where: { id, projectId },
+ });
+ }
+ },
+ );
it("should NOT delete event files from cloud storage if after expiry cutoff", async () => {
// Setup: Set retention in database to match job payload
diff --git a/worker/src/app.ts b/worker/src/app.ts
index d6c2a52ce4..d4b4d98fb2 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -101,6 +101,7 @@ import {
BATCH_DATA_RETENTION_TABLES,
} from "./features/batch-data-retention-cleaner";
import { MediaRetentionCleaner } from "./features/media-retention-cleaner";
+import { InAppAgentRetentionCleaner } from "./features/in-app-agent-retention-cleaner";
import { BatchTraceDeletionCleaner } from "./features/batch-trace-deletion-cleaner";
import { BatchProjectMediaCleaner } from "./features/batch-project-media-cleaner";
import { BatchProjectBlobCleaner } from "./features/batch-project-blob-cleaner";
@@ -804,6 +805,14 @@ if (env.LANGFUSE_BATCH_DATA_RETENTION_CLEANER_ENABLED === "true") {
}
}
+// Assistant retention runs independently of ingestion retention mode and
+// assistant enablement. Conversations with unfinished runs still need reconciliation.
+export let inAppAgentRetentionCleaner: InAppAgentRetentionCleaner | null = null;
+if (env.LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED === "true") {
+ inAppAgentRetentionCleaner = new InAppAgentRetentionCleaner();
+ inAppAgentRetentionCleaner.start();
+}
+
// Media retention cleaner for media files and blob storage
export let mediaRetentionCleaner: MediaRetentionCleaner | null = null;
@@ -854,11 +863,9 @@ if (env.LANGFUSE_TRACE_DELETE_BATCH_ACTION_RUNNER_ENABLED === "true") {
// Reconciles stale in-app agent runs that nobody reopened, then reports remainders.
export let inAppAgentIntegrityRunner: InAppAgentIntegrityRunner | null = null;
-if (
- isInAppAgentWorkerSurfaceEnabled(
- env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED,
- )
-) {
+// Reconciliation must continue after assistant opt-out so unfinished runs
+// cannot prevent retention forever. This runner never executes agent work.
+if (env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED !== "false") {
inAppAgentIntegrityRunner = new InAppAgentIntegrityRunner();
inAppAgentIntegrityRunner.start();
}
diff --git a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts
index 8a97b190eb..b8c9885ae7 100644
--- a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts
+++ b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts
@@ -9,13 +9,9 @@ import {
logger,
removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,
getCurrentSpan,
- redis,
} from "@langfuse/shared/src/server";
import { Job } from "bullmq";
import { prisma } from "@langfuse/shared/src/db";
-import { Prisma } from "@prisma/client";
-import { isMissingInAppAgentMcpApiKeyError } from "@langfuse/shared/in-app-agent/server/runLifecycle";
-import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";
import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";
import { env, v4WritesToEventsTable } from "../../env";
@@ -72,62 +68,6 @@ export const handleDataRetentionProcessingJob = async (job: Job) => {
Date.now() - currentRetention * 24 * 60 * 60 * 1000,
);
- let processedRuns = 0;
- while (processedRuns < 10_000) {
- const keyRuns = await prisma.inAppAgentRun.findMany({
- where: {
- projectId,
- conversation: { updatedAt: { lt: cutoffDate } },
- finishedAt: { not: null },
- mcpApiKeyId: { not: null },
- },
- select: { id: true, mcpApiKeyId: true },
- take: Math.min(100, 10_000 - processedRuns),
- });
- if (keyRuns.length === 0) break;
- for (const run of keyRuns) {
- // Prisma does not narrow the nullable field type from the `not: null` query filter.
- if (!run.mcpApiKeyId) continue;
- await deleteInAppAgentMcpApiKeyFromDb({
- prisma,
- id: run.mcpApiKeyId,
- projectId,
- redis,
- }).catch((error: unknown) => {
- if (!isMissingInAppAgentMcpApiKeyError(error)) throw error;
- });
- }
- await prisma.$executeRaw`
- UPDATE in_app_agent_runs AS runs
- SET mcp_api_key_id = NULL, updated_at = NOW()
- FROM (VALUES ${Prisma.join(
- keyRuns
- .filter((run): run is { id: string; mcpApiKeyId: string } =>
- Boolean(run.mcpApiKeyId),
- )
- .map((run) => Prisma.sql`(${run.id}, ${run.mcpApiKeyId})`),
- )}) AS selected(id, key_id)
- WHERE runs.id = selected.id
- AND runs.project_id = ${projectId}
- AND runs.mcp_api_key_id = selected.key_id
- `;
- processedRuns += keyRuns.length;
- }
- // Conversations with unfinished runs are skipped until those runs are reconciled.
- const deleted = await prisma.inAppAgentConversation.deleteMany({
- where: {
- projectId,
- updatedAt: { lt: cutoffDate },
- AND: [
- { runs: { none: { finishedAt: null } } },
- { runs: { none: { mcpApiKeyId: { not: null } } } },
- ],
- },
- });
- logger.info(
- `[Data Retention] Deleted ${deleted.count} expired assistant conversations for project ${projectId}`,
- );
-
// Delete media files if bucket is configured
if (env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET) {
logger.info(
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 17b74b1534..c329eb1d5a 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -388,14 +388,16 @@ const EnvSchema = z.object({
QUEUE_CONSUMER_MONITOR_QUEUE_IS_ENABLED: z
.enum(["true", "false"])
.default("true"),
- // Optional opt-outs for split-role workers. Unset follows
- // LANGFUSE_IN_APP_AGENT_ENABLED; "false" skips that surface.
+ // Run execution follows LANGFUSE_IN_APP_AGENT_ENABLED unless opted out.
QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED: z
.enum(["true", "false"])
.optional(),
LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED: z
.enum(["true", "false"])
.optional(),
+ LANGFUSE_IN_APP_AGENT_RETENTION_CLEANER_ENABLED: z
+ .enum(["true", "false"])
+ .default("true"),
LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER: z
.enum(["dangerous-docker", "lambda-microvm"])
.optional(),
diff --git a/worker/src/features/in-app-agent-retention-cleaner/index.test.ts b/worker/src/features/in-app-agent-retention-cleaner/index.test.ts
new file mode 100644
index 0000000000..f26f5d680c
--- /dev/null
+++ b/worker/src/features/in-app-agent-retention-cleaner/index.test.ts
@@ -0,0 +1,213 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { InAppAgentRetentionCleaner } from "./index";
+import { Prisma } from "@prisma/client";
+
+const mocks = vi.hoisted(() => ({
+ query: vi.fn(),
+ execute: vi.fn(),
+ findRuns: vi.fn(),
+ updateRuns: vi.fn(),
+ deleteKey: vi.fn(),
+ licensed: vi.fn(),
+ extend: vi.fn(),
+ gauge: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/db", async () => {
+ const { Prisma } = await import("@prisma/client");
+ return {
+ Prisma,
+ prisma: {
+ $queryRaw: mocks.query,
+ $executeRaw: mocks.execute,
+ inAppAgentRun: {
+ findMany: mocks.findRuns,
+ updateMany: mocks.updateRuns,
+ },
+ },
+ };
+});
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { info: vi.fn(), error: vi.fn(), debug: vi.fn() },
+ redis: null,
+ recordIncrement: vi.fn(),
+ recordGauge: mocks.gauge,
+ traceException: vi.fn(),
+ getCurrentSpan: vi.fn(),
+}));
+vi.mock("@langfuse/shared/src/server/ee/licenseCheck", () => ({
+ isEnterpriseLicenseAvailable: mocks.licensed,
+}));
+vi.mock("@langfuse/shared/src/server/auth/apiKeys", () => ({
+ deleteInAppAgentMcpApiKeyFromDb: mocks.deleteKey,
+}));
+vi.mock("../../utils/RedisLock", () => ({
+ RedisLock: class {
+ key = "test-lock";
+ extend = mocks.extend;
+ async withLock<T>(operation: () => Promise<T>) {
+ return operation();
+ }
+ },
+}));
+
+describe("InAppAgentRetentionCleaner", () => {
+ beforeEach(() => {
+ vi.resetAllMocks();
+ mocks.licensed.mockReturnValue(true);
+ mocks.extend.mockResolvedValue(true);
+ mocks.query.mockResolvedValue([
+ { id: "conversation", projectId: "project" },
+ ]);
+ mocks.findRuns.mockResolvedValue([
+ {
+ id: "run",
+ projectId: "project",
+ conversationId: "conversation",
+ mcpApiKeyId: "key",
+ },
+ ]);
+ mocks.execute.mockResolvedValue(1);
+ mocks.updateRuns.mockResolvedValue({ count: 1 });
+ mocks.deleteKey.mockResolvedValue(undefined);
+ });
+
+ it("does not inspect or delete data without a license", async () => {
+ mocks.licensed.mockReturnValue(false);
+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(600_000);
+ expect(mocks.query).not.toHaveBeenCalled();
+ expect(mocks.deleteKey).not.toHaveBeenCalled();
+ expect(mocks.execute).not.toHaveBeenCalled();
+ });
+
+ it("revokes keys before clearing pointers and deleting conversations", async () => {
+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(0);
+ expect(mocks.deleteKey).toHaveBeenCalledWith({
+ prisma: expect.objectContaining({ $queryRaw: mocks.query }),
+ id: "key",
+ projectId: "project",
+ redis: null,
+ });
+ expect(mocks.updateRuns).toHaveBeenCalledWith({
+ where: { id: "run", projectId: "project", mcpApiKeyId: "key" },
+ data: { mcpApiKeyId: null },
+ });
+ expect(mocks.deleteKey.mock.invocationCallOrder[0]).toBeLessThan(
+ mocks.updateRuns.mock.invocationCallOrder[0]!,
+ );
+ expect(mocks.updateRuns.mock.invocationCallOrder[0]).toBeLessThan(
+ mocks.execute.mock.invocationCallOrder[0]!,
+ );
+ });
+
+ it("keeps key pointers and conversations retryable when revocation fails", async () => {
+ mocks.deleteKey.mockRejectedValue(new Error("Redis unavailable"));
+ mocks.execute.mockResolvedValue(0);
+ mocks.query
+ .mockResolvedValueOnce([{ id: "conversation", projectId: "project" }])
+ .mockResolvedValueOnce([{ overdueSeconds: 864_000 }]);
+ const cleaner = new InAppAgentRetentionCleaner();
+ expect(await cleaner.processBatch()).toBe(600_000);
+ expect(mocks.gauge).toHaveBeenCalledWith(
+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",
+ 864_000,
+ { unit: "seconds" },
+ );
+ expect(mocks.updateRuns).not.toHaveBeenCalled();
+ expect(mocks.execute).toHaveBeenCalledOnce();
+
+ mocks.query.mockResolvedValueOnce([
+ { id: "conversation", projectId: "project" },
+ ]);
+ mocks.deleteKey.mockResolvedValue(undefined);
+ mocks.execute.mockResolvedValue(1);
+ expect(await cleaner.processBatch()).toBe(0);
+ expect(mocks.updateRuns).toHaveBeenCalledOnce();
+ });
+
+ it("continues revocation and deletion for another project after a key fails", async () => {
+ mocks.query.mockResolvedValue([
+ { id: "failed-conversation", projectId: "failed-project" },
+ { id: "healthy-conversation", projectId: "healthy-project" },
+ ]);
+ mocks.findRuns.mockResolvedValue([
+ {
+ id: "failed-run",
+ conversationId: "failed-conversation",
+ projectId: "failed-project",
+ mcpApiKeyId: "failed-key",
+ },
+ {
+ id: "healthy-run",
+ conversationId: "healthy-conversation",
+ projectId: "healthy-project",
+ mcpApiKeyId: "healthy-key",
+ },
+ ]);
+ mocks.deleteKey.mockRejectedValueOnce(new Error("Revocation failed"));
+
+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(0);
+
+ expect(mocks.deleteKey).toHaveBeenCalledTimes(2);
+ expect(mocks.updateRuns).toHaveBeenCalledExactlyOnceWith({
+ where: {
+ id: "healthy-run",
+ projectId: "healthy-project",
+ mcpApiKeyId: "healthy-key",
+ },
+ data: { mcpApiKeyId: null },
+ });
+ expect(mocks.execute).toHaveBeenCalledOnce();
+ });
+
+ it.each([0, 864_000])(
+ "reports maximum retention overdue seconds %s at the end of a sweep",
+ async (overdueSeconds) => {
+ mocks.query
+ .mockResolvedValueOnce([])
+ .mockResolvedValueOnce([{ overdueSeconds }]);
+
+ expect(await new InAppAgentRetentionCleaner().processBatch()).toBe(
+ 600_000,
+ );
+ expect(mocks.gauge).toHaveBeenCalledExactlyOnceWith(
+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",
+ overdueSeconds,
+ { unit: "seconds" },
+ );
+ expect(mocks.execute).not.toHaveBeenCalled();
+ },
+ );
+
+ it("does not collect retention overdue seconds for each active batch", async () => {
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(mocks.query).toHaveBeenCalledOnce();
+ expect(mocks.gauge).not.toHaveBeenCalled();
+ });
+
+ it("continues cleanup when the key was already removed", async () => {
+ mocks.deleteKey.mockRejectedValue(
+ new Prisma.PrismaClientKnownRequestError("Missing key", {
+ code: "P2025",
+ clientVersion: "test",
+ }),
+ );
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(mocks.updateRuns).toHaveBeenCalledOnce();
+ expect(mocks.execute).toHaveBeenCalledOnce();
+ });
+
+ it("stops destructive work when the lease is lost", async () => {
+ mocks.extend.mockResolvedValue(false);
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(mocks.deleteKey).not.toHaveBeenCalled();
+ expect(mocks.execute).not.toHaveBeenCalled();
+ });
+
+ it("rechecks the lease before deletion even after key cleanup made progress", async () => {
+ mocks.extend.mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+ await new InAppAgentRetentionCleaner().processBatch();
+ expect(mocks.updateRuns).toHaveBeenCalledOnce();
+ expect(mocks.execute).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/in-app-agent-retention-cleaner/index.ts b/worker/src/features/in-app-agent-retention-cleaner/index.ts
new file mode 100644
index 0000000000..1608ad6613
--- /dev/null
+++ b/worker/src/features/in-app-agent-retention-cleaner/index.ts
@@ -0,0 +1,165 @@
+import { prisma, Prisma } from "@langfuse/shared/src/db";
+import {
+ logger,
+ recordGauge,
+ recordIncrement,
+ redis,
+} from "@langfuse/shared/src/server";
+import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";
+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";
+import { cleanupRunMcpApiKey } from "@langfuse/shared/in-app-agent/server/runLifecycle";
+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+
+const BATCH_LIMIT = 100;
+const INTERVAL_MS = 600_000;
+
+export class InAppAgentRetentionCleaner extends PeriodicExclusiveRunner {
+ protected get defaultIntervalMs(): number {
+ return INTERVAL_MS;
+ }
+
+ constructor() {
+ super({
+ name: "InAppAgentRetentionCleaner",
+ metricName: "in_app_agent_retention_cleaner",
+ lockKey: "langfuse:in-app-agent-retention-cleaner",
+ lockTtlSeconds: 300,
+ onUnavailable: "fail",
+ });
+ }
+
+ protected async execute(): Promise<number> {
+ if (!isEnterpriseLicenseAvailable()) {
+ return this.defaultIntervalMs;
+ }
+
+ const nextDelay = await this.withLock(async () => {
+ // Unfinished runs belong to the integrity runner. Excluding them here
+ // prevents an unrepairable conversation from blocking actionable work.
+ const conversations = await prisma.$queryRaw<
+ Array<{ id: string; projectId: string }>
+ >`
+ SELECT c.id, c.project_id AS "projectId"
+ FROM in_app_agent_conversations c
+ JOIN projects p ON p.id = c.project_id
+ WHERE p.retention_days > 0
+ AND p.deleted_at IS NULL
+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'
+ AND NOT EXISTS (
+ SELECT 1 FROM in_app_agent_runs r
+ WHERE r.project_id = c.project_id AND r.conversation_id = c.id
+ AND r.finished_at IS NULL
+ )
+ ORDER BY c.id, c.project_id
+ LIMIT ${BATCH_LIMIT}
+ `;
+
+ if (conversations.length > 0) {
+ const keyRuns = await prisma.inAppAgentRun.findMany({
+ where: {
+ OR: conversations.map(({ id, projectId }) => ({
+ projectId,
+ conversationId: id,
+ })),
+ finishedAt: { not: null },
+ mcpApiKeyId: { not: null },
+ },
+ select: {
+ id: true,
+ projectId: true,
+ mcpApiKeyId: true,
+ },
+ take: BATCH_LIMIT,
+ });
+
+ let cleanedRuns = 0;
+ for (const run of keyRuns) {
+ if (!run.mcpApiKeyId) continue;
+ await this.extendLockOnProgress();
+ try {
+ await cleanupRunMcpApiKey({
+ prisma,
+ projectId: run.projectId,
+ runId: run.id,
+ apiKeyId: run.mcpApiKeyId,
+ deleteApiKey: async (id) => {
+ await deleteInAppAgentMcpApiKeyFromDb({
+ prisma,
+ id,
+ projectId: run.projectId,
+ redis,
+ });
+ },
+ });
+ cleanedRuns++;
+ } catch (error) {
+ this.markRunFailed(error);
+ logger.error(
+ `${this.instanceName}: Failed to clean run credentials`,
+ {
+ error,
+ id: run.id,
+ projectId: run.projectId,
+ },
+ );
+ }
+ }
+
+ await this.extendLockOnProgress(true);
+ // Re-evaluate the live policy and run guards at deletion time, after
+ // potentially slow credential cleanup. Runs and events cascade.
+ const deleted = await prisma.$executeRaw`
+ DELETE FROM in_app_agent_conversations c
+ USING projects p, (VALUES ${Prisma.join(
+ conversations.map(
+ ({ id, projectId }) => Prisma.sql`(${id}, ${projectId})`,
+ ),
+ )}) AS selected(id, project_id)
+ WHERE c.id = selected.id AND c.project_id = selected.project_id
+ AND p.id = c.project_id
+ AND p.retention_days > 0 AND p.deleted_at IS NULL
+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'
+ AND NOT EXISTS (
+ SELECT 1 FROM in_app_agent_runs r
+ WHERE r.project_id = c.project_id AND r.conversation_id = c.id
+ AND (r.finished_at IS NULL OR r.mcp_api_key_id IS NOT NULL)
+ )
+ `;
+
+ recordIncrement(
+ "langfuse.in_app_agent_retention_cleaner.conversations_deleted",
+ deleted,
+ );
+ logger.info(`${this.instanceName}: Deleted expired conversations`, {
+ deleted,
+ cleanedRuns,
+ });
+ if (cleanedRuns > 0 || deleted > 0) {
+ return 0;
+ }
+ }
+
+ // Include blocked conversations, and report even when failures prevent progress.
+ const [backlog] = await prisma.$queryRaw<
+ Array<{ overdueSeconds: number }>
+ >`
+ SELECT GREATEST(COALESCE(MAX(EXTRACT(EPOCH FROM (
+ (NOW() AT TIME ZONE 'UTC') - c.updated_at - p.retention_days * INTERVAL '1 day'
+ ))), 0), 0)::double precision AS "overdueSeconds"
+ FROM in_app_agent_conversations c
+ JOIN projects p ON p.id = c.project_id
+ WHERE p.retention_days > 0
+ AND p.deleted_at IS NULL
+ AND c.updated_at < (NOW() AT TIME ZONE 'UTC') - p.retention_days * INTERVAL '1 day'
+ `;
+ recordGauge(
+ "langfuse.in_app_agent_retention_cleaner.max_conversation_retention_lag_seconds",
+ backlog!.overdueSeconds,
+ { unit: "seconds" },
+ );
+ return this.defaultIntervalMs;
+ });
+
+ return nextDelay ?? this.defaultIntervalMs;
+ }
+}
diff --git a/worker/src/utils/PeriodicRunner.test.ts b/worker/src/utils/PeriodicRunner.test.ts
new file mode 100644
index 0000000000..9894b5611c
--- /dev/null
+++ b/worker/src/utils/PeriodicRunner.test.ts
@@ -0,0 +1,76 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { PeriodicRunner } from "./PeriodicRunner";
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { error: vi.fn(), warn: vi.fn() },
+ contextWithLangfuseProps: () => ({}),
+ instrumentAsync: async (
+ _options: unknown,
+ operation: (span: { setAttribute: () => void }) => Promise<void>,
+ ) => operation({ setAttribute: () => {} }),
+ recordDistribution: vi.fn(),
+ recordGauge: vi.fn(),
+ recordIncrement: vi.fn(),
+ traceException: vi.fn(),
+}));
+vi.mock("@opentelemetry/api", () => ({
+ context: {
+ with: (_context: unknown, operation: () => unknown) => operation(),
+ },
+ SpanKind: { INTERNAL: 0 },
+}));
+
+class TestRunner extends PeriodicRunner {
+ protected name = "test";
+ protected defaultIntervalMs = 10;
+ constructor(private readonly operation: () => Promise<void>) {
+ super("test");
+ }
+ protected execute() {
+ return this.operation();
+ }
+}
+
+afterEach(() => vi.useRealTimers());
+
+describe("PeriodicRunner.drain", () => {
+ it("waits for the active batch and does not schedule another", async () => {
+ vi.useFakeTimers();
+ let finish: () => void = () => {};
+ const operation = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ finish = resolve;
+ }),
+ );
+ const runner = new TestRunner(operation);
+ runner.start();
+ let drained = false;
+ const drain = runner.drain(5_000).then(() => {
+ drained = true;
+ });
+ await vi.advanceTimersByTimeAsync(100);
+ expect(drained).toBe(false);
+ finish();
+ await drain;
+ await vi.advanceTimersByTimeAsync(100);
+ expect(operation).toHaveBeenCalledOnce();
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it("bounds shutdown when the active batch is stuck", async () => {
+ vi.useFakeTimers();
+ const runner = new TestRunner(() => new Promise<void>(() => {}));
+ runner.start();
+ const drain = runner.drain(50);
+ await vi.advanceTimersByTimeAsync(50);
+ await drain;
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it("returns immediately when no batch started", async () => {
+ vi.useFakeTimers();
+ await new TestRunner(async () => {}).drain(5_000);
+ expect(vi.getTimerCount()).toBe(0);
+ });
+});
diff --git a/worker/src/utils/PeriodicRunner.ts b/worker/src/utils/PeriodicRunner.ts
index 0f716091cd..91d3754913 100644
--- a/worker/src/utils/PeriodicRunner.ts
+++ b/worker/src/utils/PeriodicRunner.ts
@@ -22,6 +22,7 @@ export abstract class PeriodicRunner {
private timeoutId: NodeJS.Timeout | null = null;
private isRunning = false;
private activeRun: { outcome: RunOutcome } | null = null;
+ private activeRunPromise: Promise<void> | null = null;
protected abstract get name(): string;
protected abstract get defaultIntervalMs(): number;
@@ -44,7 +45,7 @@ export abstract class PeriodicRunner {
if (this.initialDelayMs > 0) {
this.scheduleNext(this.initialDelayMs);
} else {
- this.runAndScheduleNext();
+ this.activeRunPromise = this.runAndScheduleNext();
}
}
}
@@ -57,6 +58,28 @@ export abstract class PeriodicRunner {
}
}
+ public async drain(timeoutMs: number): Promise<void> {
+ this.stop();
+ if (!this.activeRunPromise) return;
+
+ let timeout: NodeJS.Timeout | undefined;
+ try {
+ await Promise.race([
+ this.activeRunPromise,
+ new Promise<void>((resolve) => {
+ timeout = setTimeout(() => {
+ logger.warn(`${this.name}: shutdown drain timed out`, {
+ timeoutMs,
+ });
+ resolve();
+ }, timeoutMs);
+ }),
+ ]);
+ } finally {
+ clearTimeout(timeout);
+ }
+ }
+
protected markRunFailed(error: unknown): void {
if (this.activeRun) {
this.activeRun.outcome = "failed";
@@ -157,7 +180,7 @@ export abstract class PeriodicRunner {
return;
}
this.timeoutId = setTimeout(() => {
- this.runAndScheduleNext();
+ this.activeRunPromise = this.runAndScheduleNext();
}, delayMs);
}
}
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 150510c4bf..e9b3ab49b4 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -15,6 +15,7 @@ import {
batchProjectCleaners,
batchDataRetentionCleaners,
mediaRetentionCleaner,
+ inAppAgentRetentionCleaner,
batchProjectMediaCleaner,
batchProjectBlobCleaner,
batchTraceDeletionCleaner,
@@ -68,6 +69,7 @@ const runDrainAndClose = async () => {
// Stop media retention cleaner
mediaRetentionCleaner?.stop();
+ inAppAgentRetentionCleaner?.stop();
// Stop batch project media cleaner
batchProjectMediaCleaner?.stop();
@@ -117,6 +119,11 @@ const runDrainAndClose = async () => {
await ClickhouseWriter.shutdownAll();
logger.info("Clickhouse writer has been shut down.");
+ await Promise.all([
+ inAppAgentRetentionCleaner?.drain(5_000),
+ inAppAgentIntegrityRunner?.drain(5_000),
+ ]);
+
// Closes the shared client and every per-queue client in one pass. Each
// queue holds its own client; without this they stay connected, retry
// forever, and keep the event loop alive so the process never exits.