🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): emit run outcome metrics and reconcile stale runs (#16322)
Lysander Kiesele6493a82aa82
diff --git a/.env.dev.example b/.env.dev.exampleindex 0381ecf79c..69b6940ad0 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -222,6 +222,10 @@ LANGFUSE_IN_APP_AGENT_AWS_PROFILE="playground"QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED="true"# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5+# Reconciles stale runs that nobody reopened, then reports remainders.+# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED="false"+# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS=900000+# Web-only admission ceilings.# LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER=5# LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_ORG=20diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.tsindex b15291b1b8..7fbaa1b81d 100644--- a/packages/shared/src/in-app-agent/constants.ts+++ b/packages/shared/src/in-app-agent/constants.ts@@ -9,17 +9,33 @@ export const IN_APP_AGENT_GENERIC_ERROR_MESSAGE ="An error occurred. Please try again or start a new conversation.";/** Runs that are not yet in a terminal state, including parked approvals. */-export const IN_APP_AGENT_UNSETTLED_RUN_STATUSES: readonly InAppAgentRunStatus[] =- [- InAppAgentRunStatus.QUEUED,- InAppAgentRunStatus.RUNNING,- InAppAgentRunStatus.AWAITING_APPROVAL,- ];+export const IN_APP_AGENT_UNSETTLED_RUN_STATUSES = [+ InAppAgentRunStatus.QUEUED,+ InAppAgentRunStatus.RUNNING,+ InAppAgentRunStatus.AWAITING_APPROVAL,+] as const;++export type UnsettledInAppAgentRunStatus =+ (typeof IN_APP_AGENT_UNSETTLED_RUN_STATUSES)[number];++/** Complement of `IN_APP_AGENT_UNSETTLED_RUN_STATUSES`. */+export type SettledInAppAgentRunStatus = Exclude<+ InAppAgentRunStatus,+ UnsettledInAppAgentRunStatus+>;export function isUnsettledInAppAgentRunStatus(status: InAppAgentRunStatus,-): boolean {- return IN_APP_AGENT_UNSETTLED_RUN_STATUSES.includes(status);+): status is UnsettledInAppAgentRunStatus {+ return (+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES as readonly InAppAgentRunStatus[]+ ).includes(status);+}++export function isSettledInAppAgentRunStatus(+ status: InAppAgentRunStatus,+): status is SettledInAppAgentRunStatus {+ return !isUnsettledInAppAgentRunStatus(status);}export const IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE = "silent-mcp-output";diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.tsindex a2b9646e87..99f4b1d0c6 100644--- a/packages/shared/src/in-app-agent/server/persistence.ts+++ b/packages/shared/src/in-app-agent/server/persistence.ts@@ -12,6 +12,8 @@ import {LangfuseInternalTraceEnvironment,logger,} from "../../server";+import { isSettledInAppAgentRunStatus } from "../constants";+import { recordRunTerminalOutcome } from "./runMetrics";import { Prisma } from "../../db";import type { InAppAgentConversation, PrismaClient } from "../../db";@@ -169,7 +171,7 @@ export async function appendRunEvents(params: {errorMessage?: string;};}): Promise<boolean> {- return params.prisma.$transaction(async (tx) => {+ const appended = await params.prisma.$transaction(async (tx) => {await lockConversationRow(tx, params.projectId, params.conversationId);if (params.finish) {@@ -246,6 +248,22 @@ export async function appendRunEvents(params: {return true;});++ // Emitted after the transaction commits so a rolled-back flush cannot inflate+ // the outcome count. The fenced path returns false and is deliberately silent:+ // whichever writer won the CAS already recorded that run's outcome.+ if (+ appended &&+ params.finish &&+ isSettledInAppAgentRunStatus(params.finish.status)+ ) {+ recordRunTerminalOutcome({+ status: params.finish.status,+ errorCode: params.finish.errorCode ?? null,+ });+ }++ return appended;}export async function getConversationEvents(params: {diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts b/packages/shared/src/in-app-agent/server/runLifecycle.test.tsindex 26200ad5ba..c132e74da8 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts@@ -1,7 +1,10 @@import { EventType } from "@ag-ui/core";-import { describe, expect, it, vi } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";-import { InAppAgentRunStatus } from "../../features/inAppAgent/types";+import {+ InAppAgentRunErrorCode,+ InAppAgentRunStatus,+} from "../../features/inAppAgent/types";import type { PrismaClient } from "../../db";import {createQueuedRun,@@ -9,6 +12,19 @@ import {requestRunCancellation,} from "./runLifecycle";+const metricMocks = vi.hoisted(() => ({+ recordRunTerminalOutcome: vi.fn(),+}));++vi.mock("./runMetrics", async (importOriginal) => ({+ ...(await importOriginal<typeof import("./runMetrics")>()),+ recordRunTerminalOutcome: metricMocks.recordRunTerminalOutcome,+}));++beforeEach(() => {+ metricMocks.recordRunTerminalOutcome.mockClear();+});+describe("in-app agent run lifecycle races", () => {it("does not admit a run after the conversation is deleted", async () => {const tx = {@@ -148,5 +164,177 @@ describe("in-app agent run lifecycle races", () => {where: expect.objectContaining({ heartbeatAt: observedHeartbeat }),}),);+ expect(metricMocks.recordRunTerminalOutcome).not.toHaveBeenCalled();+ });++ it("reports a reconciled run's outcome once, after the transition commits", async () => {+ const tx = {+ inAppAgentRun: {+ findMany: vi.fn().mockResolvedValue([+ {+ id: "run-1",+ status: InAppAgentRunStatus.QUEUED,+ createdAt: new Date(Date.now() - 10 * 60_000),+ claimedAt: null,+ heartbeatAt: null,+ finishedAt: null,+ },+ ]),+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),+ },+ };+ const prisma = {+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>+ callback(tx),+ } as unknown as PrismaClient;++ await expect(+ reconcileConversationRuns({+ prisma,+ projectId: "project-1",+ conversationId: "conversation-1",+ }),+ ).resolves.toEqual([+ { runId: "run-1", errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT },+ ]);+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({+ status: InAppAgentRunStatus.FAILED,+ errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT,+ });+ });++ it("records a queued cancel with the cancelled error code after commit", async () => {+ const tx = {+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),+ inAppAgentRun: {+ findFirst: vi+ .fn()+ .mockResolvedValue({ status: InAppAgentRunStatus.QUEUED }),+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),+ },+ };+ const prisma = {+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>+ callback(tx),+ } as unknown as PrismaClient;++ await expect(+ requestRunCancellation({+ prisma,+ projectId: "project-1",+ conversationId: "conversation-1",+ runId: "run-1",+ }),+ ).resolves.toEqual({+ cancelledImmediately: true,+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: InAppAgentRunErrorCode.CANCELLED,+ });+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: InAppAgentRunErrorCode.CANCELLED,+ });+ });++ it("records a prior stale run when admitting a new message", async () => {+ const tx = {+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),+ inAppAgentRun: {+ findMany: vi.fn().mockResolvedValue([+ {+ id: "run-old",+ status: InAppAgentRunStatus.QUEUED,+ createdAt: new Date(Date.now() - 10 * 60_000),+ claimedAt: null,+ heartbeatAt: null,+ finishedAt: null,+ },+ ]),+ updateMany: vi.fn(+ async ({ data }: { data: { status?: InAppAgentRunStatus } }) => ({+ count: data.status === InAppAgentRunStatus.FAILED ? 1 : 0,+ }),+ ),+ findFirst: vi.fn().mockResolvedValue(null),+ create: vi.fn().mockResolvedValue({ id: "run-1" }),+ },+ inAppAgentEvent: {+ findFirst: vi.fn().mockResolvedValue(null),+ create: vi.fn().mockResolvedValue({}),+ },+ inAppAgentConversation: {+ update: vi.fn().mockResolvedValue({}),+ },+ };+ const prisma = {+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>+ callback(tx),+ } as unknown as PrismaClient;++ await createQueuedRun({+ prisma,+ runId: "run-1",+ projectId: "project-1",+ conversationId: "conversation-1",+ triggeredByUserId: "user-1",+ model: "test-model",+ request: { kind: "userMessage", context: [] },+ runStartedEvent: {+ type: EventType.RUN_STARTED,+ threadId: "conversation-1",+ runId: "run-1",+ },+ });++ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({+ status: InAppAgentRunStatus.FAILED,+ errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT,+ });+ });++ it("does not record a reconciled outcome when admitting the next run fails", async () => {+ const tx = {+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),+ inAppAgentRun: {+ findMany: vi.fn().mockResolvedValue([+ {+ id: "run-old",+ status: InAppAgentRunStatus.QUEUED,+ createdAt: new Date(Date.now() - 10 * 60_000),+ claimedAt: null,+ heartbeatAt: null,+ finishedAt: null,+ },+ ]),+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),+ findFirst: vi.fn().mockResolvedValue({ id: "active-run" }),+ create: vi.fn(),+ },+ };+ const prisma = {+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>+ callback(tx),+ } as unknown as PrismaClient;++ await expect(+ createQueuedRun({+ prisma,+ runId: "run-1",+ projectId: "project-1",+ conversationId: "conversation-1",+ triggeredByUserId: "user-1",+ model: "test-model",+ request: { kind: "userMessage", context: [] },+ runStartedEvent: {+ type: EventType.RUN_STARTED,+ threadId: "conversation-1",+ runId: "run-1",+ },+ }),+ ).rejects.toThrow("Assistant is already responding in this conversation");+ expect(metricMocks.recordRunTerminalOutcome).not.toHaveBeenCalled();});});diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.tsindex d9604b70c1..4eb6896e15 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts@@ -7,6 +7,7 @@ import {import { Prisma } from "../../db";import type { InAppAgentRun, PrismaClient } from "../../db";import { logger } from "../../server";+import { recordRunTerminalOutcome } from "./runMetrics";import { buildInAppAgentApprovalDecisionEvent } from "../approvalEvents";import type { AgUiEvent } from "../schema";import type { InAppAgentPrefixedLangfuseMcpToolName } from "./mcpPolicy";@@ -15,7 +16,10 @@ import {resolveInAppAgentRootRunId,type InAppAgentRunRequest,} from "../../features/inAppAgent/types";-import { IN_APP_AGENT_UNSETTLED_RUN_STATUSES } from "../constants";+import {+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES,+ isSettledInAppAgentRunStatus,+} from "../constants";import {ACTIVE_RUN_CONFLICT_MESSAGE,lockConversation,@@ -109,6 +113,13 @@ export async function finishClaimedRun(params: {},});+ if (count > 0 && isSettledInAppAgentRunStatus(params.status)) {+ recordRunTerminalOutcome({+ status: params.status,+ errorCode: params.errorCode ?? null,+ });+ }+return count > 0;}@@ -138,61 +149,73 @@ export async function createQueuedRun(params: {request: InAppAgentRunRequest;runStartedEvent: AgUiEvent;}): Promise<InAppAgentRun> {- return params.prisma.$transaction(async (tx) => {- await lockConversation(tx, params.projectId, params.conversationId);-- await reconcileConversationRunsInTransaction({- tx,- projectId: params.projectId,- conversationId: params.conversationId,- });+ const { run, reconciled, supersededCount } = await params.prisma.$transaction(+ async (tx) => {+ await lockConversation(tx, params.projectId, params.conversationId);- // New input supersedes a parked approval.- await tx.inAppAgentRun.updateMany({- where: {+ const reconciled = await reconcileConversationRunsInTransaction({+ tx,projectId: params.projectId,conversationId: params.conversationId,- status: InAppAgentRunStatus.AWAITING_APPROVAL,- },- data: {- status: InAppAgentRunStatus.CANCELLED,- errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,- errorMessage: "Replaced by a newer message",- },- });+ });- const activeRun = await tx.inAppAgentRun.findFirst({- where: {+ // New input supersedes a parked approval.+ const superseded = await tx.inAppAgentRun.updateMany({+ where: {+ projectId: params.projectId,+ conversationId: params.conversationId,+ status: InAppAgentRunStatus.AWAITING_APPROVAL,+ },+ data: {+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,+ errorMessage: "Replaced by a newer message",+ },+ });++ const activeRun = await tx.inAppAgentRun.findFirst({+ where: {+ projectId: params.projectId,+ conversationId: params.conversationId,+ finishedAt: null,+ },+ select: { id: true },+ });++ if (activeRun) {+ throw new LangfuseConflictError(ACTIVE_RUN_CONFLICT_MESSAGE);+ }++ const run = await createRunRow(tx, {+ runId: params.runId,projectId: params.projectId,conversationId: params.conversationId,- finishedAt: null,- },- select: { id: true },- });+ triggeredByUserId: params.triggeredByUserId,+ model: params.model,+ request: params.request,+ });- if (activeRun) {- throw new LangfuseConflictError(ACTIVE_RUN_CONFLICT_MESSAGE);- }+ await appendConversationEventInTransaction({+ tx,+ projectId: params.projectId,+ conversationId: params.conversationId,+ runId: params.runId,+ event: params.runStartedEvent,+ });- const run = await createRunRow(tx, {- runId: params.runId,- projectId: params.projectId,- conversationId: params.conversationId,- triggeredByUserId: params.triggeredByUserId,- model: params.model,- request: params.request,- });+ return { run, reconciled, supersededCount: superseded.count };+ },+ );- await appendConversationEventInTransaction({- tx,- projectId: params.projectId,- conversationId: params.conversationId,- runId: params.runId,- event: params.runStartedEvent,+ recordReconciledOutcomes(reconciled);+ for (let i = 0; i < supersededCount; i++) {+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,});+ }- return run;- });+ return run;}/** Record one approval decision and create its continuation under the lock. */@@ -241,7 +264,7 @@ export async function decideToolApproval(params: {parkedAt &&Date.now() - parkedAt.getTime() > IN_APP_AGENT_APPROVAL_TTL_MS) {- await tx.inAppAgentRun.updateMany({+ const { count } = await tx.inAppAgentRun.updateMany({where: {id: params.parentRunId,projectId: params.projectId,@@ -254,7 +277,7 @@ export async function decideToolApproval(params: {},});- return { type: "expired" as const };+ return { type: "expired" as const, expired: count > 0 };}// The parent CAS is the exactly-once decision guarantee.@@ -357,15 +380,31 @@ export async function decideToolApproval(params: {});if (outcome.type === "expired") {+ if (outcome.expired) {+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.FAILED,+ errorCode: InAppAgentRunErrorCode.APPROVAL_EXPIRED,+ });+ }throw new LangfuseConflictError("The approval request expired.");}+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.SUCCEEDED,+ });+return outcome.run;}export type CancelRunResult = {cancelledImmediately: boolean;status: InAppAgentRunStatus | null;+ errorCode?: InAppAgentRunErrorCode;+};++export type ImmediateCancel = {+ runId: string;+ errorCode: InAppAgentRunErrorCode;};/** Cancel every run that is executing or waiting on user approval. */@@ -373,7 +412,7 @@ export async function cancelConversationRunsInTransaction(params: {tx: InAppAgentTx;projectId: string;conversationId: string;-}): Promise<string[]> {+}): Promise<ImmediateCancel[]> {await lockConversation(params.tx, params.projectId, params.conversationId);const runs = await params.tx.inAppAgentRun.findMany({@@ -386,7 +425,7 @@ export async function cancelConversationRunsInTransaction(params: {select: { id: true, status: true },});- const cancelledImmediately: string[] = [];+ const cancelledImmediately: ImmediateCancel[] = [];for (const run of runs) {const parsedStatus = InAppAgentRunStatusSchema.safeParse(run.status);@@ -400,14 +439,28 @@ export async function cancelConversationRunsInTransaction(params: {runStatus: parsedStatus.data,});- if (result.cancelledImmediately) {- cancelledImmediately.push(run.id);+ if (result.cancelledImmediately && result.errorCode) {+ cancelledImmediately.push({+ runId: run.id,+ errorCode: result.errorCode,+ });}}return cancelledImmediately;}+export function recordImmediateCancelOutcomes(+ cancellations: ReadonlyArray<ImmediateCancel>,+): void {+ for (const cancellation of cancellations) {+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: cancellation.errorCode,+ });+ }+}+/** Cancel idle states immediately; signal RUNNING workers cooperatively. */export async function requestRunCancellation(params: {prisma: PrismaClient;@@ -415,7 +468,7 @@ export async function requestRunCancellation(params: {conversationId: string;runId: string;}): Promise<CancelRunResult> {- return params.prisma.$transaction(async (tx) => {+ const result = await params.prisma.$transaction(async (tx) => {await lockConversation(tx, params.projectId, params.conversationId);const run = await tx.inAppAgentRun.findFirst({@@ -440,6 +493,17 @@ export async function requestRunCancellation(params: {runStatus: parsedStatus.data,});});++ // A RUNNING run is only signalled here; the worker writes its own terminal+ // state, so only an immediate cancel settles the run in this call.+ if (result.cancelledImmediately) {+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.CANCELLED,+ errorCode: result.errorCode ?? null,+ });+ }++ return result;}async function cancelRunInTransaction(params: {@@ -481,6 +545,7 @@ async function cancelRunInTransaction(params: {return {cancelledImmediately: true,status: InAppAgentRunStatus.CANCELLED,+ errorCode: immediateCancel.errorCode,};}} else if (params.runStatus !== InAppAgentRunStatus.RUNNING) {@@ -545,9 +610,20 @@ export async function reconcileConversationRuns(params: {}),);+ recordReconciledOutcomes(reconciled);+return reconciled;}+function recordReconciledOutcomes(reconciled: ReconciledRun[]): void {+ for (const run of reconciled) {+ recordRunTerminalOutcome({+ status: InAppAgentRunStatus.FAILED,+ errorCode: run.errorCode,+ });+ }+}+async function reconcileConversationRunsInTransaction(params: {tx: InAppAgentTx;projectId: string;diff --git a/packages/shared/src/in-app-agent/server/runMetrics.ts b/packages/shared/src/in-app-agent/server/runMetrics.tsnew file mode 100644index 0000000000..3f3916bf1d--- /dev/null+++ b/packages/shared/src/in-app-agent/server/runMetrics.ts@@ -0,0 +1,27 @@+import { InAppAgentRunErrorCode } from "../../features/inAppAgent/types";+import { recordIncrement } from "../../server";+import type { SettledInAppAgentRunStatus } from "../constants";++const RUN_COMPLETED_METRIC = "langfuse.in_app_agent.run.completed";++/**+ * Record a run reaching a settled state. Call this only after the terminal CAS+ * has committed, so a rolled-back transaction cannot inflate the count.+ *+ * Settledness is the complement of `IN_APP_AGENT_UNSETTLED_RUN_STATUSES` in+ * constants.ts. `AWAITING_APPROVAL` is parked, not settled; emitting on the+ * park would double-count when the parent later becomes SUCCEEDED, FAILED, or+ * CANCELLED.+ *+ * `error_code` is "none" rather than absent so Datadog can group every outcome+ * on one tag key.+ */+export function recordRunTerminalOutcome(params: {+ status: SettledInAppAgentRunStatus;+ errorCode?: InAppAgentRunErrorCode | null;+}): void {+ recordIncrement(RUN_COMPLETED_METRIC, 1, {+ status: params.status,+ error_code: params.errorCode ?? "none",+ });+}diff --git a/web/src/features/in-app-agent/server/backgroundRunService.ts b/web/src/features/in-app-agent/server/backgroundRunService.tsindex 63412b9cc8..9289a0a887 100644--- a/web/src/features/in-app-agent/server/backgroundRunService.ts+++ b/web/src/features/in-app-agent/server/backgroundRunService.ts@@ -35,6 +35,7 @@ import {createQueuedRun,decideToolApproval,reconcileConversationRuns,+ recordImmediateCancelOutcomes,requestRunCancellation,} from "@langfuse/shared/in-app-agent/server/runLifecycle";@@ -291,7 +292,7 @@ export async function deleteBackgroundConversation(params: {userId: params.userId,});- const cancelledRunIds = await params.prisma.$transaction(async (tx) => {+ const cancelledRuns = await params.prisma.$transaction(async (tx) => {const cancelledImmediately = await cancelConversationRunsInTransaction({tx,projectId: params.projectId,@@ -314,8 +315,12 @@ export async function deleteBackgroundConversation(params: {return cancelledImmediately;});+ recordImmediateCancelOutcomes(cancelledRuns);+// Avoid spending a worker slot on jobs whose runs are already cancelled.- await Promise.all(cancelledRunIds.map(removeInAppAgentRunJob));+ await Promise.all(+ cancelledRuns.map((run) => removeInAppAgentRunJob(run.runId)),+ );return { success: true };}diff --git a/worker/src/app.ts b/worker/src/app.tsindex 9c908a957f..c7cc28b521 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -101,6 +101,7 @@ import { QueueMetricsRunner } from "./features/queue-metrics-runner";import { MonitorRunner } from "./features/monitor-runner";import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";+import { InAppAgentIntegrityRunner } from "./features/in-app-agent-integrity-runner";import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";const app = express();@@ -773,6 +774,14 @@ if (env.LANGFUSE_TRACE_DELETE_BATCH_ACTION_RUNNER_ENABLED === "true") {traceDeleteBatchActionRunner.start();}+// Reconciles stale in-app agent runs that nobody reopened, then reports remainders.+export let inAppAgentIntegrityRunner: InAppAgentIntegrityRunner | null = null;++if (env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED === "true") {+ inAppAgentIntegrityRunner = new InAppAgentIntegrityRunner();+ inAppAgentIntegrityRunner.start();+}+// ClickHouse deleted-mask cleaner for physically applying lightweight delete masksexport let deletedMaskCleaner: DeletedMaskCleaner | null = null;diff --git a/worker/src/env.ts b/worker/src/env.tsindex a3ac45e9d6..2e4921950f 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -239,6 +239,11 @@ const EnvSchema = z.object({QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED: z.enum(["true", "false"]).default("false"),+ // Reconciles stale runs that nobody reopened. Off by default: one elected+ // worker per region is enough.+ LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED: z+ .enum(["true", "false"])+ .default("false"),// The ambient host profile takes precedence over the agent-specific default// so local developer credentials win when both are configured.AWS_PROFILE: z.string().optional(),@@ -651,6 +656,13 @@ const EnvSchema = z.object({.number().positive().default(5),+ // Default 15 minutes. Do not go below the heartbeat-stale window (60s) or a+ // live worker can lose the race. Faster also adds no resolution: every+ // deadline this runner reports against is 5 minutes or longer.+ LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS: z.coerce+ .number()+ .positive()+ .default(15 * 60_000),LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS: z.coerce.number().positive()diff --git a/worker/src/features/in-app-agent-integrity-runner/index.test.ts b/worker/src/features/in-app-agent-integrity-runner/index.test.tsnew file mode 100644index 0000000000..e552a717ac--- /dev/null+++ b/worker/src/features/in-app-agent-integrity-runner/index.test.ts@@ -0,0 +1,143 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import {+ InAppAgentRunErrorCode,+ InAppAgentRunStatus,+} from "@langfuse/shared/in-app-agent";+import { IN_APP_AGENT_HEARTBEAT_STALE_MS } from "@langfuse/shared/in-app-agent/server/tunables";++const mocks = vi.hoisted(() => ({+ recordGauge: vi.fn(),+ runFindMany: vi.fn(),+ runGroupBy: vi.fn(),+ apiKeyCount: vi.fn(),+ reconcileConversationRuns: vi.fn(),+ cleanupTerminalRunMcpApiKeys: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),+ logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() },+ recordGauge: mocks.recordGauge,+ redis: {},+}));++vi.mock("@langfuse/shared/src/server/auth/apiKeys", () => ({+ deleteApiKeyFromDb: vi.fn(),+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {+ inAppAgentRun: {+ findMany: mocks.runFindMany,+ groupBy: mocks.runGroupBy,+ },+ apiKey: { count: mocks.apiKeyCount },+ },+}));++vi.mock(+ "@langfuse/shared/in-app-agent/server/runLifecycle",+ async (importOriginal) => ({+ ...(await importOriginal<+ typeof import("@langfuse/shared/in-app-agent/server/runLifecycle")+ >()),+ reconcileConversationRuns: mocks.reconcileConversationRuns,+ cleanupTerminalRunMcpApiKeys: mocks.cleanupTerminalRunMcpApiKeys,+ }),+);++vi.mock("../../env", () => ({+ env: { LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS: 60_000 },+}));++import { InAppAgentIntegrityRunner } from "./index";++function runnerWithStubbedLock() {+ const runner = new InAppAgentIntegrityRunner();+ const extend = vi.fn().mockResolvedValue(true);+ (runner as unknown as { lock: unknown }).lock = {+ withLock: async (operation: () => Promise<unknown>) => operation(),+ extend,+ };+ return { runner, extend };+}++function runningRow(params: {+ id: string;+ heartbeatAt: Date;+ claimedAt?: Date;+}) {+ const claimedAt = params.claimedAt ?? params.heartbeatAt;+ return {+ id: params.id,+ projectId: "project-1",+ conversationId: "conversation-1",+ status: InAppAgentRunStatus.RUNNING,+ createdAt: claimedAt,+ claimedAt,+ heartbeatAt: params.heartbeatAt,+ finishedAt: null,+ };+}++describe("InAppAgentIntegrityRunner", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.runGroupBy.mockResolvedValue([]);+ mocks.runFindMany.mockResolvedValue([]);+ mocks.apiKeyCount.mockResolvedValue(0);+ mocks.reconcileConversationRuns.mockResolvedValue([]);+ mocks.cleanupTerminalRunMcpApiKeys.mockResolvedValue(0);+ });++ it("reconciles a stale RUNNING conversation and cleans its MCP keys", async () => {+ const heartbeatAt = new Date(+ Date.now() - IN_APP_AGENT_HEARTBEAT_STALE_MS - 1,+ );+ mocks.runFindMany+ .mockResolvedValueOnce([runningRow({ id: "run-1", heartbeatAt })])+ .mockResolvedValueOnce([]);+ mocks.reconcileConversationRuns.mockResolvedValue([+ { runId: "run-1", errorCode: InAppAgentRunErrorCode.WORKER_LOST },+ ]);++ mocks.runGroupBy.mockResolvedValue([+ { status: InAppAgentRunStatus.RUNNING, _count: { _all: 9 } },+ ]);++ const { runner, extend } = runnerWithStubbedLock();+ await runner.processBatch();++ expect(extend).toHaveBeenCalled();+ expect(mocks.reconcileConversationRuns).toHaveBeenCalledExactlyOnceWith({+ prisma: expect.anything(),+ projectId: "project-1",+ conversationId: "conversation-1",+ });+ expect(mocks.cleanupTerminalRunMcpApiKeys).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ projectId: "project-1",+ conversationId: "conversation-1",+ }),+ );+ expect(mocks.recordGauge).toHaveBeenCalledWith(+ "langfuse.in_app_agent.active_runs",+ 9,+ { status: InAppAgentRunStatus.RUNNING },+ );+ });++ it("does not reconcile a healthy RUNNING conversation", async () => {+ mocks.runFindMany+ .mockResolvedValueOnce([+ runningRow({ id: "run-1", heartbeatAt: new Date() }),+ ])+ .mockResolvedValueOnce([]);++ await runnerWithStubbedLock().runner.processBatch();++ expect(mocks.reconcileConversationRuns).not.toHaveBeenCalled();+ expect(mocks.cleanupTerminalRunMcpApiKeys).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/features/in-app-agent-integrity-runner/index.ts b/worker/src/features/in-app-agent-integrity-runner/index.tsnew file mode 100644index 0000000000..a384f2f632--- /dev/null+++ b/worker/src/features/in-app-agent-integrity-runner/index.ts@@ -0,0 +1,272 @@+import { prisma, type Prisma } from "@langfuse/shared/src/db";+import { logger, recordGauge, redis } from "@langfuse/shared/src/server";+import { deleteApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";+import {+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES,+ InAppAgentRunErrorCode,+ InAppAgentRunStatus,+} from "@langfuse/shared/in-app-agent";+import {+ classifyStaleRun,+ cleanupTerminalRunMcpApiKeys,+ reconcileConversationRuns,+} from "@langfuse/shared/in-app-agent/server/runLifecycle";+import {+ IN_APP_AGENT_APPROVAL_TTL_MS,+ IN_APP_AGENT_HEARTBEAT_STALE_MS,+ IN_APP_AGENT_QUEUE_TIMEOUT_MS,+ IN_APP_AGENT_RUN_MAX_DURATION_MS,+} from "@langfuse/shared/in-app-agent/server/tunables";++import { env } from "../../env";+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";++const METRIC_PREFIX = "langfuse.in_app_agent";++export const IN_APP_AGENT_INTEGRITY_RUNNER_LOCK_KEY =+ "langfuse:in-app-agent-integrity-runner";++const LOCK_TTL_SECONDS = 120;+const SCAN_LIMIT = 5_000;++const REPORTED_FINDINGS = [+ InAppAgentRunErrorCode.QUEUE_TIMEOUT,+ InAppAgentRunErrorCode.WORKER_LOST,+ InAppAgentRunErrorCode.RUN_TIMEOUT,+ InAppAgentRunErrorCode.APPROVAL_EXPIRED,+];++/**+ * Reconciles stale in-app agent runs that nobody reopened, then reports what+ * remains. Read paths still reconcile on attach; this runner is the backstop+ * so a conversation nobody opens cannot hold a slot forever.+ *+ * Writes go through `reconcileConversationRuns` so the verdict cannot drift+ * from web. Gauges are emitted after repair.+ */+export class InAppAgentIntegrityRunner extends PeriodicExclusiveRunner {+ private readonly intervalMs: number;++ protected get defaultIntervalMs(): number {+ return this.intervalMs;+ }++ constructor(opts: { intervalMs?: number } = {}) {+ super({+ name: "InAppAgentIntegrityRunner",+ metricName: "in_app_agent_integrity_runner",+ lockKey: IN_APP_AGENT_INTEGRITY_RUNNER_LOCK_KEY,+ lockTtlSeconds: LOCK_TTL_SECONDS,+ onUnavailable: "fail",+ });++ this.intervalMs =+ opts.intervalMs ?? env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS;+ }++ public override start(): void {+ logger.info(`Starting ${this.instanceName}`, {+ intervalMs: this.intervalMs,+ });+ super.start();+ }++ protected async execute(): Promise<number | void> {+ return await this.withLock(async () => {+ await this.reconcileAndReportUnsettledRuns();+ await this.reportOrphanedMcpApiKeys();+ });+ }++ private async reconcileAndReportUnsettledRuns(): Promise<void> {+ const now = Date.now();+ const candidates = await prisma.inAppAgentRun.findMany({+ where: staleRunWhere(now),+ select: {+ id: true,+ projectId: true,+ conversationId: true,+ status: true,+ createdAt: true,+ claimedAt: true,+ heartbeatAt: true,+ finishedAt: true,+ },+ take: SCAN_LIMIT,+ });++ if (candidates.length === SCAN_LIMIT) {+ logger.warn(+ `${this.instanceName}: stale run scan hit its limit; repair is a floor`,+ { scanLimit: SCAN_LIMIT },+ );+ }++ const staleConversations = new Map<+ string,+ { projectId: string; conversationId: string }+ >();++ for (const run of candidates) {+ if (!classifyStaleRun(run, now)) continue;+ staleConversations.set(`${run.projectId}:${run.conversationId}`, {+ projectId: run.projectId,+ conversationId: run.conversationId,+ });+ }++ const reconciledIds = new Set<string>();++ for (const { projectId, conversationId } of staleConversations.values()) {+ await this.extendLockOnProgress();++ try {+ const reconciled = await reconcileConversationRuns({+ prisma,+ projectId,+ conversationId,+ });++ for (const run of reconciled) {+ reconciledIds.add(run.runId);+ }++ if (reconciled.length === 0) continue;++ await cleanupTerminalRunMcpApiKeys({+ prisma,+ projectId,+ conversationId,+ deleteApiKey: async (apiKeyId) => {+ await deleteApiKeyFromDb({+ prisma,+ id: apiKeyId,+ entityId: projectId,+ scope: "PROJECT",+ redis,+ });+ },+ });+ } catch (error) {+ logger.error(`${this.instanceName}: failed to reconcile conversation`, {+ projectId,+ conversationId,+ error,+ });+ }+ }++ const activeByStatusRows = await prisma.inAppAgentRun.groupBy({+ by: ["status"],+ where: { status: { in: [...IN_APP_AGENT_UNSETTLED_RUN_STATUSES] } },+ _count: { _all: true },+ });++ const activeByStatus = new Map<string, number>(+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES.map((status) => [status, 0]),+ );+ for (const row of activeByStatusRows) {+ if (row.status) {+ activeByStatus.set(row.status, row._count._all);+ }+ }++ const findings = new Map<string, number>(+ REPORTED_FINDINGS.map((code) => [code, 0]),+ );+ for (const run of candidates) {+ const stale = classifyStaleRun(run, now);+ if (!stale || reconciledIds.has(run.id)) continue;+ findings.set(stale.errorCode, (findings.get(stale.errorCode) ?? 0) + 1);+ }++ for (const [status, count] of activeByStatus) {+ recordGauge(`${METRIC_PREFIX}.active_runs`, count, { status });+ }++ for (const [finding, count] of findings) {+ recordGauge(`${METRIC_PREFIX}.lifecycle_integrity`, count, { finding });+ }+ }++ /**+ * Scoped MCP keys are deleted best-effort when a run ends, and the pointer is+ * cleared only after the delete confirms. Two different problems hide behind a+ * lingering pointer, so they are counted apart: `live_key` is a credential+ * that still exists and never expires, `stuck_pointer` is a pointer whose key+ * is already gone, which is harmless but retries and logs an error on every+ * conversation read.+ */+ private async reportOrphanedMcpApiKeys(): Promise<void> {+ const runs = await prisma.inAppAgentRun.findMany({+ where: {+ finishedAt: { not: null },+ mcpApiKeyId: { not: null },+ status: { notIn: [InAppAgentRunStatus.AWAITING_APPROVAL] },+ },+ select: { mcpApiKeyId: true },+ take: SCAN_LIMIT,+ });++ if (runs.length === SCAN_LIMIT) {+ logger.warn(+ `${this.instanceName}: MCP key scan hit its limit; counts are a floor`,+ { scanLimit: SCAN_LIMIT },+ );+ }++ const pointerIds = runs+ .map((run) => run.mcpApiKeyId)+ .filter((id): id is string => Boolean(id));++ let liveKeys = 0;+ if (pointerIds.length > 0) {+ liveKeys = await prisma.apiKey.count({+ where: { id: { in: pointerIds } },+ });+ }++ recordGauge(`${METRIC_PREFIX}.orphaned_mcp_api_keys`, liveKeys, {+ kind: "live_key",+ });+ recordGauge(+ `${METRIC_PREFIX}.orphaned_mcp_api_keys`,+ pointerIds.length - liveKeys,+ { kind: "stuck_pointer" },+ );+ }+}++function staleRunWhere(now: number): Prisma.InAppAgentRunWhereInput {+ const queueCutoff = new Date(now - IN_APP_AGENT_QUEUE_TIMEOUT_MS);+ const durationCutoff = new Date(now - IN_APP_AGENT_RUN_MAX_DURATION_MS);+ const heartbeatCutoff = new Date(now - IN_APP_AGENT_HEARTBEAT_STALE_MS);+ const approvalCutoff = new Date(now - IN_APP_AGENT_APPROVAL_TTL_MS);++ return {+ OR: [+ {+ status: InAppAgentRunStatus.QUEUED,+ createdAt: { lt: queueCutoff },+ },+ {+ status: InAppAgentRunStatus.RUNNING,+ OR: [+ { claimedAt: { lt: durationCutoff } },+ { claimedAt: null, createdAt: { lt: durationCutoff } },+ { heartbeatAt: { lt: heartbeatCutoff } },+ { heartbeatAt: null, claimedAt: { lt: heartbeatCutoff } },+ {+ heartbeatAt: null,+ claimedAt: null,+ createdAt: { lt: heartbeatCutoff },+ },+ ],+ },+ {+ status: InAppAgentRunStatus.AWAITING_APPROVAL,+ finishedAt: { lt: approvalCutoff },+ },+ ],+ };+}diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.tsindex d1f29fe963..8efa10fdcf 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts@@ -507,7 +507,11 @@ describe("executeInAppAgentRun", () => {scenarioRef.current = async ({ options }) => {await options.onEvent(textChunk("Still calling tools"));- await options.onComplete({ truncatedByStepLimit: true });+ // Truncation implies the cap was reached: wrap-up fires one step earlier.+ await options.onComplete({+ reachedStepLimit: true,+ truncatedByStepLimit: true,+ });await options.onFinish();};diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex f5e20f25f1..cbf9534624 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -3,6 +3,7 @@ import { prisma } from "@langfuse/shared/src/db";import {getLangfuseAITraceSinkParams,logger,+ recordIncrement,redis,traceException,} from "@langfuse/shared/src/server";@@ -467,6 +468,14 @@ export async function executeInAppAgentRun(params: {},onMcpToolCallCompleted: sandboxToolCallFiles.processToolCall,onComplete: async (outcome) => {+ // Truncation lands on run.completed via the error code below, but that+ // only counts turns wrap-up failed to rescue. Counting every turn that+ // reached the cap is what shows the cap becoming binding before users+ // see a cut-off answer.+ if (outcome?.reachedStepLimit) {+ recordIncrement("langfuse.in_app_agent.step_limit_reached", 1);+ }+await flushPersistedRunEvents(interruptRequest? { status: InAppAgentRunStatus.AWAITING_APPROVAL }diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex c2acb2dda0..f7f7a3c87f 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -159,6 +159,8 @@ This is your final step. Do not call any more tools. Summarize what you have fou</step_limit_wrap_up>`;export type InAppAgentCompleteOutcome = {+ /** The turn reached the step cap, whether or not wrap-up rescued it. */+ reachedStepLimit: boolean;truncatedByStepLimit: boolean;};@@ -792,6 +794,7 @@ export async function createAgUiStream(params: {instrumentation.flush(),);return params.options.onComplete?.({+ reachedStepLimit: stepLimitState.wrapUp,truncatedByStepLimit,});}diff --git a/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts b/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.tsindex ed7943968d..af7576fe58 100644--- a/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts+++ b/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts@@ -1,9 +1,11 @@import { InAppAgentRateLimitErrorResponseSchema } from "@langfuse/shared/in-app-agent";-import { logger } from "@langfuse/shared/src/server";+import { logger, recordIncrement } from "@langfuse/shared/src/server";/** Total time a single MCP operation may spend waiting out rate limits. */export const IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS = 60_000;+const MCP_RATE_LIMIT_WAIT_METRIC = "langfuse.in_app_agent.mcp_rate_limit_wait";+type McpRateLimitError = {retryAfterSeconds: number;};@@ -70,11 +72,23 @@ export async function withMcpRateLimitWait<T>(params: {}const waitMs = rateLimit.retryAfterSeconds * 1_000;+ const operation = String(params.logContext.operation ?? "unknown");if (waitMs > remainingBudgetMs) {+ // The run fails from here on, so this is the tag that separates "we+ // absorbed a rate limit" from "a rate limit killed the turn".+ recordIncrement(MCP_RATE_LIMIT_WAIT_METRIC, 1, {+ outcome: "budget_exhausted",+ operation,+ });throw error;}+ recordIncrement(MCP_RATE_LIMIT_WAIT_METRIC, 1, {+ outcome: "waited",+ operation,+ });+logger.warn("Waiting out in-app agent MCP rate limit", {...params.logContext,retryAfterSeconds: rateLimit.retryAfterSeconds,diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 69c8e1672c..509370870c 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -19,6 +19,7 @@ import {batchProjectBlobCleaner,batchTraceDeletionCleaner,traceDeleteBatchActionRunner,+ inAppAgentIntegrityRunner,deletedMaskCleaner,queueMetricsRunner,monitorRunners,@@ -58,6 +59,8 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => {// Stop durable trace-delete batch action runnertraceDeleteBatchActionRunner?.stop();+ inAppAgentIntegrityRunner?.stop();+// Stop deleted-mask cleanerdeletedMaskCleaner?.stop();