Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): emit run outcome metrics and reconcile stale runs (#16322)

Lysander Kiesele6493a82aa82
diff --git a/.env.dev.example b/.env.dev.example
index 0381ecf79c..69b6940ad0 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -222,6 +222,10 @@ LANGFUSE_IN_APP_AGENT_AWS_PROFILE="playground"
QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED="true"
# LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY=5
+# Reconciles stale runs that nobody reopened, then reports remainders.
+# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED="false"
+# LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS=900000
+
# Web-only admission ceilings.
# LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER=5
# LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_ORG=20
diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.ts
index b15291b1b8..7fbaa1b81d 100644
--- a/packages/shared/src/in-app-agent/constants.ts
+++ b/packages/shared/src/in-app-agent/constants.ts
@@ -9,17 +9,33 @@ export const IN_APP_AGENT_GENERIC_ERROR_MESSAGE =
"An error occurred. Please try again or start a new conversation.";
/** Runs that are not yet in a terminal state, including parked approvals. */
-export const IN_APP_AGENT_UNSETTLED_RUN_STATUSES: readonly InAppAgentRunStatus[] =
- [
- InAppAgentRunStatus.QUEUED,
- InAppAgentRunStatus.RUNNING,
- InAppAgentRunStatus.AWAITING_APPROVAL,
- ];
+export const IN_APP_AGENT_UNSETTLED_RUN_STATUSES = [
+ InAppAgentRunStatus.QUEUED,
+ InAppAgentRunStatus.RUNNING,
+ InAppAgentRunStatus.AWAITING_APPROVAL,
+] as const;
+
+export type UnsettledInAppAgentRunStatus =
+ (typeof IN_APP_AGENT_UNSETTLED_RUN_STATUSES)[number];
+
+/** Complement of `IN_APP_AGENT_UNSETTLED_RUN_STATUSES`. */
+export type SettledInAppAgentRunStatus = Exclude<
+ InAppAgentRunStatus,
+ UnsettledInAppAgentRunStatus
+>;
export function isUnsettledInAppAgentRunStatus(
status: InAppAgentRunStatus,
-): boolean {
- return IN_APP_AGENT_UNSETTLED_RUN_STATUSES.includes(status);
+): status is UnsettledInAppAgentRunStatus {
+ return (
+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES as readonly InAppAgentRunStatus[]
+ ).includes(status);
+}
+
+export function isSettledInAppAgentRunStatus(
+ status: InAppAgentRunStatus,
+): status is SettledInAppAgentRunStatus {
+ return !isUnsettledInAppAgentRunStatus(status);
}
export const IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE = "silent-mcp-output";
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.ts
index a2b9646e87..99f4b1d0c6 100644
--- a/packages/shared/src/in-app-agent/server/persistence.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.ts
@@ -12,6 +12,8 @@ import {
LangfuseInternalTraceEnvironment,
logger,
} from "../../server";
+import { isSettledInAppAgentRunStatus } from "../constants";
+import { recordRunTerminalOutcome } from "./runMetrics";
import { Prisma } from "../../db";
import type { InAppAgentConversation, PrismaClient } from "../../db";
@@ -169,7 +171,7 @@ export async function appendRunEvents(params: {
errorMessage?: string;
};
}): Promise<boolean> {
- return params.prisma.$transaction(async (tx) => {
+ const appended = await params.prisma.$transaction(async (tx) => {
await lockConversationRow(tx, params.projectId, params.conversationId);
if (params.finish) {
@@ -246,6 +248,22 @@ export async function appendRunEvents(params: {
return true;
});
+
+ // Emitted after the transaction commits so a rolled-back flush cannot inflate
+ // the outcome count. The fenced path returns false and is deliberately silent:
+ // whichever writer won the CAS already recorded that run's outcome.
+ if (
+ appended &&
+ params.finish &&
+ isSettledInAppAgentRunStatus(params.finish.status)
+ ) {
+ recordRunTerminalOutcome({
+ status: params.finish.status,
+ errorCode: params.finish.errorCode ?? null,
+ });
+ }
+
+ return appended;
}
export async function getConversationEvents(params: {
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
index 26200ad5ba..c132e74da8 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts
@@ -1,7 +1,10 @@
import { EventType } from "@ag-ui/core";
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
-import { InAppAgentRunStatus } from "../../features/inAppAgent/types";
+import {
+ InAppAgentRunErrorCode,
+ InAppAgentRunStatus,
+} from "../../features/inAppAgent/types";
import type { PrismaClient } from "../../db";
import {
createQueuedRun,
@@ -9,6 +12,19 @@ import {
requestRunCancellation,
} from "./runLifecycle";
+const metricMocks = vi.hoisted(() => ({
+ recordRunTerminalOutcome: vi.fn(),
+}));
+
+vi.mock("./runMetrics", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("./runMetrics")>()),
+ recordRunTerminalOutcome: metricMocks.recordRunTerminalOutcome,
+}));
+
+beforeEach(() => {
+ metricMocks.recordRunTerminalOutcome.mockClear();
+});
+
describe("in-app agent run lifecycle races", () => {
it("does not admit a run after the conversation is deleted", async () => {
const tx = {
@@ -148,5 +164,177 @@ describe("in-app agent run lifecycle races", () => {
where: expect.objectContaining({ heartbeatAt: observedHeartbeat }),
}),
);
+ expect(metricMocks.recordRunTerminalOutcome).not.toHaveBeenCalled();
+ });
+
+ it("reports a reconciled run's outcome once, after the transition commits", async () => {
+ const tx = {
+ inAppAgentRun: {
+ findMany: vi.fn().mockResolvedValue([
+ {
+ id: "run-1",
+ status: InAppAgentRunStatus.QUEUED,
+ createdAt: new Date(Date.now() - 10 * 60_000),
+ claimedAt: null,
+ heartbeatAt: null,
+ finishedAt: null,
+ },
+ ]),
+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),
+ },
+ };
+ const prisma = {
+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>
+ callback(tx),
+ } as unknown as PrismaClient;
+
+ await expect(
+ reconcileConversationRuns({
+ prisma,
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ }),
+ ).resolves.toEqual([
+ { runId: "run-1", errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT },
+ ]);
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT,
+ });
+ });
+
+ it("records a queued cancel with the cancelled error code after commit", async () => {
+ const tx = {
+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),
+ inAppAgentRun: {
+ findFirst: vi
+ .fn()
+ .mockResolvedValue({ status: InAppAgentRunStatus.QUEUED }),
+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),
+ },
+ };
+ const prisma = {
+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>
+ callback(tx),
+ } as unknown as PrismaClient;
+
+ await expect(
+ requestRunCancellation({
+ prisma,
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ runId: "run-1",
+ }),
+ ).resolves.toEqual({
+ cancelledImmediately: true,
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: InAppAgentRunErrorCode.CANCELLED,
+ });
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: InAppAgentRunErrorCode.CANCELLED,
+ });
+ });
+
+ it("records a prior stale run when admitting a new message", async () => {
+ const tx = {
+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),
+ inAppAgentRun: {
+ findMany: vi.fn().mockResolvedValue([
+ {
+ id: "run-old",
+ status: InAppAgentRunStatus.QUEUED,
+ createdAt: new Date(Date.now() - 10 * 60_000),
+ claimedAt: null,
+ heartbeatAt: null,
+ finishedAt: null,
+ },
+ ]),
+ updateMany: vi.fn(
+ async ({ data }: { data: { status?: InAppAgentRunStatus } }) => ({
+ count: data.status === InAppAgentRunStatus.FAILED ? 1 : 0,
+ }),
+ ),
+ findFirst: vi.fn().mockResolvedValue(null),
+ create: vi.fn().mockResolvedValue({ id: "run-1" }),
+ },
+ inAppAgentEvent: {
+ findFirst: vi.fn().mockResolvedValue(null),
+ create: vi.fn().mockResolvedValue({}),
+ },
+ inAppAgentConversation: {
+ update: vi.fn().mockResolvedValue({}),
+ },
+ };
+ const prisma = {
+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>
+ callback(tx),
+ } as unknown as PrismaClient;
+
+ await createQueuedRun({
+ prisma,
+ runId: "run-1",
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ triggeredByUserId: "user-1",
+ model: "test-model",
+ request: { kind: "userMessage", context: [] },
+ runStartedEvent: {
+ type: EventType.RUN_STARTED,
+ threadId: "conversation-1",
+ runId: "run-1",
+ },
+ });
+
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledTimes(1);
+ expect(metricMocks.recordRunTerminalOutcome).toHaveBeenCalledWith({
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: InAppAgentRunErrorCode.QUEUE_TIMEOUT,
+ });
+ });
+
+ it("does not record a reconciled outcome when admitting the next run fails", async () => {
+ const tx = {
+ $queryRaw: vi.fn().mockResolvedValue([{ deletedAt: null }]),
+ inAppAgentRun: {
+ findMany: vi.fn().mockResolvedValue([
+ {
+ id: "run-old",
+ status: InAppAgentRunStatus.QUEUED,
+ createdAt: new Date(Date.now() - 10 * 60_000),
+ claimedAt: null,
+ heartbeatAt: null,
+ finishedAt: null,
+ },
+ ]),
+ updateMany: vi.fn().mockResolvedValue({ count: 1 }),
+ findFirst: vi.fn().mockResolvedValue({ id: "active-run" }),
+ create: vi.fn(),
+ },
+ };
+ const prisma = {
+ $transaction: async (callback: (transaction: typeof tx) => unknown) =>
+ callback(tx),
+ } as unknown as PrismaClient;
+
+ await expect(
+ createQueuedRun({
+ prisma,
+ runId: "run-1",
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ triggeredByUserId: "user-1",
+ model: "test-model",
+ request: { kind: "userMessage", context: [] },
+ runStartedEvent: {
+ type: EventType.RUN_STARTED,
+ threadId: "conversation-1",
+ runId: "run-1",
+ },
+ }),
+ ).rejects.toThrow("Assistant is already responding in this conversation");
+ expect(metricMocks.recordRunTerminalOutcome).not.toHaveBeenCalled();
});
});
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.ts
index d9604b70c1..4eb6896e15 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts
@@ -7,6 +7,7 @@ import {
import { Prisma } from "../../db";
import type { InAppAgentRun, PrismaClient } from "../../db";
import { logger } from "../../server";
+import { recordRunTerminalOutcome } from "./runMetrics";
import { buildInAppAgentApprovalDecisionEvent } from "../approvalEvents";
import type { AgUiEvent } from "../schema";
import type { InAppAgentPrefixedLangfuseMcpToolName } from "./mcpPolicy";
@@ -15,7 +16,10 @@ import {
resolveInAppAgentRootRunId,
type InAppAgentRunRequest,
} from "../../features/inAppAgent/types";
-import { IN_APP_AGENT_UNSETTLED_RUN_STATUSES } from "../constants";
+import {
+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES,
+ isSettledInAppAgentRunStatus,
+} from "../constants";
import {
ACTIVE_RUN_CONFLICT_MESSAGE,
lockConversation,
@@ -109,6 +113,13 @@ export async function finishClaimedRun(params: {
},
});
+ if (count > 0 && isSettledInAppAgentRunStatus(params.status)) {
+ recordRunTerminalOutcome({
+ status: params.status,
+ errorCode: params.errorCode ?? null,
+ });
+ }
+
return count > 0;
}
@@ -138,61 +149,73 @@ export async function createQueuedRun(params: {
request: InAppAgentRunRequest;
runStartedEvent: AgUiEvent;
}): Promise<InAppAgentRun> {
- return params.prisma.$transaction(async (tx) => {
- await lockConversation(tx, params.projectId, params.conversationId);
-
- await reconcileConversationRunsInTransaction({
- tx,
- projectId: params.projectId,
- conversationId: params.conversationId,
- });
+ const { run, reconciled, supersededCount } = await params.prisma.$transaction(
+ async (tx) => {
+ await lockConversation(tx, params.projectId, params.conversationId);
- // New input supersedes a parked approval.
- await tx.inAppAgentRun.updateMany({
- where: {
+ const reconciled = await reconcileConversationRunsInTransaction({
+ tx,
projectId: params.projectId,
conversationId: params.conversationId,
- status: InAppAgentRunStatus.AWAITING_APPROVAL,
- },
- data: {
- status: InAppAgentRunStatus.CANCELLED,
- errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,
- errorMessage: "Replaced by a newer message",
- },
- });
+ });
- const activeRun = await tx.inAppAgentRun.findFirst({
- where: {
+ // New input supersedes a parked approval.
+ const superseded = await tx.inAppAgentRun.updateMany({
+ where: {
+ projectId: params.projectId,
+ conversationId: params.conversationId,
+ status: InAppAgentRunStatus.AWAITING_APPROVAL,
+ },
+ data: {
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,
+ errorMessage: "Replaced by a newer message",
+ },
+ });
+
+ const activeRun = await tx.inAppAgentRun.findFirst({
+ where: {
+ projectId: params.projectId,
+ conversationId: params.conversationId,
+ finishedAt: null,
+ },
+ select: { id: true },
+ });
+
+ if (activeRun) {
+ throw new LangfuseConflictError(ACTIVE_RUN_CONFLICT_MESSAGE);
+ }
+
+ const run = await createRunRow(tx, {
+ runId: params.runId,
projectId: params.projectId,
conversationId: params.conversationId,
- finishedAt: null,
- },
- select: { id: true },
- });
+ triggeredByUserId: params.triggeredByUserId,
+ model: params.model,
+ request: params.request,
+ });
- if (activeRun) {
- throw new LangfuseConflictError(ACTIVE_RUN_CONFLICT_MESSAGE);
- }
+ await appendConversationEventInTransaction({
+ tx,
+ projectId: params.projectId,
+ conversationId: params.conversationId,
+ runId: params.runId,
+ event: params.runStartedEvent,
+ });
- const run = await createRunRow(tx, {
- runId: params.runId,
- projectId: params.projectId,
- conversationId: params.conversationId,
- triggeredByUserId: params.triggeredByUserId,
- model: params.model,
- request: params.request,
- });
+ return { run, reconciled, supersededCount: superseded.count };
+ },
+ );
- await appendConversationEventInTransaction({
- tx,
- projectId: params.projectId,
- conversationId: params.conversationId,
- runId: params.runId,
- event: params.runStartedEvent,
+ recordReconciledOutcomes(reconciled);
+ for (let i = 0; i < supersededCount; i++) {
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: InAppAgentRunErrorCode.APPROVAL_SUPERSEDED,
});
+ }
- return run;
- });
+ return run;
}
/** Record one approval decision and create its continuation under the lock. */
@@ -241,7 +264,7 @@ export async function decideToolApproval(params: {
parkedAt &&
Date.now() - parkedAt.getTime() > IN_APP_AGENT_APPROVAL_TTL_MS
) {
- await tx.inAppAgentRun.updateMany({
+ const { count } = await tx.inAppAgentRun.updateMany({
where: {
id: params.parentRunId,
projectId: params.projectId,
@@ -254,7 +277,7 @@ export async function decideToolApproval(params: {
},
});
- return { type: "expired" as const };
+ return { type: "expired" as const, expired: count > 0 };
}
// The parent CAS is the exactly-once decision guarantee.
@@ -357,15 +380,31 @@ export async function decideToolApproval(params: {
});
if (outcome.type === "expired") {
+ if (outcome.expired) {
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: InAppAgentRunErrorCode.APPROVAL_EXPIRED,
+ });
+ }
throw new LangfuseConflictError("The approval request expired.");
}
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.SUCCEEDED,
+ });
+
return outcome.run;
}
export type CancelRunResult = {
cancelledImmediately: boolean;
status: InAppAgentRunStatus | null;
+ errorCode?: InAppAgentRunErrorCode;
+};
+
+export type ImmediateCancel = {
+ runId: string;
+ errorCode: InAppAgentRunErrorCode;
};
/** Cancel every run that is executing or waiting on user approval. */
@@ -373,7 +412,7 @@ export async function cancelConversationRunsInTransaction(params: {
tx: InAppAgentTx;
projectId: string;
conversationId: string;
-}): Promise<string[]> {
+}): Promise<ImmediateCancel[]> {
await lockConversation(params.tx, params.projectId, params.conversationId);
const runs = await params.tx.inAppAgentRun.findMany({
@@ -386,7 +425,7 @@ export async function cancelConversationRunsInTransaction(params: {
select: { id: true, status: true },
});
- const cancelledImmediately: string[] = [];
+ const cancelledImmediately: ImmediateCancel[] = [];
for (const run of runs) {
const parsedStatus = InAppAgentRunStatusSchema.safeParse(run.status);
@@ -400,14 +439,28 @@ export async function cancelConversationRunsInTransaction(params: {
runStatus: parsedStatus.data,
});
- if (result.cancelledImmediately) {
- cancelledImmediately.push(run.id);
+ if (result.cancelledImmediately && result.errorCode) {
+ cancelledImmediately.push({
+ runId: run.id,
+ errorCode: result.errorCode,
+ });
}
}
return cancelledImmediately;
}
+export function recordImmediateCancelOutcomes(
+ cancellations: ReadonlyArray<ImmediateCancel>,
+): void {
+ for (const cancellation of cancellations) {
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: cancellation.errorCode,
+ });
+ }
+}
+
/** Cancel idle states immediately; signal RUNNING workers cooperatively. */
export async function requestRunCancellation(params: {
prisma: PrismaClient;
@@ -415,7 +468,7 @@ export async function requestRunCancellation(params: {
conversationId: string;
runId: string;
}): Promise<CancelRunResult> {
- return params.prisma.$transaction(async (tx) => {
+ const result = await params.prisma.$transaction(async (tx) => {
await lockConversation(tx, params.projectId, params.conversationId);
const run = await tx.inAppAgentRun.findFirst({
@@ -440,6 +493,17 @@ export async function requestRunCancellation(params: {
runStatus: parsedStatus.data,
});
});
+
+ // A RUNNING run is only signalled here; the worker writes its own terminal
+ // state, so only an immediate cancel settles the run in this call.
+ if (result.cancelledImmediately) {
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.CANCELLED,
+ errorCode: result.errorCode ?? null,
+ });
+ }
+
+ return result;
}
async function cancelRunInTransaction(params: {
@@ -481,6 +545,7 @@ async function cancelRunInTransaction(params: {
return {
cancelledImmediately: true,
status: InAppAgentRunStatus.CANCELLED,
+ errorCode: immediateCancel.errorCode,
};
}
} else if (params.runStatus !== InAppAgentRunStatus.RUNNING) {
@@ -545,9 +610,20 @@ export async function reconcileConversationRuns(params: {
}),
);
+ recordReconciledOutcomes(reconciled);
+
return reconciled;
}
+function recordReconciledOutcomes(reconciled: ReconciledRun[]): void {
+ for (const run of reconciled) {
+ recordRunTerminalOutcome({
+ status: InAppAgentRunStatus.FAILED,
+ errorCode: run.errorCode,
+ });
+ }
+}
+
async function reconcileConversationRunsInTransaction(params: {
tx: InAppAgentTx;
projectId: string;
diff --git a/packages/shared/src/in-app-agent/server/runMetrics.ts b/packages/shared/src/in-app-agent/server/runMetrics.ts
new file mode 100644
index 0000000000..3f3916bf1d
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/runMetrics.ts
@@ -0,0 +1,27 @@
+import { InAppAgentRunErrorCode } from "../../features/inAppAgent/types";
+import { recordIncrement } from "../../server";
+import type { SettledInAppAgentRunStatus } from "../constants";
+
+const RUN_COMPLETED_METRIC = "langfuse.in_app_agent.run.completed";
+
+/**
+ * Record a run reaching a settled state. Call this only after the terminal CAS
+ * has committed, so a rolled-back transaction cannot inflate the count.
+ *
+ * Settledness is the complement of `IN_APP_AGENT_UNSETTLED_RUN_STATUSES` in
+ * constants.ts. `AWAITING_APPROVAL` is parked, not settled; emitting on the
+ * park would double-count when the parent later becomes SUCCEEDED, FAILED, or
+ * CANCELLED.
+ *
+ * `error_code` is "none" rather than absent so Datadog can group every outcome
+ * on one tag key.
+ */
+export function recordRunTerminalOutcome(params: {
+ status: SettledInAppAgentRunStatus;
+ errorCode?: InAppAgentRunErrorCode | null;
+}): void {
+ recordIncrement(RUN_COMPLETED_METRIC, 1, {
+ status: params.status,
+ error_code: params.errorCode ?? "none",
+ });
+}
diff --git a/web/src/features/in-app-agent/server/backgroundRunService.ts b/web/src/features/in-app-agent/server/backgroundRunService.ts
index 63412b9cc8..9289a0a887 100644
--- a/web/src/features/in-app-agent/server/backgroundRunService.ts
+++ b/web/src/features/in-app-agent/server/backgroundRunService.ts
@@ -35,6 +35,7 @@ import {
createQueuedRun,
decideToolApproval,
reconcileConversationRuns,
+ recordImmediateCancelOutcomes,
requestRunCancellation,
} from "@langfuse/shared/in-app-agent/server/runLifecycle";
@@ -291,7 +292,7 @@ export async function deleteBackgroundConversation(params: {
userId: params.userId,
});
- const cancelledRunIds = await params.prisma.$transaction(async (tx) => {
+ const cancelledRuns = await params.prisma.$transaction(async (tx) => {
const cancelledImmediately = await cancelConversationRunsInTransaction({
tx,
projectId: params.projectId,
@@ -314,8 +315,12 @@ export async function deleteBackgroundConversation(params: {
return cancelledImmediately;
});
+ recordImmediateCancelOutcomes(cancelledRuns);
+
// Avoid spending a worker slot on jobs whose runs are already cancelled.
- await Promise.all(cancelledRunIds.map(removeInAppAgentRunJob));
+ await Promise.all(
+ cancelledRuns.map((run) => removeInAppAgentRunJob(run.runId)),
+ );
return { success: true };
}
diff --git a/worker/src/app.ts b/worker/src/app.ts
index 9c908a957f..c7cc28b521 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -101,6 +101,7 @@ import { QueueMetricsRunner } from "./features/queue-metrics-runner";
import { MonitorRunner } from "./features/monitor-runner";
import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";
import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";
+import { InAppAgentIntegrityRunner } from "./features/in-app-agent-integrity-runner";
import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";
const app = express();
@@ -773,6 +774,14 @@ if (env.LANGFUSE_TRACE_DELETE_BATCH_ACTION_RUNNER_ENABLED === "true") {
traceDeleteBatchActionRunner.start();
}
+// Reconciles stale in-app agent runs that nobody reopened, then reports remainders.
+export let inAppAgentIntegrityRunner: InAppAgentIntegrityRunner | null = null;
+
+if (env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED === "true") {
+ inAppAgentIntegrityRunner = new InAppAgentIntegrityRunner();
+ inAppAgentIntegrityRunner.start();
+}
+
// ClickHouse deleted-mask cleaner for physically applying lightweight delete masks
export let deletedMaskCleaner: DeletedMaskCleaner | null = null;
diff --git a/worker/src/env.ts b/worker/src/env.ts
index a3ac45e9d6..2e4921950f 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -239,6 +239,11 @@ const EnvSchema = z.object({
QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED: z
.enum(["true", "false"])
.default("false"),
+ // Reconciles stale runs that nobody reopened. Off by default: one elected
+ // worker per region is enough.
+ LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
// The ambient host profile takes precedence over the agent-specific default
// so local developer credentials win when both are configured.
AWS_PROFILE: z.string().optional(),
@@ -651,6 +656,13 @@ const EnvSchema = z.object({
.number()
.positive()
.default(5),
+ // Default 15 minutes. Do not go below the heartbeat-stale window (60s) or a
+ // live worker can lose the race. Faster also adds no resolution: every
+ // deadline this runner reports against is 5 minutes or longer.
+ LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS: z.coerce
+ .number()
+ .positive()
+ .default(15 * 60_000),
LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS: z.coerce
.number()
.positive()
diff --git a/worker/src/features/in-app-agent-integrity-runner/index.test.ts b/worker/src/features/in-app-agent-integrity-runner/index.test.ts
new file mode 100644
index 0000000000..e552a717ac
--- /dev/null
+++ b/worker/src/features/in-app-agent-integrity-runner/index.test.ts
@@ -0,0 +1,143 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+ InAppAgentRunErrorCode,
+ InAppAgentRunStatus,
+} from "@langfuse/shared/in-app-agent";
+import { IN_APP_AGENT_HEARTBEAT_STALE_MS } from "@langfuse/shared/in-app-agent/server/tunables";
+
+const mocks = vi.hoisted(() => ({
+ recordGauge: vi.fn(),
+ runFindMany: vi.fn(),
+ runGroupBy: vi.fn(),
+ apiKeyCount: vi.fn(),
+ reconcileConversationRuns: vi.fn(),
+ cleanupTerminalRunMcpApiKeys: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+ recordGauge: mocks.recordGauge,
+ redis: {},
+}));
+
+vi.mock("@langfuse/shared/src/server/auth/apiKeys", () => ({
+ deleteApiKeyFromDb: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: {
+ inAppAgentRun: {
+ findMany: mocks.runFindMany,
+ groupBy: mocks.runGroupBy,
+ },
+ apiKey: { count: mocks.apiKeyCount },
+ },
+}));
+
+vi.mock(
+ "@langfuse/shared/in-app-agent/server/runLifecycle",
+ async (importOriginal) => ({
+ ...(await importOriginal<
+ typeof import("@langfuse/shared/in-app-agent/server/runLifecycle")
+ >()),
+ reconcileConversationRuns: mocks.reconcileConversationRuns,
+ cleanupTerminalRunMcpApiKeys: mocks.cleanupTerminalRunMcpApiKeys,
+ }),
+);
+
+vi.mock("../../env", () => ({
+ env: { LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS: 60_000 },
+}));
+
+import { InAppAgentIntegrityRunner } from "./index";
+
+function runnerWithStubbedLock() {
+ const runner = new InAppAgentIntegrityRunner();
+ const extend = vi.fn().mockResolvedValue(true);
+ (runner as unknown as { lock: unknown }).lock = {
+ withLock: async (operation: () => Promise<unknown>) => operation(),
+ extend,
+ };
+ return { runner, extend };
+}
+
+function runningRow(params: {
+ id: string;
+ heartbeatAt: Date;
+ claimedAt?: Date;
+}) {
+ const claimedAt = params.claimedAt ?? params.heartbeatAt;
+ return {
+ id: params.id,
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ status: InAppAgentRunStatus.RUNNING,
+ createdAt: claimedAt,
+ claimedAt,
+ heartbeatAt: params.heartbeatAt,
+ finishedAt: null,
+ };
+}
+
+describe("InAppAgentIntegrityRunner", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.runGroupBy.mockResolvedValue([]);
+ mocks.runFindMany.mockResolvedValue([]);
+ mocks.apiKeyCount.mockResolvedValue(0);
+ mocks.reconcileConversationRuns.mockResolvedValue([]);
+ mocks.cleanupTerminalRunMcpApiKeys.mockResolvedValue(0);
+ });
+
+ it("reconciles a stale RUNNING conversation and cleans its MCP keys", async () => {
+ const heartbeatAt = new Date(
+ Date.now() - IN_APP_AGENT_HEARTBEAT_STALE_MS - 1,
+ );
+ mocks.runFindMany
+ .mockResolvedValueOnce([runningRow({ id: "run-1", heartbeatAt })])
+ .mockResolvedValueOnce([]);
+ mocks.reconcileConversationRuns.mockResolvedValue([
+ { runId: "run-1", errorCode: InAppAgentRunErrorCode.WORKER_LOST },
+ ]);
+
+ mocks.runGroupBy.mockResolvedValue([
+ { status: InAppAgentRunStatus.RUNNING, _count: { _all: 9 } },
+ ]);
+
+ const { runner, extend } = runnerWithStubbedLock();
+ await runner.processBatch();
+
+ expect(extend).toHaveBeenCalled();
+ expect(mocks.reconcileConversationRuns).toHaveBeenCalledExactlyOnceWith({
+ prisma: expect.anything(),
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ });
+ expect(mocks.cleanupTerminalRunMcpApiKeys).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ }),
+ );
+ expect(mocks.recordGauge).toHaveBeenCalledWith(
+ "langfuse.in_app_agent.active_runs",
+ 9,
+ { status: InAppAgentRunStatus.RUNNING },
+ );
+ });
+
+ it("does not reconcile a healthy RUNNING conversation", async () => {
+ mocks.runFindMany
+ .mockResolvedValueOnce([
+ runningRow({ id: "run-1", heartbeatAt: new Date() }),
+ ])
+ .mockResolvedValueOnce([]);
+
+ await runnerWithStubbedLock().runner.processBatch();
+
+ expect(mocks.reconcileConversationRuns).not.toHaveBeenCalled();
+ expect(mocks.cleanupTerminalRunMcpApiKeys).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/in-app-agent-integrity-runner/index.ts b/worker/src/features/in-app-agent-integrity-runner/index.ts
new file mode 100644
index 0000000000..a384f2f632
--- /dev/null
+++ b/worker/src/features/in-app-agent-integrity-runner/index.ts
@@ -0,0 +1,272 @@
+import { prisma, type Prisma } from "@langfuse/shared/src/db";
+import { logger, recordGauge, redis } from "@langfuse/shared/src/server";
+import { deleteApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";
+import {
+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES,
+ InAppAgentRunErrorCode,
+ InAppAgentRunStatus,
+} from "@langfuse/shared/in-app-agent";
+import {
+ classifyStaleRun,
+ cleanupTerminalRunMcpApiKeys,
+ reconcileConversationRuns,
+} from "@langfuse/shared/in-app-agent/server/runLifecycle";
+import {
+ IN_APP_AGENT_APPROVAL_TTL_MS,
+ IN_APP_AGENT_HEARTBEAT_STALE_MS,
+ IN_APP_AGENT_QUEUE_TIMEOUT_MS,
+ IN_APP_AGENT_RUN_MAX_DURATION_MS,
+} from "@langfuse/shared/in-app-agent/server/tunables";
+
+import { env } from "../../env";
+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+
+const METRIC_PREFIX = "langfuse.in_app_agent";
+
+export const IN_APP_AGENT_INTEGRITY_RUNNER_LOCK_KEY =
+ "langfuse:in-app-agent-integrity-runner";
+
+const LOCK_TTL_SECONDS = 120;
+const SCAN_LIMIT = 5_000;
+
+const REPORTED_FINDINGS = [
+ InAppAgentRunErrorCode.QUEUE_TIMEOUT,
+ InAppAgentRunErrorCode.WORKER_LOST,
+ InAppAgentRunErrorCode.RUN_TIMEOUT,
+ InAppAgentRunErrorCode.APPROVAL_EXPIRED,
+];
+
+/**
+ * Reconciles stale in-app agent runs that nobody reopened, then reports what
+ * remains. Read paths still reconcile on attach; this runner is the backstop
+ * so a conversation nobody opens cannot hold a slot forever.
+ *
+ * Writes go through `reconcileConversationRuns` so the verdict cannot drift
+ * from web. Gauges are emitted after repair.
+ */
+export class InAppAgentIntegrityRunner extends PeriodicExclusiveRunner {
+ private readonly intervalMs: number;
+
+ protected get defaultIntervalMs(): number {
+ return this.intervalMs;
+ }
+
+ constructor(opts: { intervalMs?: number } = {}) {
+ super({
+ name: "InAppAgentIntegrityRunner",
+ metricName: "in_app_agent_integrity_runner",
+ lockKey: IN_APP_AGENT_INTEGRITY_RUNNER_LOCK_KEY,
+ lockTtlSeconds: LOCK_TTL_SECONDS,
+ onUnavailable: "fail",
+ });
+
+ this.intervalMs =
+ opts.intervalMs ?? env.LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_INTERVAL_MS;
+ }
+
+ public override start(): void {
+ logger.info(`Starting ${this.instanceName}`, {
+ intervalMs: this.intervalMs,
+ });
+ super.start();
+ }
+
+ protected async execute(): Promise<number | void> {
+ return await this.withLock(async () => {
+ await this.reconcileAndReportUnsettledRuns();
+ await this.reportOrphanedMcpApiKeys();
+ });
+ }
+
+ private async reconcileAndReportUnsettledRuns(): Promise<void> {
+ const now = Date.now();
+ const candidates = await prisma.inAppAgentRun.findMany({
+ where: staleRunWhere(now),
+ select: {
+ id: true,
+ projectId: true,
+ conversationId: true,
+ status: true,
+ createdAt: true,
+ claimedAt: true,
+ heartbeatAt: true,
+ finishedAt: true,
+ },
+ take: SCAN_LIMIT,
+ });
+
+ if (candidates.length === SCAN_LIMIT) {
+ logger.warn(
+ `${this.instanceName}: stale run scan hit its limit; repair is a floor`,
+ { scanLimit: SCAN_LIMIT },
+ );
+ }
+
+ const staleConversations = new Map<
+ string,
+ { projectId: string; conversationId: string }
+ >();
+
+ for (const run of candidates) {
+ if (!classifyStaleRun(run, now)) continue;
+ staleConversations.set(`${run.projectId}:${run.conversationId}`, {
+ projectId: run.projectId,
+ conversationId: run.conversationId,
+ });
+ }
+
+ const reconciledIds = new Set<string>();
+
+ for (const { projectId, conversationId } of staleConversations.values()) {
+ await this.extendLockOnProgress();
+
+ try {
+ const reconciled = await reconcileConversationRuns({
+ prisma,
+ projectId,
+ conversationId,
+ });
+
+ for (const run of reconciled) {
+ reconciledIds.add(run.runId);
+ }
+
+ if (reconciled.length === 0) continue;
+
+ await cleanupTerminalRunMcpApiKeys({
+ prisma,
+ projectId,
+ conversationId,
+ deleteApiKey: async (apiKeyId) => {
+ await deleteApiKeyFromDb({
+ prisma,
+ id: apiKeyId,
+ entityId: projectId,
+ scope: "PROJECT",
+ redis,
+ });
+ },
+ });
+ } catch (error) {
+ logger.error(`${this.instanceName}: failed to reconcile conversation`, {
+ projectId,
+ conversationId,
+ error,
+ });
+ }
+ }
+
+ const activeByStatusRows = await prisma.inAppAgentRun.groupBy({
+ by: ["status"],
+ where: { status: { in: [...IN_APP_AGENT_UNSETTLED_RUN_STATUSES] } },
+ _count: { _all: true },
+ });
+
+ const activeByStatus = new Map<string, number>(
+ IN_APP_AGENT_UNSETTLED_RUN_STATUSES.map((status) => [status, 0]),
+ );
+ for (const row of activeByStatusRows) {
+ if (row.status) {
+ activeByStatus.set(row.status, row._count._all);
+ }
+ }
+
+ const findings = new Map<string, number>(
+ REPORTED_FINDINGS.map((code) => [code, 0]),
+ );
+ for (const run of candidates) {
+ const stale = classifyStaleRun(run, now);
+ if (!stale || reconciledIds.has(run.id)) continue;
+ findings.set(stale.errorCode, (findings.get(stale.errorCode) ?? 0) + 1);
+ }
+
+ for (const [status, count] of activeByStatus) {
+ recordGauge(`${METRIC_PREFIX}.active_runs`, count, { status });
+ }
+
+ for (const [finding, count] of findings) {
+ recordGauge(`${METRIC_PREFIX}.lifecycle_integrity`, count, { finding });
+ }
+ }
+
+ /**
+ * Scoped MCP keys are deleted best-effort when a run ends, and the pointer is
+ * cleared only after the delete confirms. Two different problems hide behind a
+ * lingering pointer, so they are counted apart: `live_key` is a credential
+ * that still exists and never expires, `stuck_pointer` is a pointer whose key
+ * is already gone, which is harmless but retries and logs an error on every
+ * conversation read.
+ */
+ private async reportOrphanedMcpApiKeys(): Promise<void> {
+ const runs = await prisma.inAppAgentRun.findMany({
+ where: {
+ finishedAt: { not: null },
+ mcpApiKeyId: { not: null },
+ status: { notIn: [InAppAgentRunStatus.AWAITING_APPROVAL] },
+ },
+ select: { mcpApiKeyId: true },
+ take: SCAN_LIMIT,
+ });
+
+ if (runs.length === SCAN_LIMIT) {
+ logger.warn(
+ `${this.instanceName}: MCP key scan hit its limit; counts are a floor`,
+ { scanLimit: SCAN_LIMIT },
+ );
+ }
+
+ const pointerIds = runs
+ .map((run) => run.mcpApiKeyId)
+ .filter((id): id is string => Boolean(id));
+
+ let liveKeys = 0;
+ if (pointerIds.length > 0) {
+ liveKeys = await prisma.apiKey.count({
+ where: { id: { in: pointerIds } },
+ });
+ }
+
+ recordGauge(`${METRIC_PREFIX}.orphaned_mcp_api_keys`, liveKeys, {
+ kind: "live_key",
+ });
+ recordGauge(
+ `${METRIC_PREFIX}.orphaned_mcp_api_keys`,
+ pointerIds.length - liveKeys,
+ { kind: "stuck_pointer" },
+ );
+ }
+}
+
+function staleRunWhere(now: number): Prisma.InAppAgentRunWhereInput {
+ const queueCutoff = new Date(now - IN_APP_AGENT_QUEUE_TIMEOUT_MS);
+ const durationCutoff = new Date(now - IN_APP_AGENT_RUN_MAX_DURATION_MS);
+ const heartbeatCutoff = new Date(now - IN_APP_AGENT_HEARTBEAT_STALE_MS);
+ const approvalCutoff = new Date(now - IN_APP_AGENT_APPROVAL_TTL_MS);
+
+ return {
+ OR: [
+ {
+ status: InAppAgentRunStatus.QUEUED,
+ createdAt: { lt: queueCutoff },
+ },
+ {
+ status: InAppAgentRunStatus.RUNNING,
+ OR: [
+ { claimedAt: { lt: durationCutoff } },
+ { claimedAt: null, createdAt: { lt: durationCutoff } },
+ { heartbeatAt: { lt: heartbeatCutoff } },
+ { heartbeatAt: null, claimedAt: { lt: heartbeatCutoff } },
+ {
+ heartbeatAt: null,
+ claimedAt: null,
+ createdAt: { lt: heartbeatCutoff },
+ },
+ ],
+ },
+ {
+ status: InAppAgentRunStatus.AWAITING_APPROVAL,
+ finishedAt: { lt: approvalCutoff },
+ },
+ ],
+ };
+}
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
index d1f29fe963..8efa10fdcf 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
@@ -507,7 +507,11 @@ describe("executeInAppAgentRun", () => {
scenarioRef.current = async ({ options }) => {
await options.onEvent(textChunk("Still calling tools"));
- await options.onComplete({ truncatedByStepLimit: true });
+ // Truncation implies the cap was reached: wrap-up fires one step earlier.
+ await options.onComplete({
+ reachedStepLimit: true,
+ truncatedByStepLimit: true,
+ });
await options.onFinish();
};
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index f5e20f25f1..cbf9534624 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -3,6 +3,7 @@ import { prisma } from "@langfuse/shared/src/db";
import {
getLangfuseAITraceSinkParams,
logger,
+ recordIncrement,
redis,
traceException,
} from "@langfuse/shared/src/server";
@@ -467,6 +468,14 @@ export async function executeInAppAgentRun(params: {
},
onMcpToolCallCompleted: sandboxToolCallFiles.processToolCall,
onComplete: async (outcome) => {
+ // Truncation lands on run.completed via the error code below, but that
+ // only counts turns wrap-up failed to rescue. Counting every turn that
+ // reached the cap is what shows the cap becoming binding before users
+ // see a cut-off answer.
+ if (outcome?.reachedStepLimit) {
+ recordIncrement("langfuse.in_app_agent.step_limit_reached", 1);
+ }
+
await flushPersistedRunEvents(
interruptRequest
? { status: InAppAgentRunStatus.AWAITING_APPROVAL }
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index c2acb2dda0..f7f7a3c87f 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -159,6 +159,8 @@ This is your final step. Do not call any more tools. Summarize what you have fou
</step_limit_wrap_up>`;
export type InAppAgentCompleteOutcome = {
+ /** The turn reached the step cap, whether or not wrap-up rescued it. */
+ reachedStepLimit: boolean;
truncatedByStepLimit: boolean;
};
@@ -792,6 +794,7 @@ export async function createAgUiStream(params: {
instrumentation.flush(),
);
return params.options.onComplete?.({
+ reachedStepLimit: stepLimitState.wrapUp,
truncatedByStepLimit,
});
}
diff --git a/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts b/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts
index ed7943968d..af7576fe58 100644
--- a/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts
+++ b/worker/src/features/in-app-agent/runtime/mcpRateLimitWait.ts
@@ -1,9 +1,11 @@
import { InAppAgentRateLimitErrorResponseSchema } from "@langfuse/shared/in-app-agent";
-import { logger } from "@langfuse/shared/src/server";
+import { logger, recordIncrement } from "@langfuse/shared/src/server";
/** Total time a single MCP operation may spend waiting out rate limits. */
export const IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS = 60_000;
+const MCP_RATE_LIMIT_WAIT_METRIC = "langfuse.in_app_agent.mcp_rate_limit_wait";
+
type McpRateLimitError = {
retryAfterSeconds: number;
};
@@ -70,11 +72,23 @@ export async function withMcpRateLimitWait<T>(params: {
}
const waitMs = rateLimit.retryAfterSeconds * 1_000;
+ const operation = String(params.logContext.operation ?? "unknown");
if (waitMs > remainingBudgetMs) {
+ // The run fails from here on, so this is the tag that separates "we
+ // absorbed a rate limit" from "a rate limit killed the turn".
+ recordIncrement(MCP_RATE_LIMIT_WAIT_METRIC, 1, {
+ outcome: "budget_exhausted",
+ operation,
+ });
throw error;
}
+ recordIncrement(MCP_RATE_LIMIT_WAIT_METRIC, 1, {
+ outcome: "waited",
+ operation,
+ });
+
logger.warn("Waiting out in-app agent MCP rate limit", {
...params.logContext,
retryAfterSeconds: rateLimit.retryAfterSeconds,
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 69c8e1672c..509370870c 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -19,6 +19,7 @@ import {
batchProjectBlobCleaner,
batchTraceDeletionCleaner,
traceDeleteBatchActionRunner,
+ inAppAgentIntegrityRunner,
deletedMaskCleaner,
queueMetricsRunner,
monitorRunners,
@@ -58,6 +59,8 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => {
// Stop durable trace-delete batch action runner
traceDeleteBatchActionRunner?.stop();
+ inAppAgentIntegrityRunner?.stop();
+
// Stop deleted-mask cleaner
deletedMaskCleaner?.stop();