Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): score message feedback on the approval-chain root run (#16236)

Lysander Kiesel1aca52633052
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.ts
index dbff305eab..87cfd49f9a 100644
--- a/packages/shared/src/features/inAppAgent/types.ts
+++ b/packages/shared/src/features/inAppAgent/types.ts
@@ -97,3 +97,15 @@ export const InAppAgentRunRequestSchema = z.discriminatedUnion("kind", [
]);
export type InAppAgentRunRequest = z.infer<typeof InAppAgentRunRequestSchema>;
+
+// Approval continuations inherit the root run's trace ids, so telemetry keyed
+// off a run must resolve the root first.
+export const resolveInAppAgentRootRunId = (
+ request: unknown,
+ runId: string,
+): string => {
+ const parsed = InAppAgentRunRequestSchema.safeParse(request);
+ return parsed.success && parsed.data.kind === "approvalDecision"
+ ? (parsed.data.rootRunId ?? parsed.data.parentRunId)
+ : runId;
+};
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.ts
index d0d7c4bcbb..2f1c87c359 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts
@@ -15,6 +15,7 @@ import type { AgUiEvent } from "../schema";
import type { InAppAgentPrefixedLangfuseMcpToolName } from "./tools";
import {
InAppAgentRunRequestSchema,
+ resolveInAppAgentRootRunId,
type InAppAgentRunRequest,
} from "../../features/inAppAgent/types";
import {
@@ -282,10 +283,10 @@ export async function decideToolApproval(params: {
parentRequest.success && parentRequest.data.kind === "approvalDecision"
? (parentRequest.data.continuationNumber ?? 1) + 1
: 1;
- const rootRunId =
- parentRequest.success && parentRequest.data.kind === "approvalDecision"
- ? (parentRequest.data.rootRunId ?? params.parentRunId)
- : params.parentRunId;
+ const rootRunId = resolveInAppAgentRootRunId(
+ parentRun.request,
+ params.parentRunId,
+ );
const traceStartedAt =
parentRequest.success &&
parentRequest.data.kind === "approvalDecision" &&
diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
index b47234d85c..d7ae70561d 100644
--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
@@ -11,21 +11,26 @@ import type { Flags } from "@/src/features/feature-flags/types";
import { EventType } from "@ag-ui/core";
import { randomUUID } from "crypto";
import { vi } from "vitest";
+import waitForExpect from "wait-for-expect";
import {
InAppAgentRunErrorCode,
InAppAgentRunStatus,
+ type InAppAgentRunRequest,
type Plan,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
createOrgProjectAndApiKey,
generateLLMText,
+ getScoreById,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import {
createInAppAgentConversationId,
createInAppAgentRunId,
+ getInAppAgentInstrumentationObservationId,
+ getInAppAgentInstrumentationTraceId,
} from "@langfuse/shared/in-app-agent";
import {
dropEmptyAssistantMessages,
@@ -154,6 +159,7 @@ describe("in-app agent persistence", () => {
conversationId: string;
userId: string;
runId?: string;
+ request?: InAppAgentRunRequest;
}) => {
const now = new Date();
@@ -168,6 +174,7 @@ describe("in-app agent persistence", () => {
status: InAppAgentRunStatus.RUNNING,
claimedAt: now,
heartbeatAt: now,
+ ...(params.request ? { request: params.request } : {}),
},
});
};
@@ -630,10 +637,10 @@ describe("in-app agent persistence", () => {
}
});
- it("requires feedback run ids to match persisted assistant messages", async () => {
+ it("scores feedback on the approval-chain root run and rejects run id mismatches", async () => {
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
- const run = await createClaimedRunFixture({
+ const rootRun = await createClaimedRunFixture({
projectId,
conversationId: conversation.id,
userId,
@@ -641,14 +648,31 @@ describe("in-app agent persistence", () => {
const events = await startCompactRun({
projectId,
conversationId: conversation.id,
- runId: run.id,
+ runId: rootRun.id,
messageId: "feedback-user",
content: "Answer me",
});
+ // The final answer comes from an approval continuation, but the trace it is
+ // recorded on belongs to the root run. The parent settles when it parks for
+ // approval, so only the continuation is active.
+ await finishConversationRun({ projectId, runId: rootRun.id });
+ const continuationRun = await createClaimedRunFixture({
+ projectId,
+ conversationId: conversation.id,
+ userId,
+ request: {
+ kind: "approvalDecision",
+ parentRunId: rootRun.id,
+ rootRunId: rootRun.id,
+ toolCallId: "tool-call-1",
+ approved: true,
+ context: [],
+ },
+ });
await appendAssistantText({
projectId,
conversationId: conversation.id,
- runId: run.id,
+ runId: continuationRun.id,
events,
messageId: "feedback-assistant",
chunks: ["Here is an answer"],
@@ -667,16 +691,37 @@ describe("in-app agent persistence", () => {
"Feedback can only be submitted for persisted assistant messages",
);
- await expect(
- caller.submitFeedback({
- projectId,
- conversationId: conversation.id,
- messageId: "feedback-assistant",
- runId: run.id,
- value: null,
- comment: null,
- }),
- ).resolves.toEqual({ feedback: null });
+ const originalAiFeaturesProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;
+ try {
+ (env as any).LANGFUSE_AI_FEATURES_PROJECT_ID = projectId;
+
+ await expect(
+ caller.submitFeedback({
+ projectId,
+ conversationId: conversation.id,
+ messageId: "feedback-assistant",
+ runId: continuationRun.id,
+ value: "thumbs_up",
+ comment: null,
+ }),
+ ).resolves.toEqual({ feedback: { value: "thumbs_up", comment: null } });
+
+ await waitForExpect(async () => {
+ const score = await getScoreById({
+ projectId,
+ scoreId: `afbs_feedback-assistant_${userId}`,
+ });
+ expect(score?.traceId).toBe(
+ getInAppAgentInstrumentationTraceId(rootRun.id),
+ );
+ expect(score?.observationId).toBe(
+ getInAppAgentInstrumentationObservationId(rootRun.id),
+ );
+ });
+ } finally {
+ (env as any).LANGFUSE_AI_FEATURES_PROJECT_ID =
+ originalAiFeaturesProjectId;
+ }
});
it("does not reduce partial assistant content before the end event", async () => {
diff --git a/web/src/features/in-app-agent/server/router.ts b/web/src/features/in-app-agent/server/router.ts
index b9bbc83db5..6747e2419b 100644
--- a/web/src/features/in-app-agent/server/router.ts
+++ b/web/src/features/in-app-agent/server/router.ts
@@ -2,6 +2,7 @@ import { z } from "zod";
import {
InvalidRequestError,
+ resolveInAppAgentRootRunId,
ScoreDataTypeEnum,
ScoreSourceEnum,
TEXT_SCORE_MAX_LENGTH,
@@ -400,15 +401,26 @@ export const inAppAgentRouter = createTRPCRouter({
const scoreProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;
if (projectAvailability.aiTelemetryEnabled && scoreProjectId) {
+ // Approval continuations trace onto the root run, so score the root too.
+ const run = await ctx.prisma.inAppAgentRun.findUnique({
+ where: {
+ id_projectId: { id: input.runId, projectId: input.projectId },
+ },
+ select: { request: true },
+ });
+ const telemetryRunId = resolveInAppAgentRootRunId(
+ run?.request,
+ input.runId,
+ );
+
await upsertScore({
id: scoreId,
timestamp: convertDateToClickhouseDateTime(now),
project_id: scoreProjectId,
environment: IN_APP_AGENT_FEEDBACK_ENVIRONMENT,
- trace_id: getInAppAgentInstrumentationTraceId(input.runId),
- observation_id: getInAppAgentInstrumentationObservationId(
- input.runId,
- ),
+ trace_id: getInAppAgentInstrumentationTraceId(telemetryRunId),
+ observation_id:
+ getInAppAgentInstrumentationObservationId(telemetryRunId),
session_id: input.conversationId,
name: IN_APP_AGENT_FEEDBACK_SCORE_NAME,
value: input.value === "thumbs_up" ? 1 : 0,