🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): score message feedback on the approval-chain root run (#16236)
Lysander Kiesel1aca52633052
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.tsindex dbff305eab..87cfd49f9a 100644--- a/packages/shared/src/features/inAppAgent/types.ts+++ b/packages/shared/src/features/inAppAgent/types.ts@@ -97,3 +97,15 @@ export const InAppAgentRunRequestSchema = z.discriminatedUnion("kind", []);export type InAppAgentRunRequest = z.infer<typeof InAppAgentRunRequestSchema>;++// Approval continuations inherit the root run's trace ids, so telemetry keyed+// off a run must resolve the root first.+export const resolveInAppAgentRootRunId = (+ request: unknown,+ runId: string,+): string => {+ const parsed = InAppAgentRunRequestSchema.safeParse(request);+ return parsed.success && parsed.data.kind === "approvalDecision"+ ? (parsed.data.rootRunId ?? parsed.data.parentRunId)+ : runId;+};diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.tsindex d0d7c4bcbb..2f1c87c359 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts@@ -15,6 +15,7 @@ import type { AgUiEvent } from "../schema";import type { InAppAgentPrefixedLangfuseMcpToolName } from "./tools";import {InAppAgentRunRequestSchema,+ resolveInAppAgentRootRunId,type InAppAgentRunRequest,} from "../../features/inAppAgent/types";import {@@ -282,10 +283,10 @@ export async function decideToolApproval(params: {parentRequest.success && parentRequest.data.kind === "approvalDecision"? (parentRequest.data.continuationNumber ?? 1) + 1: 1;- const rootRunId =- parentRequest.success && parentRequest.data.kind === "approvalDecision"- ? (parentRequest.data.rootRunId ?? params.parentRunId)- : params.parentRunId;+ const rootRunId = resolveInAppAgentRootRunId(+ parentRun.request,+ params.parentRunId,+ );const traceStartedAt =parentRequest.success &&parentRequest.data.kind === "approvalDecision" &&diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.tsindex b47234d85c..d7ae70561d 100644--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts@@ -11,21 +11,26 @@ import type { Flags } from "@/src/features/feature-flags/types";import { EventType } from "@ag-ui/core";import { randomUUID } from "crypto";import { vi } from "vitest";+import waitForExpect from "wait-for-expect";import {InAppAgentRunErrorCode,InAppAgentRunStatus,+ type InAppAgentRunRequest,type Plan,} from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import {createOrgProjectAndApiKey,generateLLMText,+ getScoreById,} from "@langfuse/shared/src/server";import { env } from "@/src/env.mjs";import {createInAppAgentConversationId,createInAppAgentRunId,+ getInAppAgentInstrumentationObservationId,+ getInAppAgentInstrumentationTraceId,} from "@langfuse/shared/in-app-agent";import {dropEmptyAssistantMessages,@@ -154,6 +159,7 @@ describe("in-app agent persistence", () => {conversationId: string;userId: string;runId?: string;+ request?: InAppAgentRunRequest;}) => {const now = new Date();@@ -168,6 +174,7 @@ describe("in-app agent persistence", () => {status: InAppAgentRunStatus.RUNNING,claimedAt: now,heartbeatAt: now,+ ...(params.request ? { request: params.request } : {}),},});};@@ -630,10 +637,10 @@ describe("in-app agent persistence", () => {}});- it("requires feedback run ids to match persisted assistant messages", async () => {+ it("scores feedback on the approval-chain root run and rejects run id mismatches", async () => {const { caller, projectId, userId } = await createCaller();const conversation = await createConversation({ projectId, userId });- const run = await createClaimedRunFixture({+ const rootRun = await createClaimedRunFixture({projectId,conversationId: conversation.id,userId,@@ -641,14 +648,31 @@ describe("in-app agent persistence", () => {const events = await startCompactRun({projectId,conversationId: conversation.id,- runId: run.id,+ runId: rootRun.id,messageId: "feedback-user",content: "Answer me",});+ // The final answer comes from an approval continuation, but the trace it is+ // recorded on belongs to the root run. The parent settles when it parks for+ // approval, so only the continuation is active.+ await finishConversationRun({ projectId, runId: rootRun.id });+ const continuationRun = await createClaimedRunFixture({+ projectId,+ conversationId: conversation.id,+ userId,+ request: {+ kind: "approvalDecision",+ parentRunId: rootRun.id,+ rootRunId: rootRun.id,+ toolCallId: "tool-call-1",+ approved: true,+ context: [],+ },+ });await appendAssistantText({projectId,conversationId: conversation.id,- runId: run.id,+ runId: continuationRun.id,events,messageId: "feedback-assistant",chunks: ["Here is an answer"],@@ -667,16 +691,37 @@ describe("in-app agent persistence", () => {"Feedback can only be submitted for persisted assistant messages",);- await expect(- caller.submitFeedback({- projectId,- conversationId: conversation.id,- messageId: "feedback-assistant",- runId: run.id,- value: null,- comment: null,- }),- ).resolves.toEqual({ feedback: null });+ const originalAiFeaturesProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;+ try {+ (env as any).LANGFUSE_AI_FEATURES_PROJECT_ID = projectId;++ await expect(+ caller.submitFeedback({+ projectId,+ conversationId: conversation.id,+ messageId: "feedback-assistant",+ runId: continuationRun.id,+ value: "thumbs_up",+ comment: null,+ }),+ ).resolves.toEqual({ feedback: { value: "thumbs_up", comment: null } });++ await waitForExpect(async () => {+ const score = await getScoreById({+ projectId,+ scoreId: `afbs_feedback-assistant_${userId}`,+ });+ expect(score?.traceId).toBe(+ getInAppAgentInstrumentationTraceId(rootRun.id),+ );+ expect(score?.observationId).toBe(+ getInAppAgentInstrumentationObservationId(rootRun.id),+ );+ });+ } finally {+ (env as any).LANGFUSE_AI_FEATURES_PROJECT_ID =+ originalAiFeaturesProjectId;+ }});it("does not reduce partial assistant content before the end event", async () => {diff --git a/web/src/features/in-app-agent/server/router.ts b/web/src/features/in-app-agent/server/router.tsindex b9bbc83db5..6747e2419b 100644--- a/web/src/features/in-app-agent/server/router.ts+++ b/web/src/features/in-app-agent/server/router.ts@@ -2,6 +2,7 @@ import { z } from "zod";import {InvalidRequestError,+ resolveInAppAgentRootRunId,ScoreDataTypeEnum,ScoreSourceEnum,TEXT_SCORE_MAX_LENGTH,@@ -400,15 +401,26 @@ export const inAppAgentRouter = createTRPCRouter({const scoreProjectId = env.LANGFUSE_AI_FEATURES_PROJECT_ID;if (projectAvailability.aiTelemetryEnabled && scoreProjectId) {+ // Approval continuations trace onto the root run, so score the root too.+ const run = await ctx.prisma.inAppAgentRun.findUnique({+ where: {+ id_projectId: { id: input.runId, projectId: input.projectId },+ },+ select: { request: true },+ });+ const telemetryRunId = resolveInAppAgentRootRunId(+ run?.request,+ input.runId,+ );+await upsertScore({id: scoreId,timestamp: convertDateToClickhouseDateTime(now),project_id: scoreProjectId,environment: IN_APP_AGENT_FEEDBACK_ENVIRONMENT,- trace_id: getInAppAgentInstrumentationTraceId(input.runId),- observation_id: getInAppAgentInstrumentationObservationId(- input.runId,- ),+ trace_id: getInAppAgentInstrumentationTraceId(telemetryRunId),+ observation_id:+ getInAppAgentInstrumentationObservationId(telemetryRunId),session_id: input.conversationId,name: IN_APP_AGENT_FEEDBACK_SCORE_NAME,value: input.value === "thumbs_up" ? 1 : 0,