Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(in-app-agent): assert feedback on an ordinary turn scores its own run (#16240)

Lysander Kieselee28d86f18fa
diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
index d7ae70561d..f561e6f26b 100644
--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
@@ -652,6 +652,14 @@ describe("in-app agent persistence", () => {
messageId: "feedback-user",
content: "Answer me",
});
+ await appendAssistantText({
+ projectId,
+ conversationId: conversation.id,
+ runId: rootRun.id,
+ events,
+ messageId: "plain-assistant",
+ chunks: ["Answering directly"],
+ });
// The final answer comes from an approval continuation, but the trace it is
// recorded on belongs to the root run. The parent settles when it parks for
// approval, so only the continuation is active.
@@ -706,17 +714,39 @@ describe("in-app agent persistence", () => {
}),
).resolves.toEqual({ feedback: { value: "thumbs_up", comment: null } });
- await waitForExpect(async () => {
- const score = await getScoreById({
+ await expect(
+ caller.submitFeedback({
projectId,
- scoreId: `afbs_feedback-assistant_${userId}`,
- });
- expect(score?.traceId).toBe(
- getInAppAgentInstrumentationTraceId(rootRun.id),
- );
- expect(score?.observationId).toBe(
- getInAppAgentInstrumentationObservationId(rootRun.id),
- );
+ conversationId: conversation.id,
+ messageId: "plain-assistant",
+ runId: rootRun.id,
+ value: "thumbs_down",
+ comment: null,
+ }),
+ ).resolves.toEqual({ feedback: { value: "thumbs_down", comment: null } });
+
+ // Both the continuation's answer and the root run's own answer score onto
+ // the single trace the agent turn was recorded on.
+ await waitForExpect(async () => {
+ const [continuationScore, plainScore] = await Promise.all([
+ getScoreById({
+ projectId,
+ scoreId: `afbs_feedback-assistant_${userId}`,
+ }),
+ getScoreById({
+ projectId,
+ scoreId: `afbs_plain-assistant_${userId}`,
+ }),
+ ]);
+
+ for (const score of [continuationScore, plainScore]) {
+ expect(score?.traceId).toBe(
+ getInAppAgentInstrumentationTraceId(rootRun.id),
+ );
+ expect(score?.observationId).toBe(
+ getInAppAgentInstrumentationObservationId(rootRun.id),
+ );
+ }
});
} finally {
(env as any).LANGFUSE_AI_FEATURES_PROJECT_ID =