🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(in-app-agent): assert feedback on an ordinary turn scores its own run (#16240)
Lysander Kieselee28d86f18fa
diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.tsindex d7ae70561d..f561e6f26b 100644--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts@@ -652,6 +652,14 @@ describe("in-app agent persistence", () => {messageId: "feedback-user",content: "Answer me",});+ await appendAssistantText({+ projectId,+ conversationId: conversation.id,+ runId: rootRun.id,+ events,+ messageId: "plain-assistant",+ chunks: ["Answering directly"],+ });// The final answer comes from an approval continuation, but the trace it is// recorded on belongs to the root run. The parent settles when it parks for// approval, so only the continuation is active.@@ -706,17 +714,39 @@ describe("in-app agent persistence", () => {}),).resolves.toEqual({ feedback: { value: "thumbs_up", comment: null } });- await waitForExpect(async () => {- const score = await getScoreById({+ await expect(+ caller.submitFeedback({projectId,- scoreId: `afbs_feedback-assistant_${userId}`,- });- expect(score?.traceId).toBe(- getInAppAgentInstrumentationTraceId(rootRun.id),- );- expect(score?.observationId).toBe(- getInAppAgentInstrumentationObservationId(rootRun.id),- );+ conversationId: conversation.id,+ messageId: "plain-assistant",+ runId: rootRun.id,+ value: "thumbs_down",+ comment: null,+ }),+ ).resolves.toEqual({ feedback: { value: "thumbs_down", comment: null } });++ // Both the continuation's answer and the root run's own answer score onto+ // the single trace the agent turn was recorded on.+ await waitForExpect(async () => {+ const [continuationScore, plainScore] = await Promise.all([+ getScoreById({+ projectId,+ scoreId: `afbs_feedback-assistant_${userId}`,+ }),+ getScoreById({+ projectId,+ scoreId: `afbs_plain-assistant_${userId}`,+ }),+ ]);++ for (const score of [continuationScore, plainScore]) {+ expect(score?.traceId).toBe(+ getInAppAgentInstrumentationTraceId(rootRun.id),+ );+ expect(score?.observationId).toBe(+ getInAppAgentInstrumentationObservationId(rootRun.id),+ );+ }});} finally {(env as any).LANGFUSE_AI_FEATURES_PROJECT_ID =