🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
test: add test expectations for transcript builder (#17611)
marliessophie52fb01c8bb4e
diff --git a/packages/shared/src/server/transcript/README.md b/packages/shared/src/server/transcript/README.mdindex 90ce3d1ddc..31d755ca67 100644--- a/packages/shared/src/server/transcript/README.md+++ b/packages/shared/src/server/transcript/README.md@@ -86,6 +86,8 @@ and observation provenance are excluded. All fields inside parts are included.thread, so replayed history disappears but additional identical copies survive (eg user responds "Thank you" twice).- **Outputs:** always append, then count them so subsequent inputs do not repeat them.- **Registered tool responses:** use call identity, not message occurrence counts.+ Across traces, reused call IDs are matched by call occurrence in complete,+ ordered input history. Ambiguous partial replays remain as input messages.Tool-observation responses take precedence over generation-provided responses.These responses are excluded from thread matching, irrespective of their source.- **References:** each message has a reference to the trace and generation ID of the object that emitted it. Replayed inputs never overwrite references on earlier occurrences.@@ -119,7 +121,7 @@ and observation provenance are excluded. All fields inside parts are included.- **Whole-message matching:** equivalent content split into different messagesor parts may not match. Reordering parts within a message also changes identity.Registered tool responses are the exception: replay is matched by call ID- within the same trace, even when grouped with other parts.+ within the thread (and occurrence for reused IDs), even when grouped with other parts.- **Name matching is best-effort:** same-name parallel executions can start in adifferent order from their calls. Names must match exactly. Metadata-only IDsare not read by the transcript builder; they require support in normalized IO.diff --git a/packages/shared/src/server/transcript/fixtures/README.md b/packages/shared/src/server/transcript/fixtures/README.mdindex 6a42eff5c9..bb35dc019f 100644--- a/packages/shared/src/server/transcript/fixtures/README.md+++ b/packages/shared/src/server/transcript/fixtures/README.md@@ -138,8 +138,9 @@ plus a one-line registration in `fixtures/index.ts`. Usewith stable stand-ins, applied to every occurrence. Seeded demo data needsno redaction.-9. **Leave the expectation open.** `expected: undefined`, always. The- requester defines the expected transcript by hand.+9. **Define the expectation.** Review message order, thread boundaries, tool+ results, and provenance against the input. Store a static `expected`+ transcript; do not derive it from the transcript builder in the test.10. **Write the description.** Two to five sentences on what makes this treeinteresting for transcript semantics: history replay or its absence,@@ -204,7 +205,7 @@ export const <camelCaseName>Fixture = {scope: "trace",description: "<step 10>",observations,- expected: undefined,+ expected: { threads: [/* reviewed threads and messages */] },} satisfies TranscriptFixture;```@@ -217,5 +218,5 @@ export const <camelCaseName>Fixture = {of a literal; nested JSON strings are still strings.- [ ] Repeated values are hoisted; shortened values are shortened everywhere.- [ ] `level`, `statusMessage`, `model` only where the rules say.-- [ ] `expected: undefined`.+- [ ] Explicit, reviewed `expected` transcript (or `null` for no transcript).- [ ] Registered in `fixtures/index.ts`; integrity test and typecheck pass.diff --git a/packages/shared/src/server/transcript/fixtures/fixture-types.ts b/packages/shared/src/server/transcript/fixtures/fixture-types.tsindex eaf264f3f3..aaf5eded27 100644--- a/packages/shared/src/server/transcript/fixtures/fixture-types.ts+++ b/packages/shared/src/server/transcript/fixtures/fixture-types.ts@@ -7,10 +7,7 @@ import type { Transcript } from "../types";* The behavior test completes each seed with `createObservation` and converts* it to a domain `Observation` with `convertObservation` before building.*- * `expected` is authored by hand and stays `undefined` until the semantics of- * the case are decided. Fixtures without an expectation are still valuable:- * they document real trees, run through the integrity checks, and turn into- * assertions the moment the expectation is filled in.+ * Expectations pin reviewed message ordering, thread boundaries, and provenance.*/export type TranscriptFixture = {/** Test name. Describes the tree shape the case exercises. */@@ -25,5 +22,5 @@ export type TranscriptFixture = {* order, so fixtures never re-sort.*/observations: Parameters<typeof createObservation>[0][];- expected: Transcript | null | undefined;+ expected: Transcript | null;};diff --git a/packages/shared/src/server/transcript/fixtures/fixtures.test.ts b/packages/shared/src/server/transcript/fixtures/fixtures.test.tsindex 06c151c762..ce67367bf0 100644--- a/packages/shared/src/server/transcript/fixtures/fixtures.test.ts+++ b/packages/shared/src/server/transcript/fixtures/fixtures.test.ts@@ -4,12 +4,10 @@ import { createObservation } from "../../test-utils";import { convertObservation } from "../../repositories/observations_converters";import { getTranscript } from "../index";import { formatTranscript } from "./format-transcript";+import { orderSupportRoutingFixture } from "./session/order-support-routing";/**- * Structural checks on the fixtures themselves, plus the behavior assertion- * against the transcript builder. Every fixture's transcript is printed- * (visible with `--disableConsoleIntercept`) so expectations can be authored- * from real output; the assertion only runs once `expected` is defined.+ * Structural checks on fixtures and exact transcript expectations.*/describe("transcript fixtures", () => {const generation = (id: string, input: string[], output: string[]) =>@@ -95,24 +93,25 @@ describe("transcript fixtures", () => {// Complete each seed into a full ClickHouse observation record and convert// it to a domain `Observation`, then build the transcript.- it("returns the expected transcript", () => {- const observations = fixture.observations.map((observation) =>- convertObservation(createObservation(observation)),- );- const transcript = getTranscript(observations);+ // Routing history is an opaque string; its transcript expectation is deferred.+ it.skipIf(fixture === orderSupportRoutingFixture)(+ "returns the expected transcript",+ () => {+ const observations = fixture.observations.map((observation) =>+ convertObservation(createObservation(observation)),+ );+ const transcript = getTranscript(observations);- // console.log("----------Transcript-------------------");- // console.log(JSON.stringify(transcript, null, 2));- console.log("----------Formatted Transcript-------------------");- console.log(- formatTranscript(fixture.name, transcript, observations, {- hideReasoning: true,- }),- );- console.log("-------------------------------------------------");+ console.log("----------Formatted Transcript-------------------");+ console.log(+ formatTranscript(fixture.name, transcript, observations, {+ hideReasoning: true,+ }),+ );+ console.log("-------------------------------------------------");- if (fixture.expected === undefined) return;- expect(transcript).toEqual(fixture.expected);- });+ expect(transcript).toEqual(fixture.expected);+ },+ );});});diff --git a/packages/shared/src/server/transcript/fixtures/index.ts b/packages/shared/src/server/transcript/fixtures/index.tsindex 6d6b04ed0a..27d23dcd71 100644--- a/packages/shared/src/server/transcript/fixtures/index.ts+++ b/packages/shared/src/server/transcript/fixtures/index.ts@@ -2,14 +2,17 @@ import type { TranscriptFixture } from "./fixture-types";import { cumulativeHistoryFixture } from "./session/cumulative-history";import { orderSupportRoutingFixture } from "./session/order-support-routing";import { reorderedHistoryFixture } from "./session/reordered-history";+import { supportCopilotFollowUpFixture } from "./session/support-copilot-follow-up";import { openaiAgentsSpanishHandoffFixture } from "./trace/openai-agents-spanish-handoff";import { vercelAiSdkDocsToolLoopFixture } from "./trace/vercel-ai-sdk-docs-tool-loop";import { openaiAgentsJokeAndRatingFixture } from "./trace/openai-agents-joke-and-rating";import { supportCopilotRefundLoopFixture } from "./trace/support-copilot-refund-loop";+import { standaloneToolObservationFixture } from "./trace/standalone-tool-observation";export type { TranscriptFixture } from "./fixture-types";const traceTranscriptFixtures: TranscriptFixture[] = [+ standaloneToolObservationFixture,openaiAgentsSpanishHandoffFixture,vercelAiSdkDocsToolLoopFixture,openaiAgentsJokeAndRatingFixture,@@ -17,6 +20,7 @@ const traceTranscriptFixtures: TranscriptFixture[] = [];const sessionTranscriptFixtures: TranscriptFixture[] = [+ supportCopilotFollowUpFixture,orderSupportRoutingFixture,cumulativeHistoryFixture,reorderedHistoryFixture,diff --git a/packages/shared/src/server/transcript/fixtures/session/cumulative-history.ts b/packages/shared/src/server/transcript/fixtures/session/cumulative-history.tsindex d8433b2b7e..d6657e55df 100644--- a/packages/shared/src/server/transcript/fixtures/session/cumulative-history.ts+++ b/packages/shared/src/server/transcript/fixtures/session/cumulative-history.ts@@ -59,5 +59,70 @@ export const cumulativeHistoryFixture = {}),},],- expected: undefined,+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "Initial request",+ },+ ],+ source: "input",+ observationId: "cumulative-history-generation-1",+ traceId: "cumulative-history-trace-1",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "First response",+ },+ ],+ source: "output",+ observationId: "cumulative-history-generation-1",+ traceId: "cumulative-history-trace-1",+ },+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "Follow-up request",+ },+ ],+ source: "input",+ observationId: "cumulative-history-generation-2",+ traceId: "cumulative-history-trace-2",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "Second response",+ },+ ],+ source: "output",+ observationId: "cumulative-history-generation-2",+ traceId: "cumulative-history-trace-2",+ },+ ],+ observations: [+ {+ id: "cumulative-history-generation-1",+ traceId: "cumulative-history-trace-1",+ },+ {+ id: "cumulative-history-generation-2",+ traceId: "cumulative-history-trace-2",+ },+ ],+ },+ ],+ },} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/session/order-support-routing.ts b/packages/shared/src/server/transcript/fixtures/session/order-support-routing.tsindex cb0ef976f1..99fb470b7f 100644--- a/packages/shared/src/server/transcript/fixtures/session/order-support-routing.ts+++ b/packages/shared/src/server/transcript/fixtures/session/order-support-routing.ts@@ -213,5 +213,7 @@ export const orderSupportRoutingFixture = {}),}),],- expected: undefined,+ // Opaque conversation strings do not replay structured message history.+ // Keep this example for reference; transcript expectations are deferred.+ expected: null,} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/session/reordered-history.ts b/packages/shared/src/server/transcript/fixtures/session/reordered-history.tsindex 5eb5804603..f63477838a 100644--- a/packages/shared/src/server/transcript/fixtures/session/reordered-history.ts+++ b/packages/shared/src/server/transcript/fixtures/session/reordered-history.ts@@ -68,5 +68,82 @@ export const reorderedHistoryFixture = {}),},],- expected: undefined,+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "A",+ },+ ],+ source: "input",+ observationId: "reordered-history-generation-1",+ traceId: "reordered-history-trace-1",+ },+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "B",+ },+ ],+ source: "input",+ observationId: "reordered-history-generation-1",+ traceId: "reordered-history-trace-1",+ },+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "C",+ },+ ],+ source: "input",+ observationId: "reordered-history-generation-1",+ traceId: "reordered-history-trace-1",+ },+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "New",+ },+ ],+ source: "input",+ observationId: "reordered-history-generation-2",+ traceId: "reordered-history-trace-2",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "Answer",+ },+ ],+ source: "output",+ observationId: "reordered-history-generation-2",+ traceId: "reordered-history-trace-2",+ },+ ],+ observations: [+ {+ id: "reordered-history-generation-1",+ traceId: "reordered-history-trace-1",+ },+ {+ id: "reordered-history-generation-2",+ traceId: "reordered-history-trace-2",+ },+ ],+ },+ ],+ },} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/session/support-copilot-follow-up.ts b/packages/shared/src/server/transcript/fixtures/session/support-copilot-follow-up.tsnew file mode 100644index 0000000000..b541d4dc58--- /dev/null+++ b/packages/shared/src/server/transcript/fixtures/session/support-copilot-follow-up.ts@@ -0,0 +1,74 @@+import type { TranscriptFixture } from "../fixture-types";+import { supportCopilotRefundLoopFixture } from "../trace/support-copilot-refund-loop";++const firstTrace = supportCopilotRefundLoopFixture;+const firstThread = firstTrace.expected.threads[0];+const draft = firstTrace.observations.find((observation) =>+ observation.id.endsWith("-drf"),+)!;+const completedHistory = [+ ...JSON.parse(draft.input!).messages,+ JSON.parse(draft.output!),+];++// Repeat the same workflow for another charge, with distinct trace/call IDs.+function followUpCopy<T>(value: T): T {+ return JSON.parse(+ JSON.stringify(value)+ .replaceAll("sa2026083102o", "sa2026083102p")+ .replaceAll("call_find_2", "call_find_3")+ .replaceAll("call_act_2", "call_act_3")+ .replaceAll("ch_3PqK", "ch_4PqK")+ .replaceAll("re_18Lk1Q", "re_28Lk1Q")+ .replaceAll("inv_20260709", "inv_20260809")+ .replaceAll(+ "Hi — I was charged twice",+ "Thanks for resolving that. I was also charged twice",+ )+ .replaceAll("2026-07", "2026-08")+ .replaceAll("2026-08-31T12:33:", "2026-08-31T12:35:"),+ );+}++const followUpObservations = firstTrace.observations.map((original) => {+ const observation = followUpCopy(original);+ // Both traces belong to the same project.+ observation.project_id = original.project_id;+ if (observation.type === "GENERATION") {+ const input = JSON.parse(observation.input!);+ observation.input = JSON.stringify({+ ...input,+ messages: [+ ...completedHistory,+ ...input.messages.filter(+ (message: { role: string }) => message.role !== "system",+ ),+ ],+ });+ }+ return observation;+});++export const supportCopilotFollowUpFixture = {+ name: "support copilot follow-up replays history across two traces",+ scope: "session",+ description:+ "Two complete refund workflows share a conversation. The second trace replays the first trace's final history in every generation; repeated history retains its first attribution, and new calls/results retain the second trace's IDs.",+ observations: [...firstTrace.observations, ...followUpObservations],+ expected: {+ threads: [+ {+ observations: [+ ...firstThread.observations,+ ...followUpCopy(firstThread.observations),+ ],+ messages: [+ ...firstThread.messages,+ ...followUpCopy(+ firstThread.messages.filter((message) => message.role !== "system"),+ ),+ ],+ },+ ],+ },+} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/trace/openai-agents-joke-and-rating.ts b/packages/shared/src/server/transcript/fixtures/trace/openai-agents-joke-and-rating.tsindex 09de0da27d..37a6e396d2 100644--- a/packages/shared/src/server/transcript/fixtures/trace/openai-agents-joke-and-rating.ts+++ b/packages/shared/src/server/transcript/fixtures/trace/openai-agents-joke-and-rating.ts@@ -368,5 +368,84 @@ export const openaiAgentsJokeAndRatingFixture = {output: payload5,},],- expected: undefined,+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "Tell me a joke",+ },+ ],+ source: "input",+ observationId: "f8fdd8164d6f8dbe",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ {+ id: "msg_05e10c6f9849abe80068db9d07270c81a0ab2f2738f61c1d11",+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "Sure! Here’s one for you:\n\nWhy did the scarecrow win an award?\n\nBecause he was outstanding in his field! 🌾😄",+ providerMetadata: {+ logprobs: [],+ },+ },+ ],+ source: "output",+ observationId: "f8fdd8164d6f8dbe",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ ],+ observations: [+ {+ id: "f8fdd8164d6f8dbe",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ ],+ },+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "Rate this joke: Sure! Here’s one for you:\n\nWhy did the scarecrow win an award?\n\nBecause he was outstanding in his field! 🌾😄",+ },+ ],+ source: "input",+ observationId: "496efdc060a5399b",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ {+ id: "msg_03e1816217fbc8a30068db9d07dbe481a28fae7f2a41f76827",+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "Great choice! That’s a classic joke—punny, wholesome, and easy to get. Here’s how I’d rate it:\n\n**Joke Rating: 8/10**\n\n**Why?**\n- **Punniness:** Strong, farm-fresh wordplay!\n- **Audience:** Universally funny—works for kids and adults.\n- **Originality:** Popular, but still a crowd-pleaser.\n\n**Bonus:** The addition of the emoji 🌾 gives it an extra playful touch! Want to hear a joke from me now?",+ providerMetadata: {+ logprobs: [],+ },+ },+ ],+ source: "output",+ observationId: "496efdc060a5399b",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ ],+ observations: [+ {+ id: "496efdc060a5399b",+ traceId: "afa1ae379b6c3a5333e2bab357f31cdc",+ },+ ],+ },+ ],+ },} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/trace/openai-agents-spanish-handoff.ts b/packages/shared/src/server/transcript/fixtures/trace/openai-agents-spanish-handoff.tsindex 01ace8395e..5024ff3804 100644--- a/packages/shared/src/server/transcript/fixtures/trace/openai-agents-spanish-handoff.ts+++ b/packages/shared/src/server/transcript/fixtures/trace/openai-agents-spanish-handoff.ts@@ -466,5 +466,83 @@ export const openaiAgentsSpanishHandoffFixture = {}),},],- expected: undefined,+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "Hola, ¿cómo estás?",+ },+ ],+ source: "input",+ observationId: "94a1dc00d7067ae8",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "tool-call",+ toolCallId: "call_i81vsF01rKH8cDZ0SLQQBUA2",+ toolName: "transfer_to_spanish_agent",+ input: {},+ toolType: "function_call",+ providerMetadata: {+ status: "completed",+ },+ },+ ],+ source: "output",+ observationId: "94a1dc00d7067ae8",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ {+ role: "tool",+ source: "output",+ parts: [+ {+ type: "tool-result",+ toolCallId: "call_i81vsF01rKH8cDZ0SLQQBUA2",+ output: {+ assistant: "Spanish agent",+ },+ },+ ],+ observationId: "a0391ae77593ad0c",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ {+ id: "msg_000230c027f4a45e0068db9cf9d14481a3a04289f3f1b0f9a1",+ role: "assistant",+ parts: [+ {+ type: "text",+ text: "¡Hola! Estoy muy bien, gracias. ¿Y tú, cómo estás?",+ providerMetadata: {+ logprobs: [],+ },+ },+ ],+ source: "output",+ observationId: "a0391ae77593ad0c",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ ],+ observations: [+ {+ id: "94a1dc00d7067ae8",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ {+ id: "a0391ae77593ad0c",+ traceId: "c376e44920527b875add9d97b4ed9312",+ },+ ],+ },+ ],+ },} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/trace/standalone-tool-observation.ts b/packages/shared/src/server/transcript/fixtures/trace/standalone-tool-observation.tsnew file mode 100644index 0000000000..3e7fc7b62a--- /dev/null+++ b/packages/shared/src/server/transcript/fixtures/trace/standalone-tool-observation.ts@@ -0,0 +1,66 @@+import type { TranscriptFixture } from "../fixture-types";++// Tool execution is recorded only as a TOOL observation, not a model tool call.+export const standaloneToolObservationFixture = {+ name: "Standalone tool execution without a generation tool call",+ scope: "trace",+ description:+ "The generation contains only text. A subsequent TOOL observation has input and output but no matching generation tool call, so its result is omitted from the transcript.",+ observations: [+ {+ project_id: "transcript-fixture-project",+ trace_id: "standalone-tool-trace",+ id: "standalone-generation",+ parent_observation_id: null,+ type: "GENERATION",+ name: "Order assistant",+ start_time: "2026-01-01T12:00:00.000Z",+ end_time: "2026-01-01T12:00:01.000Z",+ input: JSON.stringify([{ role: "user", content: "Find my orders." }]),+ output: JSON.stringify({+ role: "assistant",+ content: "I'll look up your orders.",+ }),+ },+ {+ project_id: "transcript-fixture-project",+ trace_id: "standalone-tool-trace",+ id: "standalone-tool",+ parent_observation_id: "standalone-generation",+ type: "TOOL",+ name: "findUserOrders",+ start_time: "2026-01-01T12:00:02.000Z",+ end_time: "2026-01-01T12:00:03.000Z",+ input: JSON.stringify({ customerId: "customer-1" }),+ output: JSON.stringify({+ content: [{ type: "text", text: "Order 123 is ready." }],+ isError: false,+ }),+ },+ ],+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [{ type: "text", text: "Find my orders." }],+ source: "input",+ observationId: "standalone-generation",+ traceId: "standalone-tool-trace",+ },+ {+ role: "assistant",+ parts: [{ type: "text", text: "I'll look up your orders." }],+ source: "output",+ observationId: "standalone-generation",+ traceId: "standalone-tool-trace",+ },+ ],+ observations: [+ { id: "standalone-generation", traceId: "standalone-tool-trace" },+ ],+ },+ ],+ },+} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/trace/support-copilot-refund-loop.ts b/packages/shared/src/server/transcript/fixtures/trace/support-copilot-refund-loop.tsindex 694957bcbd..41b1e35f65 100644--- a/packages/shared/src/server/transcript/fixtures/trace/support-copilot-refund-loop.ts+++ b/packages/shared/src/server/transcript/fixtures/trace/support-copilot-refund-loop.ts@@ -1,30 +1,10 @@import type { TranscriptFixture } from "../fixture-types";/**- * Support copilot resolving a duplicate-charge refund. Seeded scenario- * (`support-agent-seed`), authored as ClickHouse `observations` seed records- * for the shared `createObservation` test factory.- * url: https://cloud.langfuse.com/project/clkpwwm0m000gmm094odg11gi/traces/sa2026083102o?observation=sa2026083102o-cls×tamp=2026-08-31T12:33:16.561Z&traceId=sa2026083102o- *- * Trace tree (export order differs: the AGENT is listed before its parent SPAN)- *- * [SPAN] support-copilot t-sa2026083102o root- * [AGENT] support-copilot sa2026083102o-root- * [GUARDRAIL] guardrail.input -gin- * [GENERATION] classify-intent gpt-5.4-mini -cls in: system + user out: JSON object (intent)- * [SPAN] load-context -load- * [TOOL] crm.get-customer -crm- * [TOOL] billing.list-invoices -bill- * [TOOL] tickets.search -tix- * [GENERATION] llm.chat gpt-5.4 -llm1 in: system + user out: tool_call stripe_find_charges- * [TOOL] stripe.find-charges -find in: = llm1 call args out: charges + duplicate_confidence- * [GENERATION] llm.chat gpt-5.4 -llm2 in: system + tool(partial result, no tool_call_id)- * out: tool_call stripe_create_refund- * [TOOL] stripe.create-refund -ref in: = llm2 call args out: refund_id- * [GENERATION] llm.chat gpt-5.4 -llm3 in: system only out: "Resolution complete."- * [GENERATION] draft-response gpt-5.4 -drf in: user "Draft the reply." out: JSON object {reply}- * [GUARDRAIL] guardrail.output -gout- * [TOOL] zendesk.send-reply -send+ * Adapted support-agent seed: every generation replays the full conversation.+ * Classification, calls, results, resolution, and final reply form one thread.+ * TOOL names match model call names; history preserves tool_call_id links.+ * Context-loading tools without model calls remain outside the transcript.*/const traceId = "sa2026083102o";@@ -50,6 +30,70 @@ const finalReply =const findChargesArguments = { customer_id: customerId, period: "2026-07" };const createRefundArguments = { charge_id: "ch_3PqK9b", reason: "duplicate" };+const initialHistory = [+ { role: "system", content: copilotSystemPrompt },+ { role: "user", content: customerMessage },+];+const classification = {+ role: "assistant",+ content: JSON.stringify({+ intent: "billing.duplicate_charge",+ urgency: "medium",+ sentiment: "frustrated",+ }),+};+const findChargesCall = {+ role: "assistant",+ content: null,+ tool_calls: [+ {+ id: findChargesToolCallId,+ type: "function",+ function: {+ name: "stripe_find_charges",+ arguments: JSON.stringify(findChargesArguments),+ },+ },+ ],+};+const charges = {+ charges: [{ id: "ch_3PqK8r" }, { id: "ch_3PqK9b" }],+ duplicate_confidence: 0.98,+};+const findChargesResult = {+ role: "tool",+ tool_call_id: findChargesToolCallId,+ content: JSON.stringify(charges),+};+const refundCall = {+ role: "assistant",+ content: null,+ tool_calls: [+ {+ id: createRefundToolCallId,+ type: "function",+ function: {+ name: "stripe_create_refund",+ arguments: JSON.stringify(createRefundArguments),+ },+ },+ ],+};+const refund = { refund_id: refundId, status: "succeeded", amount_usd: 99 };+const refundResult = {+ role: "tool",+ tool_call_id: createRefundToolCallId,+ content: JSON.stringify(refund),+};+const classifiedHistory = [...initialHistory, classification];+const chargesHistory = [+ ...classifiedHistory,+ findChargesCall,+ findChargesResult,+];+const refundHistory = [...chargesHistory, refundCall, refundResult];+const resolution = { role: "assistant", content: "Resolution complete." };+const metadata = {channel: "in-app-chat",customer_id: customerId,@@ -128,16 +172,9 @@ const observations = [start_time: "2026-08-31T12:33:14.205Z",end_time: "2026-08-31T12:33:14.818Z",input: JSON.stringify({- messages: [- { role: "system", content: "Classify the support request." },- { role: "user", content: customerMessage },- ],- }),- output: JSON.stringify({- intent: "billing.duplicate_charge",- urgency: "medium",- sentiment: "frustrated",+ messages: initialHistory,}),+ output: JSON.stringify(classification),},{...common,@@ -202,10 +239,7 @@ const observations = [start_time: "2026-08-31T12:33:16.561Z",end_time: "2026-08-31T12:33:17.389Z",input: JSON.stringify({- messages: [- { role: "system", content: copilotSystemPrompt },- { role: "user", content: customerMessage },- ],+ messages: classifiedHistory,}),output: JSON.stringify({content: null,@@ -226,15 +260,12 @@ const observations = [id: "sa2026083102o-find",parent_observation_id: agentId,type: "TOOL",- name: "stripe.find-charges",+ name: "stripe_find_charges",provided_model_name: null,start_time: "2026-08-31T12:33:17.441Z",end_time: "2026-08-31T12:33:18.118Z",input: JSON.stringify(findChargesArguments),- output: JSON.stringify({- charges: [{ id: "ch_3PqK8r" }, { id: "ch_3PqK9b" }],- duplicate_confidence: 0.98,- }),+ output: JSON.stringify(charges),},{...common,@@ -246,15 +277,7 @@ const observations = [start_time: "2026-08-31T12:33:18.172Z",end_time: "2026-08-31T12:33:19.924Z",input: JSON.stringify({- messages: [- { role: "system", content: copilotSystemPrompt },- // Partial tool result, no tool_call_id: the full result lives only on- // the sibling TOOL observation above.- {- role: "tool",- content: JSON.stringify({ duplicate_confidence: 0.98 }),- },- ],+ messages: chargesHistory,}),output: JSON.stringify({content: null,@@ -275,16 +298,12 @@ const observations = [id: "sa2026083102o-ref",parent_observation_id: agentId,type: "TOOL",- name: "stripe.create-refund",+ name: "stripe_create_refund",provided_model_name: null,start_time: "2026-08-31T12:33:19.978Z",end_time: "2026-08-31T12:33:20.893Z",input: JSON.stringify(createRefundArguments),- output: JSON.stringify({- refund_id: refundId,- status: "succeeded",- amount_usd: 99,- }),+ output: JSON.stringify(refund),},{...common,@@ -296,9 +315,9 @@ const observations = [start_time: "2026-08-31T12:33:20.947Z",end_time: "2026-08-31T12:33:22.731Z",input: JSON.stringify({- messages: [{ role: "system", content: copilotSystemPrompt }],+ messages: refundHistory,}),- output: JSON.stringify({ content: "Resolution complete.", tool_calls: [] }),+ output: JSON.stringify(resolution),},{...common,@@ -310,9 +329,13 @@ const observations = [start_time: "2026-08-31T12:33:22.790Z",end_time: "2026-08-31T12:33:24.377Z",input: JSON.stringify({- messages: [{ role: "user", content: "Draft the reply." }],+ messages: [+ ...refundHistory,+ resolution,+ { role: "user", content: "Draft the reply." },+ ],}),- output: JSON.stringify({ reply: finalReply }),+ output: JSON.stringify({ role: "assistant", content: finalReply }),},{...common,@@ -341,16 +364,106 @@ const observations = [];export const supportCopilotRefundLoopFixture = {- name: "support copilot refund loop with tool results only on TOOL observations",+ name: "support copilot refund loop with cumulative conversation history",scope: "trace",- description: [- "Five generations under one AGENT, none of which replays earlier assistant output in its input.",- "Tool results exist only as sibling TOOL observations whose input equals the preceding generation's tool-call arguments;",- "the following generation carries a partial result in a `tool` message without a tool_call_id.",- "`classify-intent` and `draft-response` output bare JSON objects rather than messages.",- "The same system message repeats across the three `llm.chat` generations.",- "Non-contributing types (SPAN, GUARDRAIL) are present, and the export lists the AGENT before its parent SPAN.",- ].join(" "),+ description:+ "Five generations replay all prior messages. Matching TOOL observations supply authoritative results, while replayed history is deduplicated and retains first-emitter provenance.",observations,- expected: undefined,+ expected: {+ threads: [+ {+ observations: ["cls", "llm1", "find", "llm2", "ref", "llm3", "drf"].map(+ (suffix) => ({ id: `sa2026083102o-${suffix}`, traceId }),+ ),+ messages: [+ {+ role: "system",+ source: "input",+ observationId: "sa2026083102o-cls",+ traceId,+ parts: [{ type: "text", text: copilotSystemPrompt }],+ },+ {+ role: "user",+ source: "input",+ observationId: "sa2026083102o-cls",+ traceId,+ parts: [{ type: "text", text: customerMessage }],+ },+ {+ role: "assistant",+ source: "output",+ observationId: "sa2026083102o-cls",+ traceId,+ parts: [{ type: "text", text: classification.content }],+ },+ {+ role: "assistant",+ source: "output",+ observationId: "sa2026083102o-llm1",+ traceId,+ parts: [+ {+ type: "tool-call",+ toolCallId: findChargesToolCallId,+ toolName: "stripe_find_charges",+ input: findChargesArguments,+ toolType: "function",+ },+ ],+ },+ {+ role: "tool",+ source: "output",+ observationId: "sa2026083102o-find",+ traceId,+ parts: [{ type: "data", value: charges }],+ },+ {+ role: "assistant",+ source: "output",+ observationId: "sa2026083102o-llm2",+ traceId,+ parts: [+ {+ type: "tool-call",+ toolCallId: createRefundToolCallId,+ toolName: "stripe_create_refund",+ input: createRefundArguments,+ toolType: "function",+ },+ ],+ },+ {+ role: "tool",+ source: "output",+ observationId: "sa2026083102o-ref",+ traceId,+ parts: [{ type: "data", value: refund }],+ },+ {+ role: "assistant",+ source: "output",+ observationId: "sa2026083102o-llm3",+ traceId,+ parts: [{ type: "text", text: "Resolution complete." }],+ },+ {+ role: "user",+ source: "input",+ observationId: "sa2026083102o-drf",+ traceId,+ parts: [{ type: "text", text: "Draft the reply." }],+ },+ {+ role: "assistant",+ source: "output",+ observationId: "sa2026083102o-drf",+ traceId,+ parts: [{ type: "text", text: finalReply }],+ },+ ],+ },+ ],+ },} satisfies TranscriptFixture;diff --git a/packages/shared/src/server/transcript/fixtures/trace/vercel-ai-sdk-docs-tool-loop.ts b/packages/shared/src/server/transcript/fixtures/trace/vercel-ai-sdk-docs-tool-loop.tsindex 9e4bb5a282..8f06fdf35d 100644--- a/packages/shared/src/server/transcript/fixtures/trace/vercel-ai-sdk-docs-tool-loop.ts+++ b/packages/shared/src/server/transcript/fixtures/trace/vercel-ai-sdk-docs-tool-loop.ts@@ -702,5 +702,113 @@ export const vercelAiSdkDocsToolLoopFixture: TranscriptFixture = {output: payload13,},],- expected: undefined,+ expected: {+ threads: [+ {+ messages: [+ {+ role: "user",+ parts: [+ {+ type: "text",+ text: "How do I attach a user score to a trace?",+ },+ ],+ source: "input",+ observationId: "1bbf3edb8ef95ab6",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: '**Searching documents for scores**\n\nI need to use tools to search the documentation. It looks like Langfuse has a "scores" API that can be linked to traces or observations. There are "Score" entities to consider, which include details like name, value, data type, comments, source, traceId, observationId, and timestamp. The SDKs have methods such as client.score or langfuse.score. I plan to search Langfuse docs to get more information on this.',+ },+ },+ {+ type: "text",+ text: "I’m going to search the Langfuse docs for how to add or attach a score to a trace, including minimal code examples.",+ },+ {+ type: "tool-call",+ toolCallId: "call_OQNWWW4PjixkLIhkuHJRwiFV",+ toolName: "searchLangfuseDocs",+ input: {+ query:+ "How to attach a score to a trace user score to a trace Langfuse SDK example",+ },+ toolType: "tool_call",+ },+ ],+ finishReason: {+ type: "tool-calls",+ raw: "tool_call",+ },+ source: "output",+ observationId: "1bbf3edb8ef95ab6",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ {+ role: "tool",+ source: "output",+ parts: [+ {+ type: "text",+ text: '{"content": [{"type": "document", "record_type": "documentation", "url": "https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk", "title": "Scores via API/SDK", "source": {"content": [{"type": "text", "text": "Scores via API/SDK - Langfuse\\n[Docs](/docs)\\n[Evaluation](/docs/evaluation/overview)\\n[Evaluation Methods](/docs/evaluation/evaluation-methods/llm-as-a-judge)\\n[Scores via API/SDK](/docs/evaluation/evaluation-methods/scores-via-sdk)\\n# Scores via API/SDK\\nYou can use the Langfuse SDKs or API to add [scores](/docs/evaluation/scores/overview) to traces, observations, sessions and dataset runs. This is an evaluation method that allows you to set up custom evaluation workflows and extend the scoring capabilities of Langfuse. See the [data model](/docs/evaluation/scores/data-model#scores) for full details on the score object.\\nIf you want Langfuse to run deterministic Python or TypeScript logic for you, use [code evaluators](/docs/evaluation/evaluation-methods/code-evaluators). Use this page when your application, pipeline, or CI job computes scores and sends them to Langfuse via API or SDK.\\n## Ingesting Scores via API/SDKs\\nScores can be attached at different levels of granularity: to individual traces, to specific observations within a trace, or to full sessions.\\nSee the [API reference](/docs/api) for full details on the POST and GET endpoints for scores and score configs.\\n### Trace or Observation-level Scores\\nYou can add scores via the Langfuse SDKs or API. Scores can take one of four data types: **Numeric** , **Categorical** , **Boolean** , or **Text**. See [Score Types](/docs/evaluation/scores/overview#score-types) for details.\\nIf a score is ingested manually using a `trace_id` to link the score to a trace, it is not necessary to wait until the trace has been created. The score will show up in the scores table and will be linked to the trace once the trace with the same `trace_id` is created.\\nHere are examples by `Score` data types."}, {"type": "text", "text": "\\nFor trace and observation scores, `trace_id`/`traceId` is required and `observation_id`/`observationId` is optional. If you attach a score to an observation, always provide both the observation ID and the corresponding trace ID.\\nPython SDKJS/TS SDKAPI\\nNumericCategoricalBooleanText\\nNumeric score values must be provided as float.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"correctness\\",\\n value=0.9,\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n data_type=\\"NUMERIC\\", ///# optional, inferred if not provided\\n comment=\\"Factually correct\\", ///# optional\\n)\\n///# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"correctness\\",\\n value=0.9,\\n data_type=\\"NUMERIC\\",\\n comment=\\"Factually correct\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=0.95,\\n data_type=\\"NUMERIC\\"\\n )\\n///# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"correctness\\",\\n value=0.9,\\n data_type=\\"NUMERIC\\",\\n comment=\\"Factually correct\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=0.95,\\n data_type=\\"NUMERIC\\"\\n )\\n```\\nCategorical score values must be provided as strings.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n data_type=\\"CATEGORICAL\\", ///# optional, inferred if not provided\\n comment=\\"Some factual errors\\", ///# optional\\n)"}, {"type": "text", "text": "\\n///# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\",\\n comment=\\"Some factual errors\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\"\\n )\\n///# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\",\\n comment=\\"Some factual errors\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\"\\n )\\n```\\nBoolean scores must be provided as a float, where `1` is true and `0` is false. When read via the [v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores), the value is returned as a boolean. See [API reference](/docs/api) for more details on POST/GET scores endpoints.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"helpfulness\\",\\n value=0, ///# 0 or 1\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n data_type=\\"BOOLEAN\\", ///# required, numeric values without data type would be inferred as NUMERIC\\n comment=\\"Incorrect answer\\", ///# optional\\n)\\n///# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span"}, {"type": "text", "text": "\\n span.score(\\n name=\\"helpfulness\\",\\n value=1, ///# 0 or 1\\n data_type=\\"BOOLEAN\\",\\n comment=\\"Very helpful response\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=1, ///# 0 or 1\\n data_type=\\"BOOLEAN\\"\\n )\\n///# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"helpfulness\\",\\n value=1, ///# 0 or 1\\n data_type=\\"BOOLEAN\\",\\n comment=\\"Very helpful response\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=1, ///# 0 or 1\\n data_type=\\"BOOLEAN\\"\\n )\\n```\\nText score values must be provided as strings between 1 and 500 characters.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n data_type=\\"TEXT\\", ///# optional, inferred if not provided\\n comment=\\"Reviewed by QA team\\", ///# optional\\n)\\n///# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n data_type=\\"TEXT\\",\\n comment=\\"Reviewed by QA team\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_notes\\",\\n value=\\"Good quality overall, minor formatting issues.\\",\\n data_type=\\"TEXT\\"\\n )\\n///# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span"}, {"type": "text", "text": "\\n langfuse.score_current_span(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n data_type=\\"TEXT\\",\\n comment=\\"Reviewed by QA team\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_notes\\",\\n value=\\"Good quality overall, minor formatting issues.\\",\\n data_type=\\"TEXT\\"\\n )\\n```\\nNumericCategoricalBooleanText\\nNumeric score values must be provided as float.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"correctness\\",\\n value: 0.9,\\n dataType: \\"NUMERIC\\", // optional, inferred if not provided\\n comment: \\"Factually correct\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nCategorical score values must be provided as strings.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"accuracy\\",\\n value: \\"partially correct\\",\\n dataType: \\"CATEGORICAL\\", // optional, inferred if not provided\\n comment: \\"Factually correct\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```"}, {"type": "text", "text": "\\nBoolean scores must be provided as a float, where `1` is true and `0` is false. When read via the [v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores), the value is returned as a boolean. See [API reference](/docs/api) for more details on POST/GET scores endpoints.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"helpfulness\\",\\n value: 0, // 0 or 1\\n dataType: \\"BOOLEAN\\", // required, numeric values without data type would be inferred as NUMERIC\\n comment: \\"Incorrect answer\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nText score values must be provided as strings between 1 and 500 characters.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"reviewer_notes\\",\\n value: \\"The response was helpful but could be more concise.\\",\\n dataType: \\"TEXT\\", // optional, inferred if not provided\\n comment: \\"Reviewed by QA team\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```"}, {"type": "text", "text": "\\nYou can also create scores directly via the [REST API](https://api.reference.langfuse.com/#tag/legacyscorev1/POST/api/public/scores). Authenticate using HTTP Basic Auth with your Langfuse Public Key as username and Secret Key as password.\\nNumericCategoricalBooleanText\\nNumeric score values must be provided as float.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"correctness\\",\\n \\"value\\": 0.9,\\n \\"dataType\\": \\"NUMERIC\\",\\n \\"comment\\": \\"Factually correct\\"\\n }\'\\n```\\nCategorical score values must be provided as strings.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"accuracy\\",\\n \\"value\\": \\"partially correct\\",\\n \\"dataType\\": \\"CATEGORICAL\\",\\n \\"comment\\": \\"Some factual errors\\"\\n }\'\\n```\\nBoolean scores must be provided as a float, where `1` is true and `0` is false. When read via the [v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores), the value is returned as a boolean.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"helpfulness\\",\\n \\"value\\": 0,\\n \\"dataType\\": \\"BOOLEAN\\",\\n \\"comment\\": \\"Incorrect answer\\"\\n }\'\\n```\\nText score values must be provided as strings between 1 and 500 characters.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\"}, {"type": "text", "text": "\\n comment=\\"Factually correct\\",\\n config_id=\\"12345-6565-3453654-43543\\",\\n data_type=\\"CATEGORICAL\\"\\n )\\n```\\nWhen ingesting boolean scores, you can provide the value as a float. If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n name=\\"helpfulness\\",\\n value=1,\\n comment=\\"Factually correct\\", ///# optional\\n score_id=\\"unique_id\\", ///# optional, can be used as an idempotency key to update the score subsequently\\n config_id=\\"93547-6565-3453654-43543\\", ///# optional, can be used to infer the score data type and validate the score value\\n data_type=\\"BOOLEAN\\" # optional, possibly inferred\\n)\\n///# Method 2: Score within context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n span.score(\\n name=\\"helpfulness\\",\\n value=1,\\n comment=\\"Factually correct\\",\\n config_id=\\"93547-6565-3453654-43543\\",\\n data_type=\\"BOOLEAN\\"\\n )\\n```\\nWhen ingesting text scores, provide the value as a string (1–500 characters). If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n///# Method 1: Score via low-level method\\nlangfuse.create_score(\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", ///# optional\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n comment=\\"Reviewed by QA team\\", ///# optional"}, {"type": "text", "text": "\\n score_id=\\"unique_id\\", ///# optional, can be used as an idempotency key to update the score subsequently\\n config_id=\\"24680-6565-3453654-43543\\", ///# optional\\n data_type=\\"TEXT\\" # optional, possibly inferred\\n)\\n///# Method 2: Score within context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n span.score(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n comment=\\"Reviewed by QA team\\",\\n config_id=\\"24680-6565-3453654-43543\\",\\n data_type=\\"TEXT\\"\\n )\\n```\\nNumeric ScoresCategorical ScoresBoolean ScoresText Scores\\nWhen ingesting numeric scores, you can provide the value as a float. If you provide a configId, the score value will be validated against the config\'s numeric range, which might be defined by a minimum and/or maximum value.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"accuracy\\",\\n value: 0.9,\\n comment: \\"Factually correct\\", // optional\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n configId: \\"78545-6565-3453654-43543\\", // optional, to ensure that the score follows a specific min/max value range\\n dataType: \\"NUMERIC\\", // optional, possibly inferred\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nCategorical scores are used to evaluate data that falls into specific categories. When ingesting categorical scores, you can provide the value as a string. If you provide a configId, the score value will be validated against the config\'s categories.\\n \\n \\n```"}, {"type": "text", "text": "\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"correctness\\",\\n value: \\"correct\\",\\n comment: \\"Factually correct\\", // optional\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n configId: \\"12345-6565-3453654-43543\\", // optional, to ensure that a score maps to a specific category defined in a score config\\n dataType: \\"CATEGORICAL\\", // optional, possibly inferred\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nWhen ingesting boolean scores, you can provide the value as a float. If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"helpfulness\\",\\n value: 1,\\n comment: \\"Factually correct\\", // optional\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n configId: \\"93547-6565-3453654-43543\\", // optional, can be used to infer the score data type and validate the score value\\n dataType: \\"BOOLEAN\\", // optional, possibly inferred\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nWhen ingesting text scores, provide the value as a string (1–500 characters). If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"reviewer_notes\\",\\n value: \\"The response was helpful but could be more concise.\\","}, {"type": "text", "text": "\\nWhen ingesting boolean scores, you can provide the value as a float. If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"helpfulness\\",\\n \\"value\\": 1,\\n \\"dataType\\": \\"BOOLEAN\\",\\n \\"configId\\": \\"93547-6565-3453654-43543\\",\\n \\"comment\\": \\"Factually correct\\"\\n }\'\\n```\\nWhen ingesting text scores, provide the value as a string (1–500 characters). If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"reviewer_notes\\",\\n \\"value\\": \\"The response was helpful but could be more concise.\\",\\n \\"dataType\\": \\"TEXT\\",\\n \\"configId\\": \\"24680-6565-3453654-43543\\",\\n \\"comment\\": \\"Reviewed by QA team\\"\\n }\'\\n```\\nSee [API reference](/docs/api) for more details on POST/GET score configs endpoints.\\n### Inferred Score Properties\\nCertain score properties might be inferred based on your input:\\n * **If you don\'t provide a score data type** it will always be inferred. See tables below for details."}, {"type": "text", "text": "\\n * **For boolean and categorical scores** , we will provide the score value in both numerical and string format where possible. The score value format that is not provided as input, i.e. the translated value is referred to as the inferred value in the tables below.\\n * **On read via the[v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores)**, boolean scores are returned as a single boolean `value`.\\n * **For categorical scores** , the string representation is always provided and a numerical mapping of the category will be produced only if a `ScoreConfig` was provided.\\nDetailed Examples:\\nNumeric ScoresCategorical ScoresBoolean ScoresText Scores\\nFor example, let\'s assume you\'d like to ingest a numeric score to measure **accuracy**. We have included a table of possible score ingestion scenarios below.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Valid \\n---|---|---|---|---|--- \\n`0.9`| `Null`| `Null`| Data type is inferred| `NUMERIC`| Yes \\n`0.9`| `NUMERIC`| `Null`| No properties inferred| | Yes \\n`depth`| `NUMERIC`| `Null`| Error: data type of value does not match provided data type| | No \\n`0.9`| `NUMERIC`| `78545`| No properties inferred| | Conditional on config validation \\n`0.9`| `Null`| `78545`| Data type inferred| `NUMERIC`| Conditional on config validation \\n`depth`| `NUMERIC`| `78545`| Error: data type of value does not match provided data type| | No \\n \\nFor example, let\'s assume you\'d like to ingest a categorical score to measure **correctness**. We have included a table of possible score ingestion scenarios below."}, {"type": "text", "text": "Value| Data Type| Config Id| Description| Inferred Data Type| Inferred Value representation| Valid \\n---|---|---|---|---|---|--- \\n`correct`| `Null`| `Null`| Data type is inferred| `CATEGORICAL`| | Yes \\n`correct`| `CATEGORICAL`| `Null`| No properties inferred| | | Yes \\n`1`| `CATEGORICAL`| `Null`| Error: data type of value does not match provided data type| | | No \\n`correct`| `CATEGORICAL`| `12345`| Numeric value inferred| | `4` numeric config category mapping| Conditional on config validation \\n`correct`| `NULL`| `12345`| Data type inferred| `CATEGORICAL`| | Conditional on config validation \\n`1`| `CATEGORICAL`| `12345`| Error: data type of value does not match provided data type| | | No \\n \\nFor example, let\'s assume you\'d like to ingest a boolean score to measure **helpfulness**. We have included a table of possible score ingestion scenarios below.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Inferred Value representation| Valid \\n---|---|---|---|---|---|--- \\n`1`| `BOOLEAN`| `Null`| Value\'s string equivalent inferred| | `True`| Yes \\n`true`| `BOOLEAN`| `Null`| Error: data type of value does not match provided data type| | | No \\n`3`| `BOOLEAN`| `Null`| Error: boolean data type expects `0` or `1` as input value| | | No \\n`0.9`| `Null`| `93547`| Data type and value\'s string equivalent inferred| `BOOLEAN`| `True`| Conditional on config validation \\n`depth`| `BOOLEAN`| `93547`| Error: data type of value does not match provided data type| | | No \\n \\nFor example, let\'s assume you\'d like to ingest a text score to capture **reviewer notes**. Text scores must be non-empty strings of at most 500 characters.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Valid "}, {"type": "text", "text": "---|---|---|---|---|--- \\n`\\"Good response\\"`| `Null`| `Null`| Data type is inferred| `TEXT`| Yes \\n`\\"Good response\\"`| `TEXT`| `Null`| No properties inferred| | Yes \\n`0.9`| `TEXT`| `Null`| Error: data type of value does not match provided data type| | No \\n`\\"Good response\\"`| `TEXT`| `24680`| No properties inferred| | Conditional on config validation \\n`\\"Good response\\"`| `Null`| `24680`| Data type inferred| `TEXT`| Conditional on config validation \\n`\\"\\"`| `TEXT`| `Null`| Error: text scores must be non-empty| | No \\n \\n## Update Existing Scores via API/SDKs\\nWhen creating a score, you can provide an optional `id` (JS/TS) / `score_id` (Python) parameter. A re-ingested score overwrites an existing one only when its `id`, `name`, and `timestamp` (at date granularity, `toDate(timestamp)`) all match — a matching `id` alone is not sufficient, and any difference creates a duplicate instead of an update.\\nTo overwrite a score without fetching existing scores first, set a stable `id` as an idempotency key when initially creating the score, and keep the `name` and `timestamp` unchanged on subsequent calls. See Preventing Duplicate Scores for details.\\n## Related guides\\nGuides\\n * [Collecting user feedback\\nCapture explicit and implicit user feedback as scores on your traces, including in-app ratings sent via our Browser SDK.\\n](/guides/user-feedback-loop)\\n * [Custom evaluation data pipeline\\nContinuously monitor quality by fetching traces, running custom evaluations, and ingesting scores back into Langfuse.\\n](/guides/cookbook/example_external_evaluation_pipelines)\\n * [Guardrails and security checks"}, {"type": "text", "text": "\\nCheck whether output contains a certain keyword, matches a required structure or format, or exceeds a length limit.\\n](/guides/cookbook/security-and-guardrails)\\n * [Custom internal workflow tooling\\nBuild internal tooling for human-in-the-loop workflows, ingesting scores back into Langfuse following your custom schema.\\n](/guides/human-in-the-loop-scoring)\\n * [Session-level quality tracking\\nScore full conversations, such as support chats or agent threads, by attaching scores via sessionId in the SDK or API.\\n](/guides/cookbook/example_evaluating_multi_turn_conversations)\\n* * *\\n* * *\\nLast edited 7/22/2026\\n"}], "type": "content"}}, {"type": "document", "record_type": "documentation", "url": "https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk", "title": "Scores via API/SDK", "source": {"content": [{"type": "text", "text": "# Scores via API/SDK\\nYou can use the Langfuse SDKs or API to add [scores](/docs/evaluation/scores/overview) to traces, observations, sessions and dataset runs. This is an evaluation method that allows you to set up custom evaluation workflows and extend the scoring capabilities of Langfuse. See the [data model](/docs/evaluation/scores/data-model#scores) for full details on the score object.\\nIf you want Langfuse to run deterministic Python or TypeScript logic for you, use [code evaluators](/docs/evaluation/evaluation-methods/code-evaluators). Use this page when your application, pipeline, or CI job computes scores and sends them to Langfuse via API or SDK.\\n\\n## Ingesting Scores via API/SDKs\\nScores can be attached at different levels of granularity: to individual traces, to specific observations within a trace, or to full sessions.\\nSee the [API reference](/docs/api) for full details on the POST and GET endpoints for scores and score configs.\\n### Trace or Observation-level Scores\\nYou can add scores via the Langfuse SDKs or API. Scores can take one of four data types: **Numeric** , **Categorical** , **Boolean** , or **Text**. See [Score Types](/docs/evaluation/scores/overview#score-types) for details.\\nIf a score is ingested manually using a `trace_id` to link the score to a trace, it is not necessary to wait until the trace has been created. The score will show up in the scores table and will be linked to the trace once the trace with the same `trace_id` is created.\\nHere are examples by `Score` data types.\\nFor trace and observation scores, `trace_id`/`traceId` is required and `observation_id`/`observationId` is optional. If you attach a score to an observation, always provide both the observation ID and the corresponding trace ID.\\nPython SDKJS/TS SDKAPI\\nNumericCategoricalBooleanText\\nNumeric score values must be provided as float.\\n \\n \\n```"}, {"type": "text", "text": "\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"correctness\\",\\n value=0.9,\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", # optional\\n data_type=\\"NUMERIC\\", # optional, inferred if not provided\\n comment=\\"Factually correct\\", # optional\\n)\\n# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"correctness\\",\\n value=0.9,\\n data_type=\\"NUMERIC\\",\\n comment=\\"Factually correct\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=0.95,\\n data_type=\\"NUMERIC\\"\\n )\\n# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"correctness\\",\\n value=0.9,\\n data_type=\\"NUMERIC\\",\\n comment=\\"Factually correct\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=0.95,\\n data_type=\\"NUMERIC\\"\\n )\\n```\\nCategorical score values must be provided as strings.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", # optional\\n data_type=\\"CATEGORICAL\\", # optional, inferred if not provided\\n comment=\\"Some factual errors\\", # optional\\n)\\n# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\",\\n comment=\\"Some factual errors\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=\\"partially correct\\","}, {"type": "text", "text": "\\n data_type=\\"CATEGORICAL\\"\\n )\\n# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"accuracy\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\",\\n comment=\\"Some factual errors\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=\\"partially correct\\",\\n data_type=\\"CATEGORICAL\\"\\n )\\n```\\nBoolean scores must be provided as a float, where `1` is true and `0` is false. When read via the [v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores), the value is returned as a boolean. See [API reference](/docs/api) for more details on POST/GET scores endpoints.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"helpfulness\\",\\n value=0, # 0 or 1\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", # optional\\n data_type=\\"BOOLEAN\\", # required, numeric values without data type would be inferred as NUMERIC\\n comment=\\"Incorrect answer\\", # optional\\n)\\n# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"helpfulness\\",\\n value=1, # 0 or 1\\n data_type=\\"BOOLEAN\\",\\n comment=\\"Very helpful response\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_quality\\",\\n value=1, # 0 or 1\\n data_type=\\"BOOLEAN\\"\\n )\\n# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"helpfulness\\","}, {"type": "text", "text": "\\n value=1, # 0 or 1\\n data_type=\\"BOOLEAN\\",\\n comment=\\"Very helpful response\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_quality\\",\\n value=1, # 0 or 1\\n data_type=\\"BOOLEAN\\"\\n )\\n```\\nText score values must be provided as strings between 1 and 500 characters.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n# Method 1: Score via low-level method\\nlangfuse.create_score(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", # optional\\n data_type=\\"TEXT\\", # optional, inferred if not provided\\n comment=\\"Reviewed by QA team\\", # optional\\n)\\n# Method 2: Score current span/generation (within context)\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n # Score the current span\\n span.score(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n data_type=\\"TEXT\\",\\n comment=\\"Reviewed by QA team\\"\\n )\\n # Score the trace\\n span.score_trace(\\n name=\\"overall_notes\\",\\n value=\\"Good quality overall, minor formatting issues.\\",\\n data_type=\\"TEXT\\"\\n )\\n# Method 3: Score via the current context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\"):\\n # Score the current span\\n langfuse.score_current_span(\\n name=\\"reviewer_notes\\",\\n value=\\"The response was helpful but could be more concise.\\",\\n data_type=\\"TEXT\\",\\n comment=\\"Reviewed by QA team\\"\\n )\\n # Score the trace\\n langfuse.score_current_trace(\\n name=\\"overall_notes\\",\\n value=\\"Good quality overall, minor formatting issues.\\",\\n data_type=\\"TEXT\\"\\n )\\n```\\nNumericCategoricalBooleanText\\nNumeric score values must be provided as float.\\n \\n \\n```"}, {"type": "text", "text": "\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"correctness\\",\\n value: 0.9,\\n dataType: \\"NUMERIC\\", // optional, inferred if not provided\\n comment: \\"Factually correct\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nCategorical score values must be provided as strings.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"accuracy\\",\\n value: \\"partially correct\\",\\n dataType: \\"CATEGORICAL\\", // optional, inferred if not provided\\n comment: \\"Factually correct\\", // optional\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nBoolean scores must be provided as a float, where `1` is true and `0` is false. When read via the [v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores), the value is returned as a boolean. See [API reference](/docs/api) for more details on POST/GET scores endpoints.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n traceId: message.traceId,\\n observationId: message.generationId, // optional"}, {"type": "text", "text": "\\nnpm install @langfuse/browser\\n```\\n \\n \\n```\\nimport { LangfuseBrowserClient } from \\"@langfuse/browser\\";\\nconst langfuse = new LangfuseBrowserClient({\\n publicKey: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY!,\\n baseUrl: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL, // optional, defaults to https://cloud.langfuse.com\\n});\\nawait langfuse.score({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n id: `user-feedback-${message.traceId}`, // optional, use as an idempotency key\\n name: \\"user-feedback\\",\\n value: 1, // 1 for positive, 0 for negative\\n dataType: \\"BOOLEAN\\",\\n comment: \\"Helpful answer\\", // optional\\n});\\n```\\n`score()` returns `{ id }` after ingestion succeeds. No `flush()` call is required.\\n### Session-level Scores\\nTo score an entire session (without attaching the score to a trace or observation), provide only `session_id` (Python SDK) or `sessionId` (JS/TS SDK and API).\\nPython SDKJS/TS SDKAPI\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\nlangfuse.create_score(\\n name=\\"session_quality\\",\\n value=0.85,\\n session_id=\\"session_id_here\\",\\n data_type=\\"NUMERIC\\",\\n comment=\\"Overall conversation quality\\"\\n)\\n```\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n name: \\"session_quality\\",\\n value: 0.85,\\n sessionId: \\"session_id_here\\",\\n dataType: \\"NUMERIC\\",\\n comment: \\"Overall conversation quality\\",\\n});\\nawait langfuse.flush();\\n```\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"sessionId\\": \\"session_id_here\\",\\n \\"name\\": \\"session_quality\\",\\n \\"value\\": 0.85,\\n \\"dataType\\": \\"NUMERIC\\",\\n \\"comment\\": \\"Overall conversation quality\\"\\n }\'\\n```\\n"}], "type": "content"}}, {"type": "document", "record_type": "documentation", "url": "https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk", "title": "Scores via API/SDK", "source": {"content": [{"type": "text", "text": "## Advanced\\n### Preventing Duplicate Scores\\nBy default, Langfuse allows for multiple scores of the same `name` on the same trace. This is useful if you\'d like to track the evolution of a score over time or if e.g. you\'ve received multiple user feedback scores on the same trace.\\nIn some cases, you want to prevent this behavior or overwrite an existing score. A score is identified by three fields — its `id`, its `name`, and its date (`toDate(timestamp)`) — and a new score only replaces an existing one when all three match. To achieve this, create an **idempotency key** and pass it as the `id` (JS/TS) / `score_id` (Python) when creating the score (for example `trace_id-score_name`), and keep the `name` and `timestamp` unchanged across calls. If any of the three differs — for example the same `id` with a different `name` — you get an additional score rather than a replacement.\\nDo not send partial score updates (only the changed fields) and rely on Langfuse to merge them into the existing record. This merge only happens for a limited window after creation, is deprecated, and will be removed. Always send the complete score.\\nSee [How to update traces, observations, and scores](/faq/all/tracing-data-updates#updating-scores) for more details on Langfuse\'s immutable data model.\\n### Enforcing a Score Config\\nScore configs are helpful when you want to standardize your scores for future analysis.\\nTo enforce a score config, you can provide a `configId` when creating a score to reference a `ScoreConfig` that was previously created. `Score Configs` can be defined in the Langfuse UI or via our API. [See our guide on how to create and manage score configs](/faq/all/manage-score-configs).\\nWhenever you provide a `ScoreConfig`, the score data will be validated against the config. The following rules apply:\\n * **Score Name** : Must equal the config\'s name"}, {"type": "text", "text": "\\n * **Score Data Type** : When provided, must match the config\'s data type\\n * **Score Value when Type is`NUMERIC`**: Value must be within the min and max values defined in the config (if provided, min and max are optional and otherwise are assumed as -∞ and +∞ respectively)\\n * **Score Value when Type is`CATEGORICAL`**: Value must map to one of the categories defined in the config\\n * **Score Value when Type is`BOOLEAN`**: Value must equal `0` or `1`\\n * **Score Value when Type is`TEXT`**: Value must be a non-empty string of at most 500 characters\\nPython SDKJS/TS SDKAPI\\nNumeric ScoresCategorical ScoresBoolean ScoresText Scores\\nWhen ingesting numeric scores, you can provide the value as a float. If you provide a configId, the score value will be validated against the config\'s numeric range, which might be defined by a minimum and/or maximum value.\\n \\n \\n```\\nfrom langfuse import get_client\\nlangfuse = get_client()\\n# Method 1: Score via low-level method\\nlangfuse.create_score(\\n trace_id=\\"trace_id_here\\",\\n observation_id=\\"observation_id_here\\", # optional\\n session_id=\\"session_id_here\\", # optional, ID of the session the score relates to\\n name=\\"accuracy\\",\\n value=0.9,\\n comment=\\"Factually correct\\", # optional\\n score_id=\\"unique_id\\", # optional, can be used as an idempotency key to update the score subsequently\\n config_id=\\"78545-6565-3453654-43543\\", # optional, to ensure that the score follows a specific min/max value range\\n data_type=\\"NUMERIC\\" # optional, possibly inferred\\n)\\n# Method 2: Score within context\\nwith langfuse.start_as_current_observation(as_type=\\"span\\", name=\\"my-operation\\") as span:\\n span.score(\\n name=\\"accuracy\\",\\n value=0.9,\\n comment=\\"Factually correct\\",\\n config_id=\\"78545-6565-3453654-43543\\",\\n data_type=\\"NUMERIC\\"\\n )\\n```"}, {"type": "text", "text": "\\nWhen ingesting numeric scores, you can provide the value as a float. If you provide a configId, the score value will be validated against the config\'s numeric range, which might be defined by a minimum and/or maximum value.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"accuracy\\",\\n value: 0.9,\\n comment: \\"Factually correct\\", // optional\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n configId: \\"78545-6565-3453654-43543\\", // optional, to ensure that the score follows a specific min/max value range\\n dataType: \\"NUMERIC\\", // optional, possibly inferred\\n});\\n// Flush the scores in short-lived environments\\nawait langfuse.flush();\\n```\\nCategorical scores are used to evaluate data that falls into specific categories. When ingesting categorical scores, you can provide the value as a string. If you provide a configId, the score value will be validated against the config\'s categories.\\n \\n \\n```\\nimport { LangfuseClient } from \\"@langfuse/client\\";\\nconst langfuse = new LangfuseClient();\\nlangfuse.score.create({\\n traceId: message.traceId,\\n observationId: message.generationId, // optional\\n name: \\"correctness\\",\\n value: \\"correct\\",\\n comment: \\"Factually correct\\", // optional\\n id: \\"unique_id\\", // optional, can be used as an idempotency key to update the score subsequently\\n configId: \\"12345-6565-3453654-43543\\", // optional, to ensure that a score maps to a specific category defined in a score config\\n dataType: \\"CATEGORICAL\\", // optional, possibly inferred\\n});"}, {"type": "text", "text": "\\nYou can also enforce score configs via the [REST API](https://api.reference.langfuse.com/#tag/legacyscorev1/POST/api/public/scores) by providing a `configId`.\\nNumeric ScoresCategorical ScoresBoolean ScoresText Scores\\nWhen ingesting numeric scores, you can provide the value as a float. If you provide a configId, the score value will be validated against the config\'s numeric range.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"accuracy\\",\\n \\"value\\": 0.9,\\n \\"dataType\\": \\"NUMERIC\\",\\n \\"configId\\": \\"78545-6565-3453654-43543\\",\\n \\"comment\\": \\"Factually correct\\"\\n }\'\\n```\\nCategorical scores are used to evaluate data that falls into specific categories. If you provide a configId, the score value will be validated against the config\'s categories.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"correctness\\",\\n \\"value\\": \\"correct\\",\\n \\"dataType\\": \\"CATEGORICAL\\",\\n \\"configId\\": \\"12345-6565-3453654-43543\\",\\n \\"comment\\": \\"Factually correct\\"\\n }\'\\n```\\nWhen ingesting boolean scores, you can provide the value as a float. If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"helpfulness\\","}, {"type": "text", "text": "\\n \\"value\\": 1,\\n \\"dataType\\": \\"BOOLEAN\\",\\n \\"configId\\": \\"93547-6565-3453654-43543\\",\\n \\"comment\\": \\"Factually correct\\"\\n }\'\\n```\\nWhen ingesting text scores, provide the value as a string (1–500 characters). If you provide a configId, the score\'s name and config\'s name must match as well as their data types.\\n \\n \\n```\\ncurl -X POST https://cloud.langfuse.com/api/public/scores \\\\\\n -u \\"pk-lf-...\\":\\"sk-lf-...\\" \\\\\\n -H \\"Content-Type: application/json\\" \\\\\\n -d \'{\\n \\"id\\": \\"unique_id\\",\\n \\"traceId\\": \\"trace_id_here\\",\\n \\"observationId\\": \\"observation_id_here\\",\\n \\"name\\": \\"reviewer_notes\\",\\n \\"value\\": \\"The response was helpful but could be more concise.\\",\\n \\"dataType\\": \\"TEXT\\",\\n \\"configId\\": \\"24680-6565-3453654-43543\\",\\n \\"comment\\": \\"Reviewed by QA team\\"\\n }\'\\n```\\nSee [API reference](/docs/api) for more details on POST/GET score configs endpoints.\\n### Inferred Score Properties\\nCertain score properties might be inferred based on your input:\\n * **If you don\'t provide a score data type** it will always be inferred. See tables below for details.\\n * **For boolean and categorical scores** , we will provide the score value in both numerical and string format where possible. The score value format that is not provided as input, i.e. the translated value is referred to as the inferred value in the tables below.\\n * **On read via the[v3 scores API](https://api.reference.langfuse.com/#tag/scoresv3/GET/api/public/v3/scores)**, boolean scores are returned as a single boolean `value`.\\n * **For categorical scores** , the string representation is always provided and a numerical mapping of the category will be produced only if a `ScoreConfig` was provided.\\nDetailed Examples:"}, {"type": "text", "text": "\\nNumeric ScoresCategorical ScoresBoolean ScoresText Scores\\nFor example, let\'s assume you\'d like to ingest a numeric score to measure **accuracy**. We have included a table of possible score ingestion scenarios below.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Valid \\n---|---|---|---|---|--- \\n`0.9`| `Null`| `Null`| Data type is inferred| `NUMERIC`| Yes \\n`0.9`| `NUMERIC`| `Null`| No properties inferred| | Yes \\n`depth`| `NUMERIC`| `Null`| Error: data type of value does not match provided data type| | No \\n`0.9`| `NUMERIC`| `78545`| No properties inferred| | Conditional on config validation \\n`0.9`| `Null`| `78545`| Data type inferred| `NUMERIC`| Conditional on config validation \\n`depth`| `NUMERIC`| `78545`| Error: data type of value does not match provided data type| | No \\n \\nFor example, let\'s assume you\'d like to ingest a categorical score to measure **correctness**. We have included a table of possible score ingestion scenarios below.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Inferred Value representation| Valid \\n---|---|---|---|---|---|--- \\n`correct`| `Null`| `Null`| Data type is inferred| `CATEGORICAL`| | Yes \\n`correct`| `CATEGORICAL`| `Null`| No properties inferred| | | Yes \\n`1`| `CATEGORICAL`| `Null`| Error: data type of value does not match provided data type| | | No \\n`correct`| `CATEGORICAL`| `12345`| Numeric value inferred| | `4` numeric config category mapping| Conditional on config validation \\n`correct`| `NULL`| `12345`| Data type inferred| `CATEGORICAL`| | Conditional on config validation \\n`1`| `CATEGORICAL`| `12345`| Error: data type of value does not match provided data type| | | No \\n "}, {"type": "text", "text": "For example, let\'s assume you\'d like to ingest a boolean score to measure **helpfulness**. We have included a table of possible score ingestion scenarios below.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Inferred Value representation| Valid \\n---|---|---|---|---|---|--- \\n`1`| `BOOLEAN`| `Null`| Value\'s string equivalent inferred| | `True`| Yes \\n`true`| `BOOLEAN`| `Null`| Error: data type of value does not match provided data type| | | No \\n`3`| `BOOLEAN`| `Null`| Error: boolean data type expects `0` or `1` as input value| | | No \\n`0.9`| `Null`| `93547`| Data type and value\'s string equivalent inferred| `BOOLEAN`| `True`| Conditional on config validation \\n`depth`| `BOOLEAN`| `93547`| Error: data type of value does not match provided data type| | | No \\n \\nFor example, let\'s assume you\'d like to ingest a text score to capture **reviewer notes**. Text scores must be non-empty strings of at most 500 characters.\\nValue| Data Type| Config Id| Description| Inferred Data Type| Valid \\n---|---|---|---|---|--- \\n`\\"Good response\\"`| `Null`| `Null`| Data type is inferred| `TEXT`| Yes \\n`\\"Good response\\"`| `TEXT`| `Null`| No properties inferred| | Yes \\n`0.9`| `TEXT`| `Null`| Error: data type of value does not match provided data type| | No \\n`\\"Good response\\"`| `TEXT`| `24680`| No properties inferred| | Conditional on config validation \\n`\\"Good response\\"`| `Null`| `24680`| Data type inferred| `TEXT`| Conditional on config validation \\n`\\"\\"`| `TEXT`| `Null`| Error: text scores must be non-empty| | No \\n \\n\\n## Update Existing Scores via API/SDKs"}, {"type": "text", "text": "When creating a score, you can provide an optional `id` (JS/TS) / `score_id` (Python) parameter. A re-ingested score overwrites an existing one only when its `id`, `name`, and `timestamp` (at date granularity, `toDate(timestamp)`) all match — a matching `id` alone is not sufficient, and any difference creates a duplicate instead of an update.\\nTo overwrite a score without fetching existing scores first, set a stable `id` as an idempotency key when initially creating the score, and keep the `name` and `timestamp` unchanged on subsequent calls. See Preventing Duplicate Scores for details.\\n## Related guides\\nGuides\\n * [Collecting user feedback\\nCapture explicit and implicit user feedback as scores on your traces, including in-app ratings sent via our Browser SDK.\\n](/guides/user-feedback-loop)\\n * [Custom evaluation data pipeline\\nContinuously monitor quality by fetching traces, running custom evaluations, and ingesting scores back into Langfuse.\\n](/guides/cookbook/example_external_evaluation_pipelines)\\n * [Guardrails and security checks\\nCheck whether output contains a certain keyword, matches a required structure or format, or exceeds a length limit.\\n](/guides/cookbook/security-and-guardrails)\\n * [Custom internal workflow tooling\\nBuild internal tooling for human-in-the-loop workflows, ingesting scores back into Langfuse following your custom schema.\\n](/guides/human-in-the-loop-scoring)\\n * [Session-level quality tracking\\nScore full conversations, such as support chats or agent threads, by attaching scores via sessionId in the SDK or API.\\n](/guides/cookbook/example_evaluating_multi_turn_conversations)\\n* * *\\n* * *\\nLast edited 7/22/2026\\n"}], "type": "content"}}, {"type": "document", "record_type": "documentation", "url": "https://langfuse.com/docs/evaluation/scores/overview", "title": "Scores", "source": {"content": [{"type": "text", "text": "Overview - Langfuse\\n[Docs](/docs)\\n[Evaluation](/docs/evaluation/overview)\\n[Scores](/docs/evaluation/scores/overview)\\n[Overview](/docs/evaluation/scores/overview)\\n# Scores\\nScores are Langfuse\'s universal data object for storing evaluation results. Any time you want to assign a quality judgment to an LLM output, whether by a [human annotation](/docs/evaluation/evaluation-methods/scores-via-ui), an [LLM judge](/docs/evaluation/evaluation-methods/llm-as-a-judge), a [programmatic check](/docs/evaluation/evaluation-methods/scores-via-sdk), or end-user feedback, the result is stored as a score.\\nEvery score has a **name** (like `\\"correctness\\"` or `\\"helpfulness\\"`), a **value** , and a **data type**. Scores also support an optional **comment** for additional context.\\nScores can be attached to [traces](/docs/observability/data-model#observations-and-traces), [observations](/docs/observability/data-model#observations-and-traces), [sessions](/docs/observability/data-model#sessions), or [dataset runs](/docs/evaluation/experiments/data-model). Most commonly, scores are attached to traces to evaluate a single end-to-end interaction.\\nOnce you have scores, they show up in [score analytics](/docs/evaluation/scores/score-analytics), can be visualized in [custom dashboards](/docs/metrics/features/custom-dashboards), and can be queried via the [API](/docs/api).\\n## When to Use Scores\\nScores become useful when you want to go beyond observing what your application does and start measuring how well it does it. Common use cases:\\n * **Collecting user feedback** : Capture thumbs up/down or star ratings from your users and attach them to traces. See the [user feedback guide](/docs/observability/features/user-feedback)."}, {"type": "text", "text": "\\n * **Monitoring production quality** : Set up automated evaluators (like [LLM-as-a-Judge](/docs/evaluation/evaluation-methods/llm-as-a-judge)) to continuously score live traces for things like hallucination, relevance, or tone.\\n * **Running guardrails** : Score whether outputs pass safety checks like PII detection, format validation, or content policy compliance.\\n * **Comparing changes with experiments** : When you change a prompt, model, or pipeline, run an [experiment](/docs/evaluation/experiments) to score the new version against a dataset.\\n## Score Types\\nLangfuse supports four score data types:\\nType| Value| Use when \\n---|---|--- \\n`NUMERIC`| Float (e.g. `0.9`)| Continuous judgments like accuracy, relevance, or similarity scores \\n`CATEGORICAL`| String from predefined categories (e.g. `\\"correct\\"`, `\\"partially correct\\"`)| Discrete classifications where the set of possible values is known upfront \\n`BOOLEAN`| `0` or `1`| Pass/fail checks like hallucination detection or format validation \\n`TEXT`| Free-form string (1-500 characters)| Open-ended annotations like reviewer notes or qualitative feedback. Often used for [open coding](https://en.wikipedia.org/wiki/Open_coding) before formalizing into quantifiable scores via [axial coding](https://en.wikipedia.org/wiki/Axial_coding). \\n \\nText scores are designed for qualitative, open-ended scoring. Because free-form text cannot be meaningfully aggregated or compared, text scores are not supported in [experiments](/docs/evaluation/core-concepts#experiments), [LLM-as-a-Judge](/docs/evaluation/evaluation-methods/llm-as-a-judge), or [score analytics](/docs/evaluation/scores/score-analytics).\\n## How to Create Scores"}, {"type": "text", "text": "\\nThere are five ways to add scores:\\n * **LLM-as-a-Judge** : Set up [automated evaluators](/docs/evaluation/evaluation-methods/llm-as-a-judge) that score traces based on custom criteria (e.g. hallucination, tone, relevance). These can return numeric or categorical scores plus reasoning, and can run on live production traces or on experiment results.\\n * **Code evaluators** : Run [custom Python or TypeScript evaluators](/docs/evaluation/evaluation-methods/code-evaluators) in Langfuse for deterministic checks such as exact match, JSON validation, or custom business rules.\\n * **Scores via UI** : Team members [manually score](/docs/evaluation/evaluation-methods/scores-via-ui) traces, observations, or sessions directly in the Langfuse UI. Requires a [score config](/faq/all/manage-score-configs) to be set up first.\\n * **Annotation Queues** : Set up [structured review workflows](/docs/evaluation/evaluation-methods/annotation-queues) where reviewers work through batches of traces.\\n * **Scores via API/SDK** : [Programmatically add scores](/docs/evaluation/evaluation-methods/scores-via-sdk) from your application code. This is the way to go for user feedback (thumbs up/down, star ratings), guardrail results, or custom evaluation pipelines.\\n## Should I Use Scores or Tags?\\n| Scores| Tags \\n---|---|--- \\n **Purpose**| Measure _how good_ something is| Describe _what_ something is \\n **Data**| Numeric, categorical, boolean, or text value| Simple string label \\n **When added**| Can be added at any time, including long after the trace was created| Set during tracing and cannot be changed afterwards \\n **Used for**| Quality measurement, analytics, experiments| Filtering, segmentation, organizing \\n "}, {"type": "text", "text": "As a rule of thumb: if you already know the category at tracing time (e.g. which feature or API endpoint triggered the trace), use a [tag](/docs/observability/features/tags). If you need to classify or evaluate traces later, use a score.\\n## Score Comments\\nEvery score supports an optional **comment** field. Use it to capture reasoning (e.g. why an LLM judge assigned a particular score), reviewer notes, or context that helps others understand the score value. Comments are shown alongside scores in the Langfuse UI.\\nUse a `TEXT` score instead of comments to capture standalone qualitative feedback -- comments are best for additional reasoning on an existing score.\\n* * *\\n* * *\\nLast edited 7/23/2026\\n"}], "type": "content"}}, {"type": "document", "record_type": "github_discussion", "url": "https://github.com/orgs/langfuse/discussions/6016", "title": "Using scores in data sets from the SDK?", "source": {"content": [{"type": "text", "text": "Title: Using scores in data sets from the SDK?\\nCategory: Support\\nAuthor: @howinator\\nTimestamp: March 14, 2025\\nBody:\\nHere is a workflow that I want, but I can\'t figure out how to accomplish it without a lot of manual work.\\r\\n\\r\\n1. Collect traces\\r\\n2. Annotate traces with either manual annotation or programmatic feedback from users. For a simple classification, I would use a boolean score where True means the model returned the correct answer and False when it returned the incorrect answer.\\r\\n4. Create a \\"false positive\\" data set composed of all traces where the score was False. <-- I can\'t figure out how to do this without a lot of clicking.\\r\\n5. Run an eval from that data set and use the score to determine if the eval was correct, e.g., something like `(dataset.item.score && dataset.item.output === evalOutput.output) || (!dataset.item.score && dataset.item.output !== evalOutput.output)`. <-- I can\'t figure out how to do this programmatically at all.\\r\\n\\r\\nIs my workflow completely off here? Am I missing something? Or is this flow just not possible in Langfuse today?\\nComments:\\n\\n Comment #1 from @dosubot on March 14, 2025:\\n\\n <!-- Greeting -->\\nHey there, @howinator! I\'m here to help you out with any bugs, questions, or contributions you have in mind. Let\'s get to work!\\n\\n<!-- Answer -->\\nYour workflow is mostly feasible in Langfuse, and you can achieve it programmatically with some adjustments:\\n\\n1. **Collect Traces**: This step is straightforward and can be done using Langfuse\'s SDKs to capture and store traces.\\n\\n2. **Annotate Traces**: You can programmatically annotate traces with scores using the `langfuse_context.score_current_trace` function. This allows you to attach a boolean score to each trace based on user feedback or other criteria <sup>[[1]](https://langfuse.com/guides/cookbook/integration_langgraph)</sup>.\\n"}, {"type": "text", "text": "\\n3. **Create a \\"False Positive\\" Dataset**: While there isn\'t a direct method to filter dataset items based on scores, you can programmatically create a dataset by iterating over your traces and selecting those with a score of `False`. Use the SDK to create dataset items for these traces <sup>[[2]](https://langfuse.com/docs/datasets/get-started)</sup>.\\n\\n4. **Run an Evaluation**: You can automate the creation of evaluation jobs for your dataset using the `createEvalJobs` function. This function allows you to specify configurations and run evaluations on specific subsets of dataset items <sup>[[3]](https://github.com/langfuse/langfuse/blob/eecf0d680b13a4bde9fe8fed617acc328029307d/worker/src/__tests__/evalService.test.ts)</sup>.\\n\\nFor the evaluation logic, you can implement custom logic to compare the dataset item scores and outputs with the evaluation outputs. This might require some additional scripting to handle the specific logic you described.\\n\\nOverall, your workflow is possible with Langfuse, but it may require some custom scripting to fully automate the process of filtering and evaluating dataset items based on scores.\\n\\n\\n<!-- Dosu Comment Footer -->\\n*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*\\n\\n---"}, {"type": "text", "text": "1. Create a \\"false positive\\" data set composed of all traces where the score was False -> use scores api to fetch all of these\\r\\n2. Run your own eval on them and add it to the same trace via the sdk\\r\\n3. Compare the evals by fetching all of them via the sdk again.\\r\\n\\r\\nWe currently plan to add more configurable dashboards which will allow for the comparison of these evals more easily in the UI.\\n"}], "type": "content"}}, {"type": "document", "record_type": "documentation", "url": "https://langfuse.com/docs/observability/troubleshooting-and-faq", "title": "Troubleshooting and FAQ", "source": {"content": [{"type": "text", "text": "\\n[LLM as a Session-Level Judge with Support for LangGraph Session](https://github.com/orgs/langfuse/discussions/8303 \\"Langfuse Ideas: LLM as a Session-Level Judge with Support for LangGraph Session\\")\\n[Score Configs: Allow editing the categories of a categorical score](https://github.com/orgs/langfuse/discussions/8259 \\"Langfuse Ideas: Score Configs: Allow editing the categories of a categorical score\\")\\n[feat(trace-detail-view): for each span, show the sum of nested observations\' token usage details](https://github.com/orgs/langfuse/discussions/8201 \\"Langfuse Ideas: feat\\\\(trace-detail-view\\\\): for each span, show the sum of nested observations\' token usage details\\")\\n[UI-LLM as a Jury](https://github.com/orgs/langfuse/discussions/8195 \\"Langfuse Ideas: UI-LLM as a Jury\\")\\n[Add Gemini 2.5 Pro model pricing](https://github.com/orgs/langfuse/discussions/8146 \\"Langfuse Ideas: Add Gemini 2.5 Pro model pricing\\")\\n[Deeplinking for Spans inside of a Trace](https://github.com/orgs/langfuse/discussions/8112 \\"Langfuse Ideas: Deeplinking for Spans inside of a Trace\\")\\n[feat(comments): allow attaching images](https://github.com/orgs/langfuse/discussions/8111 \\"Langfuse Ideas: feat\\\\(comments\\\\): allow attaching images\\")\\n[feat(sessions): support session-level tags](https://github.com/orgs/langfuse/discussions/8041 \\"Langfuse Ideas: feat\\\\(sessions\\\\): support session-level tags\\")\\n[Enable Immediate Score Management for User Feedback](https://github.com/orgs/langfuse/discussions/7686 \\"Langfuse Ideas: Enable Immediate Score Management for User Feedback\\")\\n[Allow for structured filtering on inputs / outputs](https://github.com/orgs/langfuse/discussions/7663 \\"Langfuse Ideas: Allow for structured filtering on inputs / outputs\\")"}, {"type": "text", "text": "\\n[Allow for lazy updating of traces in Python SDK (+ possible workaround)](https://github.com/orgs/langfuse/discussions/7655 \\"Langfuse Ideas: Allow for lazy updating of traces in Python SDK \\\\(+ possible workaround\\\\)\\")\\n[Filter by scores in session view](https://github.com/orgs/langfuse/discussions/7528 \\"Langfuse Ideas: Filter by scores in session view\\")\\n[Support break lines on evaluation run tooltip hint](https://github.com/orgs/langfuse/discussions/7452 \\"Langfuse Ideas: Support break lines on evaluation run tooltip hint\\")\\n[Enable sending Langfuse SDK OTEL spans to third party observability backend](https://github.com/orgs/langfuse/discussions/7365 \\"Langfuse Ideas: Enable sending Langfuse SDK OTEL spans to third party observability backend\\")\\n[Add Thinking config in playground for anthropic models](https://github.com/orgs/langfuse/discussions/7267 \\"Langfuse Ideas: Add Thinking config in playground for anthropic models\\")\\n[Better support for big int in metadata.](https://github.com/orgs/langfuse/discussions/7167 \\"Langfuse Ideas: Better support for big int in metadata.\\")\\n[Ability to filter/search comments on traces](https://github.com/orgs/langfuse/discussions/7109 \\"Langfuse Ideas: Ability to filter/search comments on traces\\")\\n[Image / File / Multimedia rendering in dataset items using mediaId](https://github.com/orgs/langfuse/discussions/6883 \\"Langfuse Ideas: Image / File / Multimedia rendering in dataset items using mediaId\\")\\n[Trace Comments support upload image](https://github.com/orgs/langfuse/discussions/6857 \\"Langfuse Ideas: Trace Comments support upload image\\")"}], "type": "content"}}, {"type": "document", "record_type": "github_issue", "url": "https://github.com/langfuse/langfuse/issues/3380", "title": "feat: add fetch_score(s) in the sdks", "source": {"content": [{"type": "text", "text": "Github Issue by @marcklingen created on September 17, 2024. The current state of the issue is: closed\\n\\n### Discussed in https://github.com/orgs/langfuse/discussions/3379\\r\\n\\r\\n<div type=\'discussions-op-text\'>\\r\\n\\r\\n<sup>Originally posted by **Preet1234-ux** September 17, 2024</sup>\\r\\nI want to fetch the score provided by the user from langfuse in my python code . Are there any functionality by which I can fetch the score for a trace ? </div>\\r\\n\\r\\n\\r\\nCurrently, we need to use either langfuse.client.*(potentially breaking) or the api.\\n---\\nComment #1 by @hassiebp:\\n\\nAlready shipped in Python SDK v3\\n\\n"}], "type": "content"}}, {"type": "document", "record_type": "github_discussion", "url": "https://github.com/orgs/langfuse/discussions/6210", "title": "Next.js + Vercel AI SDK: How to get `observationId` and attach it to scores?", "source": {"content": [{"type": "text", "text": "Title: Next.js + Vercel AI SDK: How to get `observationId` and attach it to scores?\\nCategory: Support\\nAuthor: @natterstefan\\nTimestamp: March 26, 2025\\nBody:\\nDear langfuse-Community,\\r\\n\\r\\nI need your help please, as I do not know how to obtain the `observationId` and attach it to a `score`.\\r\\n\\r\\nContext: this is how my `experimental_telemetry` looks like:\\r\\n\\r\\n```js\\r\\n// src: https://langfuse.com/docs/integrations/vercel-ai-sdk#group-multiple-executions-in-one-trace\\r\\n\\r\\nconst parentTraceId = getRandomId()\\r\\n\\r\\nexport async function POST(req: Request) {\\r\\n\\t// ... other code\\r\\n\\r\\n\\tconst trace = langfuse.trace({\\r\\n\\t id: parentTraceId,\\r\\n\\t name: \'cd-chat-messages\',\\r\\n\\t})\\r\\n\\r\\n\\tconsole.log(\'trace\', trace.observationId) // <<-- this returns `null`\\r\\n\\t\\r\\n\\tconst result = streamText({\\r\\n\\t\\t// ....\\r\\n\\t\\texperimental_telemetry: {\\r\\n\\t\\t isEnabled: true,\\r\\n\\t\\t functionId: \'cd-chat-message-\' + messages.length,\\r\\n\\t\\t metadata: {\\r\\n\\t\\t langfuseTraceId: parentTraceId,\\r\\n\\t\\t\\t // Do not update the parent trace with execution results\\r\\n\\t\\t langfuseUpdateParent: false,\\r\\n\\t\\t },\\r\\n\\t\\t },\\r\\n\\t})\\r\\n\\r\\n\\t// ... other code\\r\\n}\\r\\n```\\r\\n\\r\\nThe docs state\\r\\n\\r\\n> The id of the observation to which the score should be attached. Pass from backend trace. E.g. {generation, event, span}.id\\r\\n\\r\\nBut how and where do I get the observationId from, taking into account that I use `streamText` (<https://sdk.vercel.ai/docs/reference/ai-sdk-core/stream-text>)\\r\\n\\r\\n```js\\r\\n// source: https://langfuse.com/docs/sdk/typescript/guide-web#create-scores\\r\\n\\r\\n// pass traceId and observationId to front end\\r\\nawait langfuseWeb.score({\\r\\n traceId: message.traceId,\\r\\n observationId: message.observationId, <<<-- where do I get this?\\r\\n name: \\"user-feedback\\",\\r\\n value: 1,\\r\\n comment: \\"I like how personalized the response is\\",\\r\\n});\\r\\n```\\r\\n\\r"}, {"type": "text", "text": "Another question I could not answer myself is where (= which \\"layer\\". I guess generation, event, span is the correct wording) a score should be attached to. Is there any best practice recommendation or should I always go for the observation (`ai.streamText.doStream` in this example _if_ that\'s the observationId I can obtain)?\\r\\n\\r\\n\\r\\n\\r\\nThank you very much for your time and help!\\nComments:\\n\\n Comment #1 from @dosubot on March 26, 2025:\\n\\n <!-- Greeting -->\\nHey @natterstefan! I\'m here to help you with any bugs, questions, or even becoming a contributor. Let\'s squash those bugs together! 🤖\\n\\n<!-- Answer -->\\nTo obtain the `observationId` when using the `streamText` function in the Vercel AI SDK, you should look for it in the response from the `streamText` function. The SDK typically uses the ID from the provider response if available, or it generates one if not. Ensure that your telemetry setup is configured to pass the `observationId` from the backend trace, specifically from the context of the `streamText` execution.\\n\\nRegarding where to attach the score, it is generally recommended to attach it to the most relevant observation. In your case, if `ai.streamText.doStream` is the observation you can obtain, it would be appropriate to attach the score there. This aligns with best practices, as it ensures the score is associated with the specific operation or event you are evaluating.\\n"}, {"type": "text", "text": "\\nFor more detailed guidance, you might want to refer to the Langfuse documentation on creating scores and handling observations <sup>[[1]](https://langfuse.com/docs/sdk/typescript)</sup><sup>[[2]](https://langfuse.com/docs/sdk/typescript/guide)</sup>.\\n\\n\\n<!-- Dosu Comment Footer -->\\n*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*\\n\\n---\\n<sup>Help Dosu learn! Give it feedback: [Great Response](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=great_response) | [Irrelevant Answer](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=irrelevant_answer) | [Incorrect Sources](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=incorrect_sources) | [Too Verbose](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=too_verbose) | [Hallucination](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=hallucination) | [Bug Report](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=bug_report) | [Other](https://app.dosu.dev/response-feedback/68b78af0-2c34-4867-b75c-008bcf449b81?feedback_type=other)</sup>\\n\\n\\n Comment #2 from @marcklingen on March 26, 2025:\\n\\n This seems like a feature request for the AI SDK as you\'d need the SDK to expose the span ids of each of the requests.\\n\\n Replies to comment #2:\\n\\n Reply #1 to comment #2 from marcklingen on March 27, 2025:\\n\\n Hi @marcklingen,\\r\\n\\r\\nthanks for your help. Alright, I [moved (aka copied) the discussion to their repo](https://github.com/vercel/ai/discussions/5415) and hope they can help me.\\r\\n\\r\\nIs @dosubot correct with \\"_(...) In your case, if ai.streamText.doStream is the observation you can obtain, it would be appropriate to attach the score there._\\"\\r\\n\\r"}, {"type": "text", "text": "\\n<sup>Help Dosu learn! Give it feedback: [Great Response](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=great_response) | [Irrelevant Answer](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=irrelevant_answer) | [Incorrect Sources](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=incorrect_sources) | [Too Verbose](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=too_verbose) | [Hallucination](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=hallucination) | [Bug Report](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=bug_report) | [Other](https://app.dosu.dev/response-feedback/d9fcd303-8526-48dc-85b4-1432b3d12429?feedback_type=other)</sup>\\n\\n\\n Reply #3 to comment #2 from marcklingen on March 27, 2025:\\n\\n Hi @marcklingen and @dosubot,\\r\\n\\r\\nFollow up question (once I get the `generation.id`/`response.id` whatever vercel/ai calls it): how do I attach it to the current trace to make sure the observation in langfuse has the same id as the score in my code uses?\\n\\n Reply #4 to comment #2 from marcklingen on March 27, 2025:\\n\\n <!-- Answer -->\\nTo attach the `generation.id` or `response.id` from Vercel AI to the current trace in Langfuse, use the `generation.id` as the `observationId` when logging user feedback. This ensures that the observation has the same ID as the score used in your code. Here\'s how you can do it:\\n\\n1. Add the `generation.id` to the streaming response as a custom header:\\n\\n ```javascript\\n return new StreamingTextResponse(stream, {\\n headers: {\\n \\"X-Message-Id\\": lfGeneration.id,"}], "type": "content"}}]}',+ },+ ],+ observationId: "3dbf50cbf2036387",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ {+ role: "assistant",+ parts: [+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: "**Crafting user score attachment instructions**\n\nI've gathered enough info to help the user with attaching a user score to a trace. They want minimal examples in TypeScript, Python, and a Browser client. I think providing a basic snippet that adds a boolean score named \"user-feedback\" to the traceId would be effective. It's also essential to note that the data type BOOLEAN requires values of 0 or 1. I’ll include concise runnable examples and useful links about scores via API/SDK and user feedback. We should also consider concluding with a follow-up question.",+ },+ },+ {+ type: "reasoning",+ content: {+ kind: "text",+ text: "**Creating minimal code examples**\n\nI want to create concise code samples while mentioning that you need a traceId, and optionally an observationId. It's important to highlight that you can send user feedback even before the trace exists, which is quite interesting! This way, the user gets a clear understanding while also seeing how to implement everything in a few lines of code. Let's keep this straightforward and focus on clarity in the examples!",+ },+ },+ {+ type: "text",+ text: 'You can add a user score to a trace via the SDKs or API by creating a Score linked to the traceId (optionally to a specific observationId). For thumbs up/down, use dataType "BOOLEAN" with value 1 or 0.\n\nJS/TS (server)\n- Docs: https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk\n- Example:\n - npm i @langfuse/client\n - \n import { LangfuseClient } from "@langfuse/client";\n const langfuse = new LangfuseClient();\n await langfuse.score.create({\n traceId: "<trace_id>",\n name: "user-feedback",\n value: 1, // 1 = up, 0 = down\n dataType: "BOOLEAN", // required for boolean\n comment: "Helpful answer", // optional\n });\n await langfuse.flush();\n\nBrowser (collect in-app feedback)\n- Docs: https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk\n- Example:\n - npm i @langfuse/browser\n - \n import { LangfuseBrowserClient } from "@langfuse/browser";\n const lf = new LangfuseBrowserClient({ publicKey: "<public_key>", baseUrl: "https://cloud.langfuse.com" });\n await lf.score({\n traceId: "<trace_id>",\n // observationId: "<generation_or_span_id>", // optional if you want to score a specific step\n id: `user-feedback-<trace_id>`, // optional idempotency key\n name: "user-feedback",\n value: 1,\n dataType: "BOOLEAN",\n comment: "Great result!",\n });\n\nPython\n- Docs: https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk\n- Example:\n - \n from langfuse import get_client\n lf = get_client()\n lf.create_score(\n name="user-feedback",\n value=1.0, # float\n data_type="BOOLEAN", # required for boolean\n trace_id="<trace_id>",\n # observation_id="<observation_id>", # optional\n comment="User liked it", # optional\n )\n\nNotes\n- You only need traceId to attach to a trace; observationId is optional.\n- You can send the score before the trace exists; it links once the trace is created.\n- Score types supported: NUMERIC, CATEGORICAL, BOOLEAN (0/1), TEXT.\n- More: Scores overview: https://langfuse.com/docs/evaluation/scores/overview\n\nWant an example of passing traceId/observationId from your backend to the frontend for a “thumbs up/down” widget?',+ },+ ],+ finishReason: {+ type: "stop",+ raw: "stop",+ },+ source: "output",+ observationId: "0c13b686b4cf6ccc",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ ],+ observations: [+ {+ id: "1bbf3edb8ef95ab6",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ {+ id: "3dbf50cbf2036387",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ {+ id: "0c13b686b4cf6ccc",+ traceId: "75b336f98e39e275a823b0a7ec1ba794",+ },+ ],+ },+ ],+ },};diff --git a/packages/shared/src/server/transcript/threads.ts b/packages/shared/src/server/transcript/threads.tsindex 3f3be5d857..7ea2398467 100644--- a/packages/shared/src/server/transcript/threads.ts+++ b/packages/shared/src/server/transcript/threads.ts@@ -56,6 +56,18 @@ export function append() {const { thread, messages, shownCounts } = state;const inputCounts = new Map<string, number>();+ const replayCalls = new Map<string, number>();+ const replayTotals = new Map<string, number>();+ // Count first so a truncated replay cannot attach a result to the wrong call.+ for (const { message } of input) {+ for (const part of message.parts) {+ if (part.type === "tool-call" && part.toolCallId)+ replayTotals.set(+ part.toolCallId,+ (replayTotals.get(part.toolCallId) ?? 0) + 1,+ );+ }+ }for (const { message, key: originalKey } of [...input, ...output]) {const isOutput = message.source === "output";const emitted: ThreadMessage = {@@ -67,7 +79,16 @@ export function append(// Anchor output calls before attaching any results carried by the same message.if (isOutput) thread.messages.push(emitted);for (const part of message.parts) {- if (!toolCalls.consumePart(observation, part, thread, emitted))+ if (+ !toolCalls.consumePart(+ observation,+ part,+ thread,+ emitted,+ replayCalls,+ replayTotals,+ )+ )emitted.parts.push(part);}if (!emitted.parts.length) {diff --git a/packages/shared/src/server/transcript/tool-calls.ts b/packages/shared/src/server/transcript/tool-calls.tsindex 47142c46d6..dbef39061c 100644--- a/packages/shared/src/server/transcript/tool-calls.ts+++ b/packages/shared/src/server/transcript/tool-calls.ts@@ -19,6 +19,7 @@ const key = (traceId: string, id: string) => JSON.stringify([traceId, id]);export function createToolCallRegistry() {const calls = new Map<string, Call>();+ const callsByThread = new WeakMap<Thread, Map<string, Call[]>>();const pending = new Map<string, { calls: Call[]; next: number }>();const responseTails = new WeakMap<ThreadMessage, ThreadMessage>();@@ -34,6 +35,13 @@ export function createToolCallRegistry() {if (id && calls.has(id)) return;const call: Call = { thread, message };if (id) calls.set(id, call);+ if (part.toolCallId) {+ if (!callsByThread.has(thread)) callsByThread.set(thread, new Map());+ const threadCalls = callsByThread.get(thread)!;+ const matches = threadCalls.get(part.toolCallId) ?? [];+ matches.push(call);+ threadCalls.set(part.toolCallId, matches);+ }const name = key(observation.traceId, part.toolName);if (!pending.has(name)) pending.set(name, { calls: [], next: 0 });pending.get(name)!.calls.push(call);@@ -74,11 +82,39 @@ export function createToolCallRegistry() {part: NormalizedMessage["parts"][number],thread: Thread,message: ThreadMessage,+ replayCalls: Map<string, number>,+ replayTotals: Map<string, number>,): boolean {+ if (+ part.type === "tool-call" &&+ message.source === "input" &&+ part.toolCallId+ )+ replayCalls.set(+ part.toolCallId,+ (replayCalls.get(part.toolCallId) ?? 0) + 1,+ );if (part.type === "tool-call" && message.source === "output")register(observation, part, thread, message);if (part.type !== "tool-result" || !part.toolCallId) return false;- const call = calls.get(key(observation.traceId, part.toolCallId));+ const matches = callsByThread.get(thread)?.get(part.toolCallId);+ // Reused IDs need complete, ordered call history to disambiguate results.+ const occurrence = replayCalls.get(part.toolCallId) ?? 0;+ let call: Call | undefined;+ if (message.source === "output") {+ call = calls.get(key(observation.traceId, part.toolCallId));+ } else if (+ matches?.length === 1 &&+ (replayTotals.get(part.toolCallId) ?? 0) <= 1+ ) {+ call = matches[0];+ } else if (+ matches &&+ replayTotals.get(part.toolCallId) === matches.length &&+ occurrence > 0+ ) {+ call = matches[occurrence - 1];+ }if (!call) return false;setResponse(call, observation, [part]);return true;