πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(normalized-io): enhance additional input handling (#17718)
marliessophie55ce03dc8f23
diff --git a/packages/shared/src/utils/normalized-io/README.md b/packages/shared/src/utils/normalized-io/README.mdindex 542310f166..286429d2dc 100644--- a/packages/shared/src/utils/normalized-io/README.md+++ b/packages/shared/src/utils/normalized-io/README.md@@ -212,6 +212,7 @@ or the fixture documents it.on` input, assistant on output) with the raw string preserved as senderName.The parser does not correlate role strings with known tool names, so suchturns surface as regular messages, not tool turns.+- Bare message inputs (for example, `{ role, content, ... }`) do not retain unrecognized sibling fields in `additionalInput`; these remain available in raw `span.input`.### Intentional lossesdiff --git a/packages/shared/src/utils/normalized-io/conventions/providers/ai-sdk/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/ai-sdk/index.tsindex c7b4782b70..79038c1e12 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/ai-sdk/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/ai-sdk/index.ts@@ -3,6 +3,7 @@ import {asRecord,compact,optionalString,+ recordKeyAsParsed,toJsonValue,} from "../../../core/utils/json";import {@@ -197,16 +198,17 @@ export const aiSdkProvider = {name: "ai-sdk",claimMessages: (root, kind): MessageSource[] => {if (kind !== "input" || root.messages !== undefined) return [];- if (typeof root.prompt === "string") {- return [{ kind: "single", value: root.prompt, fallbackRole: "user" }];+ const prompt = root.prompt;+ if (typeof prompt === "string") {+ recordKeyAsParsed(root, "prompt");+ return [{ kind: "single", value: prompt, fallbackRole: "user" }];}if (- Array.isArray(root.prompt) &&- root.prompt.every(- (message) => typeof asRecord(message)?.role === "string",- )+ Array.isArray(prompt) &&+ prompt.every((message) => typeof asRecord(message)?.role === "string")) {- return [{ kind: "sequence", values: root.prompt, fallbackRole: "user" }];+ recordKeyAsParsed(root, "prompt");+ return [{ kind: "sequence", values: prompt, fallbackRole: "user" }];}return [];},diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.tsindex 699da04a5b..05a306c0ab 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/anthropic/index.ts@@ -4,6 +4,7 @@ import {asRecord,compact,optionalString,+ recordKeyAsParsed,toJsonValue,toProviderMetadata,} from "../../../core/utils/json";@@ -207,7 +208,7 @@ function anthropicSystemMessage(if (kind !== "input" || !("system" in root)) return undefined;return {kind: "single",- value: { content: root.system },+ value: { content: recordKeyAsParsed(root, "system") },fallbackRole: "user",roleOverride: "system",};diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/gemini/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/gemini/fixtures.tsindex f5dc622665..0f4f812626 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/gemini/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/gemini/fixtures.ts@@ -314,6 +314,17 @@ export const capturedTraceFixtures: NormalizedIOFixture[] = ['{"attributes":{"session.id":"demo-session","user.id":"demo-user","gen_ai.system":"gcp.vertex.agent","gen_ai.request.model":"gemini-2.0-flash","gcp.vertex.agent.invocation_id":"e-5022fa36-69c5-4fe7-bb37-662704dcb713","gcp.vertex.agent.session_id":"demo-session","gcp.vertex.agent.event_id":"f2596747-d905-44bc-b85c-1e9692c3249c","gcp.vertex.agent.llm_request":"{\\"model\\": \\"gemini-2.0-flash\\", \\"config\\": {\\"http_options\\": {\\"headers\\": {\\"x-goog-api-client\\": \\"google-adk/1.12.0 gl-python/3.12.11\\", \\"user-agent\\": \\"google-adk/1.12.0 gl-python/3.12.11\\"}}, \\"system_instruction\\": \\"Always greet using the say_hello tool.\\\\n\\\\nYou are an agent. Your internal name is \\\\\\"hello_agent\\\\\\".\\", \\"tools\\": [{\\"function_declarations\\": [{\\"name\\": \\"say_hello\\"}]}]}, \\"contents\\": [{\\"parts\\": [{\\"text\\": \\"hi\\"}], \\"role\\": \\"user\\"}, {\\"parts\\": [{\\"function_call\\": {\\"args\\": {}, \\"name\\": \\"say_hello\\"}}], \\"role\\": \\"model\\"}, {\\"parts\\": [{\\"function_response\\": {\\"name\\": \\"say_hello\\", \\"response\\": {\\"greeting\\": \\"Hello Langfuse π\\"}}}], \\"role\\": \\"user\\"}]}","gcp.vertex.agent.llm_response":"{\\"content\\":{\\"parts\\":[{\\"text\\":\\"Hello Langfuse π!\\\\n\\"}],\\"role\\":\\"model\\"},\\"finish_reason\\":\\"STOP\\",\\"usage_metadata\\":{\\"candidates_token_count\\":6,\\"candidates_tokens_details\\":[{\\"modality\\":\\"TEXT\\",\\"token_count\\":6}],\\"prompt_token_count\\":39,\\"prompt_tokens_details\\":[{\\"modality\\":\\"TEXT\\",\\"token_count\\":39}],\\"total_token_count\\":45}}","gen_ai.usage.input_tokens":"39","gen_ai.usage.output_tokens":"6","gen_ai.response.finish_reasons":"[\\"stop\\"]","llm.provider":"google","input.value":"{\\"model\\": \\"gemini-2.0-flash\\", \\"config\\": {\\"http_options\\": {\\"headers\\": {\\"x-goog-api-client\\": \\"google-adk/1.12.0 gl-python/3.12.11\\", \\"user-agent\\": \\"google-adk/1.12.0 gl-python/3.12.11\\"}}, \\"system_instruction\\": \\"Always greet using the say_hello tool.\\\\n\\\\nYou are an agent. Your internal name is \\\\\\"hello_agent\\\\\\".\\", \\"tools\\": [{\\"function_declarations\\": [{\\"name\\": \\"say_hello\\"}]}]}, \\"contents\\": [{\\"parts\\": [{\\"text\\": \\"hi\\"}], \\"role\\": \\"user\\"}, {\\"parts\\": [{\\"function_call\\": {\\"args\\": {}, \\"name\\": \\"say_hello\\"}}], \\"role\\": \\"model\\"}, {\\"parts\\": [{\\"function_response\\": {\\"name\\": \\"say_hello\\", \\"response\\": {\\"greeting\\": \\"Hello Langfuse π\\"}}}], \\"role\\": \\"user\\"}]}","input.mime_type":"application/json","llm.tools.0.tool.json_schema":"{\\"name\\":\\"say_hello\\"}","llm.model_name":"gemini-2.0-flash","llm.invocation_parameters":"{\\"http_options\\":{\\"headers\\":{\\"x-goog-api-client\\":\\"google-adk/1.12.0 gl-python/3.12.11\\",\\"user-agent\\":\\"google-adk/1.12.0 gl-python/3.12.11\\"}},\\"system_instruction\\":\\"Always greet using the say_hello tool.\\\\n\\\\nYou are an agent. Your internal name is \\\\\\"hello_agent\\\\\\".\\",\\"tools\\":[{\\"function_declarations\\":[{\\"name\\":\\"say_hello\\"}]}]}","llm.input_messages.0.message.role":"system","llm.input_messages.0.message.content":"Always greet using the say_hello tool.\\n\\nYou are an agent. Your internal name is \\"hello_agent\\".","llm.input_messages.1.message.role":"user","llm.input_messages.1.message.contents.0.message_content.text":"hi","llm.input_messages.1.message.contents.0.message_content.type":"text","llm.input_messages.2.message.role":"model","llm.input_messages.2.message.tool_calls.0.tool_call.function.name":"say_hello","llm.input_messages.3.message.role":"tool","llm.input_messages.3.message.name":"say_hello","llm.input_messages.3.message.content":"{\\"greeting\\": \\"Hello Langfuse π\\"}","output.value":"{\\"content\\":{\\"parts\\":[{\\"text\\":\\"Hello Langfuse π!\\\\n\\"}],\\"role\\":\\"model\\"},\\"finish_reason\\":\\"STOP\\",\\"usage_metadata\\":{\\"candidates_token_count\\":6,\\"candidates_tokens_details\\":[{\\"modality\\":\\"TEXT\\",\\"token_count\\":6}],\\"prompt_token_count\\":39,\\"prompt_tokens_details\\":[{\\"modality\\":\\"TEXT\\",\\"token_count\\":39}],\\"total_token_count\\":45}}","output.mime_type":"application/json","llm.token_count.total":"45","llm.token_count.prompt":"39","llm.token_count.completion":"6","llm.output_messages.0.message.role":"model","llm.output_messages.0.message.contents.0.message_content.text":"Hello Langfuse π!\\n","llm.output_messages.0.message.contents.0.message_content.type":"text","openinference.span.kind":"LLM"},"resourceAttributes":{"telemetry.sdk.language":"python","telemetry.sdk.name":"opentelemetry","telemetry.sdk.version":"1.36.0","service.name":"unknown_service"},"scope":{"name":"openinference.instrumentation.google_adk","version":"0.1.3","attributes":{}}}',},expected: {+ additionalInput: {+ config: {+ http_options: {+ headers: {+ "user-agent": "google-adk/1.12.0 gl-python/3.12.11",+ "x-goog-api-client": "google-adk/1.12.0 gl-python/3.12.11",+ },+ },+ },+ model: "gemini-2.0-flash",+ },messages: [{role: "system",@@ -394,6 +405,7 @@ export const capturedTraceFixtures: NormalizedIOFixture[] = ['{"attributes":{"llm.input_messages.0.message.content":"What is Langfuse?","llm.input_messages.0.message.role":"user","output.value":"{\\"sdk_http_response\\":{\\"headers\\":{\\"content-type\\":\\"application/json; charset=UTF-8\\",\\"vary\\":\\"Origin, X-Origin, Referer\\",\\"content-encoding\\":\\"gzip\\",\\"date\\":\\"Fri, 01 Aug 2025 13:22:12 GMT\\",\\"server\\":\\"scaffolding on HTTPServer2\\",\\"x-xss-protection\\":\\"0\\",\\"x-frame-options\\":\\"SAMEORIGIN\\",\\"x-content-type-options\\":\\"nosniff\\",\\"server-timing\\":\\"gfet4t7; dur=11935\\",\\"alt-svc\\":\\"h3=\\\\\\":443\\\\\\"; ma=2592000,h3-29=\\\\\\":443\\\\\\"; ma=2592000\\",\\"transfer-encoding\\":\\"chunked\\"}},\\"candidates\\":[{\\"content\\":{\\"parts\\":[{\\"text\\":\\"**Langfuse** is an **open-source observability and evaluation platform specifically designed for Large Language Model (LLM) applications.**\\\\n\\\\nIn essence, it acts as a \\\\\\"flight recorder\\\\\\" or \\\\\\"black box\\\\\\" for your AI applications, giving you deep insights into how they are performing in real-time and helping you iterate and improve them.\\\\n\\\\nHere\'s a breakdown of what Langfuse is and why it\'s important:\\\\n\\\\n### The Problem Langfuse Solves\\\\n\\\\nBuilding applications powered by LLMs (like chatbots, summarizers, RAG systems) presents unique challenges:\\\\n\\\\n1. **Non-Determinism:** LLMs aren\'t like traditional code; their outputs can vary, making debugging difficult.\\\\n2. **Opacity:** It\'s hard to see what\'s happening \\\\\\"inside\\\\\\" the LLM call β what prompt was sent, what response was received, how long it took, and what tools were used.\\\\n3. **Performance & Cost Tracking:** Monitoring token usage, latency, and costs across various LLM calls and chains is complex.\\\\n4. **Evaluation & Improvement:** How do you know if a change to your prompt or RAG system actually made your application better? How do you collect user feedback?\\\\n5. **Debugging Production Issues:** When something goes wrong in production, tracing the exact cause (e.g., a bad prompt, a hallucination, a failed tool call) is crucial.\\\\n\\\\n### How Langfuse Works & Its Key Features\\\\n\\\\nLangfuse addresses these challenges by providing:\\\\n\\\\n1. **Tracing & Observability:**\\\\n * **End-to-end visibility:** It captures every step of an LLM application\'s lifecycle, from user input, through various LLM calls, tool usage, RAG retrievals, and final output.\\\\n * **Detailed Spans:** Each step (e.g., an OpenAI call, a LlamaIndex query, a custom function) is recorded as a \\\\\\"span,\\\\\\" showing inputs, outputs, timestamps, costs (tokens, price), and associated metadata.\\\\n * **Visualizations:** Presents these traces in a clear, waterfall-style UI, making it easy to understand the flow and identify bottlenecks or errors.\\\\n\\\\n2. **Debugging & Monitoring:**\\\\n * **Real-time Dashboards:** Provides metrics on latency, cost, token usage, and error rates across your application.\\\\n * **Error Identification:** Quickly pinpoint exactly which LLM call or step failed within a complex chain.\\\\n * **Search & Filtering:** Easily find specific traces based on user IDs, session IDs, error status, or custom tags.\\\\n\\\\n3. **Evaluation & Feedback Loops:**\\\\n * **Manual Feedback:** Allows users (or developers) to provide direct feedback (e.g., \\\\\\"good,\\\\\\" \\\\\\"bad,\\\\\\" custom scores) on specific traces. This is invaluable for human-in-the-loop evaluation.\\\\n * **Automated Evaluation:** Integrates with evaluation frameworks (e.g., RAGAS) or custom scripts to automatically score LLM outputs, enabling A/B testing of prompts or models.\\\\n * **Prompt Comparison:** Helps you compare the performance of different prompt versions side-by-side based on collected metrics and evaluations.\\\\n\\\\n4. **Prompt Management:**\\\\n * Store and version control your prompts.\\\\n * Attach evaluations to specific prompt versions to track their effectiveness over time.\\\\n\\\\n5. **User & Session Tracking:**\\\\n * Associate traces with specific users or sessions, allowing you to understand individual user journeys and identify patterns.\\\\n\\\\n### Key Benefits\\\\n\\\\n* **Faster Debugging:** Quickly diagnose and fix issues in your LLM applications.\\\\n* **Improved Performance:** Identify and optimize slow or costly parts of your LLM chains.\\\\n* **Data-Driven Iteration:** Make informed decisions about prompt engineering, model selection, and architecture based on real usage data and evaluations.\\\\n* **Cost Control:** Monitor token usage and costs to prevent budget overruns.\\\\n* **Enhanced Reliability:** Build more robust and production-ready LLM applications.\\\\n\\\\n### Integrations\\\\n\\\\nLangfuse integrates with popular LLM frameworks and libraries, including:\\\\n\\\\n* LangChain\\\\n* LlamaIndex\\\\n* OpenAI API\\\\n* Anthropic API\\\\n* And can be used with any custom LLM integration.\\\\n\\\\nIn summary, Langfuse is a critical tool for any developer or team serious about building, deploying, and maintaining high-quality, performant, and reliable LLM-powered applications in production.\\"}],\\"role\\":\\"model\\"},\\"finish_reason\\":\\"STOP\\",\\"index\\":0}],\\"model_version\\":\\"gemini-2.5-flash\\",\\"usage_metadata\\":{\\"candidates_token_count\\":951,\\"prompt_token_count\\":6,\\"prompt_tokens_details\\":[{\\"modality\\":\\"TEXT\\",\\"token_count\\":6}],\\"thoughts_token_count\\":1104,\\"total_token_count\\":2061},\\"automatic_function_calling_history\\":[]}","output.mime_type":"application/json","llm.provider":"google","input.value":"{\\"model\\": \\"gemini-2.5-flash\\", \\"contents\\": \\"What is Langfuse?\\"}","input.mime_type":"application/json","llm.model_name":"gemini-2.5-flash","llm.token_count.total":"2061","llm.token_count.prompt":"6","llm.token_count.completion_details.reasoning":"1104","llm.token_count.completion":"2055","llm.output_messages.0.message.content":"**Langfuse** is an **open-source observability and evaluation platform specifically designed for Large Language Model (LLM) applications.**\\n\\nIn essence, it acts as a \\"flight recorder\\" or \\"black box\\" for your AI applications, giving you deep insights into how they are performing in real-time and helping you iterate and improve them.\\n\\nHere\'s a breakdown of what Langfuse is and why it\'s important:\\n\\n### The Problem Langfuse Solves\\n\\nBuilding applications powered by LLMs (like chatbots, summarizers, RAG systems) presents unique challenges:\\n\\n1. **Non-Determinism:** LLMs aren\'t like traditional code; their outputs can vary, making debugging difficult.\\n2. **Opacity:** It\'s hard to see what\'s happening \\"inside\\" the LLM call β what prompt was sent, what response was received, how long it took, and what tools were used.\\n3. **Performance & Cost Tracking:** Monitoring token usage, latency, and costs across various LLM calls and chains is complex.\\n4. **Evaluation & Improvement:** How do you know if a change to your prompt or RAG system actually made your application better? How do you collect user feedback?\\n5. **Debugging Production Issues:** When something goes wrong in production, tracing the exact cause (e.g., a bad prompt, a hallucination, a failed tool call) is crucial.\\n\\n### How Langfuse Works & Its Key Features\\n\\nLangfuse addresses these challenges by providing:\\n\\n1. **Tracing & Observability:**\\n * **End-to-end visibility:** It captures every step of an LLM application\'s lifecycle, from user input, through various LLM calls, tool usage, RAG retrievals, and final output.\\n * **Detailed Spans:** Each step (e.g., an OpenAI call, a LlamaIndex query, a custom function) is recorded as a \\"span,\\" showing inputs, outputs, timestamps, costs (tokens, price), and associated metadata.\\n * **Visualizations:** Presents these traces in a clear, waterfall-style UI, making it easy to understand the flow and identify bottlenecks or errors.\\n\\n2. **Debugging & Monitoring:**\\n * **Real-time Dashboards:** Provides metrics on latency, cost, token usage, and error rates across your application.\\n * **Error Identification:** Quickly pinpoint exactly which LLM call or step failed within a complex chain.\\n * **Search & Filtering:** Easily find specific traces based on user IDs, session IDs, error status, or custom tags.\\n\\n3. **Evaluation & Feedback Loops:**\\n * **Manual Feedback:** Allows users (or developers) to provide direct feedback (e.g., \\"good,\\" \\"bad,\\" custom scores) on specific traces. This is invaluable for human-in-the-loop evaluation.\\n * **Automated Evaluation:** Integrates with evaluation frameworks (e.g., RAGAS) or custom scripts to automatically score LLM outputs, enabling A/B testing of prompts or models.\\n * **Prompt Comparison:** Helps you compare the performance of different prompt versions side-by-side based on collected metrics and evaluations.\\n\\n4. **Prompt Management:**\\n * Store and version control your prompts.\\n * Attach evaluations to specific prompt versions to track their effectiveness over time.\\n\\n5. **User & Session Tracking:**\\n * Associate traces with specific users or sessions, allowing you to understand individual user journeys and identify patterns.\\n\\n### Key Benefits\\n\\n* **Faster Debugging:** Quickly diagnose and fix issues in your LLM applications.\\n* **Improved Performance:** Identify and optimize slow or costly parts of your LLM chains.\\n* **Data-Driven Iteration:** Make informed decisions about prompt engineering, model selection, and architecture based on real usage data and evaluations.\\n* **Cost Control:** Monitor token usage and costs to prevent budget overruns.\\n* **Enhanced Reliability:** Build more robust and production-ready LLM applications.\\n\\n### Integrations\\n\\nLangfuse integrates with popular LLM frameworks and libraries, including:\\n\\n* LangChain\\n* LlamaIndex\\n* OpenAI API\\n* Anthropic API\\n* And can be used with any custom LLM integration.\\n\\nIn summary, Langfuse is a critical tool for any developer or team serious about building, deploying, and maintaining high-quality, performant, and reliable LLM-powered applications in production.","llm.output_messages.0.message.role":"model","openinference.span.kind":"LLM"},"resourceAttributes":{"telemetry.sdk.language":"python","telemetry.sdk.name":"opentelemetry","telemetry.sdk.version":"1.34.1","service.name":"unknown_service"},"scope":{"name":"openinference.instrumentation.google_genai","version":"0.1.5","attributes":{}}}',},expected: {+ additionalInput: { model: "gemini-2.5-flash" },messages: [{role: "user",@@ -435,6 +447,12 @@ export const capturedTraceFixtures: NormalizedIOFixture[] = ['{"attributes":{"input.value":"{\\"contents\\": [{\\"role\\": \\"user\\", \\"parts\\": [{\\"text\\": \\"What is Langfuse?\\"}]}], \\"model\\": \\"projects/vertex-ai-integration-467711/locations/europe-central2/publishers/google/models/gemini-2.5-flash\\", \\"tools\\": [], \\"labels\\": {}, \\"safety_settings\\": []}","input.mime_type":"application/json","llm.model_name":"projects/vertex-ai-integration-467711/locations/europe-central2/publishers/google/models/gemini-2.5-flash","llm.invocation_parameters":"{\\"stop_sequences\\": [], \\"response_mime_type\\": \\"\\"}","llm.input_messages.0.message.role":"user","llm.input_messages.0.message.contents.0.message_content.text":"What is Langfuse?","output.value":"{\\"candidates\\": [{\\"content\\": {\\"role\\": \\"model\\", \\"parts\\": [{\\"text\\": \\"**Langfuse** is an **open-source observability and evaluation platform specifically designed for Large Language Model (LLM) applications.**\\\\n\\\\nThink of it as an Application Performance Management (APM) tool, but tailored to the unique challenges of building and running applications powered by LLMs, such as chatbots, summarizers, code generators, and more.\\\\n\\\\n### Why Langfuse Exists (The Problem It Solves)\\\\n\\\\nDeveloping with LLMs introduces new complexities:\\\\n\\\\n1. **Non-Determinism:** LLM outputs can vary even with the same input, making debugging difficult.\\\\n2. **Opacity:** It\'s hard to see *why* an LLM responded a certain way, especially with complex prompt chains or tool usage.\\\\n3. **Cost & Latency:** LLM API calls can be expensive and slow, requiring careful monitoring.\\\\n4. **Quality Evaluation:** Subjective and difficult to measure if an LLM is truly \\\\\\"good\\\\\\" or \\\\\\"helpful\\\\\\" for specific tasks.\\\\n5. **Iteration Speed:** Testing prompt changes, different models, or retrieval-augmented generation (RAG) strategies requires robust tooling.\\\\n6. **User Feedback:** Connecting user-reported issues or satisfaction back to specific LLM interactions.\\\\n\\\\n### What Langfuse Does (Key Features)\\\\n\\\\nLangfuse helps address these problems by providing:\\\\n\\\\n1. **Tracing & Debugging:**\\\\n * **End-to-end visibility:** It captures every step of an LLM call or a chain of LLM calls (e.g., initial user query, prompt engineering, internal thoughts of the LLM, tool calls, final response).\\\\n * **Detailed data:** Records prompts, responses, models used, tokens consumed, latency, cost, errors, and custom metadata.\\\\n * **Visual timelines:** Allows developers to inspect the flow of an interaction, pinpointing where issues occurred.\\\\n\\\\n2. **Evaluation & Feedback:**\\\\n * **Manual Annotations:** Allows human reviewers to rate the quality of LLM outputs directly within the platform.\\\\n * **Automated Evaluations:** Integrates with evaluation frameworks to run automated checks (e.g., factual consistency, toxicity, helpfulness).\\\\n * **Feedback Integration:** Connects user feedback (e.g., thumbs up/down, specific comments) directly to the trace of the interaction.\\\\n\\\\n3. **Monitoring & Analytics:**\\\\n * **Dashboards:** Provides metrics on key performance indicators like latency, cost, token usage, error rates, and throughput.\\\\n * **Trend Analysis:** Helps identify regressions or improvements over time.\\\\n * **Experimentation:** Facilitates A/B testing of different prompts, models, or configurations by comparing their performance metrics and evaluations.\\\\n\\\\n4. **Prompt Management:** While not a dedicated prompt engineering tool, its tracing capabilities allow you to see how different prompts perform in production and iterate on them effectively.\\\\n\\\\n### How It Works\\\\n\\\\n* **SDKs:** You integrate Langfuse into your application code using their SDKs (e.g., Python, JavaScript).\\\\n* **Data Capture:** The SDKs automatically capture relevant data about your LLM calls and send it to the Langfuse backend.\\\\n* **Backend:** Langfuse provides a hosted cloud service, or you can self-host their open-source backend (which uses PostgreSQL and ClickHouse).\\\\n* **Web UI:** All the captured data is visualized in a user-friendly web interface for analysis, debugging, and evaluation.\\\\n\\\\n### Who Uses Langfuse?\\\\n\\\\n* **LLM Developers:** For debugging, testing, and improving their LLM-powered applications.\\\\n* **ML Engineers:** To monitor the performance and reliability of LLMs in production.\\\\n* **Product Managers:** To understand user experience and the impact of LLM changes.\\\\n* **Anyone building and deploying Generative AI applications.**\\\\n\\\\nIn essence, Langfuse provides the necessary tools for developers to bring LLM applications from prototype to production with confidence, ensuring they are observable, debuggable, and continuously improving.\\"}]}, \\"finish_reason\\": 1, \\"avg_logprobs\\": -0.7123595830251479, \\"index\\": 0, \\"score\\": 0.0, \\"safety_ratings\\": []}], \\"usage_metadata\\": {\\"prompt_token_count\\": 5, \\"candidates_token_count\\": 845, \\"total_token_count\\": 2023}, \\"model_version\\": \\"gemini-2.5-flash\\"}","output.mime_type":"application/json","llm.token_count.prompt":"5","llm.token_count.completion":"845","llm.token_count.total":"2023","llm.output_messages.0.message.role":"assistant","llm.output_messages.0.message.contents.0.message_content.text":"**Langfuse** is an **open-source observability and evaluation platform specifically designed for Large Language Model (LLM) applications.**\\n\\nThink of it as an Application Performance Management (APM) tool, but tailored to the unique challenges of building and running applications powered by LLMs, such as chatbots, summarizers, code generators, and more.\\n\\n### Why Langfuse Exists (The Problem It Solves)\\n\\nDeveloping with LLMs introduces new complexities:\\n\\n1. **Non-Determinism:** LLM outputs can vary even with the same input, making debugging difficult.\\n2. **Opacity:** It\'s hard to see *why* an LLM responded a certain way, especially with complex prompt chains or tool usage.\\n3. **Cost & Latency:** LLM API calls can be expensive and slow, requiring careful monitoring.\\n4. **Quality Evaluation:** Subjective and difficult to measure if an LLM is truly \\"good\\" or \\"helpful\\" for specific tasks.\\n5. **Iteration Speed:** Testing prompt changes, different models, or retrieval-augmented generation (RAG) strategies requires robust tooling.\\n6. **User Feedback:** Connecting user-reported issues or satisfaction back to specific LLM interactions.\\n\\n### What Langfuse Does (Key Features)\\n\\nLangfuse helps address these problems by providing:\\n\\n1. **Tracing & Debugging:**\\n * **End-to-end visibility:** It captures every step of an LLM call or a chain of LLM calls (e.g., initial user query, prompt engineering, internal thoughts of the LLM, tool calls, final response).\\n * **Detailed data:** Records prompts, responses, models used, tokens consumed, latency, cost, errors, and custom metadata.\\n * **Visual timelines:** Allows developers to inspect the flow of an interaction, pinpointing where issues occurred.\\n\\n2. **Evaluation & Feedback:**\\n * **Manual Annotations:** Allows human reviewers to rate the quality of LLM outputs directly within the platform.\\n * **Automated Evaluations:** Integrates with evaluation frameworks to run automated checks (e.g., factual consistency, toxicity, helpfulness).\\n * **Feedback Integration:** Connects user feedback (e.g., thumbs up/down, specific comments) directly to the trace of the interaction.\\n\\n3. **Monitoring & Analytics:**\\n * **Dashboards:** Provides metrics on key performance indicators like latency, cost, token usage, error rates, and throughput.\\n * **Trend Analysis:** Helps identify regressions or improvements over time.\\n * **Experimentation:** Facilitates A/B testing of different prompts, models, or configurations by comparing their performance metrics and evaluations.\\n\\n4. **Prompt Management:** While not a dedicated prompt engineering tool, its tracing capabilities allow you to see how different prompts perform in production and iterate on them effectively.\\n\\n### How It Works\\n\\n* **SDKs:** You integrate Langfuse into your application code using their SDKs (e.g., Python, JavaScript).\\n* **Data Capture:** The SDKs automatically capture relevant data about your LLM calls and send it to the Langfuse backend.\\n* **Backend:** Langfuse provides a hosted cloud service, or you can self-host their open-source backend (which uses PostgreSQL and ClickHouse).\\n* **Web UI:** All the captured data is visualized in a user-friendly web interface for analysis, debugging, and evaluation.\\n\\n### Who Uses Langfuse?\\n\\n* **LLM Developers:** For debugging, testing, and improving their LLM-powered applications.\\n* **ML Engineers:** To monitor the performance and reliability of LLMs in production.\\n* **Product Managers:** To understand user experience and the impact of LLM changes.\\n* **Anyone building and deploying Generative AI applications.**\\n\\nIn essence, Langfuse provides the necessary tools for developers to bring LLM applications from prototype to production with confidence, ensuring they are observable, debuggable, and continuously improving.","openinference.span.kind":"LLM"},"resourceAttributes":{"telemetry.sdk.language":"python","telemetry.sdk.name":"opentelemetry","telemetry.sdk.version":"1.34.1","service.name":"unknown_service"},"scope":{"name":"openinference.instrumentation.vertexai","version":"0.1.11","attributes":{}}}',},expected: {+ additionalInput: {+ labels: {},+ model:+ "projects/vertex-ai-integration-467711/locations/europe-central2/publishers/google/models/gemini-2.5-flash",+ safety_settings: [],+ },messages: [{role: "user",diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/gemini/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/gemini/index.tsindex db3abbb3fe..9130aa6d51 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/gemini/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/gemini/index.ts@@ -4,6 +4,7 @@ import {compact,optionalString,parseArray,+ recordKeyAsParsed,toJsonValue,} from "../../../core/utils/json";import {@@ -177,8 +178,10 @@ function normalizeGeminiPart(function geminiToolDefinitionSources(carrier: ToolDefinitionCarrier,): ToolDefinitionSource[] {- const tools = parseArray(asRecord(carrier.root?.config)?.tools);+ if (!carrier.root) return [];+ const tools = parseArray(asRecord(carrier.root.config)?.tools);if (!tools) return [];+ recordKeyAsParsed(carrier.root, "config", "tools");return tools.map((tool, index) => {const toolGroup = asRecord(tool);@@ -210,12 +213,11 @@ function geminiSystemMessage(): MessageSource | undefined {if (kind !== "input") return undefined;- const config = asRecord(root.config);const systemInstruction =- root.systemInstruction ??- root.system_instruction ??- config?.system_instruction ??- config?.systemInstruction;+ recordKeyAsParsed(root, "systemInstruction") ??+ recordKeyAsParsed(root, "system_instruction") ??+ recordKeyAsParsed(root, "config", "system_instruction") ??+ recordKeyAsParsed(root, "config", "systemInstruction");if (!systemInstruction) return undefined;return {@@ -236,16 +238,15 @@ function geminiMessages(if ("new_message" in root) {sources.push({kind: "single",- value: root.new_message,+ value: recordKeyAsParsed(root, "new_message"),fallbackRole: "user",});return sources;}if ("contents" in root) {- const contents = Array.isArray(root.contents)- ? root.contents- : [root.contents];+ const rawContents = recordKeyAsParsed(root, "contents");+ const contents = Array.isArray(rawContents) ? rawContents : [rawContents];sources.push({kind: "sequence",values: contents,diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.tsindex c9cb419df5..3931ba8b7e 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts@@ -17,6 +17,7 @@ export const documentedResponsesFixtures: NormalizedIOFixture[] = [metadata: undefined,},expected: {+ additionalInput: { model: "gpt-4.1" },messages: [{source: "input",diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.tsindex 01972eae99..943259f22a 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/index.ts@@ -8,6 +8,7 @@ import {optionalString,parseArray,parseIfString,+ recordKeyAsParsed,toJsonValue,toProviderMetadata,} from "../../../core/utils/json";@@ -430,6 +431,23 @@ function openAiReasoningParts(return parts;}+function isResponsesRequest(root: Record<string, unknown>): boolean {+ if (typeof root.input === "string" && typeof root.model === "string") {+ return true;+ }+ const responseInput = parseArray(root.input);+ return Boolean(+ responseInput?.length &&+ responseInput.every((item) => {+ const record = asRecord(item);+ return (+ record &&+ (typeof record.role === "string" || typeof record.type === "string")+ );+ }),+ );+}+/*** OpenAI response envelopes: Chat Completions `choices[]` (finish reason* per choice) and Responses' `output[]` item array.@@ -440,25 +458,17 @@ function openAiMessages(): MessageSource[] {// Responses API uses `input` instead of Chat Completions' `messages`:if (kind === "input") {- if (typeof root.input === "string" && typeof root.model === "string") {- return [{ kind: "single", value: root.input, fallbackRole: "user" }];- }- const responseInput = parseArray(root.input);- if (- responseInput?.length &&- responseInput.every((item) => {- const record = asRecord(item);- return (- record &&- (typeof record.role === "string" || typeof record.type === "string")- );- })- ) {- return [- { kind: "sequence", values: responseInput, fallbackRole: "user" },- ];- }- return [];+ if (!isResponsesRequest(root)) return [];+ const input = recordKeyAsParsed(root, "input");+ return typeof input === "string" && typeof root.model === "string"+ ? [{ kind: "single", value: input, fallbackRole: "user" }]+ : [+ {+ kind: "sequence",+ values: parseArray(input) ?? [],+ fallbackRole: "user",+ },+ ];}if (kind !== "output") return [];@@ -640,10 +650,10 @@ export const openAiProvider: IOConvention = {getSystemMessage: (root, kind) =>kind === "input" &&typeof root.instructions === "string" &&- openAiMessages(root, kind).length > 0+ isResponsesRequest(root)? {kind: "single",- value: root.instructions,+ value: recordKeyAsParsed(root, "instructions"),fallbackRole: "user",roleOverride: "system",}diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts b/packages/shared/src/utils/normalized-io/core/accumulator/collect.tsindex 2246bc04d1..7fc4a86c58 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/collect.ts@@ -1,7 +1,13 @@/* eslint-disable no-nested-ternary */import { registeredProviders } from "../../conventions";import type { MessageSource } from "../../conventions/io-convention";-import { asRecord, parseIfString, parseRecord } from "../utils/json";+import {+ asRecord,+ parseArray,+ parseIfString,+ parseRecord,+ recordKeyAsParsed,+} from "../utils/json";import type { NormalizedMessage, NormalizedMessagePart } from "../../types";import { addMessage, addToolDefinitionValue } from "./helpers";import type { NormalizedIOAccumulator } from "./interface";@@ -16,14 +22,13 @@ import { isMessageLike, isToolDefinitionMessage } from "../utils/format";export type ParsedIOValue = {value: unknown;record?: Record<string, unknown>;- messages?: unknown[];};function collectToolDefinitionsFromRecord(record: Record<string, unknown>,accumulator: NormalizedIOAccumulator,-): void {- addToolDefinitionValue(accumulator, record.tools, {+): boolean {+ const toolsFullyParsed = addToolDefinitionValue(accumulator, record.tools, {allowProviderToolWithoutName: true,allowToolMap: true,});@@ -31,13 +36,16 @@ function collectToolDefinitionsFromRecord(if (isToolDefinitionMessage(record)) {addToolDefinitionValue(accumulator, record.content);}+ return toolsFullyParsed;}function collectRootToolDefinitions(root: Record<string, unknown>,accumulator: NormalizedIOAccumulator,): void {- collectToolDefinitionsFromRecord(root, accumulator);+ if (collectToolDefinitionsFromRecord(root, accumulator)) {+ recordKeyAsParsed(root, "tools");+ }// Definitions are independent of the message claim. A record can contain// an OpenAI `choices` carrier and, next to it, a framework `messages` or@@ -199,48 +207,67 @@ function emitRootSource();}+function findSystemMessageSources(+ root: Record<string, unknown>,+ source: "input" | "output",+): MessageSource[] {+ const sources: MessageSource[] = [];+ for (const provider of registeredProviders) {+ const systemMessage = provider.getSystemMessage?.(root, source);+ if (systemMessage) sources.push(systemMessage);+ }+ return sources;+}++function claimMessages(+ root: Record<string, unknown>,+ source: "input" | "output",+ parserContext: ParserContext,+): MessageSource[] {+ for (const provider of registeredProviders) {+ const sources = provider.claimMessages?.(root, source) ?? [];+ if (sources.length > 0) {+ parserContext.preferredProvider = provider;+ return sources;+ }+ }+ return [];+}+function collectRecordMessages(- parsedValue: ParsedIOValue,+ root: Record<string, unknown>,parserContext: ParserContext,accumulator: NormalizedIOAccumulator,): void {- const record = parsedValue.record;- if (!record) return;-const { source } = parserContext;const fallbackRole = source === "input" ? "user" : "assistant";- const providerClaims = registeredProviders.map((provider) => ({- provider,- sources: provider.claimMessages?.(record, source) ?? [],- systemMessage: provider.getSystemMessage?.(record, source),- }));- const selectedClaim = providerClaims.find(- ({ sources }) => sources.length > 0,- );-- if (selectedClaim) {- parserContext.preferredProvider = selectedClaim.provider;- }+ const systemSources =+ source === "input" ? findSystemMessageSources(root, source) : [];+ const claimedSources = claimMessages(root, source, parserContext);+ const rootMessages = parseArray(root.messages);const messages: NormalizedMessage[] = [];- const claimedSources = selectedClaim?.sources;- if (claimedSources && claimedSources.length > 0) {+ if (claimedSources.length > 0) {for (const rootSource of claimedSources) {emitRootSource(rootSource, parserContext, messages, accumulator);}- } else if (parsedValue.messages) {+ } else if (rootMessages) {+ recordKeyAsParsed(root, "messages");collectMessageSequence(- parsedValue.messages,+ rootMessages,fallbackRole,parserContext,messages,accumulator,);} else {- const recordMessage = normalizeMessage(record, fallbackRole, parserContext);- if (recordMessage) addMessage(messages, recordMessage, parserContext);+ const recordMessage = normalizeMessage(root, fallbackRole, parserContext);+ if (recordMessage) {+ addMessage(messages, recordMessage, parserContext);+ delete accumulator.unparsedKeys[source];+ }}// System instructions supplement the selected input conversation. Defer@@ -248,16 +275,9 @@ function collectRecordMessages(// suppresses every top-level system sidecar.if (source === "input" && !parserContext.hasSystemMessage) {const systemMessages: NormalizedMessage[] = [];- for (const { systemMessage } of providerClaims) {- if (systemMessage) {- emitRootSource(- systemMessage,- parserContext,- systemMessages,- accumulator,- );- }- if (parserContext.hasSystemMessage) break;+ for (const systemSource of systemSources) {+ emitRootSource(systemSource, parserContext, systemMessages, accumulator);+ if (systemMessages.length > 0) break;}messages.unshift(...systemMessages);}@@ -285,7 +305,7 @@ export function collectIO(if (Array.isArray(parsedValue.value)) {collectMessageSequence(- parsedValue.messages ?? parsedValue.value,+ parsedValue.value,fallbackRole,context,accumulator.messages,@@ -295,8 +315,10 @@ export function collectIO(}if (parsedValue.record) {- collectRootToolDefinitions(parsedValue.record, accumulator);- collectRecordMessages(parsedValue, context, accumulator);+ const unparsedKeys = { ...parsedValue.record };+ accumulator.unparsedKeys[source] = unparsedKeys;+ collectRootToolDefinitions(unparsedKeys, accumulator);+ collectRecordMessages(unparsedKeys, context, accumulator);return;}diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/helpers.ts b/packages/shared/src/utils/normalized-io/core/accumulator/helpers.tsindex c894e0339d..fea5da6f3c 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/helpers.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/helpers.ts@@ -1,4 +1,4 @@-import { normalizeToolDefinitionValue } from "../normalize/tool-definitions";+import { parseToolDefinitionValue } from "../normalize/tool-definitions";import {NormalizedMessage,NormalizedMessagePart,@@ -13,10 +13,12 @@ export function addToolDefinitionValue(accumulator: NormalizedIOAccumulator,value: unknown,options: ToolDefinitionOptions = {},-): void {- for (const definition of normalizeToolDefinitionValue(value, options)) {+): boolean {+ const { definitions, fullyParsed } = parseToolDefinitionValue(value, options);+ for (const definition of definitions) {addToolDefinition(accumulator, definition);}+ return fullyParsed;}function getToolCallKey(part: NormalizedMessagePart): string | undefined {diff --git a/packages/shared/src/utils/normalized-io/core/accumulator/interface.ts b/packages/shared/src/utils/normalized-io/core/accumulator/interface.tsindex f19500fec9..c4b835ee94 100644--- a/packages/shared/src/utils/normalized-io/core/accumulator/interface.ts+++ b/packages/shared/src/utils/normalized-io/core/accumulator/interface.ts@@ -4,6 +4,10 @@ export type NormalizedIOAccumulator = {messages: NormalizedMessage[];toolDefinitions: ToolDefinition[];toolDefinitionIndexByName: Map<string, number>;+ unparsedKeys: {+ input?: Record<string, unknown>;+ output?: Record<string, unknown>;+ };};export function createAccumulator(): NormalizedIOAccumulator {@@ -11,5 +15,6 @@ export function createAccumulator(): NormalizedIOAccumulator {messages: [],toolDefinitions: [],toolDefinitionIndexByName: new Map(),+ unparsedKeys: {},};}diff --git a/packages/shared/src/utils/normalized-io/core/normalize/tool-definitions.ts b/packages/shared/src/utils/normalized-io/core/normalize/tool-definitions.tsindex e6c7a7e798..bd966dc328 100644--- a/packages/shared/src/utils/normalized-io/core/normalize/tool-definitions.ts+++ b/packages/shared/src/utils/normalized-io/core/normalize/tool-definitions.ts@@ -96,39 +96,46 @@ function normalizeDefinitionItem(return normalizeLooseToolDefinition(value, options);}-/**- * Normalizes one tool-definition carrier value: an array, a single- * definition record, or (with `allowToolMap`) a map keyed by tool name.- */-export function normalizeToolDefinitionValue(+export function parseToolDefinitionValue(value: unknown,options: ToolDefinitionOptions = {},-): ToolDefinition[] {+): { definitions: ToolDefinition[]; fullyParsed: boolean } {const parsed = parseIfString(value);if (Array.isArray(parsed)) {- return parsed- .map((item) => normalizeDefinitionItem(parseIfString(item), options))- .filter((def): def is ToolDefinition => def !== null);+ const results = parsed.map((item) =>+ normalizeDefinitionItem(parseIfString(item), options),+ );+ return {+ definitions: results.filter((def): def is ToolDefinition => def !== null),+ fullyParsed: results.every((def) => def !== null),+ };}- if (!isRecord(parsed)) return [];+ if (!isRecord(parsed)) return { definitions: [], fullyParsed: false };const singleDefinition = normalizeDefinitionItem(parsed, options);- if (singleDefinition) return [singleDefinition];+ if (singleDefinition) {+ return { definitions: [singleDefinition], fullyParsed: true };+ }- if (!options.allowToolMap) return [];+ if (!options.allowToolMap) return { definitions: [], fullyParsed: false };// Some instrumentation exports definitions as a map keyed by tool name.const definitions: ToolDefinition[] = [];+ let fullyParsed = true;for (const [name, rawDefinition] of Object.entries(parsed)) {const definition = parseRecord(rawDefinition);- if (!definition) continue;+ if (!definition) {+ fullyParsed = false;+ continue;+ }const normalized = normalizeDefinitionItem({ name, ...definition }, {});if (normalized) definitions.push(normalized);+ else fullyParsed = false;}- return definitions;+ return { definitions, fullyParsed };}export type ToolDefinitionFields = {diff --git a/packages/shared/src/utils/normalized-io/core/utils/json.ts b/packages/shared/src/utils/normalized-io/core/utils/json.tsindex ee30fc156f..76a68a8f54 100644--- a/packages/shared/src/utils/normalized-io/core/utils/json.ts+++ b/packages/shared/src/utils/normalized-io/core/utils/json.ts@@ -128,3 +128,27 @@ export function omitKeys(Object.entries(record).filter(([key]) => !keys.includes(key)),);}++export function recordKeyAsParsed(+ unparsedKeys: Record<string, unknown>,+ key: string,+ ...nestedPath: string[]+): unknown {+ if (nestedPath.length === 0) {+ const value = unparsedKeys[key];+ delete unparsedKeys[key];+ return value;+ }++ const nested = asRecord(unparsedKeys[key]);+ const [nestedKey, ...rest] = nestedPath;+ if (!nested || !(nestedKey in nested)) return undefined;+ const nestedCopy = { ...nested };+ const value = recordKeyAsParsed(nestedCopy, nestedKey, ...rest);+ if (Object.keys(nestedCopy).length === 0) {+ delete unparsedKeys[key];+ } else {+ unparsedKeys[key] = nestedCopy;+ }+ return value;+}diff --git a/packages/shared/src/utils/normalized-io/parser.ts b/packages/shared/src/utils/normalized-io/parser.tsindex 156d5d4137..5d0f57306f 100644--- a/packages/shared/src/utils/normalized-io/parser.ts+++ b/packages/shared/src/utils/normalized-io/parser.ts@@ -1,5 +1,4 @@-/* eslint-disable no-nested-ternary */-import { asRecord, parseArray, parseIfString } from "./core/utils/json";+import { asRecord, parseIfString } from "./core/utils/json";import type { NormalizedIO, SpanIO } from "./types";import {collectMetadata,@@ -17,17 +16,7 @@ type ParsedSpanIO = {function parseIOValue(value: unknown): ParsedIOValue {const parsed = parseIfString(value);- const record = asRecord(parsed);-- return {- value: parsed,- record,- messages: Array.isArray(parsed)- ? parsed- : record- ? parseArray(record.messages)- : undefined,- };+ return { value: parsed, record: asRecord(parsed) };}function parseSpanIO(span: SpanIO): ParsedSpanIO {@@ -52,9 +41,14 @@ export function normalizeSpanIO(span: SpanIO): NormalizedIO {collectIO(output, createParserContext("output"), accumulator);collectMetadata(metadata, accumulator);+ const additionalInput = accumulator.unparsedKeys.input;+ const hasAdditionalInput =+ additionalInput !== undefined && Object.keys(additionalInput).length > 0;+return {messages: accumulator.messages,toolDefinitions: accumulator.toolDefinitions,span,+ ...(hasAdditionalInput ? { additionalInput } : {}),};}diff --git a/packages/shared/src/utils/normalized-io/types.ts b/packages/shared/src/utils/normalized-io/types.tsindex cc88b49fff..1ec41a2f38 100644--- a/packages/shared/src/utils/normalized-io/types.ts+++ b/packages/shared/src/utils/normalized-io/types.ts@@ -172,6 +172,7 @@ export type NormalizedIO = {// Input messages followed by output messages. Message and part order is preserved.messages: NormalizedMessage[];toolDefinitions: ToolDefinition[];+ additionalInput?: Record<string, unknown>;/** Raw pre-normalization values compiled from the source; always returned for now. */span: SpanIO;};diff --git a/web/src/features/sessions/SessionConversationTimeline/fns/prepareSessionTimelineObservations.clienttest.ts b/web/src/features/sessions/SessionConversationTimeline/fns/prepareSessionTimelineObservations.clienttest.tsindex a4d130287a..dd032f1c2a 100644--- a/web/src/features/sessions/SessionConversationTimeline/fns/prepareSessionTimelineObservations.clienttest.ts+++ b/web/src/features/sessions/SessionConversationTimeline/fns/prepareSessionTimelineObservations.clienttest.ts@@ -67,7 +67,9 @@ describe("prepareSessionTimelineObservations", () => {});it("preserves skipped and failed observations in their original positions", () => {+ // Force a parsing failure even when the parser copies enumerable fields.const invalidInput = Object.defineProperty({}, "messages", {+ enumerable: true,get() {throw new Error("Cannot parse input");},diff --git a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts b/web/src/features/traces/hooks/useChatMLParser.clienttest.tsindex eea254e90a..04a39f536e 100644--- a/web/src/features/traces/hooks/useChatMLParser.clienttest.ts+++ b/web/src/features/traces/hooks/useChatMLParser.clienttest.ts@@ -18,10 +18,7 @@ const parserImplementations = [{name: "normalized",parse: (input: unknown, output: unknown) =>- toIOPreview(- normalizeSpanIO({ input, output, metadata: undefined }),- input,- ),+ toIOPreview(normalizeSpanIO({ input, output, metadata: undefined })),// The normalized parser canonicalizes: JSON-string arguments are decoded// once, so consumers always receive the parsed value.expectedArguments: (raw: string): unknown => JSON.parse(raw),diff --git a/web/src/features/traces/hooks/useIOPreviewParser.ts b/web/src/features/traces/hooks/useIOPreviewParser.tsindex ad71e664bc..c9f77d5d6f 100644--- a/web/src/features/traces/hooks/useIOPreviewParser.ts+++ b/web/src/features/traces/hooks/useIOPreviewParser.ts@@ -115,7 +115,7 @@ export function useIOPreviewParser(output: parsedOutput,metadata: parsedMetadata,});- normalizedResult = toIOPreview(normalized, parsedInput);+ normalizedResult = toIOPreview(normalized);} catch {// The legacy parser remains the visible fallback for malformed input.}diff --git a/web/src/features/traces/parsers/toIOPreview.clienttest.ts b/web/src/features/traces/parsers/toIOPreview.clienttest.tsindex 8e515e8cd1..e68126e818 100644--- a/web/src/features/traces/parsers/toIOPreview.clienttest.ts+++ b/web/src/features/traces/parsers/toIOPreview.clienttest.ts@@ -28,7 +28,7 @@ describe("toIOPreview", () => {],};- expect(toIOPreview(io, {}).allMessages[0]?.content).toEqual([+ expect(toIOPreview(io).allMessages[0]?.content).toEqual([{ type: "text", text: "Look at this" },{ type: "image_url", image_url: { url: mediaReference } },]);@@ -69,7 +69,7 @@ describe("toIOPreview", () => {],};- const result = toIOPreview(io, {});+ const result = toIOPreview(io);expect(result.allMessages).toEqual([{@@ -135,7 +135,7 @@ describe("toIOPreview", () => {],};- const result = toIOPreview(io, {});+ const result = toIOPreview(io);expect(result.allMessages).toEqual([{diff --git a/web/src/features/traces/parsers/toIOPreview.ts b/web/src/features/traces/parsers/toIOPreview.tsindex e54a2b6aaa..9c4e060045 100644--- a/web/src/features/traces/parsers/toIOPreview.ts+++ b/web/src/features/traces/parsers/toIOPreview.ts@@ -7,7 +7,6 @@ import type {NormalizedMessagePart,ToolDefinition as NormalizedToolDefinition,} from "@langfuse/shared/src/utils/normalized-io";-import { extractAdditionalInput } from "@/src/utils/chatml";import {computeToolCallBookkeeping,type ChatMlMessage,@@ -228,10 +227,7 @@ function toChatMessages(message: NormalizedMessage): ProjectedMessage {* This is intentionally a web-only compatibility layer; the normalized parser* remains independent of React and the current rendering components.*/-export function toIOPreview(- io: NormalizedIO,- parsedInput: unknown,-): ChatMLParserResult {+export function toIOPreview(io: NormalizedIO): ChatMLParserResult {const projected = io.messages.map((message) => ({source: message.source,...toChatMessages(message),@@ -290,10 +286,9 @@ export function toIOPreview(return {canDisplayAsChat: allMessages.length > 0,allMessages,- // computes additionalInput via the legacy extractAdditionalInput(parsedInput) helper, which strips- // only the top-level keys the OLD provider adapters used to populate. This projection must be- // updated to manage only the keys that are additional in the context of the new provider adapters.- additionalInput: extractAdditionalInput(parsedInput),+ // Input fields the parser did not turn into messages or tool definitions.+ // The parser removes each field as it consumes it, so this is the leftover.+ additionalInput: io.additionalInput,inputMessageCount,...bookkeeping,};