🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(otel): map livekit-agents 1.8 lk.pii.* content attributes (#17467)
Jannik Maierhöfer77e3418aec6e
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex c5b25da912..ffa5c532a3 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -1828,13 +1828,22 @@ export class OtelIngestionProcessor {"prompt","all_messages_events","events",- // LiveKit+ // LiveKit (livekit-agents >= 1.8 marks content attributes with `lk.pii.`)"lk.input_text","lk.user_transcript","lk.chat_ctx","lk.user_input","lk.function_tool.output","lk.response.text",+ "lk.pii.input_text",+ "lk.pii.user_transcript",+ "lk.pii.chat_ctx",+ "lk.pii.user_input",+ "lk.pii.instructions",+ "lk.pii.function_tool.arguments",+ "lk.pii.function_tool.output",+ "lk.pii.response.text",+ "lk.pii.response.function_calls",// MLFlow"mlflow.spanInputs","mlflow.spanOutputs",@@ -2154,13 +2163,42 @@ export class OtelIngestionProcessor {return { input, output, filteredAttributes };}- // LiveKit+ // LiveKit. livekit-agents >= 1.8 marks content attributes with a `pii`+ // segment (`lk.pii.<name>`); older versions use the bare `lk.<name>`.+ const livekitInstructions = attributes["lk.pii.instructions"] || undefined;+ // the agent may speak first, in which case user_input is empty+ const livekitUserInput =+ attributes["lk.user_input"] ||+ attributes["lk.pii.user_input"] ||+ undefined;input =attributes["lk.input_text"] ??+ attributes["lk.pii.input_text"] ??attributes["lk.user_transcript"] ??- attributes["lk.chat_ctx"];+ attributes["lk.pii.user_transcript"] ??+ attributes["lk.chat_ctx"] ??+ attributes["lk.pii.chat_ctx"] ??+ // agent_turn spans carry the system instructions and the user message+ // separately; combine whatever is present into a chat-style input+ (livekitInstructions+ ? [+ { role: "system", content: livekitInstructions },+ ...(livekitUserInput+ ? [{ role: "user", content: livekitUserInput }]+ : []),+ ]+ : livekitUserInput) ??+ attributes["lk.pii.function_tool.arguments"];+ const livekitFunctionCalls = attributes["lk.pii.response.function_calls"];output =- attributes["lk.function_tool.output"] || attributes["lk.response.text"];+ attributes["lk.function_tool.output"] ||+ attributes["lk.pii.function_tool.output"] ||+ attributes["lk.response.text"] ||+ attributes["lk.pii.response.text"] ||+ // a turn that only produced tool calls has no response text+ (livekitFunctionCalls && livekitFunctionCalls !== "[]"+ ? livekitFunctionCalls+ : undefined);if (input || output) {return { input, output, filteredAttributes };}diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 681986e74c..198155bb40 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -1981,6 +1981,81 @@ describe("OTel Resource Span Mapping", () => {},);+ it("should build chat input from lk.pii.instructions and lk.pii.user_input on a realtime agent_turn", async () => {+ const events = await convertOtelSpanToIngestionEvent(+ createLivekitSpan({+ name: "agent_turn",+ scopeName: "livekit-agents",+ attributes: [+ {+ key: "lk.pii.instructions",+ value: { stringValue: "Your name is Marin." },+ },+ {+ key: "lk.pii.user_input",+ value: { stringValue: "What is Langfuse?" },+ },+ {+ key: "lk.pii.response.text",+ value: {+ stringValue: "Langfuse is an LLM engineering platform.",+ },+ },+ {+ key: "lk.pii.response.function_calls",+ value: { stringValue: "[]" },+ },+ ],+ }),+ new Set([livekitTraceId]),+ );++ const observation = findObservationCreateEvent(events);+ expect(observation?.body.input).toEqual([+ { role: "system", content: "Your name is Marin." },+ { role: "user", content: "What is Langfuse?" },+ ]);+ expect(observation?.body.output).toBe(+ "Langfuse is an LLM engineering platform.",+ );+ expect(+ observation?.body.metadata?.attributes?.["lk.pii.response.text"],+ ).toBeUndefined();+ });++ it("should keep lk.pii.instructions as system input when the agent speaks first", async () => {+ const events = await convertOtelSpanToIngestionEvent(+ createLivekitSpan({+ name: "agent_turn",+ scopeName: "livekit-agents",+ attributes: [+ {+ key: "lk.pii.instructions",+ value: { stringValue: "Your name is Marin." },+ },+ {+ key: "lk.pii.user_input",+ value: { stringValue: "" },+ },+ {+ key: "lk.pii.response.text",+ value: { stringValue: "Hi, I am Marin." },+ },+ ],+ }),+ new Set([livekitTraceId]),+ );++ const observation = findObservationCreateEvent(events);+ expect(observation?.body.input).toEqual([+ { role: "system", content: "Your name is Marin." },+ ]);+ expect(observation?.body.output).toBe("Hi, I am Marin.");+ expect(observation?.body.metadata?.attributes ?? {}).not.toHaveProperty(+ "lk.pii.instructions",+ );+ });+it("should not map LiveKit span names without livekit-agents scope", async () => {const events = await convertOtelSpanToIngestionEvent(createLivekitSpan({@@ -3944,6 +4019,105 @@ describe("OTel Resource Span Mapping", () => {entityAttributeValue: '{"temperature": 75, "condition": "sunny"}',},],+ // livekit-agents >= 1.8 marks content attributes with a `pii` segment+ [+ "should map lk.pii.input_text to input",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.input_text",+ otelAttributeValue: {+ stringValue: "What is the weather today?",+ },+ entityAttributeKey: "input",+ entityAttributeValue: "What is the weather today?",+ },+ ],+ [+ "should map lk.pii.user_transcript to input",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.user_transcript",+ otelAttributeValue: {+ stringValue: "Hey there. What's the weather?",+ },+ entityAttributeKey: "input",+ entityAttributeValue: "Hey there. What's the weather?",+ },+ ],+ [+ "should map lk.pii.chat_ctx to input",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.chat_ctx",+ otelAttributeValue: {+ stringValue: '{"items": [{"role": "user", "content": ["Hi"]}]}',+ },+ entityAttributeKey: "input",+ entityAttributeValue:+ '{"items": [{"role": "user", "content": ["Hi"]}]}',+ },+ ],+ [+ "should map lk.pii.user_input to input",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.user_input",+ otelAttributeValue: {+ stringValue: "What is Langfuse?",+ },+ entityAttributeKey: "input",+ entityAttributeValue: "What is Langfuse?",+ },+ ],+ [+ "should map lk.pii.function_tool.arguments to input",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.function_tool.arguments",+ otelAttributeValue: {+ stringValue: '{"query": "langfuse sessions"}',+ },+ entityAttributeKey: "input",+ entityAttributeValue: '{"query": "langfuse sessions"}',+ },+ ],+ [+ "should map lk.pii.response.text to output",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.response.text",+ otelAttributeValue: {+ stringValue: "The weather is sunny with a high of 75°F.",+ },+ entityAttributeKey: "output",+ entityAttributeValue: "The weather is sunny with a high of 75°F.",+ },+ ],+ [+ "should map lk.pii.function_tool.output to output",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.function_tool.output",+ otelAttributeValue: {+ stringValue: '{"temperature": 75, "condition": "sunny"}',+ },+ entityAttributeKey: "output",+ entityAttributeValue: '{"temperature": 75, "condition": "sunny"}',+ },+ ],+ [+ "should map lk.pii.response.function_calls to output",+ {+ entity: "observation",+ otelAttributeKey: "lk.pii.response.function_calls",+ otelAttributeValue: {+ stringValue: '[{"name": "searchLangfuseDocs", "arguments": "{}"}]',+ },+ entityAttributeKey: "output",+ entityAttributeValue:+ '[{"name": "searchLangfuseDocs", "arguments": "{}"}]',+ },+ ],[// OTLP/JSON encodes int64 attribute values as decimal strings (e.g. "100")// rather than the Long object produced by the protobuf decoder. See