Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(otel): map livekit-agents 1.8 lk.pii.* content attributes (#17467)

Jannik Maierhöfer77e3418aec6e
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index c5b25da912..ffa5c532a3 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -1828,13 +1828,22 @@ export class OtelIngestionProcessor {
"prompt",
"all_messages_events",
"events",
- // LiveKit
+ // LiveKit (livekit-agents >= 1.8 marks content attributes with `lk.pii.`)
"lk.input_text",
"lk.user_transcript",
"lk.chat_ctx",
"lk.user_input",
"lk.function_tool.output",
"lk.response.text",
+ "lk.pii.input_text",
+ "lk.pii.user_transcript",
+ "lk.pii.chat_ctx",
+ "lk.pii.user_input",
+ "lk.pii.instructions",
+ "lk.pii.function_tool.arguments",
+ "lk.pii.function_tool.output",
+ "lk.pii.response.text",
+ "lk.pii.response.function_calls",
// MLFlow
"mlflow.spanInputs",
"mlflow.spanOutputs",
@@ -2154,13 +2163,42 @@ export class OtelIngestionProcessor {
return { input, output, filteredAttributes };
}
- // LiveKit
+ // LiveKit. livekit-agents >= 1.8 marks content attributes with a `pii`
+ // segment (`lk.pii.<name>`); older versions use the bare `lk.<name>`.
+ const livekitInstructions = attributes["lk.pii.instructions"] || undefined;
+ // the agent may speak first, in which case user_input is empty
+ const livekitUserInput =
+ attributes["lk.user_input"] ||
+ attributes["lk.pii.user_input"] ||
+ undefined;
input =
attributes["lk.input_text"] ??
+ attributes["lk.pii.input_text"] ??
attributes["lk.user_transcript"] ??
- attributes["lk.chat_ctx"];
+ attributes["lk.pii.user_transcript"] ??
+ attributes["lk.chat_ctx"] ??
+ attributes["lk.pii.chat_ctx"] ??
+ // agent_turn spans carry the system instructions and the user message
+ // separately; combine whatever is present into a chat-style input
+ (livekitInstructions
+ ? [
+ { role: "system", content: livekitInstructions },
+ ...(livekitUserInput
+ ? [{ role: "user", content: livekitUserInput }]
+ : []),
+ ]
+ : livekitUserInput) ??
+ attributes["lk.pii.function_tool.arguments"];
+ const livekitFunctionCalls = attributes["lk.pii.response.function_calls"];
output =
- attributes["lk.function_tool.output"] || attributes["lk.response.text"];
+ attributes["lk.function_tool.output"] ||
+ attributes["lk.pii.function_tool.output"] ||
+ attributes["lk.response.text"] ||
+ attributes["lk.pii.response.text"] ||
+ // a turn that only produced tool calls has no response text
+ (livekitFunctionCalls && livekitFunctionCalls !== "[]"
+ ? livekitFunctionCalls
+ : undefined);
if (input || output) {
return { input, output, filteredAttributes };
}
diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
index 681986e74c..198155bb40 100644
--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
@@ -1981,6 +1981,81 @@ describe("OTel Resource Span Mapping", () => {
},
);
+ it("should build chat input from lk.pii.instructions and lk.pii.user_input on a realtime agent_turn", async () => {
+ const events = await convertOtelSpanToIngestionEvent(
+ createLivekitSpan({
+ name: "agent_turn",
+ scopeName: "livekit-agents",
+ attributes: [
+ {
+ key: "lk.pii.instructions",
+ value: { stringValue: "Your name is Marin." },
+ },
+ {
+ key: "lk.pii.user_input",
+ value: { stringValue: "What is Langfuse?" },
+ },
+ {
+ key: "lk.pii.response.text",
+ value: {
+ stringValue: "Langfuse is an LLM engineering platform.",
+ },
+ },
+ {
+ key: "lk.pii.response.function_calls",
+ value: { stringValue: "[]" },
+ },
+ ],
+ }),
+ new Set([livekitTraceId]),
+ );
+
+ const observation = findObservationCreateEvent(events);
+ expect(observation?.body.input).toEqual([
+ { role: "system", content: "Your name is Marin." },
+ { role: "user", content: "What is Langfuse?" },
+ ]);
+ expect(observation?.body.output).toBe(
+ "Langfuse is an LLM engineering platform.",
+ );
+ expect(
+ observation?.body.metadata?.attributes?.["lk.pii.response.text"],
+ ).toBeUndefined();
+ });
+
+ it("should keep lk.pii.instructions as system input when the agent speaks first", async () => {
+ const events = await convertOtelSpanToIngestionEvent(
+ createLivekitSpan({
+ name: "agent_turn",
+ scopeName: "livekit-agents",
+ attributes: [
+ {
+ key: "lk.pii.instructions",
+ value: { stringValue: "Your name is Marin." },
+ },
+ {
+ key: "lk.pii.user_input",
+ value: { stringValue: "" },
+ },
+ {
+ key: "lk.pii.response.text",
+ value: { stringValue: "Hi, I am Marin." },
+ },
+ ],
+ }),
+ new Set([livekitTraceId]),
+ );
+
+ const observation = findObservationCreateEvent(events);
+ expect(observation?.body.input).toEqual([
+ { role: "system", content: "Your name is Marin." },
+ ]);
+ expect(observation?.body.output).toBe("Hi, I am Marin.");
+ expect(observation?.body.metadata?.attributes ?? {}).not.toHaveProperty(
+ "lk.pii.instructions",
+ );
+ });
+
it("should not map LiveKit span names without livekit-agents scope", async () => {
const events = await convertOtelSpanToIngestionEvent(
createLivekitSpan({
@@ -3944,6 +4019,105 @@ describe("OTel Resource Span Mapping", () => {
entityAttributeValue: '{"temperature": 75, "condition": "sunny"}',
},
],
+ // livekit-agents >= 1.8 marks content attributes with a `pii` segment
+ [
+ "should map lk.pii.input_text to input",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.input_text",
+ otelAttributeValue: {
+ stringValue: "What is the weather today?",
+ },
+ entityAttributeKey: "input",
+ entityAttributeValue: "What is the weather today?",
+ },
+ ],
+ [
+ "should map lk.pii.user_transcript to input",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.user_transcript",
+ otelAttributeValue: {
+ stringValue: "Hey there. What's the weather?",
+ },
+ entityAttributeKey: "input",
+ entityAttributeValue: "Hey there. What's the weather?",
+ },
+ ],
+ [
+ "should map lk.pii.chat_ctx to input",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.chat_ctx",
+ otelAttributeValue: {
+ stringValue: '{"items": [{"role": "user", "content": ["Hi"]}]}',
+ },
+ entityAttributeKey: "input",
+ entityAttributeValue:
+ '{"items": [{"role": "user", "content": ["Hi"]}]}',
+ },
+ ],
+ [
+ "should map lk.pii.user_input to input",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.user_input",
+ otelAttributeValue: {
+ stringValue: "What is Langfuse?",
+ },
+ entityAttributeKey: "input",
+ entityAttributeValue: "What is Langfuse?",
+ },
+ ],
+ [
+ "should map lk.pii.function_tool.arguments to input",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.function_tool.arguments",
+ otelAttributeValue: {
+ stringValue: '{"query": "langfuse sessions"}',
+ },
+ entityAttributeKey: "input",
+ entityAttributeValue: '{"query": "langfuse sessions"}',
+ },
+ ],
+ [
+ "should map lk.pii.response.text to output",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.response.text",
+ otelAttributeValue: {
+ stringValue: "The weather is sunny with a high of 75°F.",
+ },
+ entityAttributeKey: "output",
+ entityAttributeValue: "The weather is sunny with a high of 75°F.",
+ },
+ ],
+ [
+ "should map lk.pii.function_tool.output to output",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.function_tool.output",
+ otelAttributeValue: {
+ stringValue: '{"temperature": 75, "condition": "sunny"}',
+ },
+ entityAttributeKey: "output",
+ entityAttributeValue: '{"temperature": 75, "condition": "sunny"}',
+ },
+ ],
+ [
+ "should map lk.pii.response.function_calls to output",
+ {
+ entity: "observation",
+ otelAttributeKey: "lk.pii.response.function_calls",
+ otelAttributeValue: {
+ stringValue: '[{"name": "searchLangfuseDocs", "arguments": "{}"}]',
+ },
+ entityAttributeKey: "output",
+ entityAttributeValue:
+ '[{"name": "searchLangfuseDocs", "arguments": "{}"}]',
+ },
+ ],
[
// OTLP/JSON encodes int64 attribute values as decimal strings (e.g. "100")
// rather than the Long object produced by the protobuf decoder. See