SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
feat(tracedetail): format openllmetry legacy prompt and completion keys (#13135)
Nityananda Gohain81ff25b2f84e
diff --git a/pkg/types/aiobservabilitytypes/message_format.go b/pkg/types/aiobservabilitytypes/message_format.goindex 89f6a2bbac..f68e5b5c68 100644--- a/pkg/types/aiobservabilitytypes/message_format.go+++ b/pkg/types/aiobservabilitytypes/message_format.go@@ -44,7 +44,8 @@ type FormattedMessages struct {// FormatMessages builds the OTel GenAI view of a span's messages from the formats SDKs write. attributes// is the span's JSON attribute document, nested by key segment, so an object value is whole. Each// message value is a JSON string or a structured value. OpenInference's flattened messages come first,-// then a tool call and its result, then the message attributes. gen_ai.system_instructions leads the+// then a tool call and its result, then the message attributes, then openllmetry's flattened prompts+// and completions. gen_ai.system_instructions leads the// input and gen_ai.response.finish_reasons overrides the finish reasons the converter found, whichever// converter ran.func FormatMessages(attributes map[string]any) FormattedMessages {@@ -55,6 +56,7 @@ func FormatMessages(attributes map[string]any) FormattedMessages {f.tool()case attribute(f.attributes, GenAIInputMessages) != nil || attribute(f.attributes, GenAIOutputMessages) != nil:f.messages()+ case f.traceloop():}f.prependSystemInstructions()f.fillFinishReasons()@@ -80,6 +82,7 @@ func (f *formatting) messages() {}if output := attribute(f.attributes, GenAIOutputMessages); output != nil {f.out.Output, outputLabel = f.side(decode(output), RoleAssistant)+ f.completionToolCalls()}f.warnMixedFormats(inputLabel, outputLabel)}@@ -207,6 +210,12 @@ func indexed(doc map[string]any, key string) []map[string]any {return out}+// indexedAt reads one entry of a flattened list by its index.+func indexedAt(doc map[string]any, key string, i int) map[string]any {+ m, _ := attribute(doc, key+"."+strconv.Itoa(i)).(map[string]any)+ return m+}+func firstObject(list []any) (map[string]any, bool) {if len(list) == 0 {return nil, falsediff --git a/pkg/types/aiobservabilitytypes/message_format_traceloop.go b/pkg/types/aiobservabilitytypes/message_format_traceloop.gonew file mode 100644index 0000000000..5bd4bd5cc8--- /dev/null+++ b/pkg/types/aiobservabilitytypes/message_format_traceloop.go@@ -0,0 +1,87 @@+package aiobservabilitytypes++// Format: the flattened gen_ai.prompt.N.* and gen_ai.completion.N.* keys of openllmetry's instrumentations+// from before their semconv rewrite, role, content, tool_call_id, tool_calls.M.{id,name,arguments} and+// finish_reason, with list content as its JSON:+// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/opentelemetry-semantic-conventions-ai/opentelemetry/semconv_ai/__init__.py#L159+// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L408+// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L460+// Written by: crewai through the openai instrumentation 0.45.6, mistralai 0.61.0 (https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L83),+// litellm beside its semconv value.++const (+ traceloopPrompts = "gen_ai.prompt"+ traceloopCompletions = "gen_ai.completion"+)++func (f *formatting) traceloop() bool {+ prompts := indexed(f.attributes, traceloopPrompts)+ completions := indexed(f.attributes, traceloopCompletions)+ if len(prompts) == 0 && len(completions) == 0 {+ return false+ }+ f.out.Input = inputMessages(f.traceloopMessages(prompts))+ f.out.Output = f.traceloopMessages(completions)+ return true+}++// completionToolCalls fills an output message left without parts from the gen_ai.completion.N keys+// litellm writes beside its semconv value, whose parts never carry the tool call:+// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2365+// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2770+func (f *formatting) completionToolCalls() {+ for i := range f.out.Output {+ if len(f.out.Output[i].Parts) == 0 {+ f.out.Output[i].Parts = append(f.out.Output[i].Parts, traceloopToolCalls(indexedAt(f.attributes, traceloopCompletions, i))...)+ }+ }+}++func (f *formatting) traceloopMessages(buckets []map[string]any) OutputMessages {+ out := make(OutputMessages, 0, len(buckets))+ for i, m := range buckets {+ msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: Parts{}, Name: optionalString(m["name"]), FinishReason: finishReason(stringOf(m["finish_reason"]))}+ content, hasContent := m["content"]+ if id := stringOf(m["tool_call_id"]); msg.Role == RoleTool && id != "" {+ msg.Parts = Parts{part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: &id, Response: decode(content)})}+ out = append(out, msg)+ continue+ }+ if hasContent {+ msg.Parts = append(msg.Parts, traceloopContent(content)...)+ }+ msg.Parts = append(msg.Parts, traceloopToolCalls(m)...)+ if len(msg.Parts) == 0 {+ f.warn("message %d has no content", i)+ }+ out = append(out, msg)+ }+ return out+}++// traceloopContent reads a content value, a string or the JSON of an OpenAI content block list.+func traceloopContent(content any) Parts {+ if blocks, ok := decode(content).([]any); ok {+ parts := make(Parts, 0, len(blocks))+ for _, block := range blocks {+ parts = append(parts, contentPart(block))+ }+ return parts+ }+ if text := stringOf(content); text != "" {+ return Parts{part(textPart(text))}+ }+ return nil+}++// traceloopToolCalls reads tool_calls.M.{id,name,arguments} and litellm's function_call object.+func traceloopToolCalls(m map[string]any) Parts {+ var parts Parts+ for _, call := range indexed(m, "tool_calls") {+ parts = append(parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["name"]), ID: optionalString(call["id"]), Arguments: decode(call["arguments"])}))+ }+ if call, ok := m["function_call"].(map[string]any); ok {+ parts = append(parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["name"]), Arguments: decode(call["arguments"])}))+ }+ return parts+}diff --git a/pkg/types/aiobservabilitytypes/message_format_traceloop_test.go b/pkg/types/aiobservabilitytypes/message_format_traceloop_test.gonew file mode 100644index 0000000000..a816ffa80f--- /dev/null+++ b/pkg/types/aiobservabilitytypes/message_format_traceloop_test.go@@ -0,0 +1,106 @@+package aiobservabilitytypes++import "testing"++// Captured cases are the gen_ai.prompt.N.* and gen_ai.completion.N.* keys as the collector stored+// them for the SigNoz/scripts crewai, mistral and litellm captures.+func TestFormat_Traceloop(t *testing.T) {+ runFormatCases(t, []formatCase{+ {+ // crewai capture, openai.chat span of a tool round, as the openai instrumentation 0.45.6 wrote+ // prompts and completions before its semconv rewrite+ // https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L408+ name: "ToolRound_ToolCallAndToolMessages",+ attributes: map[string]any{+ "gen_ai.prompt.0.role": "system",+ "gen_ai.prompt.0.content": "You are Weather Reporter. Call get_weather and summarise the result.",+ "gen_ai.prompt.1.role": "user",+ "gen_ai.prompt.1.content": "Current Task: What's the weather in Bengaluru?",+ "gen_ai.prompt.2.role": "assistant",+ "gen_ai.prompt.2.tool_calls.0.id": "call_bbVgWPe8AMBol7JlAhDkA6R7",+ "gen_ai.prompt.2.tool_calls.0.name": "get_weather",+ "gen_ai.prompt.2.tool_calls.0.arguments": `{"city":"Bengaluru"}`,+ "gen_ai.prompt.3.role": "tool",+ "gen_ai.prompt.3.tool_call_id": "call_bbVgWPe8AMBol7JlAhDkA6R7",+ "gen_ai.prompt.3.content": `{"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}`,+ "gen_ai.completion.0.role": "assistant",+ "gen_ai.completion.0.finish_reason": "stop",+ "gen_ai.completion.0.content": "The current weather in Bengaluru is clear with a temperature of 18°C.",+ },+ wantInput: `[+ {"role": "system", "parts": [{"type": "text", "content": "You are Weather Reporter. Call get_weather and summarise the result."}]},+ {"role": "user", "parts": [{"type": "text", "content": "Current Task: What's the weather in Bengaluru?"}]},+ {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_bbVgWPe8AMBol7JlAhDkA6R7", "arguments": {"city": "Bengaluru"}}]},+ {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_bbVgWPe8AMBol7JlAhDkA6R7", "response": {"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}}]}+ ]`,+ wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The current weather in Bengaluru is clear with a temperature of 18°C."}], "finish_reason": "stop"}]`,+ },+ {+ // crewai capture, openai.chat span answering with two tool calls+ // https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L460+ name: "CompletionToolCalls_ToolCallParts",+ attributes: map[string]any{+ "gen_ai.prompt.0.role": "user",+ "gen_ai.prompt.0.content": "What's the weather in Paris and in London?",+ "gen_ai.completion.0.role": "assistant",+ "gen_ai.completion.0.finish_reason": "tool_calls",+ "gen_ai.completion.0.tool_calls.0.id": "call_1",+ "gen_ai.completion.0.tool_calls.0.name": "get_weather",+ "gen_ai.completion.0.tool_calls.0.arguments": `{"city":"Paris"}`,+ "gen_ai.completion.0.tool_calls.1.id": "call_2",+ "gen_ai.completion.0.tool_calls.1.name": "get_weather",+ "gen_ai.completion.0.tool_calls.1.arguments": `{"city":"London"}`,+ },+ wantInput: `[{"role": "user", "parts": [{"type": "text", "content": "What's the weather in Paris and in London?"}]}]`,+ wantOutput: `[{"role": "assistant", "parts": [+ {"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris"}},+ {"type": "tool_call", "name": "get_weather", "id": "call_2", "arguments": {"city": "London"}}+ ], "finish_reason": "tool_call"}]`,+ },+ {+ // mistral capture, mistralai.chat span: the instrumentation writes the role for every message but+ // drops a content that is not a plain string+ // https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L75+ // https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L88+ name: "RoleWithoutContent_EmptyMessageWarns",+ attributes: map[string]any{+ "gen_ai.prompt.0.role": "user",+ "gen_ai.completion.0.role": "assistant",+ "gen_ai.completion.0.finish_reason": "stop",+ "gen_ai.completion.0.content": "Observability is the ability to understand a system's internal state by analyzing its external outputs.",+ },+ warnings: []string{"message 0 has no content"},+ wantInput: `[{"role": "user", "parts": []}]`,+ wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Observability is the ability to understand a system's internal state by analyzing its external outputs."}], "finish_reason": "stop"}]`,+ },+ {+ // a vision prompt: list content is written as its JSON, the OpenAI content block list+ // https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L431+ name: "ContentBlockList_TextAndBlobParts",+ attributes: map[string]any{+ "gen_ai.prompt.0.role": "user",+ "gen_ai.prompt.0.content": `[{"type":"text","text":"What animal is this?"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,/9j/4AAQ"}}]`,+ },+ wantInput: `[{"role": "user", "parts": [+ {"type": "text", "content": "What animal is this?"},+ {"type": "blob", "modality": "image", "mime_type": "image/jpeg", "content": "/9j/4AAQ"}+ ]}]`,+ wantOutput: `[]`,+ },+ {+ // litellm capture, litellm_request span: the semconv output has no part for the tool call, the+ // gen_ai.completion.0.function_call keys beside it still carry it+ // https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2365+ // https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2770+ name: "SemconvEmptyParts_ToolCallFromCompletionKeys",+ attributes: map[string]any{+ GenAIInputMessages: `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,+ GenAIOutputMessages: `[{"role": "assistant", "parts": [], "finish_reason": "tool_calls"}]`,+ "gen_ai.completion.0.function_call.name": "get_current_weather",+ "gen_ai.completion.0.function_call.arguments": `{"city":"Bengaluru"}`,+ },+ wantInput: `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,+ wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "arguments": {"city": "Bengaluru"}}], "finish_reason": "tool_call"}]`,+ },+ })+}diff --git a/tests/integration/tests/tracedetail/02_thread.py b/tests/integration/tests/tracedetail/02_thread.pyindex 97154ad9f2..a79cda61f1 100644--- a/tests/integration/tests/tracedetail/02_thread.py+++ b/tests/integration/tests/tracedetail/02_thread.py@@ -21,7 +21,7 @@ def test_thread_returns_gen_ai_spans_in_order(seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)now = datetime.now(tz=UTC).replace(microsecond=0)trace_id = TraceIdGenerator.trace_id()- root_id, first_llm_id, tool_id, agent_id, second_llm_id, third_llm_id, flattened_llm_id = (TraceIdGenerator.span_id() for _ in range(7))+ root_id, first_llm_id, tool_id, agent_id, second_llm_id, legacy_llm_id, third_llm_id, flattened_llm_id = (TraceIdGenerator.span_id() for _ in range(8))resources = {"service.name": "tracedetail-thread"}first_input = json.dumps([{"role": "user", "parts": [{"type": "text", "content": "weather in Bangalore?"}]}])first_output = json.dumps([{"role": "assistant", "parts": [{"type": "tool_call", "id": "call_1", "name": "get_weather", "arguments": {"city": "Bangalore"}}]}])@@ -53,6 +53,16 @@ def test_thread_returns_gen_ai_spans_in_order(),Traces(timestamp=now - timedelta(seconds=5), trace_id=trace_id, span_id=agent_id, parent_span_id=root_id, name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}, attribute_write_mode="json_only"),Traces(timestamp=now - timedelta(seconds=4), trace_id=trace_id, span_id=second_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": second_input}, attribute_write_mode="json_only"),+ Traces(+ timestamp=now - timedelta(seconds=3),+ trace_id=trace_id,+ span_id=legacy_llm_id,+ parent_span_id=root_id,+ name="openai.chat",+ resources=resources,+ attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.prompt.0.role": "user", "gen_ai.prompt.0.content": "and tomorrow?", "gen_ai.completion.0.role": "assistant", "gen_ai.completion.0.content": "Also sunny.", "gen_ai.completion.0.finish_reason": "stop"},+ attribute_write_mode="json_only",+ ),Traces(timestamp=now - timedelta(seconds=2), trace_id=trace_id, span_id=third_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": "It is sunny in Bangalore."}, attribute_write_mode="json_only"),Traces(timestamp=now - timedelta(seconds=1),@@ -79,10 +89,10 @@ def test_thread_returns_gen_ai_spans_in_order(assert response.status_code == HTTPStatus.OK, response.textthread = response.json()["data"]- assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, agent_id, second_llm_id, third_llm_id, flattened_llm_id]+ assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, agent_id, second_llm_id, legacy_llm_id, third_llm_id, flattened_llm_id]assert "nextCursor" not in thread- first, tool, agent, input_only, output_only, flattened = thread["spans"]+ first, tool, agent, input_only, legacy, output_only, flattened = thread["spans"]assert agent["attributes"]["gen_ai.agent.name"] == "planner"assert agent["formattedInput"] == []assert agent["formattedOutput"] == []@@ -108,6 +118,9 @@ def test_thread_returns_gen_ai_spans_in_order(assert output_only["formattedInput"] == []assert output_only["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "It is sunny in Bangalore."}]}]assert output_only["formatterWarnings"] == ["bare assistant text, role assumed"]+ assert legacy["attributes"]["gen_ai.prompt.0.content"] == "and tomorrow?"+ assert legacy["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "and tomorrow?"}]}]+ assert legacy["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "Also sunny."}], "finish_reason": "stop"}]assert flattened["attributes"]["llm.input_messages.0.message.content"] == "thanks"assert flattened["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "thanks"}]}]assert flattened["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "You're welcome."}], "finish_reason": "stop"}]