SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
feat: thread rows by the gen_ai gate, formatted tool spans (#13115)
Nityananda Gohain441f7e2927f7
diff --git a/docs/api/openapi.yml b/docs/api/openapi.ymlindex f39d62f5a0..f6cf85d530 100644--- a/docs/api/openapi.yml+++ b/docs/api/openapi.yml@@ -16440,12 +16440,12 @@ paths:/api/v1/traces/{traceID}/thread:get:deprecated: false- description: Returns the spans carrying gen_ai input or output messages, or- a tool execution, in timestamp order. Messages already in the OTel GenAI shape- are decoded into formatted_input and formatted_output, with formatter naming- the converter and formatter_warnings what it could not resolve. Pass nextCursor- as after or prevCursor as before to page, or spanId to open the page around- a span.+ description: Returns the gen_ai spans of a trace, model calls, tool calls and+ agent spans, in timestamp order. The messages a span carries are decoded into+ formatted_input and formatted_output, with formatter naming the converter+ and formatter_warnings what it could not resolve; a span without messages+ has both empty. Pass nextCursor as after or prevCursor as before to page,+ or spanId to open the page around a span.operationId: GetTraceThreadparameters:- description: Page size, at most 100. 0 means 20.diff --git a/frontend/src/api/generated/services/tracedetail/index.ts b/frontend/src/api/generated/services/tracedetail/index.tsindex d3b7f63dd2..9bf78a15f1 100644--- a/frontend/src/api/generated/services/tracedetail/index.ts+++ b/frontend/src/api/generated/services/tracedetail/index.ts@@ -261,7 +261,7 @@ export const invalidateGetTraceSummary = async (};/**- * Returns the spans carrying gen_ai input or output messages, or a tool execution, in timestamp order. Messages already in the OTel GenAI shape are decoded into formatted_input and formatted_output, with formatter naming the converter and formatter_warnings what it could not resolve. Pass nextCursor as after or prevCursor as before to page, or spanId to open the page around a span.+ * Returns the gen_ai spans of a trace, model calls, tool calls and agent spans, in timestamp order. The messages a span carries are decoded into formatted_input and formatted_output, with formatter naming the converter and formatter_warnings what it could not resolve; a span without messages has both empty. Pass nextCursor as after or prevCursor as before to page, or spanId to open the page around a span.* @summary Get thread view for a trace*/export const getTraceThread = (diff --git a/pkg/apiserver/signozapiserver/tracedetail.go b/pkg/apiserver/signozapiserver/tracedetail.goindex 3241df83c8..a582b140c5 100644--- a/pkg/apiserver/signozapiserver/tracedetail.go+++ b/pkg/apiserver/signozapiserver/tracedetail.go@@ -90,7 +90,7 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {ID: "GetTraceThread",Tags: []string{"tracedetail"},Summary: "Get thread view for a trace",- Description: "Returns the spans carrying gen_ai input or output messages, or a tool execution, in timestamp order. Messages already in the OTel GenAI shape are decoded into formatted_input and formatted_output, with formatter naming the converter and formatter_warnings what it could not resolve. Pass nextCursor as after or prevCursor as before to page, or spanId to open the page around a span.",+ Description: "Returns the gen_ai spans of a trace, model calls, tool calls and agent spans, in timestamp order. The messages a span carries are decoded into formatted_input and formatted_output, with formatter naming the converter and formatter_warnings what it could not resolve; a span without messages has both empty. Pass nextCursor as after or prevCursor as before to page, or spanId to open the page around a span.",RequestQuery: new(spantypes.GetTraceThreadParams),Response: new(spantypes.GettableTraceThread),ResponseContentType: "application/json",diff --git a/pkg/modules/tracedetail/impltracedetail/store.go b/pkg/modules/tracedetail/impltracedetail/store.goindex 6c905a1780..70f8975bf4 100644--- a/pkg/modules/tracedetail/impltracedetail/store.go+++ b/pkg/modules/tracedetail/impltracedetail/store.go@@ -297,11 +297,11 @@ func (s *traceStore) GetThreadSpans(ctx context.Context, orgID valuer.UUID, tracsb.SelectMore("attributes")}sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))- keys, err := s.genAIFieldKeys(ctx, orgID, aiobservabilitytypes.GenAIThreadKeys)+ keys, err := s.genAIFieldKeys(ctx, orgID, aiobservabilitytypes.GenAISpanGateKeys)if err != nil {return nil, err}- isThreadSpan, err := s.anyExistsCondition(ctx, q, aiobservabilitytypes.GenAIThreadKeys, keys, sb)+ isThreadSpan, err := s.anyExistsCondition(ctx, q, aiobservabilitytypes.GenAISpanGateKeys, keys, sb)if err != nil {return nil, err}diff --git a/pkg/modules/tracedetail/impltracedetail/store_test.go b/pkg/modules/tracedetail/impltracedetail/store_test.goindex 175d70730a..1acf1ddce6 100644--- a/pkg/modules/tracedetail/impltracedetail/store_test.go+++ b/pkg/modules/tracedetail/impltracedetail/store_test.go@@ -220,14 +220,14 @@ func TestGetThreadSpans(t *testing.T) {name: "FlagOff_ReadsAndFiltersLegacyMaps",jsonRelease: &jsonInsideTrace,selectSQL: selectSQL,- whereSQL: "(mapContains(attributes_string, 'gen_ai.input.messages') OR mapContains(attributes_string, 'gen_ai.output.messages') OR mapContains(attributes_string, 'gen_ai.tool.name'))",+ whereSQL: "(mapContains(attributes_string, 'gen_ai.request.model') OR mapContains(attributes_string, 'gen_ai.tool.name') OR mapContains(attributes_string, 'gen_ai.agent.name'))",},{name: "FlagOn_ReleasedDuringTrace_ReadsJSONFiltersJSONThenMaps",jsonOn: true,jsonRelease: &jsonInsideTrace,selectSQL: selectSQL + ", attributes",- whereSQL: "(multiIf(attributes.`gen_ai.input.messages` IS NOT NULL, attributes.`gen_ai.input.messages`::String, mapContains(attributes_string, 'gen_ai.input.messages'), attributes_string['gen_ai.input.messages'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.output.messages` IS NOT NULL, attributes.`gen_ai.output.messages`::String, mapContains(attributes_string, 'gen_ai.output.messages'), attributes_string['gen_ai.output.messages'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.tool.name` IS NOT NULL, attributes.`gen_ai.tool.name`::String, mapContains(attributes_string, 'gen_ai.tool.name'), attributes_string['gen_ai.tool.name'], NULL) IS NOT NULL)",+ whereSQL: "(multiIf(attributes.`gen_ai.request.model` IS NOT NULL, attributes.`gen_ai.request.model`::String, mapContains(attributes_string, 'gen_ai.request.model'), attributes_string['gen_ai.request.model'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.tool.name` IS NOT NULL, attributes.`gen_ai.tool.name`::String, mapContains(attributes_string, 'gen_ai.tool.name'), attributes_string['gen_ai.tool.name'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.agent.name` IS NOT NULL, attributes.`gen_ai.agent.name`::String, mapContains(attributes_string, 'gen_ai.agent.name'), attributes_string['gen_ai.agent.name'], NULL) IS NOT NULL)",},}diff --git a/pkg/types/aiobservabilitytypes/keys.go b/pkg/types/aiobservabilitytypes/keys.goindex e7e4635204..ac4475dd31 100644--- a/pkg/types/aiobservabilitytypes/keys.go+++ b/pkg/types/aiobservabilitytypes/keys.go@@ -19,6 +19,10 @@ const (GenAIInputMessages = "gen_ai.input.messages"GenAIOutputMessages = "gen_ai.output.messages"++ GenAIToolCallID = "gen_ai.tool.call.id"+ GenAIToolCallArguments = "gen_ai.tool.call.arguments"+ GenAIToolCallResult = "gen_ai.tool.call.result")// Per-span costs the SigNoz LLM pricing processor attaches; SigNoz semconv@@ -32,12 +36,10 @@ const ()// GenAISpanGateKeys mark a span as gen_ai: an LLM call, a tool call, or an-// agent span. A trace belongs to the AI explorer when any span carries one.+// agent span. A trace belongs to the AI explorer when any span carries one,+// and the thread of a trace is these spans.var GenAISpanGateKeys = []string{GenAIRequestModel, GenAIToolName, GenAIAgentName}-// GenAIThreadKeys mark a span as a thread row: a model call with its messages, or a tool execution.-var GenAIThreadKeys = []string{GenAIInputMessages, GenAIOutputMessages, GenAIToolName}-// GenAISpanFilterExpression renders the gate as a query-builder filter// expression: each gate key ORed on EXISTS.func GenAISpanFilterExpression() string {diff --git a/pkg/types/aiobservabilitytypes/message_format.go b/pkg/types/aiobservabilitytypes/message_format.goindex a35bc0e9c1..ba93bcc7ad 100644--- a/pkg/types/aiobservabilitytypes/message_format.go+++ b/pkg/types/aiobservabilitytypes/message_format.go@@ -11,6 +11,7 @@ import (const (FormatterSemconv = "semconv"+ FormatterTool = "tool"FormatterOpenAIChat = "openai.chat"FormatterText = "text"FormatterGeneric = "generic"@@ -34,19 +35,42 @@ type FormattedMessages struct {Warnings []string}-// FormatMessages builds the OTel GenAI view of gen_ai.input.messages and gen_ai.output.messages from the-// formats SDKs write. Each value is a JSON string or a structured value; nil means absent.-func FormatMessages(input, output any) FormattedMessages {- f := &formatting{out: FormattedMessages{Input: InputMessages{}, Output: OutputMessages{}, Warnings: []string{}}}+// FormatMessages builds the OTel GenAI view of a span's messages from the formats SDKs write. attributes+// is the span's JSON attribute document, nested by key segment, so an object value is whole. Each+// message value is a JSON string or a structured value. A tool call and its result take precedence over+// messages beside them.+func FormatMessages(attributes map[string]any) FormattedMessages {+ f := &formatting{attributes: attributes, out: FormattedMessages{Input: InputMessages{}, Output: OutputMessages{}, Warnings: []string{}}}+ switch {+ case attribute(f.attributes, GenAIToolCallArguments) != nil || attribute(f.attributes, GenAIToolCallResult) != nil:+ f.tool()+ case attribute(f.attributes, GenAIInputMessages) != nil || attribute(f.attributes, GenAIOutputMessages) != nil:+ f.messages()+ }+ return f.out+}++type formatting struct {+ attributes map[string]any+ out FormattedMessages+}++func (f *formatting) warn(format string, args ...any) {+ f.out.Warnings = append(f.out.Warnings, fmt.Sprintf(format, args...))+}++// messages converts gen_ai.input.messages and gen_ai.output.messages, each by its own shape, and names+// the formatter after the input side, or the output side when the input has none.+func (f *formatting) messages() {var inputLabel, outputLabel string- if input != nil {+ if input := attribute(f.attributes, GenAIInputMessages); input != nil {var msgs OutputMessagesmsgs, inputLabel = f.side(decode(input), RoleUser)for _, m := range msgs {f.out.Input = append(f.out.Input, ChatMessage{Role: m.Role, Parts: m.Parts, Name: m.Name})}}- if output != nil {+ if output := attribute(f.attributes, GenAIOutputMessages); output != nil {f.out.Output, outputLabel = f.side(decode(output), RoleAssistant)}f.out.Formatter = inputLabel@@ -55,15 +79,6 @@ func FormatMessages(input, output any) FormattedMessages {} else if outputLabel != "" && outputLabel != inputLabel {f.warn("output formatted as %s, input as %s", outputLabel, inputLabel)}- return f.out-}--type formatting struct {- out FormattedMessages-}--func (f *formatting) warn(format string, args ...any) {- f.out.Warnings = append(f.out.Warnings, fmt.Sprintf(format, args...))}// side converts one attribute value by its shape; role is what a bare string on this side is@@ -97,6 +112,21 @@ func (f *formatting) side(value any, role Role) (OutputMessages, string) {return OutputMessages{genericMessage(value)}, FormatterGeneric}+// attribute walks the document along the dotted key; a nested object comes back whole. Not handled: a+// scalar at a prefix of the path, such as gen_ai.input beside gen_ai.input.messages, which ends the+// walk early.+func attribute(doc map[string]any, key string) any {+ var current any = doc+ for segment := range strings.SplitSeq(key, ".") {+ object, ok := current.(map[string]any)+ if !ok {+ return nil+ }+ current = object[segment]+ }+ return current+}+func firstObject(list []any) (map[string]any, bool) {if len(list) == 0 {return nil, falsediff --git a/pkg/types/aiobservabilitytypes/message_format_test.go b/pkg/types/aiobservabilitytypes/message_format_test.goindex 03ff0ff0ba..3b40d15fcc 100644--- a/pkg/types/aiobservabilitytypes/message_format_test.go+++ b/pkg/types/aiobservabilitytypes/message_format_test.go@@ -2,6 +2,8 @@ package aiobservabilitytypesimport ("encoding/json"+ "maps"+ "strings""testing""github.com/stretchr/testify/assert"@@ -12,6 +14,7 @@ type formatCase struct {name stringinput anyoutput any+ attributes map[string]any // flat dotted keys beside the two message attributesformatter stringwarnings []stringwantInput string@@ -22,7 +25,9 @@ func runFormatCases(t *testing.T, testCases []formatCase) {t.Helper()for _, testCase := range testCases {t.Run(testCase.name, func(t *testing.T) {- formatted := FormatMessages(testCase.input, testCase.output)+ attributes := map[string]any{GenAIInputMessages: testCase.input, GenAIOutputMessages: testCase.output}+ maps.Copy(attributes, testCase.attributes)+ formatted := FormatMessages(document(attributes))assert.Equal(t, testCase.formatter, formatted.Formatter)assert.Equal(t, append([]string{}, testCase.warnings...), formatted.Warnings)@@ -36,6 +41,25 @@ func runFormatCases(t *testing.T, testCases []formatCase) {}}+// document nests flat dotted keys the way the JSON column stores them.+func document(flat map[string]any) map[string]any {+ doc := map[string]any{}+ for key, value := range flat {+ segments := strings.Split(key, ".")+ current := doc+ for _, segment := range segments[:len(segments)-1] {+ child, ok := current[segment].(map[string]any)+ if !ok {+ child = map[string]any{}+ current[segment] = child+ }+ current = child+ }+ current[segments[len(segments)-1]] = value+ }+ return doc+}+func TestFormat(t *testing.T) {runFormatCases(t, []formatCase{{@@ -93,7 +117,8 @@ So, the ball costs 5 cents.`,wantOutput: `[]`,},{- name: "NoAttributes_EmptyListsNoFormatter",+ name: "NoMessageAttributes_EmptyListsNoFormatter",+ attributes: map[string]any{GenAIRequestModel: "gpt-4o", GenAIToolName: "get_weather"},wantInput: `[]`,wantOutput: `[]`,},diff --git a/pkg/types/aiobservabilitytypes/message_format_tool.go b/pkg/types/aiobservabilitytypes/message_format_tool.gonew file mode 100644index 0000000000..9fb4595eca--- /dev/null+++ b/pkg/types/aiobservabilitytypes/message_format_tool.go@@ -0,0 +1,69 @@+package aiobservabilitytypes++// Format: gen_ai.tool.call.arguments and gen_ai.tool.call.result on an execute_tool span, each an+// object or its JSON string (open-telemetry/semantic-conventions-genai@06ec68e, model/gen-ai/registry.yaml).+// Written by: opentelemetry-instrumentation-langchain, traceloop @tool, OpenInference TOOL spans and+// Vercel ai.toolCall.* through the span mapper.+// Envelopes unwrapped: LangChain's callback payloads, {input_str, tags, metadata, inputs, kwargs} and+// {output, kwargs} (openllmetry callback_handler.py on_tool_start/on_tool_end), and the traceloop+// decorator's {args, kwargs} (traceloop-sdk decorators/base.py _handle_span_input).++// tool renders the call as an assistant tool_call and the result as a tool tool_call_response, both+// carrying gen_ai.tool.call.id so the pair can be matched.+func (f *formatting) tool() {+ id := optionalString(attribute(f.attributes, GenAIToolCallID))+ if raw := attribute(f.attributes, GenAIToolCallArguments); raw != nil {+ call := ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(attribute(f.attributes, GenAIToolName)), ID: id, Arguments: toolArguments(decode(raw))}+ f.out.Input = InputMessages{{Role: RoleAssistant, Parts: Parts{part(call)}}}+ }+ if raw := attribute(f.attributes, GenAIToolCallResult); raw != nil {+ result := ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: id, Response: toolResult(decode(raw))}+ f.out.Output = OutputMessages{{Role: RoleTool, Parts: Parts{part(result)}}}+ }+ f.out.Formatter = FormatterTool+}++func toolArguments(value any) any {+ m, ok := value.(map[string]any)+ if !ok {+ return value+ }+ if _, isLangChain := m["input_str"]; isLangChain && hasKeys(m, "inputs", "kwargs") {+ if inputs := m["inputs"]; inputs != nil {+ return inputs+ }+ return decode(m["input_str"])+ }+ if len(m) == 2 && hasKeys(m, "args", "kwargs") {+ args, _ := m["args"].([]any)+ kwargs, _ := m["kwargs"].(map[string]any)+ switch {+ case len(args) == 0:+ return m["kwargs"]+ case len(kwargs) == 0:+ return args+ }+ }+ return value+}++// toolResult unwraps the LangChain envelope; the traceloop decorator encodes a tool returning+// nothing as the literal null.+func toolResult(value any) any {+ if m, ok := value.(map[string]any); ok && len(m) == 2 && hasKeys(m, "output", "kwargs") {+ return decode(m["output"])+ }+ if value == "null" {+ return nil+ }+ return value+}++func hasKeys(m map[string]any, keys ...string) bool {+ for _, key := range keys {+ if _, ok := m[key]; !ok {+ return false+ }+ }+ return true+}diff --git a/pkg/types/aiobservabilitytypes/message_format_tool_test.go b/pkg/types/aiobservabilitytypes/message_format_tool_test.gonew file mode 100644index 0000000000..9b37f5e23c--- /dev/null+++ b/pkg/types/aiobservabilitytypes/message_format_tool_test.go@@ -0,0 +1,110 @@+package aiobservabilitytypes++import "testing"++// Captured cases are the tool span attributes as the collector stored them for the SigNoz/scripts+// static-telemetry-generator captures, after the span mapper filled the gen_ai.tool.call.* keys.+func TestFormat_Tool(t *testing.T) {+ runFormatCases(t, []formatCase{+ {+ // langchain capture, execute_tool get_weather span; opentelemetry-instrumentation-langchain+ // callback_handler.py on_tool_start/on_tool_end envelopes+ name: "LangChainEnvelopes_InputsAndOutputUnwrapped",+ attributes: map[string]any{+ GenAIToolName: "get_weather",+ GenAIToolCallArguments: `{"input_str": "{'city': 'Bengaluru'}", "tags": [], "metadata": {}, "inputs": {"city": "Bengaluru"}, "kwargs": {"color": "green", "name": null, "tool_call_id": null}}`,+ GenAIToolCallResult: `{"output": "{\"city\": \"Bengaluru\", \"temp_c\": 18, \"summary\": \"Clear\"}", "kwargs": {"tags": [], "color": "green", "name": "get_weather"}}`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "arguments": {"city": "Bengaluru"}}]}]`,+ wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": {"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}}]}]`,+ },+ {+ // openllmetry capture, get_weather.tool span; traceloop-sdk decorators/base.py+ // _handle_span_input {args, kwargs} and the result encoded as a JSON string twice+ name: "TraceloopDecorator_KwargsUnwrapped_ResultDecodedTwice",+ attributes: map[string]any{+ GenAIToolName: "get_weather",+ GenAIToolCallArguments: `{"args": [], "kwargs": {"city": "Bengaluru"}}`,+ GenAIToolCallResult: `"{\"city\": \"Bengaluru\", \"temp_c\": 28, \"summary\": \"Partly cloudy\"}"`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "arguments": {"city": "Bengaluru"}}]}]`,+ wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": {"city": "Bengaluru", "temp_c": 28, "summary": "Partly cloudy"}}]}]`,+ },+ {+ // a decorated function returning None is json.dumps(None); the openllmetry capture shows it on+ // workflow spans, with this positional-args input+ name: "TraceloopDecorator_PositionalArgsOnly_ListKept",+ attributes: map[string]any{+ GenAIToolName: "lookup",+ GenAIToolCallArguments: `{"args": ["OpenAI", "What's the weather in Paris?"], "kwargs": {}}`,+ GenAIToolCallResult: `null`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "lookup", "arguments": ["OpenAI", "What's the weather in Paris?"]}]}]`,+ wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": null}]}]`,+ },+ {+ name: "TraceloopDecorator_BothArgsAndKwargs_EnvelopeKept",+ attributes: map[string]any{+ GenAIToolName: "lookup",+ GenAIToolCallArguments: `{"args": ["Paris"], "kwargs": {"unit": "c"}}`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "lookup", "arguments": {"args": ["Paris"], "kwargs": {"unit": "c"}}}]}]`,+ wantOutput: `[]`,+ },+ {+ // openinference-openai-agents capture, get_weather TOOL span: input.value and output.value+ // copied by the mapper (Arize-ai/openinference@12f2cc03, spec/traces.md)+ name: "OpenInferenceToolSpan_PlainJSONAndText",+ attributes: map[string]any{+ GenAIToolName: "get_weather",+ GenAIToolCallArguments: `{"city":"Bangalore"}`,+ GenAIToolCallResult: `27C, partly cloudy, light breeze from the south.`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "arguments": {"city": "Bangalore"}}]}]`,+ wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": "27C, partly cloudy, light breeze from the south."}]}]`,+ },+ {+ // vercel capture, ai.toolCall span: ai.toolCall.{id,name,args,result} copied by the mapper+ // (https://ai-sdk.dev/docs/ai-sdk-core/telemetry)+ name: "VercelToolCall_IDOnBothParts",+ attributes: map[string]any{+ GenAIToolName: "get_current_weather",+ GenAIToolCallID: "call_l0ycKcmt1uPCdGWR4SkwoLBv",+ GenAIToolCallArguments: `{"city":"Bengaluru","unit":"c"}`,+ GenAIToolCallResult: `{"city":"Bengaluru","unit":"c","temperature":27,"condition":"partly cloudy"}`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "id": "call_l0ycKcmt1uPCdGWR4SkwoLBv", "arguments": {"city": "Bengaluru", "unit": "c"}}]}]`,+ wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_l0ycKcmt1uPCdGWR4SkwoLBv", "response": {"city": "Bengaluru", "unit": "c", "temperature": 27, "condition": "partly cloudy"}}]}]`,+ },+ {+ // the object form the OTel registry asks for, whole in the JSON column+ name: "StructuredArguments_ObjectKept",+ attributes: map[string]any{+ GenAIToolName: "get_weather",+ GenAIToolCallID: "call_1",+ GenAIToolCallArguments: map[string]any{"city": "Paris", "days": []any{1.0, 2.0}},+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris", "days": [1, 2]}}]}]`,+ wantOutput: `[]`,+ },+ {+ // the mapper groups never fill both; the tool call wins when an SDK does+ name: "MessagesBesideToolCall_ToolCallFormatted",+ attributes: map[string]any{+ GenAIInputMessages: `[{"role": "user", "parts": [{"type": "text", "content": "hi"}]}]`,+ GenAIToolName: "lookup",+ GenAIToolCallArguments: `{"city": "Paris"}`,+ },+ formatter: FormatterTool,+ wantInput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "lookup", "arguments": {"city": "Paris"}}]}]`,+ wantOutput: `[]`,+ },+ })+}diff --git a/pkg/types/spantypes/thread.go b/pkg/types/spantypes/thread.goindex 91fa8e1919..8ce601dc35 100644--- a/pkg/types/spantypes/thread.go+++ b/pkg/types/spantypes/thread.go@@ -4,7 +4,6 @@ import ("encoding/base64""encoding/json""maps"- "strings""github.com/SigNoz/signoz/pkg/errors""github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"@@ -180,7 +179,9 @@ func newThreadSpan(traceID string, storable *StorableSpan) *ThreadSpan {maps.Copy(resources, storable.ResourcesString)timeUnixNano := uint64(storable.StartTime.UnixNano())attributes := threadAttributes(storable)- formatted := aiobservabilitytypes.FormatMessages(rawAttribute(storable, aiobservabilitytypes.GenAIInputMessages), rawAttribute(storable, aiobservabilitytypes.GenAIOutputMessages))+ // Spans from before the JSON rollout have no document and are not decoded: the legacy maps+ // hold an object split into one key per field.+ formatted := aiobservabilitytypes.FormatMessages(storable.AttributesJSON)return &ThreadSpan{SpanID: storable.SpanID,TraceID: traceID,@@ -214,20 +215,3 @@ func threadAttributes(storable *StorableSpan) map[string]any {storable.AttributesJSON.FlattenInto("", attributes)return attributes}--// rawAttribute reads one attribute for decoding from the JSON document, where an object value is-// still whole. The legacy maps hold objects split into one key per field, so spans from before-// the JSON rollout are not decoded. Not handled: a list of JSON strings, which formats as generic,-// and a scalar at a prefix of the path, such as gen_ai.input beside gen_ai.input.messages, which-// ends the walk early.-func rawAttribute(storable *StorableSpan, key string) any {- var current any = map[string]any(storable.AttributesJSON)- for segment := range strings.SplitSeq(key, ".") {- object, ok := current.(map[string]any)- if !ok {- return nil- }- current = object[segment]- }- return current-}diff --git a/pkg/types/spantypes/thread_test.go b/pkg/types/spantypes/thread_test.goindex c897400cb6..6670d99346 100644--- a/pkg/types/spantypes/thread_test.go+++ b/pkg/types/spantypes/thread_test.go@@ -73,35 +73,3 @@ func TestThreadAttributes(t *testing.T) {})}}--func TestRawAttribute(t *testing.T) {- arguments := map[string]any{"city": "Paris", "days": []any{1.0, 2.0}}- testCases := []struct {- name string- storable StorableSpan- key string- want any- }{- {- name: "JSONColumn_ObjectReturnedWhole",- storable: StorableSpan{AttributesJSON: telemetrystoretypes.JSONValue{"gen_ai": map[string]any{"tool": map[string]any{"call": map[string]any{"arguments": arguments}}}}},- key: "gen_ai.tool.call.arguments",- want: arguments,- },- {- name: "JSONColumn_MissingPath_Nil",- storable: StorableSpan{AttributesJSON: telemetrystoretypes.JSONValue{"gen_ai": map[string]any{"tool": map[string]any{"name": "get_weather"}}}},- key: "gen_ai.tool.call.arguments",- },- {- name: "LegacyMapsOnly_Nil",- storable: StorableSpan{AttributesString: map[string]string{"gen_ai.output.messages": "sunny", "gen_ai.tool.call.arguments.city": "Paris"}},- key: "gen_ai.output.messages",- },- }- for _, testCase := range testCases {- t.Run(testCase.name, func(t *testing.T) {- assert.Equal(t, testCase.want, rawAttribute(&testCase.storable, testCase.key))- })- }-}diff --git a/tests/integration/tests/tracedetail/02_thread.py b/tests/integration/tests/tracedetail/02_thread.pyindex d5f0011769..d19f3261ff 100644--- a/tests/integration/tests/tracedetail/02_thread.py+++ b/tests/integration/tests/tracedetail/02_thread.py@@ -11,7 +11,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORDfrom fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, TraceIdGenerator, Traces, TracesKind-def test_thread_returns_message_spans_in_order(+def test_thread_returns_gen_ai_spans_in_order(signoz: types.SigNoz,create_user_admin: None, # pylint: disable=unused-argumentget_token: Callable[[str, str], str],@@ -54,10 +54,14 @@ def test_thread_returns_message_spans_in_order(assert response.status_code == HTTPStatus.OK, response.textthread = response.json()["data"]- assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, second_llm_id, third_llm_id]+ assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, agent_id, second_llm_id, third_llm_id]assert "nextCursor" not in thread- first, tool, input_only, output_only = thread["spans"]+ first, tool, agent, input_only, output_only = thread["spans"]+ assert agent["attributes"]["gen_ai.agent.name"] == "planner"+ assert agent["formatter"] == ""+ assert agent["formatted_input"] == []+ assert agent["formatted_output"] == []assert tool["attributes"]["gen_ai.tool.call.arguments"] == json.dumps({"city": "Bangalore"})assert tool["attributes"]["gen_ai.tool.call.result"] == "sunny"assert first["time_unix"] == int((now - timedelta(seconds=8)).timestamp() * 1000)@@ -74,10 +78,9 @@ def test_thread_returns_message_spans_in_order(assert first["formatted_input"] == [{"role": "user", "parts": [{"type": "text", "content": "weather in Bangalore?"}]}]assert first["formatted_output"] == [{"role": "assistant", "parts": [tool_call], "finish_reason": "tool_call"}]assert first["formatter_warnings"] == []- # tool spans carry no messages until their converter lands- assert tool["formatter"] == ""- assert tool["formatted_input"] == []- assert tool["formatted_output"] == []+ assert tool["formatter"] == "tool"+ assert tool["formatted_input"] == [{"role": "assistant", "parts": [tool_call]}]+ assert tool["formatted_output"] == [{"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_1", "response": "sunny"}]}]assert input_only["formatter"] == "openai.chat"assert input_only["formatted_input"] == [{"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_1", "response": "sunny"}]}]assert input_only["formatted_output"] == []@@ -102,7 +105,7 @@ def test_thread_paginates_with_cursors(timestamps = [now - timedelta(seconds=6), now - timedelta(seconds=3), now - timedelta(seconds=3)]insert_traces([- Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-pages"}, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")+ Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-pages"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")for span_id, timestamp in zip(span_ids, timestamps, strict=True)])@@ -164,7 +167,7 @@ def test_thread_paginates_across_buckets(span_ids = [TraceIdGenerator.span_id() for _ in timestamps]insert_traces([- Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-buckets"}, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")+ Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-buckets"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")for span_id, timestamp in zip(span_ids, timestamps, strict=True)])@@ -209,16 +212,25 @@ def test_thread_opens_around_span(resources = {"service.name": "tracedetail-thread-anchor"}root_id = TraceIdGenerator.span_id()llm_ids = [TraceIdGenerator.span_id() for _ in range(5)]- agent_id = TraceIdGenerator.span_id()- # agent span, not a thread row, sits between the third and fourth llm spans+ http_id = TraceIdGenerator.span_id()+ # http span, not a thread row, sits between the third and fourth llm spansinsert_traces([Traces(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=19), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources, attribute_write_mode="json_only"),*(- Traces(timestamp=now - timedelta(seconds=18 - 3 * i), trace_id=trace_id, span_id=span_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")+ Traces(+ timestamp=now - timedelta(seconds=18 - 3 * i),+ trace_id=trace_id,+ span_id=span_id,+ parent_span_id=root_id,+ name="chat gpt-4o",+ resources=resources,+ attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])},+ attribute_write_mode="json_only",+ )for i, span_id in enumerate(llm_ids)),- Traces(timestamp=now - timedelta(seconds=11), trace_id=trace_id, span_id=agent_id, parent_span_id=root_id, name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}, attribute_write_mode="json_only"),+ Traces(timestamp=now - timedelta(seconds=11), trace_id=trace_id, span_id=http_id, parent_span_id=root_id, name="GET /weather", resources=resources, attributes={"http.request.method": "GET"}, attribute_write_mode="json_only"),])@@ -231,7 +243,7 @@ def test_thread_opens_around_span(assert response.status_code == HTTPStatus.OK, f"{params}: {response.text}"return response.json()["data"]- # span with messages: included with its neighbours+ # thread span: included with its neighboursaround = get_page({"limit": 3, "spanId": llm_ids[2]})assert [span["span_id"] for span in around["spans"]] == llm_ids[1:4]assert around["prevCursor"]@@ -239,11 +251,11 @@ def test_thread_opens_around_span(assert [span["span_id"] for span in get_page({"limit": 3, "before": around["prevCursor"]})["spans"]] == llm_ids[:1]assert [span["span_id"] for span in get_page({"limit": 3, "after": around["nextCursor"]})["spans"]] == llm_ids[4:]- # span without messages: only its neighbours- around_agent = get_page({"limit": 2, "spanId": agent_id})- assert [span["span_id"] for span in around_agent["spans"]] == llm_ids[2:4]- assert around_agent["prevCursor"]- assert around_agent["nextCursor"]+ # span outside the thread: only its neighbours+ around_http = get_page({"limit": 2, "spanId": http_id})+ assert [span["span_id"] for span in around_http["spans"]] == llm_ids[2:4]+ assert around_http["prevCursor"]+ assert around_http["nextCursor"]# near the start, the short side gives its room to the otherat_start = get_page({"limit": 3, "spanId": llm_ids[0]})@@ -287,11 +299,11 @@ def test_thread_reads_spans_across_json_rollout(legacy_id, json_id = TraceIdGenerator.span_id(), TraceIdGenerator.span_id()insert_traces([- Traces(timestamp=rollout - timedelta(minutes=10), trace_id=before_trace_id, span_id=before_ids[0], name="chat gpt-4o", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "first"}])}, attribute_write_mode="legacy_only"),- Traces(timestamp=rollout - timedelta(minutes=8), trace_id=before_trace_id, span_id=TraceIdGenerator.span_id(), name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}, attribute_write_mode="legacy_only"),- Traces(timestamp=rollout - timedelta(minutes=5), trace_id=before_trace_id, span_id=before_ids[1], name="chat gpt-4o", resources=resources, attributes={"gen_ai.output.messages": json.dumps([{"role": "assistant", "content": "second"}])}, attribute_write_mode="legacy_only"),- Traces(timestamp=rollout - timedelta(minutes=5), trace_id=straddle_trace_id, span_id=legacy_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "legacy"}])}, attribute_write_mode="legacy_only"),- Traces(timestamp=rollout + timedelta(minutes=5), trace_id=straddle_trace_id, span_id=json_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "json"}])}, attribute_write_mode="json_only"),+ Traces(timestamp=rollout - timedelta(minutes=10), trace_id=before_trace_id, span_id=before_ids[0], name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": "first"}])}, attribute_write_mode="legacy_only"),+ Traces(timestamp=rollout - timedelta(minutes=8), trace_id=before_trace_id, span_id=TraceIdGenerator.span_id(), name="GET /weather", resources=resources, attributes={"http.request.method": "GET"}, attribute_write_mode="legacy_only"),+ Traces(timestamp=rollout - timedelta(minutes=5), trace_id=before_trace_id, span_id=before_ids[1], name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": json.dumps([{"role": "assistant", "content": "second"}])}, attribute_write_mode="legacy_only"),+ Traces(timestamp=rollout - timedelta(minutes=5), trace_id=straddle_trace_id, span_id=legacy_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": "legacy"}])}, attribute_write_mode="legacy_only"),+ Traces(timestamp=rollout + timedelta(minutes=5), trace_id=straddle_trace_id, span_id=json_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": "json"}])}, attribute_write_mode="json_only"),])@@ -322,7 +334,7 @@ def test_thread_reads_spans_across_json_rollout(assert straddle_spans[1]["formatted_input"] == [{"role": "user", "parts": [{"type": "text", "content": "json"}]}]-def test_thread_without_messages_is_empty(+def test_thread_without_gen_ai_spans_is_empty(signoz: types.SigNoz,create_user_admin: None, # pylint: disable=unused-argumentget_token: Callable[[str, str], str],@@ -331,7 +343,14 @@ def test_thread_without_messages_is_empty() -> None:seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)trace_id = TraceIdGenerator.trace_id()- insert_traces([Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="GET /health", resources={"service.name": "tracedetail-thread-empty"}, attribute_write_mode="json_only")])+ resources = {"service.name": "tracedetail-thread-empty"}+ # messages without a gate key, as a failed call or an unmapped SDK writes them, are not a row either+ insert_traces(+ [+ Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="GET /health", resources=resources, attribute_write_mode="json_only"),+ Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=4), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="chat", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "hi"}])}, attribute_write_mode="json_only"),+ ]+ )token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/thread"), headers={"Authorization": f"Bearer {token}"}, timeout=10)