Skip to content

The only tool you need to know what is happening and how to fix it.

27.4 KB
Raw Blame
package otelcontrollers

import (
	"context"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/tracewayapp/traceway/backend/app/controllers/clientcontrollers"
	"github.com/tracewayapp/traceway/backend/app/models"
	"github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"
	"github.com/tracewayapp/traceway/backend/app/services/contentflag"
	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
	commonpb "go.opentelemetry.io/proto/otlp/common/v1"
	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
)

type aiTraceConversation struct {
	StorageKey string
	Content    []byte
	Input      string
	Output     string
}

type entityKind int

const (
	entityNone entityKind = iota
	entityEndpoint
	entityTask
	entityAiTrace
)

func (k entityKind) traceType() string {
	switch k {
	case entityEndpoint:
		return "endpoint"
	case entityTask:
		return "task"
	case entityAiTrace:
		return "ai_trace"
	}
	return ""
}

type convertedTraces struct {
	Spans           []models.OtelSpan
	Endpoints       []models.Endpoint
	Tasks           []models.Task
	Exceptions      []models.ExceptionStackTrace
	AiTraces        []models.AiTrace
	AiConversations []aiTraceConversation
	InvalidSpanIDs  int64
}

func convertTraces(ctx context.Context, existingProject *models.Project, projectId uuid.UUID, req *coltracepb.ExportTraceServiceRequest) (result convertedTraces) {
	suppressEntities := existingProject != nil && clientcontrollers.IsFrontendFramework(existingProject.Framework)

	// nil languages falls back to the default pack; a project that disabled
	// every pack carries an empty (non-nil) slice and scans custom terms only.
	var flagLanguages, customFlagTerms []string
	if existingProject != nil {
		flagLanguages = existingProject.AiFlaggedLanguages
		customFlagTerms = existingProject.AiFlaggedTerms
	}
	flagMatcher := contentflag.NewMatcher(flagLanguages, customFlagTerms)
	ingestedAt := time.Now().UTC()

	for _, rs := range req.ResourceSpans {
		resourceAttrs := rs.GetResource().GetAttributes()
		serverName := getStringAttribute(resourceAttrs, "service.name")
		appVersion := getStringAttribute(resourceAttrs, "service.version")
		language := getStringAttribute(resourceAttrs, "telemetry.sdk.language")
		proguardUuid := getStringAttribute(resourceAttrs, "app.debug.proguard_uuid")
		if appVersion == "" {
			if scriptVersionId := getStringAttribute(resourceAttrs, "cloudflare.script_version.id"); scriptVersionId != "" {
				if idx := strings.LastIndex(scriptVersionId, "-"); idx != -1 {
					appVersion = scriptVersionId[idx+1:]
				}
			}
		}

		type spanEntry struct {
			span     *tracepb.Span
			metadata *tracepb.ResourceSpans
		}
		var allSpans []spanEntry
		spanByKey := map[string]*tracepb.Span{}
		for _, ss := range rs.ScopeSpans {
			metadata := &tracepb.ResourceSpans{Resource: rs.Resource, SchemaUrl: rs.SchemaUrl,
				ScopeSpans: []*tracepb.ScopeSpans{{Scope: ss.Scope, SchemaUrl: ss.SchemaUrl}}}
			metadata.ProtoReflect().SetUnknown(rs.ProtoReflect().GetUnknown())
			metadata.ScopeSpans[0].ProtoReflect().SetUnknown(ss.ProtoReflect().GetUnknown())
			for _, span := range ss.Spans {
				if !validSourceSpanIDs(span) {
					result.InvalidSpanIDs++
					continue
				}
				allSpans = append(allSpans, spanEntry{span: span, metadata: metadata})
				spanByKey[otelSpanKey(span)] = span
			}
		}
		parentInBatch := func(span *tracepb.Span) *tracepb.Span {
			if len(span.ParentSpanId) == 0 {
				return nil
			}
			return spanByKey[string(span.TraceId)+string(span.ParentSpanId)]
		}

		for _, entry := range allSpans {
			span := entry.span
			scopeName := entry.metadata.ScopeSpans[0].GetScope().GetName()
			traceId := hex.EncodeToString(span.TraceId)
			spanId := hex.EncodeToString(span.SpanId)
			parentSpanId := hex.EncodeToString(span.ParentSpanId)
			start := shared.OtelNanosToTime(span.StartTimeUnixNano)
			duration := shared.OtelDuration(span.StartTimeUnixNano, span.EndTimeUnixNano)
			result.Spans = append(result.Spans, models.OtelSpan{
				OTLP: span, Context: entry.metadata,
				Span: models.Span{
					ProjectId: projectId, TraceId: traceId, SpanId: spanId, ParentSpanId: parentSpanId,
					Name: span.Name, StartTime: start, RecordedAt: start, Duration: duration,
					SpanKind: int32(span.Kind), StatusCode: int32(span.GetStatus().GetCode()),
					ServiceName: serverName, ScopeName: scopeName,
				},
			})
			kind := entityNone
			if !suppressEntities {
				kind = classifySpan(span, parentInBatch)
			}
			if kind == entityNone && !spanCarriesException(span) {
				continue
			}

			spanAttrs := span.Attributes
			allAttrs := extractAttributes(spanAttrs)
			// A span carrying exception.stacktrace always yields an exception
			// record (from the attribute path, or from the event when both are
			// present), so the raw blob is never duplicated onto the
			// endpoint/task/span rows or the exception's attribute map.
			delete(allAttrs, "exception.stacktrace")
			recordedStart, _, _ := shared.OtelStorageTimes(span.StartTimeUnixNano, ingestedAt)
			id := otelOccurrenceID(projectId, span)

			switch kind {
			case entityEndpoint:
				ep := buildEndpoint(
					id, projectId, span, spanAttrs, allAttrs,
					recordedStart, duration, serverName, appVersion,
				)
				ep.TraceId, ep.SpanId, ep.ParentSpanId = traceId, spanId, parentSpanId
				ep.IsRoot = parentSpanId == ""
				result.Endpoints = append(result.Endpoints, ep)
			case entityTask:
				t := buildTask(
					id, projectId, span, allAttrs,
					recordedStart, duration, serverName, appVersion,
				)
				t.TraceId, t.SpanId, t.ParentSpanId = traceId, spanId, parentSpanId
				t.IsRoot = parentSpanId == ""
				result.Tasks = append(result.Tasks, t)
			case entityAiTrace:
				aiTrace := buildAiTrace(
					id, projectId, span, spanAttrs, allAttrs,
					recordedStart, duration, serverName, appVersion,
				)
				aiTrace.TraceId, aiTrace.SpanId, aiTrace.ParentSpanId = traceId, spanId, parentSpanId
				aiTrace.IsRoot = parentSpanId == ""
				aiTrace.ConversationId = resolveConversationId(spanAttrs, resourceAttrs, traceId)
				var convInput, convOutput string
				if conv := extractConversation(spanAttrs, projectId, id); conv != nil {
					convInput, convOutput = conv.Input, conv.Output
					result.AiConversations = append(result.AiConversations, *conv)
				}
				aiTrace.ToolCallCount, aiTrace.ToolNames = extractToolCalls(spanAttrs, convOutput)
				if terms := flagMatcher.Scan(convInput, convOutput); len(terms) > 0 {
					aiTrace.Flagged = true
					aiTrace.FlaggedTerms = terms
				}
				result.AiTraces = append(result.AiTraces, aiTrace)
			}

			appendException := func(attrs []*commonpb.KeyValue, timeUnixNano uint64) {
				exc := buildException(
					ctx, existingProject, projectId, attrs, timeUnixNano,
					allAttrs, serverName, appVersion, language, proguardUuid, scopeName,
				)
				// The kind is stored only when this span says it. An exception further down the request finds its entity at read time.
				exc.TraceId, exc.SpanId, exc.TraceType = traceId, spanId, kind.traceType()
				result.Exceptions = append(result.Exceptions, exc)
			}

			hadExceptionEvent := false
			for _, event := range span.Events {
				if event.Name == "exception" {
					hadExceptionEvent = true
					appendException(event.Attributes, event.TimeUnixNano)
				}
			}
			if !hadExceptionEvent && hasExceptionAttributes(span.Attributes) {
				appendException(span.Attributes, span.StartTimeUnixNano)
			}
		}
	}
	return
}

const (
	spanFlagHasIsRemote = uint32(tracepb.SpanFlags_SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK)
	spanFlagIsRemote    = uint32(tracepb.SpanFlags_SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK)
	// Bounds the walk when malformed input makes the parent chain a cycle.
	maxEntryPointDepth = 64
)

func isHTTPServerSpan(span *tracepb.Span) bool {
	return span.Kind == tracepb.Span_SPAN_KIND_SERVER && hasHTTPAttributes(span.Attributes)
}

// An HTTP SERVER span is a request of its own unless a local span above it already is one (Next.js under the HTTP
// instrumentation). The walk stays inside the process: it stops at a remote parent, which OTLP flags mark however
// the trace is batched, and at a CLIENT or PRODUCER span, which is the calling side. A plain wrapper span above
// the request does not make it nested. Missing parents have unknown kinds, even
// when flags establish locality, so they cannot suppress an HTTP entry point.
func isEntryPoint(span *tracepb.Span, parentOf func(*tracepb.Span) *tracepb.Span) bool {
	for current, depth := span, 0; depth < maxEntryPointDepth; depth++ {
		if len(current.ParentSpanId) == 0 {
			return true
		}
		localityKnown := current.Flags&spanFlagHasIsRemote != 0
		if localityKnown && current.Flags&spanFlagIsRemote != 0 {
			return true
		}
		parent := parentOf(current)
		if parent == nil {
			return true
		}
		if parent.Kind == tracepb.Span_SPAN_KIND_CLIENT || parent.Kind == tracepb.Span_SPAN_KIND_PRODUCER {
			return true
		}
		if isHTTPServerSpan(parent) {
			return false
		}
		current = parent
	}
	return true
}

func classifySpan(span *tracepb.Span, parentOf func(*tracepb.Span) *tracepb.Span) entityKind {
	attrs := span.Attributes
	// Honeycomb's browser SDK stamps page context (url.path etc.) on every
	// span, including the zero-duration INTERNAL `exception` spans emitted by
	// its global-errors instrumentation. Those must become exception rows,
	// not endpoints, so exception-bearing INTERNAL spans are never promoted.
	if span.Kind == tracepb.Span_SPAN_KIND_INTERNAL && hasExceptionAttributes(attrs) {
		return entityNone
	}
	if (isHTTPServerSpan(span) && isEntryPoint(span, parentOf)) ||
		(span.Kind == tracepb.Span_SPAN_KIND_INTERNAL && len(span.ParentSpanId) == 0 && hasHTTPAttributes(attrs)) {
		return entityEndpoint
	}

	if span.Kind == tracepb.Span_SPAN_KIND_CONSUMER {
		return entityTask
	}
	// keepsuit's ConsoleInstrumentation (and equivalents) emits a root INTERNAL
	// span with `console.command` set. Only promote when it is a root span —
	// otherwise we would scoop up arbitrary manual roots from other code paths.
	if span.Kind == tracepb.Span_SPAN_KIND_INTERNAL && len(span.ParentSpanId) == 0 && getStringAttribute(attrs, "console.command") != "" {
		return entityTask
	}
	if hasGenAiAttributes(attrs) {
		return entityAiTrace
	}
	return entityNone
}

func hasHTTPAttributes(attrs []*commonpb.KeyValue) bool {
	for _, kv := range attrs {
		switch kv.Key {
		case "http.request.method", "http.method", "http.route", "url.path":
			return true
		}
	}
	return false
}

func buildEndpoint(
	id, projectId uuid.UUID,
	span *tracepb.Span,
	attrs []*commonpb.KeyValue,
	allAttrs map[string]string,
	startTime time.Time,
	duration time.Duration,
	serverName, appVersion string,
) models.Endpoint {
	endpoint := getHTTPEndpoint(attrs, span.Name)

	statusCode := int16(0)
	if code, ok := getIntAttribute(attrs, "http.response.status_code"); ok {
		statusCode = int16(code)
	} else if code, ok := getIntAttribute(attrs, "http.status_code"); ok {
		statusCode = int16(code)
	}

	// A 404 collapses to UNMATCHED only when no real route matched: http.route
	// is missing/invalid, or a catch-all made of only slashes and wildcards
	// ("/", "/*", "/**", "*/*") — what Express middleware, Spring resource
	// handlers and not-found handlers report for unmatched requests. A concrete
	// matched route returning 404 is a deliberate response and keeps its identity.
	route := getStringAttribute(attrs, "http.route")
	if statusCode == 404 && (!strings.HasPrefix(route, "/") || strings.Trim(route, "/*") == "") {
		endpoint = "UNMATCHED"
	}

	bodySize := int32(0)
	if size, ok := getIntAttribute(attrs, "http.response.body.size"); ok {
		bodySize = int32(size)
	} else if size, ok := getIntAttribute(attrs, "http.response_content_length"); ok {
		bodySize = int32(size)
	}

	clientIP := getStringAttribute(attrs, "client.address")
	if clientIP == "" {
		clientIP = getStringAttribute(attrs, "net.peer.ip")
	}

	return models.Endpoint{
		Id:         id,
		ProjectId:  projectId,
		Endpoint:   endpoint,
		Duration:   duration,
		RecordedAt: startTime,
		StatusCode: statusCode,
		BodySize:   bodySize,
		ClientIP:   clientIP,
		Attributes: allAttrs,
		AppVersion: appVersion,
		ServerName: serverName,
		IsStream:   isOtelStreamingEndpoint(attrs, statusCode),
	}
}

// isOtelStreamingEndpoint detects long-lived streaming responses on OTel spans:
//   - status 101 (WebSocket upgrade)
//   - http.response.header.content-type contains text/event-stream (SSE)
//
// OTel has no standard `is_stream` attribute, so we sniff the captured headers.
// Clients that don't capture `http.response.header.content-type` won't trigger
// SSE detection — they can fall back to a vendor extension attribute
// `traceway.is_stream` (boolean) or the WebSocket signal.
func isOtelStreamingEndpoint(attrs []*commonpb.KeyValue, statusCode int16) bool {
	if statusCode == http.StatusSwitchingProtocols {
		return true
	}
	if b, ok := getBoolAttribute(attrs, "traceway.is_stream"); ok && b {
		return true
	}
	for _, ct := range getStringValues(attrs, "http.response.header.content-type") {
		if strings.HasPrefix(strings.ToLower(strings.TrimSpace(ct)), "text/event-stream") {
			return true
		}
	}
	return false
}

func getBoolAttribute(attrs []*commonpb.KeyValue, key string) (bool, bool) {
	for _, kv := range attrs {
		if kv.Key == key && kv.Value != nil {
			if bv, ok := kv.Value.Value.(*commonpb.AnyValue_BoolValue); ok {
				return bv.BoolValue, true
			}
		}
	}
	return false, false
}

func getHTTPEndpoint(attrs []*commonpb.KeyValue, fallback string) string {
	method := getStringAttribute(attrs, "http.request.method")
	if method == "" {
		method = getStringAttribute(attrs, "http.method")
	}
	route := getStringAttribute(attrs, "http.route")
	if route != "" && !strings.HasPrefix(route, "/") {
		route = ""
	}
	if route == "" {
		route = getStringAttribute(attrs, "url.path")
	}

	if method != "" && route != "" {
		return method + " " + route
	}
	if method != "" {
		return method + " " + fallback
	}
	return fallback
}

func buildTask(
	id, projectId uuid.UUID,
	span *tracepb.Span,
	allAttrs map[string]string,
	startTime time.Time,
	duration time.Duration,
	serverName, appVersion string,
) models.Task {
	return models.Task{
		Id:         id,
		ProjectId:  projectId,
		TaskName:   span.Name,
		Duration:   duration,
		RecordedAt: startTime,
		Attributes: allAttrs,
		AppVersion: appVersion,
		ServerName: serverName,
	}
}

func buildException(
	ctx context.Context,
	existingProject *models.Project,
	projectId uuid.UUID,
	excAttrs []*commonpb.KeyValue,
	timeUnixNano uint64,
	spanAttrs map[string]string,
	serverName, appVersion, language, proguardUuid, scopeName string,
) models.ExceptionStackTrace {
	excType := getStringAttribute(excAttrs, "exception.type")
	excMessage := getStringAttribute(excAttrs, "exception.message")

	stackTrace, ok := buildHoneycombStackTrace(excType, excMessage, excAttrs)
	if !ok {
		excStacktrace := getStringAttribute(excAttrs, "exception.stacktrace")
		if excStacktrace == "" {
			if frames, aok := buildAndroidStructuredStackTrace(excAttrs); aok {
				excStacktrace = frames
			}
		}
		stackTrace = formatExceptionStackTrace(excType, excMessage, excStacktrace)
	}

	stackTrace = otelSymbolicateJs(existingProject, projectId, ctx, stackTrace, language, scopeName)
	stackTrace = otelSymbolicateAndroid(existingProject, projectId, ctx, stackTrace, language, proguardUuid)

	hash := clientcontrollers.ComputeExceptionHash(stackTrace, false)
	recordedAt, _, _ := shared.OtelStorageTimes(timeUnixNano, time.Now())

	attrs := spanAttrs
	if isJsLanguage(language) || isAndroidLanguage(language) {
		attrs = cloneStringMap(spanAttrs)
		attrs["telemetry.sdk.language"] = language
	}

	return models.ExceptionStackTrace{
		Id:            uuid.New(),
		ProjectId:     projectId,
		ExceptionHash: hash,
		StackTrace:    stackTrace,
		RecordedAt:    recordedAt,
		Attributes:    attrs,
		AppVersion:    appVersion,
		ServerName:    serverName,
	}
}

func hasExceptionAttributes(attrs []*commonpb.KeyValue) bool {
	return getStringAttribute(attrs, "exception.type") != "" ||
		getStringAttribute(attrs, "exception.message") != "" ||
		getStringAttribute(attrs, "exception.stacktrace") != ""
}

func spanCarriesException(span *tracepb.Span) bool {
	for _, event := range span.Events {
		if event.Name == "exception" {
			return true
		}
	}
	return hasExceptionAttributes(span.Attributes)
}

func cloneStringMap(m map[string]string) map[string]string {
	out := make(map[string]string, len(m)+1)
	for k, v := range m {
		out[k] = v
	}
	return out
}

func isJsLanguage(lang string) bool {
	switch strings.ToLower(lang) {
	case "webjs", "nodejs", "javascript", "typescript":
		return true
	}
	return false
}

func isAndroidLanguage(lang string) bool {
	switch strings.ToLower(strings.TrimSpace(lang)) {
	case "android", "java", "kotlin":
		return true
	}
	return false
}

func buildAndroidStructuredStackTrace(eventAttrs []*commonpb.KeyValue) (string, bool) {
	classes := getStringArray(eventAttrs, "exception.structured_stacktrace.classes")
	if len(classes) == 0 {
		return "", false
	}
	methods := getStringArray(eventAttrs, "exception.structured_stacktrace.methods")
	files := getStringArray(eventAttrs, "exception.structured_stacktrace.source_files")
	lines := getIntArray(eventAttrs, "exception.structured_stacktrace.lines")

	var b strings.Builder
	for i := range classes {
		method := ""
		if i < len(methods) {
			method = methods[i]
		}
		file := ""
		if i < len(files) {
			file = files[i]
		}
		loc := file
		if i < len(lines) && lines[i] > 0 {
			loc = fmt.Sprintf("%s:%d", file, lines[i])
		}
		fmt.Fprintf(&b, "\tat %s.%s(%s)\n", classes[i], method, loc)
	}
	return strings.TrimRight(b.String(), "\n"), true
}

// JS instrumentations are also recognizable by their scope name when
// telemetry.sdk.language is absent: npm scoped packages ("@scope/pkg") are
// an npm-only naming convention across OTel SDKs, and Next.js's built-in
// tracer reports the unscoped scope name "next.js".
func isJsTelemetry(language, scopeName string) bool {
	if isJsLanguage(language) {
		return true
	}
	if strings.HasPrefix(scopeName, "@") && strings.Contains(scopeName, "/") {
		return true
	}
	return scopeName == "next.js"
}

func buildHoneycombStackTrace(excType, excMessage string, eventAttrs []*commonpb.KeyValue) (string, bool) {
	urls := getStringArray(eventAttrs, "exception.structured_stacktrace.urls")
	if len(urls) == 0 {
		return "", false
	}
	functions := getStringArray(eventAttrs, "exception.structured_stacktrace.functions")
	lines := getIntArray(eventAttrs, "exception.structured_stacktrace.lines")
	columns := getIntArray(eventAttrs, "exception.structured_stacktrace.columns")

	at := func(s []string, i int) string {
		if i < len(s) {
			return s[i]
		}
		return ""
	}
	atInt := func(s []int64, i int) int64 {
		if i < len(s) {
			return s[i]
		}
		return 0
	}

	var b strings.Builder
	if header := formatExceptionStackTrace(excType, excMessage, ""); header != "unknown exception" {
		b.WriteString(header)
		b.WriteByte('\n')
	}
	for i, url := range urls {
		if fn := at(functions, i); fn != "" {
			b.WriteString(fn)
			b.WriteString("()\n")
		}
		fmt.Fprintf(&b, "    %s:%d:%d\n", url, atInt(lines, i), atInt(columns, i))
	}
	return strings.TrimRight(b.String(), "\n"), true
}

func hasGenAiAttributes(attrs []*commonpb.KeyValue) bool {
	for _, kv := range attrs {
		if strings.HasPrefix(kv.Key, "gen_ai.") {
			return true
		}
	}
	return false
}

func buildAiTrace(
	id, projectId uuid.UUID,
	span *tracepb.Span,
	attrs []*commonpb.KeyValue,
	allAttrs map[string]string,
	startTime time.Time,
	duration time.Duration,
	serverName, appVersion string,
) models.AiTrace {
	model := getStringAttribute(attrs, "gen_ai.request.model")
	responseModel := getStringAttribute(attrs, "gen_ai.response.model")
	provider := getStringAttribute(attrs, "gen_ai.system")
	if provider == "" {
		provider = getStringAttribute(attrs, "gen_ai.provider.name")
	}
	operation := getStringAttribute(attrs, "gen_ai.operation.name")

	inputTokens, _ := getIntAttribute(attrs, "gen_ai.usage.input_tokens")
	outputTokens, _ := getIntAttribute(attrs, "gen_ai.usage.output_tokens")
	totalTokens, hasTotalTokens := getIntAttribute(attrs, "gen_ai.usage.total_tokens")
	if !hasTotalTokens {
		totalTokens = inputTokens + outputTokens
	}
	cachedTokens, _ := getIntAttribute(attrs, "gen_ai.usage.input_tokens.cached")
	reasoningTokens, _ := getIntAttribute(attrs, "gen_ai.usage.output_tokens.reasoning")

	inputCost := getFloatAttribute(attrs, "gen_ai.usage.input_cost")
	outputCost := getFloatAttribute(attrs, "gen_ai.usage.output_cost")
	totalCost := getFloatAttribute(attrs, "gen_ai.usage.total_cost")
	if totalCost == 0 {
		totalCost = inputCost + outputCost
	}

	traceName := getStringAttribute(attrs, "trace.name")
	if traceName == "" {
		traceName = span.Name
	}

	userId := getStringAttribute(attrs, "user.id")
	finishReason := getStringAttribute(attrs, "gen_ai.response.finish_reason")
	if finishReason == "" {
		// finish_reasons is an array attribute in the OTel gen_ai conventions;
		// getStringValues handles both the scalar and array encodings.
		finishReason = strings.Join(getStringValues(attrs, "gen_ai.response.finish_reasons"), ",")
	}

	statusCode := uint8(span.Status.GetCode())
	storageKey := fmt.Sprintf("ai-traces/%s/%s.json", projectId, id)

	filteredAttrs := filterNonStandardAiAttrs(allAttrs)

	return models.AiTrace{
		Id:              id,
		ProjectId:       projectId,
		RecordedAt:      startTime,
		Duration:        duration,
		StatusCode:      statusCode,
		Model:           model,
		ResponseModel:   responseModel,
		Provider:        provider,
		Operation:       operation,
		InputTokens:     inputTokens,
		OutputTokens:    outputTokens,
		TotalTokens:     totalTokens,
		CachedTokens:    cachedTokens,
		ReasoningTokens: reasoningTokens,
		InputCost:       inputCost,
		OutputCost:      outputCost,
		TotalCost:       totalCost,
		TraceName:       traceName,
		UserId:          userId,
		FinishReason:    finishReason,
		ServerName:      serverName,
		AppVersion:      appVersion,
		StorageKey:      storageKey,
		Attributes:      filteredAttrs,
	}
}

var standardAiAttrPrefixes = []string{
	"gen_ai.request.model",
	"gen_ai.response.model",
	"gen_ai.system",
	"gen_ai.provider.name",
	"gen_ai.operation.name",
	"gen_ai.usage.",
	"gen_ai.prompt",
	"gen_ai.completion",
	"gen_ai.response.finish_reason",
	"gen_ai.response.finish_reasons",
	"gen_ai.conversation.id",
	"gen_ai.tool.",
	"trace.name",
	"trace.input",
	"trace.output",
	"span.input",
	"span.output",
	"user.id",
}

func filterNonStandardAiAttrs(allAttrs map[string]string) map[string]string {
	if len(allAttrs) == 0 {
		return nil
	}
	result := make(map[string]string)
	for k, v := range allAttrs {
		standard := false
		for _, prefix := range standardAiAttrPrefixes {
			if k == prefix || strings.HasPrefix(k, prefix) {
				standard = true
				break
			}
		}
		if !standard {
			result[k] = v
		}
	}
	if len(result) == 0 {
		return nil
	}
	return result
}

func extractConversation(attrs []*commonpb.KeyValue, projectId, traceId uuid.UUID) *aiTraceConversation {
	input := getStringAttribute(attrs, "gen_ai.prompt")
	if input == "" {
		input = getStringAttribute(attrs, "trace.input")
	}
	if input == "" {
		input = getStringAttribute(attrs, "span.input")
	}

	output := getStringAttribute(attrs, "gen_ai.completion")
	if output == "" {
		output = getStringAttribute(attrs, "trace.output")
	}
	if output == "" {
		output = getStringAttribute(attrs, "span.output")
	}

	if input == "" && output == "" {
		return nil
	}

	content := map[string]string{
		"input":  input,
		"output": output,
	}
	data, err := json.Marshal(content)
	if err != nil {
		return nil
	}

	return &aiTraceConversation{
		StorageKey: fmt.Sprintf("ai-traces/%s/%s.json", projectId, traceId),
		Content:    data,
		Input:      input,
		Output:     output,
	}
}

// resolveConversationId picks the conversation grouping key for an AI trace:
// an explicit gen_ai.conversation.id, else session.id (span first, then
// resource, where browser SDKs stamp it), else the trace id so a single agent
// run still groups its calls.
func resolveConversationId(spanAttrs, resourceAttrs []*commonpb.KeyValue, traceId string) string {
	if id := getStringAttribute(spanAttrs, "gen_ai.conversation.id"); id != "" {
		return id
	}
	if id := getStringAttribute(spanAttrs, "session.id"); id != "" {
		return id
	}
	if id := getStringAttribute(resourceAttrs, "session.id"); id != "" {
		return id
	}
	// Keep fallback conversation keys compatible with history written before V2.
	if id, err := uuid.Parse(traceId); err == nil {
		return id.String()
	}
	return traceId
}

const maxToolNames = 50

// extractToolCalls pulls tool-call telemetry out of the completion payload
// (OpenAI choices/tool_calls, Anthropic content/tool_use, OTel gen_ai output
// messages with tool_call parts), falling back to the execute_tool span
// attributes when no payload is available. Names are deduplicated in
// first-seen order; commas are stripped because the names are persisted as a
// comma-separated column.
func extractToolCalls(attrs []*commonpb.KeyValue, output string) (int64, []string) {
	count, names := parseToolCallsFromOutput(output)
	if count == 0 && getStringAttribute(attrs, "gen_ai.operation.name") == "execute_tool" {
		count = 1
		if name := sanitizeToolName(getStringAttribute(attrs, "gen_ai.tool.name")); name != "" {
			names = []string{name}
		}
	}
	return count, names
}

func parseToolCallsFromOutput(output string) (int64, []string) {
	if output == "" {
		return 0, nil
	}

	var count int64
	var names []string
	seen := map[string]struct{}{}
	record := func(name string) {
		count++
		name = sanitizeToolName(name)
		if name == "" || len(names) >= maxToolNames {
			return
		}
		if _, dup := seen[name]; dup {
			return
		}
		seen[name] = struct{}{}
		names = append(names, name)
	}

	type contentPart struct {
		Type string `json:"type"`
		Name string `json:"name"`
	}
	var objectShape struct {
		// OpenAI-style chat completion response.
		Choices []struct {
			Message struct {
				ToolCalls []struct {
					Function struct {
						Name string `json:"name"`
					} `json:"function"`
				} `json:"tool_calls"`
			} `json:"message"`
		} `json:"choices"`
		// Anthropic-style response content blocks.
		Content []contentPart `json:"content"`
	}
	if err := json.Unmarshal([]byte(output), &objectShape); err == nil {
		for _, choice := range objectShape.Choices {
			for _, call := range choice.Message.ToolCalls {
				record(call.Function.Name)
			}
		}
		for _, part := range objectShape.Content {
			if part.Type == "tool_use" {
				record(part.Name)
			}
		}
		return count, names
	}

	// OTel gen_ai output messages: a top-level array of messages with parts.
	var messagesShape []struct {
		Parts []contentPart `json:"parts"`
	}
	if err := json.Unmarshal([]byte(output), &messagesShape); err == nil {
		for _, msg := range messagesShape {
			for _, part := range msg.Parts {
				if part.Type == "tool_call" || part.Type == "tool_use" {
					record(part.Name)
				}
			}
		}
	}
	return count, names
}

func sanitizeToolName(name string) string {
	return strings.TrimSpace(strings.ReplaceAll(name, ",", ""))
}

func formatExceptionStackTrace(excType, excMessage, excStacktrace string) string {
	header := excType
	if excMessage != "" {
		if header != "" {
			header += ": " + excMessage
		} else {
			header = excMessage
		}
	}
	if excStacktrace != "" {
		// JVM OTel agents embed the exception class name as the first line of the
		// stacktrace (e.g. "java.lang.RuntimeException: msg\n\tat ..."). Skip
		// prepending the header when it's already there to avoid a duplicate line.
		if header != "" && (excType == "" || !strings.HasPrefix(excStacktrace, excType)) {
			return fmt.Sprintf("%s\n%s", header, excStacktrace)
		}
		return excStacktrace
	}
	if header != "" {
		return header
	}
	return "unknown exception"
}