Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

feat(traces-qb): gate JSON span attribute reads behind a feature flag (#12966)

Nikhil Sonibcf96f28161c
diff --git a/pkg/flagger/registry.go b/pkg/flagger/registry.go
index 709607ad5e..a9c54085b1 100644
--- a/pkg/flagger/registry.go
+++ b/pkg/flagger/registry.go
@@ -11,6 +11,7 @@ var (
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
FeatureResolveSemconvFamilies = featuretypes.MustNewName("resolve_semconv_families")
+ FeatureUseTraceAttributesJSON = featuretypes.MustNewName("use_trace_attributes_json")
)
func MustNewRegistry() featuretypes.Registry {
@@ -79,6 +80,14 @@ func MustNewRegistry() featuretypes.Registry {
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
+ &featuretypes.Feature{
+ Name: FeatureUseTraceAttributesJSON,
+ Kind: featuretypes.KindBoolean,
+ Stage: featuretypes.StageExperimental,
+ Description: "Controls whether trace queries read span attributes from the JSON columns",
+ DefaultVariant: featuretypes.MustNewName("disabled"),
+ Variants: featuretypes.NewBooleanVariants(),
+ },
)
if err != nil {
panic(err)
diff --git a/pkg/querybuilder/resolve.go b/pkg/querybuilder/resolve.go
index 86c72d2f67..e28b149ebb 100644
--- a/pkg/querybuilder/resolve.go
+++ b/pkg/querybuilder/resolve.go
@@ -24,7 +24,9 @@ func NewQueryInfo(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, si
FamiliesOn: semconvFamiliesEnabled(ctx, orgID, fl),
}
if fl != nil {
- q.BodyJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))
+ evalCtx := featuretypes.NewFlaggerEvaluationContext(orgID)
+ q.BodyJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, evalCtx)
+ q.TraceAttrsJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseTraceAttributesJSON, evalCtx)
}
return q
}
diff --git a/pkg/statementbuilder/tracesstatementbuilder/bulk_attributes_test.go b/pkg/statementbuilder/tracesstatementbuilder/bulk_attributes_test.go
index 500e0c7dbe..982a5f23f9 100644
--- a/pkg/statementbuilder/tracesstatementbuilder/bulk_attributes_test.go
+++ b/pkg/statementbuilder/tracesstatementbuilder/bulk_attributes_test.go
@@ -7,6 +7,7 @@ import (
"testing"
"time"
+ "github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -24,7 +25,7 @@ var jsonAttrColRe = regexp.MustCompile(`,\s*attributes\s*(,| FROM )`)
func newBulkTestBuilder(t *testing.T, releaseTime time.Time) *traceQueryStatementBuilder {
t.Helper()
- fl := flaggertest.New(t)
+ fl := flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureUseTraceAttributesJSON.String(): true})
storage := tracestelemetryschema.NewStorage()
store := telemetrytypestest.NewMockMetadataStore()
store.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
diff --git a/pkg/telemetryschema/tracestelemetryschema/storage.go b/pkg/telemetryschema/tracestelemetryschema/storage.go
index 08b7b573e9..de139169cf 100644
--- a/pkg/telemetryschema/tracestelemetryschema/storage.go
+++ b/pkg/telemetryschema/tracestelemetryschema/storage.go
@@ -172,7 +172,7 @@ func NewStorage() qbtypes.Storage {
func (m *storage) getColumn(
_ context.Context,
- _, _ uint64,
+ q qbtypes.QueryInfo,
key *telemetrytypes.TelemetryFieldKey,
) ([]*schema.Column, error) {
switch key.FieldContext {
@@ -194,8 +194,8 @@ func (m *storage) getColumn(
default:
return nil, qbtypes.ErrColumnNotFound
}
- // The `attributes` evolution entry is the rollout control.
- if attributeColumnEvolutionRegistered(key, SpanAttributesColumn) {
+ // The use_trace_attributes_json flag and the `attributes` evolution entry are the rollout control.
+ if q.TraceAttrsJSONOn && attributeColumnEvolutionRegistered(key, SpanAttributesColumn) {
cols := make([]*schema.Column, 0, 3)
if attributeColumnEvolutionRegistered(key, SpanAttributesPromotedColumn) {
cols = append(cols, indexV3Columns["attributes_promoted"])
@@ -233,15 +233,15 @@ func (m *storage) getColumn(
// (after evolution selection); existExprs only carries guards for guardable column types.
func (m *storage) resolveColumnExprs(
ctx context.Context,
- startNs, endNs uint64,
+ q qbtypes.QueryInfo,
key *telemetrytypes.TelemetryFieldKey,
) (exprs []string, existExprs []string, columns []*schema.Column, err error) {
- columns, err = m.getColumn(ctx, startNs, endNs, key)
+ columns, err = m.getColumn(ctx, q, key)
if err != nil {
return nil, nil, nil, err
}
- newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+ newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, q.StartNs, q.EndNs)
if err != nil {
return nil, nil, nil, err
}
@@ -355,12 +355,12 @@ func attributeJSONValueExpr(path string, dataType telemetrytypes.FieldDataType)
// columnIsTemporal reports whether key resolves to a single time column, after evolution
// selection. Multiple columns mean an attribute-map union, which is never temporal.
-func (m *storage) columnIsTemporal(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
- columns, err := m.getColumn(ctx, startNs, endNs, key)
+func (m *storage) columnIsTemporal(ctx context.Context, q qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
+ columns, err := m.getColumn(ctx, q, key)
if err != nil {
return false, err
}
- newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+ newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, q.StartNs, q.EndNs)
if err != nil {
return false, err
}
@@ -394,7 +394,7 @@ func (m *storage) read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
return key.Name, nil
}
- exprs, existExpr, columns, err := m.resolveColumnExprs(ctx, q.StartNs, q.EndNs, key)
+ exprs, existExpr, columns, err := m.resolveColumnExprs(ctx, q, key)
if err != nil {
return "", err
}
@@ -457,7 +457,7 @@ func (m *storage) Read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
if isSpanSearchScopeField(key.Name) {
return qbtypes.Read{SQL: key.Name, Presence: "true", Absence: "false", WhenAbsent: qbtypes.AlwaysPresent}, nil
}
- exprs, existExprs, columns, err := m.resolveColumnExprs(ctx, q.StartNs, q.EndNs, key)
+ exprs, existExprs, columns, err := m.resolveColumnExprs(ctx, q, key)
if err != nil {
return qbtypes.Read{}, err
}
@@ -469,7 +469,7 @@ func (m *storage) Read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
if err != nil {
return qbtypes.Read{}, err
}
- temporal, err := m.columnIsTemporal(ctx, q.StartNs, q.EndNs, key)
+ temporal, err := m.columnIsTemporal(ctx, q, key)
if err != nil {
return qbtypes.Read{}, err
}
@@ -532,18 +532,18 @@ func foldAbsentJSONReadToTypeDefault(key *telemetrytypes.TelemetryFieldKey, oper
// and corrects to the attribute maps when it names no column. A strict
// context synthesizes its type variants under the stripped and the literal
// spelling.
-func (m *storage) Fallback(ctx context.Context, _ qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, value any) ([]*telemetrytypes.LogicalField, error) {
+func (m *storage) Fallback(ctx context.Context, q qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, value any) ([]*telemetrytypes.LogicalField, error) {
var keys []*telemetrytypes.TelemetryFieldKey
switch key.FieldContext {
case telemetrytypes.FieldContextUnspecified:
probe := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextSpan, key.FieldDataType)
- if columns, err := m.getColumn(ctx, 0, 0, probe); err == nil {
+ if columns, err := m.getColumn(ctx, q, probe); err == nil {
keys = []*telemetrytypes.TelemetryFieldKey{stampColumnType(probe, columns)}
} else {
keys = querybuilder.SynthesizeKeys(key, value)
}
case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace:
- if columns, err := m.getColumn(ctx, 0, 0, key); err == nil {
+ if columns, err := m.getColumn(ctx, q, key); err == nil {
column := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType)
keys = []*telemetrytypes.TelemetryFieldKey{stampColumnType(column, columns)}
} else {
diff --git a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
index f0543c2387..0d7d642a10 100644
--- a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
+++ b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
@@ -22,7 +22,7 @@ var (
)
func readSQL(ctx context.Context, storage qbtypes.Storage, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
- read, err := storage.Read(ctx, qbtypes.QueryInfo{StartNs: startNs, EndNs: endNs}, key)
+ read, err := storage.Read(ctx, qbtypes.QueryInfo{StartNs: startNs, EndNs: endNs, TraceAttrsJSONOn: true}, key)
return read.SQL, err
}
@@ -96,6 +96,59 @@ func TestFieldForAttributeNoEvolutionParity(t *testing.T) {
}
}
+// TestAttributeJSONFlagOffParity proves the evolution entry alone does not switch reads to the
+// JSON column: with use_trace_attributes_json off, the read and every condition are the Map's for
+// every window.
+func TestAttributeJSONFlagOffParity(t *testing.T) {
+ ctx := context.Background()
+ storage := NewStorage()
+ evo := MockAttributeEvolutionData(attrJSONRelease)
+ windows := map[string][2]uint64{
+ "BeforeRelease": attrWindowBefore,
+ "AfterRelease": attrWindowAfter,
+ "StraddlingRelease": attrWindowStraddle,
+ }
+
+ testCases := []struct {
+ name string
+ operator qbtypes.FilterOperator
+ value any
+ expected string
+ }{
+ {"Equal", qbtypes.FilterOperatorEqual, float64(1), "WHERE (toFloat64(attributes_number['user.id']) = ? AND mapContains(attributes_number, 'user.id'))"},
+ {"NotEqual", qbtypes.FilterOperatorNotEqual, float64(1), "WHERE toFloat64(ifNull(attributes_number['user.id'], 0)) <> ?"},
+ {"Exists", qbtypes.FilterOperatorExists, nil, "WHERE mapContains(attributes_number, 'user.id')"},
+ {"NotExists", qbtypes.FilterOperatorNotExists, nil, "WHERE NOT mapContains(attributes_number, 'user.id')"},
+ }
+
+ for windowName, window := range windows {
+ q := qbtypes.QueryInfo{StartNs: window[0], EndNs: window[1]}
+ key := attrKey("user.id", telemetrytypes.FieldDataTypeNumber, evo)
+
+ t.Run(windowName+"_Read", func(t *testing.T) {
+ read, err := storage.Read(ctx, q, &key)
+ require.NoError(t, err)
+ assert.Equal(t, qbtypes.Read{
+ SQL: "attributes_number['user.id']",
+ Presence: "mapContains(attributes_number, 'user.id')",
+ Absence: "NOT mapContains(attributes_number, 'user.id')",
+ WhenAbsent: qbtypes.AbsentIsSentinel,
+ }, read)
+ })
+
+ for _, testCase := range testCases {
+ t.Run(windowName+"_"+testCase.name, func(t *testing.T) {
+ sb := sqlbuilder.NewSelectBuilder()
+ conds, _, err := querybuilder.Conditions(ctx, q, storage, &key, testCase.operator, testCase.value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ require.NoError(t, err)
+ sb.Where(conds...)
+ sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+ assert.Equal(t, testCase.expected, sql)
+ })
+ }
+ }
+}
+
// TestConditionForAttributeJSON asserts the emitted WHERE fragment per operator against the JSON
// column (window fully after release). Positive operators carry the raw-path existence guard;
// numeric comparisons keep numeric semantics; existence never tests the ::String cast.
@@ -170,7 +223,7 @@ func TestConditionForAttributeJSON(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &tc.key, tc.operator, tc.value, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &tc.key, tc.operator, tc.value, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -189,7 +242,7 @@ func TestConditionForAttributeJSONNotExistsDualRead(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1]}, storage, &key, qbtypes.FilterOperatorNotExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorNotExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -209,14 +262,14 @@ func TestColumnExpressionForAttributeJSON(t *testing.T) {
t.Run("group by string", func(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
- got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, telemetrytypes.FieldDataTypeString, nil)
+ got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, telemetrytypes.FieldDataTypeString, nil)
require.NoError(t, err)
assert.Equal(t, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::String, mapContains(attributes_string, 'attribute.user.id'), attributes_string['attribute.user.id'], NULL)", got)
})
t.Run("aggregation numeric", func(t *testing.T) {
key := attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo)
- got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, telemetrytypes.FieldDataTypeFloat64, nil)
+ got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, telemetrytypes.FieldDataTypeFloat64, nil)
require.NoError(t, err)
assert.Equal(t, "multiIf(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL)), mapContains(attributes_number, 'attribute.latency'), toFloat64(attributes_number['attribute.latency']), NULL)", got)
})
@@ -232,7 +285,7 @@ func TestAttributeJSONNoAmbiguityWarning(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
sb := sqlbuilder.NewSelectBuilder()
- _, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, qbtypes.FilterOperatorEqual, "x", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ _, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorEqual, "x", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
assert.Empty(t, warnings, "a plain attribute filter must not emit an ambiguity warning")
}
@@ -254,7 +307,7 @@ func TestConditionForAttributeJSONTypeCollision(t *testing.T) {
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
sb := sqlbuilder.NewSelectBuilder()
- conds, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
+ conds, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
require.NoError(t, err)
require.Len(t, conds, 2, "a colliding name must build one condition per data type")
@@ -283,7 +336,7 @@ func TestColumnExpressionForAttributeJSONTypeCollision(t *testing.T) {
}
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
- got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, telemetrytypes.FieldDataTypeString, fieldKeys)
+ got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeString, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
"multiIf(attributes.`http.status_code` IS NOT NULL, attributes.`http.status_code`::String, if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toString(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), NULL)",
@@ -307,7 +360,7 @@ func TestColumnExpressionForAttributeJSONTypeCollisionNumericAgg(t *testing.T) {
}
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
- got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, telemetrytypes.FieldDataTypeFloat64, fieldKeys)
+ got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeFloat64, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
"multiIf(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), attributes.`http.status_code` IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), NULL)",
@@ -330,7 +383,7 @@ func TestConditionForAttributeMapTypeCollisionParity(t *testing.T) {
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowBefore[0], EndNs: attrWindowBefore[1]}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowBefore[0], EndNs: attrWindowBefore[1], TraceAttrsJSONOn: true}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
require.NoError(t, err)
require.Len(t, conds, 2, "a colliding name must build one condition per data type")
@@ -351,7 +404,7 @@ func TestColumnForUnspecifiedAttributeNoBranchFlip(t *testing.T) {
evo := MockAttributeEvolutionData(attrJSONRelease)
key := attrKey("user.id", telemetrytypes.FieldDataTypeUnspecified, evo)
- _, err := (&storage{}).getColumn(ctx, attrWindowAfter[0], attrWindowAfter[1], &key)
+ _, err := (&storage{}).getColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, &key)
assert.ErrorIs(t, err, qbtypes.ErrColumnNotFound)
}
@@ -369,7 +422,7 @@ func TestConditionForAttributeJSONNegativeOperatorParity(t *testing.T) {
build := func(t *testing.T, key telemetrytypes.TelemetryFieldKey, window [2]uint64, op qbtypes.FilterOperator, value any) string {
t.Helper()
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: window[0], EndNs: window[1]}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: window[0], EndNs: window[1], TraceAttrsJSONOn: true}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -449,7 +502,7 @@ func TestConditionForAttributeJSONStraddleAbsentKeyExclusion(t *testing.T) {
build := func(t *testing.T, key telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) string {
t.Helper()
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1]}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1], TraceAttrsJSONOn: true}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
diff --git a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_promoted_test.go b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_promoted_test.go
index c99ed65710..73c1eb6a92 100644
--- a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_promoted_test.go
+++ b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_promoted_test.go
@@ -81,7 +81,7 @@ func TestConditionForAttributePromoted(t *testing.T) {
t.Run("equal reads promoted column only", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1]}, storage, &key, qbtypes.FilterOperatorEqual, "GET", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorEqual, "GET", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -91,7 +91,7 @@ func TestConditionForAttributePromoted(t *testing.T) {
t.Run("exists uses promoted raw path", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
- conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1]}, storage, &key, qbtypes.FilterOperatorExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go b/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go
index 536231cdaa..a820555965 100644
--- a/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go
+++ b/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go
@@ -13,6 +13,7 @@ import (
// SelectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
// Logic:
+// - Ignores evolutions of columns outside the candidate columns
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
// - Rejects all evolutions before this latest base evolution
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
@@ -23,6 +24,11 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
return columns, nil, nil
}
+ columnLookUpMap := make(map[string]*schema.Column, len(columns))
+ for _, column := range columns {
+ columnLookUpMap[column.Name] = column
+ }
+
// Derive the base column from the candidate columns.
seen := make(map[string]struct{}, len(evolutions))
for _, e := range evolutions {
@@ -64,6 +70,9 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
if evolution.ReleaseTime.After(tsStartTime) {
break
}
+ if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
+ continue
+ }
latestBaseEvolutionAcrossAll = evolution
}
@@ -72,11 +81,6 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
}
- columnLookUpMap := make(map[string]*schema.Column)
- for _, column := range columns {
- columnLookUpMap[column.Name] = column
- }
-
// Collect column-evolution pairs
type colEvoPair struct {
column *schema.Column
@@ -95,7 +99,7 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
}
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
- return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
+ continue
}
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go b/pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go
index 98c93bf431..9a550e2e3a 100644
--- a/pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go
+++ b/pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go
@@ -208,7 +208,7 @@ func TestSelectEvolutionsForColumns(t *testing.T) {
expectedColumns: []string{},
expectedEvols: []string{},
expectedError: true,
- errorStr: "column resources_string not found",
+ errorStr: "no base evolution found",
},
{
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
@@ -441,6 +441,26 @@ func TestSelectEvolutionsForColumns(t *testing.T) {
expectedColumns: []string{"attributes"},
expectedEvols: []string{"attributes"},
},
+ {
+ name: "Non-candidate evolution ignored - JSON released before window keeps the map",
+ columns: []*schema.Column{
+ attributes_string,
+ },
+ evolutions: []*telemetrytypes.EvolutionEntry{
+ {
+ Signal: telemetrytypes.SignalTraces,
+ ColumnName: "attributes",
+ ColumnType: "JSON()",
+ FieldContext: telemetrytypes.FieldContextAttribute,
+ FieldName: "__all__",
+ ReleaseTime: time.Date(2024, 2, 10, 0, 0, 0, 0, time.UTC),
+ },
+ },
+ tsStart: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+ tsEnd: uint64(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixNano()),
+ expectedColumns: []string{"attributes_string"},
+ expectedEvols: []string{"attributes_string"},
+ },
}
for _, tc := range testCases {
diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go
index fa8c39e44a..b4bc6f2056 100644
--- a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go
+++ b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go
@@ -39,14 +39,16 @@ var (
// QueryInfo is the query's context as one value. It holds the time range
// every read needs, the signal and queried metric that family admission
// needs, and the query-path flags evaluated one time per request. The
-// generic flows read FamiliesOn. Only the logs storage reads BodyJSONOn.
+// generic flows read FamiliesOn. Only the logs storage reads BodyJSONOn, and
+// only the traces storage reads TraceAttrsJSONOn.
type QueryInfo struct {
- StartNs uint64
- EndNs uint64
- Signal telemetrytypes.Signal
- Metric *telemetrytypes.MetricContext
- FamiliesOn bool
- BodyJSONOn bool
+ StartNs uint64
+ EndNs uint64
+ Signal telemetrytypes.Signal
+ Metric *telemetrytypes.MetricContext
+ FamiliesOn bool
+ BodyJSONOn bool
+ TraceAttrsJSONOn bool
}
// Absent is how a field key reads for a row that does not carry it, with
diff --git a/tests/integration/tests/queriertraces/conftest.py b/tests/integration/tests/queriertraces/conftest.py
new file mode 100644
index 0000000000..93efe4e8cb
--- /dev/null
+++ b/tests/integration/tests/queriertraces/conftest.py
@@ -0,0 +1,30 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.signoz import create_signoz
+
+
[email protected](name="signoz", scope="package")
+def signoz_trace_attributes_json(
+ network: Network,
+ zeus: types.TestContainerDocker,
+ gateway: types.TestContainerDocker,
+ sqlstore: types.TestContainerSQL,
+ clickhouse: types.TestContainerClickhouse,
+ request: pytest.FixtureRequest,
+ pytestconfig: pytest.Config,
+) -> types.SigNoz:
+ return create_signoz(
+ network=network,
+ zeus=zeus,
+ gateway=gateway,
+ sqlstore=sqlstore,
+ clickhouse=clickhouse,
+ request=request,
+ pytestconfig=pytestconfig,
+ cache_key="signoz-trace-attributes-json",
+ env_overrides={
+ "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_USE__TRACE__ATTRIBUTES__JSON": True,
+ },
+ )