Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

feat(querier): read traces json numbers as toFloat64OrNull and index that (#13101)

Nikhil Sonib770c249a05e
diff --git a/pkg/modules/promote/implpromote/module.go b/pkg/modules/promote/implpromote/module.go
index cc2723b4f4..bd543d919d 100644
--- a/pkg/modules/promote/implpromote/module.go
+++ b/pkg/modules/promote/implpromote/module.go
@@ -253,7 +253,7 @@ func (m *module) promotePaths(ctx context.Context, target promotetypes.Target, p
return err
}
indexes = append(indexes, schemamigrator.Index{
- Name: schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, index.JSONDataType.StringValue(), typeIndex),
+ Name: schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, strings.ToLower(index.JSONDataType.StringValue()), typeIndex),
Expression: target.IndexExpression(parentColumn, it.Path, index.JSONDataType.StringValue()),
Type: index.Type,
Granularity: index.Granularity,
diff --git a/pkg/modules/promote/implpromote/module_test.go b/pkg/modules/promote/implpromote/module_test.go
index 600849d1d0..6a7d6afd7a 100644
--- a/pkg/modules/promote/implpromote/module_test.go
+++ b/pkg/modules/promote/implpromote/module_test.go
@@ -4,15 +4,20 @@ import (
"context"
"regexp"
"testing"
+ "time"
sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/huandu/go-sqlbuilder"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/SigNoz/signoz/pkg/factory/factorytest"
+ "github.com/SigNoz/signoz/pkg/querybuilder"
+ "github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
+ qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
)
@@ -150,7 +155,7 @@ func TestPromotePathsCreatesIndexes(t *testing.T) {
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1, JSONDataType: telemetrytypes.String},
},
},
- wantDDLColumn: "`attributes_promoted.http.method_String_ngrambf_v1` attributes_promoted.`http.method`::String",
+ wantDDLColumn: "`attributes_promoted.http.method_string_ngrambf_v1` attributes_promoted.`http.method`::String",
},
{
name: "TracesUnpromotedPath_IndexesBaseColumn",
@@ -162,7 +167,7 @@ func TestPromotePathsCreatesIndexes(t *testing.T) {
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1, JSONDataType: telemetrytypes.String},
},
},
- wantDDLColumn: "`attributes.http.method_String_ngrambf_v1` attributes.`http.method`::String",
+ wantDDLColumn: "`attributes.http.method_string_ngrambf_v1` attributes.`http.method`::String",
},
{
name: "BloomFilterIndex_FullTypeInDDL",
@@ -174,7 +179,7 @@ func TestPromotePathsCreatesIndexes(t *testing.T) {
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "bloom_filter(0.01)", Granularity: 1, JSONDataType: telemetrytypes.String},
},
},
- wantDDLColumn: "`body_v2.user.name_String_bloom_filter` lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) TYPE bloom_filter(0.01)",
+ wantDDLColumn: "`body_v2.user.name_string_bloom_filter` lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) TYPE bloom_filter(0.01)",
},
{
name: "SetIndex_FullTypeInDDL",
@@ -186,7 +191,7 @@ func TestPromotePathsCreatesIndexes(t *testing.T) {
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "set(100)", Granularity: 1, JSONDataType: telemetrytypes.String},
},
},
- wantDDLColumn: "`body_v2.user.name_String_set` lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) TYPE set(100)",
+ wantDDLColumn: "`body_v2.user.name_string_set` lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) TYPE set(100)",
},
}
@@ -419,8 +424,8 @@ func TestIndexMaterializedPaths(t *testing.T) {
{Name: "retry.count", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber, Indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{{IndexType: "minmax", Granularity: 1}}},
{Name: "k8s.pod.name", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString, Indexes: bloomIndex},
}
- userIDDDL := "`attributes.user.id_String_bloom_filter` attributes.`user.id`::String TYPE bloom_filter GRANULARITY 4"
- retryCountDDL := "`attributes.retry.count_Float64_minmax` attributes.`retry.count`::Float64 TYPE minmax GRANULARITY 1"
+ userIDDDL := "`attributes.user.id_string_bloom_filter` attributes.`user.id`::String TYPE bloom_filter GRANULARITY 4"
+ retryCountDDL := "`attributes.retry.count_float64_minmax` toFloat64OrNull(attributes.`retry.count`::String) TYPE minmax GRANULARITY 1"
testCases := []struct {
name string
@@ -510,3 +515,34 @@ func TestIndexMaterializedPaths(t *testing.T) {
})
}
}
+
+// An index is used only when a filter's left-hand side is its exact expression.
+func TestTracesNumberIndexExpr_MatchesQuerierFilters(t *testing.T) {
+ release := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+ queryInfo := qbtypes.QueryInfo{StartNs: uint64(release.AddDate(0, 6, 0).UnixNano()), EndNs: uint64(release.AddDate(0, 7, 0).UnixNano()), TraceAttrsJSONOn: true}
+ key := telemetrytypes.TelemetryFieldKey{Name: "http.status_code", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber, Evolutions: tracestelemetryschema.MockAttributeEvolutionData(release)}
+ indexExpr := promotetypes.NewTracesAttributesTarget().IndexExpression("attributes", "http.status_code", telemetrytypes.Float64.StringValue())
+
+ testCases := []struct {
+ name string
+ operator qbtypes.FilterOperator
+ value any
+ want string
+ }{
+ {name: "Equal", operator: qbtypes.FilterOperatorEqual, value: float64(200), want: indexExpr + " = ?"},
+ {name: "In", operator: qbtypes.FilterOperatorIn, value: []any{float64(200), float64(404)}, want: indexExpr + " = ? OR " + indexExpr + " = ?"},
+ {name: "GreaterThan", operator: qbtypes.FilterOperatorGreaterThan, value: float64(200), want: indexExpr + " > ?"},
+ {name: "Between", operator: qbtypes.FilterOperatorBetween, value: []any{float64(200), float64(299)}, want: indexExpr + " BETWEEN ? AND ?"},
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ sb := sqlbuilder.NewSelectBuilder()
+ conditions, _, err := querybuilder.Conditions(context.Background(), queryInfo, tracestelemetryschema.NewStorage(), &key, testCase.operator, testCase.value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
+ require.NoError(t, err)
+ sb.Where(conditions...)
+ sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+ assert.Contains(t, sql, testCase.want)
+ })
+ }
+}
diff --git a/pkg/modules/tracedetail/impltracedetail/store_test.go b/pkg/modules/tracedetail/impltracedetail/store_test.go
index 1acf1ddce6..b19d6f298a 100644
--- a/pkg/modules/tracedetail/impltracedetail/store_test.go
+++ b/pkg/modules/tracedetail/impltracedetail/store_test.go
@@ -96,12 +96,12 @@ func TestGetTraceStats(t *testing.T) {
{
name: "JSONEvolutionInsideTrace_ReadsBothColumns",
jsonRelease: &jsonInsideTrace,
- expectedSQL: "SELECT toUInt64(min(span_start_ns)) AS start_ns, toUInt64(max(span_end_ns)) AS end_ns, count() AS total_spans, countIf(has_error) AS total_error_spans, countIf(has_missing_parent) > 0 AS has_missing_spans, argMinIf(root_service, (span_start_ns, root_name), is_root) AS root_service_name, argMinIf(root_name, (span_start_ns, root_name), is_root) AS root_entry_point, argMinIf(root_status_code, (span_start_ns, root_name), is_root) AS root_span_status_code, countIf(is_gen_ai) AS gen_ai_span_count, toUInt64(coalesce(sum(input_tokens_value), 0)) AS input_tokens, toUInt64(coalesce(sum(output_tokens_value), 0)) AS output_tokens, toUInt64(coalesce(sum(cache_read_tokens_value), 0)) AS cache_read_tokens, toUInt64(coalesce(sum(cache_write_tokens_value), 0)) AS cache_write_tokens, toUInt64(coalesce(sum(reasoning_tokens_value), 0)) AS reasoning_tokens, sum(total_cost_value) AS total_cost FROM (SELECT toUnixTimestamp64Nano(timestamp) AS span_start_ns, span_start_ns + duration_nano AS span_end_ns, span_id, has_error, (parent_span_id <> '' AND parent_span_id GLOBAL NOT IN (SELECT span_id FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?)) AS has_missing_parent, (parent_span_id = '' OR has_missing_parent) AS is_root, if(parent_span_id = '', name, 'Missing Span') AS root_name, if(parent_span_id = '', resource_string_service$$name, '') AS root_service, if(parent_span_id = '', response_status_code, '') AS root_status_code, (multiIf(attributes.`gen_ai.request.model` IS NOT NULL, attributes.`gen_ai.request.model`::String, mapContains(attributes_string, 'gen_ai.request.model'), attributes_string['gen_ai.request.model'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.tool.name` IS NOT NULL, attributes.`gen_ai.tool.name`::String, mapContains(attributes_string, 'gen_ai.tool.name'), attributes_string['gen_ai.tool.name'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.agent.name` IS NOT NULL, attributes.`gen_ai.agent.name`::String, mapContains(attributes_string, 'gen_ai.agent.name'), attributes_string['gen_ai.agent.name'], NULL) IS NOT NULL) AS is_gen_ai, toFloat64(multiIf(if(dynamicType(attributes.`gen_ai.usage.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.input_tokens`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`gen_ai.usage.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.input_tokens`, 'Float64'), NULL), mapContains(attributes_number, 'gen_ai.usage.input_tokens'), attributes_number['gen_ai.usage.input_tokens'], NULL)) AS input_tokens_value, toFloat64(multiIf(if(dynamicType(attributes.`gen_ai.usage.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.output_tokens`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`gen_ai.usage.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.output_tokens`, 'Float64'), NULL), mapContains(attributes_number, 'gen_ai.usage.output_tokens'), attributes_number['gen_ai.usage.output_tokens'], NULL)) AS output_tokens_value, toFloat64(multiIf(if(dynamicType(attributes.`gen_ai.usage.cache_read.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_read.input_tokens`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`gen_ai.usage.cache_read.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_read.input_tokens`, 'Float64'), NULL), mapContains(attributes_number, 'gen_ai.usage.cache_read.input_tokens'), attributes_number['gen_ai.usage.cache_read.input_tokens'], NULL)) AS cache_read_tokens_value, toFloat64(multiIf(if(dynamicType(attributes.`gen_ai.usage.cache_creation.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`gen_ai.usage.cache_creation.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`, 'Float64'), NULL), mapContains(attributes_number, 'gen_ai.usage.cache_creation.input_tokens'), attributes_number['gen_ai.usage.cache_creation.input_tokens'], NULL)) AS cache_write_tokens_value, toFloat64(multiIf(if(dynamicType(attributes.`gen_ai.usage.reasoning.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.reasoning.output_tokens`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`gen_ai.usage.reasoning.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.reasoning.output_tokens`, 'Float64'), NULL), mapContains(attributes_number, 'gen_ai.usage.reasoning.output_tokens'), attributes_number['gen_ai.usage.reasoning.output_tokens'], NULL)) AS reasoning_tokens_value, toFloat64(multiIf(if(dynamicType(attributes.`signoz.gen_ai.usage.tokens.cost`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`signoz.gen_ai.usage.tokens.cost`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`signoz.gen_ai.usage.tokens.cost`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`signoz.gen_ai.usage.tokens.cost`, 'Float64'), NULL), mapContains(attributes_number, 'signoz.gen_ai.usage.tokens.cost'), attributes_number['signoz.gen_ai.usage.tokens.cost'], NULL)) AS total_cost_value FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT 1 BY span_id) AS spans",
+ expectedSQL: "SELECT toUInt64(min(span_start_ns)) AS start_ns, toUInt64(max(span_end_ns)) AS end_ns, count() AS total_spans, countIf(has_error) AS total_error_spans, countIf(has_missing_parent) > 0 AS has_missing_spans, argMinIf(root_service, (span_start_ns, root_name), is_root) AS root_service_name, argMinIf(root_name, (span_start_ns, root_name), is_root) AS root_entry_point, argMinIf(root_status_code, (span_start_ns, root_name), is_root) AS root_span_status_code, countIf(is_gen_ai) AS gen_ai_span_count, toUInt64(coalesce(sum(input_tokens_value), 0)) AS input_tokens, toUInt64(coalesce(sum(output_tokens_value), 0)) AS output_tokens, toUInt64(coalesce(sum(cache_read_tokens_value), 0)) AS cache_read_tokens, toUInt64(coalesce(sum(cache_write_tokens_value), 0)) AS cache_write_tokens, toUInt64(coalesce(sum(reasoning_tokens_value), 0)) AS reasoning_tokens, sum(total_cost_value) AS total_cost FROM (SELECT toUnixTimestamp64Nano(timestamp) AS span_start_ns, span_start_ns + duration_nano AS span_end_ns, span_id, has_error, (parent_span_id <> '' AND parent_span_id GLOBAL NOT IN (SELECT span_id FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?)) AS has_missing_parent, (parent_span_id = '' OR has_missing_parent) AS is_root, if(parent_span_id = '', name, 'Missing Span') AS root_name, if(parent_span_id = '', resource_string_service$$name, '') AS root_service, if(parent_span_id = '', response_status_code, '') AS root_status_code, (multiIf(attributes.`gen_ai.request.model` IS NOT NULL, attributes.`gen_ai.request.model`::String, mapContains(attributes_string, 'gen_ai.request.model'), attributes_string['gen_ai.request.model'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.tool.name` IS NOT NULL, attributes.`gen_ai.tool.name`::String, mapContains(attributes_string, 'gen_ai.tool.name'), attributes_string['gen_ai.tool.name'], NULL) IS NOT NULL OR multiIf(attributes.`gen_ai.agent.name` IS NOT NULL, attributes.`gen_ai.agent.name`::String, mapContains(attributes_string, 'gen_ai.agent.name'), attributes_string['gen_ai.agent.name'], NULL) IS NOT NULL) AS is_gen_ai, toFloat64(multiIf(toFloat64OrNull(attributes.`gen_ai.usage.input_tokens`::String) IS NOT NULL, toFloat64OrNull(attributes.`gen_ai.usage.input_tokens`::String), mapContains(attributes_number, 'gen_ai.usage.input_tokens'), attributes_number['gen_ai.usage.input_tokens'], NULL)) AS input_tokens_value, toFloat64(multiIf(toFloat64OrNull(attributes.`gen_ai.usage.output_tokens`::String) IS NOT NULL, toFloat64OrNull(attributes.`gen_ai.usage.output_tokens`::String), mapContains(attributes_number, 'gen_ai.usage.output_tokens'), attributes_number['gen_ai.usage.output_tokens'], NULL)) AS output_tokens_value, toFloat64(multiIf(toFloat64OrNull(attributes.`gen_ai.usage.cache_read.input_tokens`::String) IS NOT NULL, toFloat64OrNull(attributes.`gen_ai.usage.cache_read.input_tokens`::String), mapContains(attributes_number, 'gen_ai.usage.cache_read.input_tokens'), attributes_number['gen_ai.usage.cache_read.input_tokens'], NULL)) AS cache_read_tokens_value, toFloat64(multiIf(toFloat64OrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`::String) IS NOT NULL, toFloat64OrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`::String), mapContains(attributes_number, 'gen_ai.usage.cache_creation.input_tokens'), attributes_number['gen_ai.usage.cache_creation.input_tokens'], NULL)) AS cache_write_tokens_value, toFloat64(multiIf(toFloat64OrNull(attributes.`gen_ai.usage.reasoning.output_tokens`::String) IS NOT NULL, toFloat64OrNull(attributes.`gen_ai.usage.reasoning.output_tokens`::String), mapContains(attributes_number, 'gen_ai.usage.reasoning.output_tokens'), attributes_number['gen_ai.usage.reasoning.output_tokens'], NULL)) AS reasoning_tokens_value, toFloat64(multiIf(toFloat64OrNull(attributes.`signoz.gen_ai.usage.tokens.cost`::String) IS NOT NULL, toFloat64OrNull(attributes.`signoz.gen_ai.usage.tokens.cost`::String), mapContains(attributes_number, 'signoz.gen_ai.usage.tokens.cost'), attributes_number['signoz.gen_ai.usage.tokens.cost'], NULL)) AS total_cost_value FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT 1 BY span_id) AS spans",
},
{
name: "JSONEvolutionBeforeTrace_ReadsJSONOnly",
jsonRelease: &jsonBeforeTrace,
- expectedSQL: "SELECT toUInt64(min(span_start_ns)) AS start_ns, toUInt64(max(span_end_ns)) AS end_ns, count() AS total_spans, countIf(has_error) AS total_error_spans, countIf(has_missing_parent) > 0 AS has_missing_spans, argMinIf(root_service, (span_start_ns, root_name), is_root) AS root_service_name, argMinIf(root_name, (span_start_ns, root_name), is_root) AS root_entry_point, argMinIf(root_status_code, (span_start_ns, root_name), is_root) AS root_span_status_code, countIf(is_gen_ai) AS gen_ai_span_count, toUInt64(coalesce(sum(input_tokens_value), 0)) AS input_tokens, toUInt64(coalesce(sum(output_tokens_value), 0)) AS output_tokens, toUInt64(coalesce(sum(cache_read_tokens_value), 0)) AS cache_read_tokens, toUInt64(coalesce(sum(cache_write_tokens_value), 0)) AS cache_write_tokens, toUInt64(coalesce(sum(reasoning_tokens_value), 0)) AS reasoning_tokens, sum(total_cost_value) AS total_cost FROM (SELECT toUnixTimestamp64Nano(timestamp) AS span_start_ns, span_start_ns + duration_nano AS span_end_ns, span_id, has_error, (parent_span_id <> '' AND parent_span_id GLOBAL NOT IN (SELECT span_id FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?)) AS has_missing_parent, (parent_span_id = '' OR has_missing_parent) AS is_root, if(parent_span_id = '', name, 'Missing Span') AS root_name, if(parent_span_id = '', resource_string_service$$name, '') AS root_service, if(parent_span_id = '', response_status_code, '') AS root_status_code, (attributes.`gen_ai.request.model` IS NOT NULL OR attributes.`gen_ai.tool.name` IS NOT NULL OR attributes.`gen_ai.agent.name` IS NOT NULL) AS is_gen_ai, toFloat64(if(dynamicType(attributes.`gen_ai.usage.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.input_tokens`, 'Float64'), NULL)) AS input_tokens_value, toFloat64(if(dynamicType(attributes.`gen_ai.usage.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.output_tokens`, 'Float64'), NULL)) AS output_tokens_value, toFloat64(if(dynamicType(attributes.`gen_ai.usage.cache_read.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_read.input_tokens`, 'Float64'), NULL)) AS cache_read_tokens_value, toFloat64(if(dynamicType(attributes.`gen_ai.usage.cache_creation.input_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`, 'Float64'), NULL)) AS cache_write_tokens_value, toFloat64(if(dynamicType(attributes.`gen_ai.usage.reasoning.output_tokens`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`gen_ai.usage.reasoning.output_tokens`, 'Float64'), NULL)) AS reasoning_tokens_value, toFloat64(if(dynamicType(attributes.`signoz.gen_ai.usage.tokens.cost`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`signoz.gen_ai.usage.tokens.cost`, 'Float64'), NULL)) AS total_cost_value FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT 1 BY span_id) AS spans",
+ expectedSQL: "SELECT toUInt64(min(span_start_ns)) AS start_ns, toUInt64(max(span_end_ns)) AS end_ns, count() AS total_spans, countIf(has_error) AS total_error_spans, countIf(has_missing_parent) > 0 AS has_missing_spans, argMinIf(root_service, (span_start_ns, root_name), is_root) AS root_service_name, argMinIf(root_name, (span_start_ns, root_name), is_root) AS root_entry_point, argMinIf(root_status_code, (span_start_ns, root_name), is_root) AS root_span_status_code, countIf(is_gen_ai) AS gen_ai_span_count, toUInt64(coalesce(sum(input_tokens_value), 0)) AS input_tokens, toUInt64(coalesce(sum(output_tokens_value), 0)) AS output_tokens, toUInt64(coalesce(sum(cache_read_tokens_value), 0)) AS cache_read_tokens, toUInt64(coalesce(sum(cache_write_tokens_value), 0)) AS cache_write_tokens, toUInt64(coalesce(sum(reasoning_tokens_value), 0)) AS reasoning_tokens, sum(total_cost_value) AS total_cost FROM (SELECT toUnixTimestamp64Nano(timestamp) AS span_start_ns, span_start_ns + duration_nano AS span_end_ns, span_id, has_error, (parent_span_id <> '' AND parent_span_id GLOBAL NOT IN (SELECT span_id FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?)) AS has_missing_parent, (parent_span_id = '' OR has_missing_parent) AS is_root, if(parent_span_id = '', name, 'Missing Span') AS root_name, if(parent_span_id = '', resource_string_service$$name, '') AS root_service, if(parent_span_id = '', response_status_code, '') AS root_status_code, (attributes.`gen_ai.request.model` IS NOT NULL OR attributes.`gen_ai.tool.name` IS NOT NULL OR attributes.`gen_ai.agent.name` IS NOT NULL) AS is_gen_ai, toFloat64OrNull(attributes.`gen_ai.usage.input_tokens`::String) AS input_tokens_value, toFloat64OrNull(attributes.`gen_ai.usage.output_tokens`::String) AS output_tokens_value, toFloat64OrNull(attributes.`gen_ai.usage.cache_read.input_tokens`::String) AS cache_read_tokens_value, toFloat64OrNull(attributes.`gen_ai.usage.cache_creation.input_tokens`::String) AS cache_write_tokens_value, toFloat64OrNull(attributes.`gen_ai.usage.reasoning.output_tokens`::String) AS reasoning_tokens_value, toFloat64OrNull(attributes.`signoz.gen_ai.usage.tokens.cost`::String) AS total_cost_value FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT 1 BY span_id) AS spans",
},
}
diff --git a/pkg/querybuilder/fallback_expr.go b/pkg/querybuilder/fallback_expr.go
index c3918dabf3..42a2f20628 100644
--- a/pkg/querybuilder/fallback_expr.go
+++ b/pkg/querybuilder/fallback_expr.go
@@ -7,6 +7,7 @@ import (
"reflect"
"regexp"
"strconv"
+ "strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/clickhousesql"
@@ -135,6 +136,8 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
// This following is one way to workaround it
// if the key is a number, the value is a string, we will let clickHouse handle the conversion
+ // and clickhouse float64 has a 53-bit mantissa, so values above 2^53 rounds to the nearest representable double
+ // breaking the equality filters for large integers
case float32, float64:
tblFieldName = castFloatHack(tblFieldName)
case string:
@@ -207,14 +210,22 @@ func ColumnIsTemporal(col *schema.Column) bool {
return false
}
-func castFloat(col string) string { return fmt.Sprintf("toFloat64OrNull(%s)", col) }
-func castFloatHack(col string) string { return fmt.Sprintf("toFloat64(%s)", col) }
-func castString(col string) string { return fmt.Sprintf("toString(%s)", col) }
-func castBool(col string) string { return fmt.Sprintf("accurateCastOrNull(%s, 'Bool')", col) }
+func castFloat(col string) string { return fmt.Sprintf("toFloat64OrNull(%s)", col) }
+func castString(col string) string { return fmt.Sprintf("toString(%s)", col) }
+func castBool(col string) string { return fmt.Sprintf("accurateCastOrNull(%s, 'Bool')", col) }
func accurateCastFloat(col string) string {
return fmt.Sprintf("accurateCastOrNull(%s, 'Float64')", col)
}
+// castFloatHack leaves a read already cast to Float64 bare: the wrapper would hide it from
+// its skip index.
+func castFloatHack(col string) string {
+ if strings.HasPrefix(col, "toFloat64OrNull(") {
+ return col
+ }
+ return fmt.Sprintf("toFloat64(%s)", col)
+}
+
func allFloats(in []any) bool {
for _, x := range in {
if _, ok := x.(float64); !ok {
diff --git a/pkg/telemetrymetadata/body_json_metadata.go b/pkg/telemetrymetadata/body_json_metadata.go
index e405eb5582..20006974d0 100644
--- a/pkg/telemetrymetadata/body_json_metadata.go
+++ b/pkg/telemetrymetadata/body_json_metadata.go
@@ -42,12 +42,18 @@ var logsBodyIndexLookup = promotetypes.NewLogsBodyTarget().JSONIndexLookup()
// a backtick-quoted path may hold any character, parentheses included.
var simpleJSONSubColumnIndexExprRe = regexp.MustCompile("^CAST\\((?P<expr>(?:[^()`]|`[^`]*`)+), '(?P<type>.+)'\\)$")
-// unfoldJSONSubColumnIndexExpr accepts both the folded (lower/assumeNotNull)
-// and the bare type-cast expression forms.
+// ClickHouse stores a traces number index expression with its ::String unfolded to CAST.
+var jsonNumberSubColumnIndexExprRe = regexp.MustCompile("^toFloat64OrNull\\(CAST\\((?P<expr>(?:[^()`]|`[^`]*`)+), 'String'\\)\\)$")
+
+// unfoldJSONSubColumnIndexExpr accepts the folded (lower/assumeNotNull), the
+// traces number and the bare type-cast expression forms.
func unfoldJSONSubColumnIndexExpr(expr string) (string, string, error) {
if columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(expr); err == nil {
return columnExpr, columnType, nil
}
+ if matches := jsonNumberSubColumnIndexExprRe.FindStringSubmatch(expr); matches != nil {
+ return matches[1], telemetrytypes.Float64.StringValue(), nil
+ }
matches := simpleJSONSubColumnIndexExprRe.FindStringSubmatch(expr)
if matches == nil {
return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid expression: %s", expr)
diff --git a/pkg/telemetrymetadata/body_json_metadata_test.go b/pkg/telemetrymetadata/body_json_metadata_test.go
index ed6d9a4e2d..ec272f2d72 100644
--- a/pkg/telemetrymetadata/body_json_metadata_test.go
+++ b/pkg/telemetrymetadata/body_json_metadata_test.go
@@ -92,6 +92,23 @@ func TestUnfoldJSONSubColumnIndexExpr(t *testing.T) {
wantColumn: "attributes_promoted.status",
wantType: "Int64",
},
+ {
+ name: "TracesNumber",
+ expr: "toFloat64OrNull(CAST(attributes.`http.status_code`, 'String'))",
+ wantColumn: "attributes.`http.status_code`",
+ wantType: "Float64",
+ },
+ {
+ name: "TracesNumber_BacktickedParentheses",
+ expr: "toFloat64OrNull(CAST(attributes.`latency(ms)`, 'String'))",
+ wantColumn: "attributes.`latency(ms)`",
+ wantType: "Float64",
+ },
+ {
+ name: "TracesNumber_NotAStringCast",
+ expr: "toFloat64OrNull(CAST(attributes.a, 'Int64'))",
+ wantErr: true,
+ },
{
name: "UnrecognizedForm",
expr: "toString(attributes.status)",
@@ -137,9 +154,25 @@ func TestUnfoldJSONSubColumnIndexExprTracesRoundTrip(t *testing.T) {
name: "PlainKey_UnquotedInStorage",
column: "attributes_promoted",
path: "status",
- jsonDataType: "Int64",
- wantExpr: "attributes_promoted.`status`::Int64",
- storedExpr: "CAST(attributes_promoted.status, 'Int64')",
+ jsonDataType: "String",
+ wantExpr: "attributes_promoted.`status`::String",
+ storedExpr: "CAST(attributes_promoted.status, 'String')",
+ },
+ {
+ name: "DottedNumber_BackticksPreservedInStorage",
+ column: "attributes",
+ path: "http.status_code",
+ jsonDataType: "Float64",
+ wantExpr: "toFloat64OrNull(attributes.`http.status_code`::String)",
+ storedExpr: "toFloat64OrNull(CAST(attributes.`http.status_code`, 'String'))",
+ },
+ {
+ name: "PlainNumber_UnquotedInStorage",
+ column: "attributes_promoted",
+ path: "latency",
+ jsonDataType: "Float64",
+ wantExpr: "toFloat64OrNull(attributes_promoted.`latency`::String)",
+ storedExpr: "toFloat64OrNull(CAST(attributes_promoted.latency, 'String'))",
},
}
diff --git a/pkg/telemetryschema/tracestelemetryschema/storage.go b/pkg/telemetryschema/tracestelemetryschema/storage.go
index de139169cf..fb259f7fb0 100644
--- a/pkg/telemetryschema/tracestelemetryschema/storage.go
+++ b/pkg/telemetryschema/tracestelemetryschema/storage.go
@@ -273,7 +273,7 @@ func (m *storage) resolveColumnExprs(
}
case telemetrytypes.FieldContextAttribute:
path := fmt.Sprintf("%s.%s", columnName, clickhousesql.Identifier(key.Name))
- expr, existExpr := attributeJSONValueExpr(path, key.FieldDataType)
+ expr, existExpr := AttributeJSONValueExpr(path, key.FieldDataType)
exprs = append(exprs, expr)
existExprs = append(existExprs, existExpr)
default:
@@ -330,20 +330,22 @@ func attributeColumnEvolutionRegistered(key *telemetrytypes.TelemetryFieldKey, c
return false
}
-// attributeJSONValueExpr renders the value expression for a span attribute read from the JSON
+// AttributeJSONValueExpr renders the value expression for a span attribute read from the JSON
// column along with its per-type existence guard.
-// Numeric and bool gate a crash-safe accurateCastOrNull by dynamicType: the cast alone coerces
-// across domains (bool true reads 1, '200' reads 200, 200.5 reads true), so the read is
-// restricted to values stored as that type — the per-type separation the typed maps gave
-// structurally. Being NULL-capable, the gated read itself is the existence guard (present AS
-// THIS TYPE). Other reads are total (::String folds absent to ” on the raw path), so the
-// guard is presence on the raw path.
-func attributeJSONValueExpr(path string, dataType telemetrytypes.FieldDataType) (string, string) {
+// Numeric reads through the string form so that a value stored as a number or as a numeric
+// string reads alike, like the attributes_number map did. Bool gates a crash-safe
+// accurateCastOrNull by dynamicType: the cast alone coerces across domains (200.5 reads
+// true), so the read is restricted to values stored as that type — the per-type separation the
+// typed maps gave structurally. Being NULL-capable, the numeric and bool reads are their own
+// existence guard (present AS THIS TYPE). Other reads are total (::String folds absent to ” on
+// the raw path), so the guard is presence on the raw path. Skip indexes are built over the value
+// expression.
+func AttributeJSONValueExpr(path string, dataType telemetrytypes.FieldDataType) (string, string) {
switch dataType {
case telemetrytypes.FieldDataTypeInt64,
telemetrytypes.FieldDataTypeFloat64,
telemetrytypes.FieldDataTypeNumber:
- expr := fmt.Sprintf("if(dynamicType(%s) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(%s, 'Float64'), NULL)", path, path) // all numeric types to float64 like attributes_number map.
+ expr := fmt.Sprintf("toFloat64OrNull(%s::String)", path)
return expr, expr + " IS NOT NULL"
case telemetrytypes.FieldDataTypeBool:
expr := fmt.Sprintf("if(dynamicType(%s) = 'Bool', accurateCastOrNull(%s, 'Bool'), NULL)", path, path)
diff --git a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
index 0d7d642a10..7e01d46cdc 100644
--- a/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
+++ b/pkg/telemetryschema/tracestelemetryschema/storage_attributes_json_test.go
@@ -57,9 +57,9 @@ func TestFieldForAttributeJSONEvolution(t *testing.T) {
{"string after -> json", telemetrytypes.FieldDataTypeString, attrWindowAfter, "attributes.`user.id`::String"},
{"string straddle -> dual", telemetrytypes.FieldDataTypeString, attrWindowStraddle, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::String, mapContains(attributes_string, 'user.id'), attributes_string['user.id'], NULL)"},
{"number before -> map", telemetrytypes.FieldDataTypeNumber, attrWindowBefore, "attributes_number['user.id']"},
- {"number after -> json", telemetrytypes.FieldDataTypeNumber, attrWindowAfter, "if(dynamicType(attributes.`user.id`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`user.id`, 'Float64'), NULL)"},
- {"number straddle -> dual", telemetrytypes.FieldDataTypeNumber, attrWindowStraddle, "multiIf(if(dynamicType(attributes.`user.id`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`user.id`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`user.id`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`user.id`, 'Float64'), NULL), mapContains(attributes_number, 'user.id'), attributes_number['user.id'], NULL)"},
- {"int64 after -> json", telemetrytypes.FieldDataTypeInt64, attrWindowAfter, "if(dynamicType(attributes.`user.id`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`user.id`, 'Float64'), NULL)"},
+ {"number after -> json", telemetrytypes.FieldDataTypeNumber, attrWindowAfter, "toFloat64OrNull(attributes.`user.id`::String)"},
+ {"number straddle -> dual", telemetrytypes.FieldDataTypeNumber, attrWindowStraddle, "multiIf(toFloat64OrNull(attributes.`user.id`::String) IS NOT NULL, toFloat64OrNull(attributes.`user.id`::String), mapContains(attributes_number, 'user.id'), attributes_number['user.id'], NULL)"},
+ {"int64 after -> json", telemetrytypes.FieldDataTypeInt64, attrWindowAfter, "toFloat64OrNull(attributes.`user.id`::String)"},
{"bool before -> map", telemetrytypes.FieldDataTypeBool, attrWindowBefore, "attributes_bool['user.id']"},
{"bool after -> json", telemetrytypes.FieldDataTypeBool, attrWindowAfter, "if(dynamicType(attributes.`user.id`) = 'Bool', accurateCastOrNull(attributes.`user.id`, 'Bool'), NULL)"},
{"bool straddle -> dual", telemetrytypes.FieldDataTypeBool, attrWindowStraddle, "multiIf(if(dynamicType(attributes.`user.id`) = 'Bool', accurateCastOrNull(attributes.`user.id`, 'Bool'), NULL) IS NOT NULL, if(dynamicType(attributes.`user.id`) = 'Bool', accurateCastOrNull(attributes.`user.id`, 'Bool'), NULL), mapContains(attributes_bool, 'user.id'), attributes_bool['user.id'], NULL)"},
@@ -180,7 +180,7 @@ func TestConditionForAttributeJSON(t *testing.T) {
name: "greater than number",
key: attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo),
operator: qbtypes.FilterOperatorGreaterThan, value: float64(200),
- expected: "toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)) > ?",
+ expected: "toFloat64OrNull(attributes.`http.status_code`::String) > ?",
},
{
name: "ilike string",
@@ -216,7 +216,31 @@ func TestConditionForAttributeJSON(t *testing.T) {
name: "between number",
key: attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo),
operator: qbtypes.FilterOperatorBetween, value: []any{float64(1), float64(9)},
- expected: "toFloat64(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL)) BETWEEN ? AND ?",
+ expected: "toFloat64OrNull(attributes.`latency`::String) BETWEEN ? AND ?",
+ },
+ {
+ name: "In_Number",
+ key: attrKey("http.status_code", telemetrytypes.FieldDataTypeNumber, evo),
+ operator: qbtypes.FilterOperatorIn, value: []any{float64(200), float64(404)},
+ expected: "(toFloat64OrNull(attributes.`http.status_code`::String) = ? OR toFloat64OrNull(attributes.`http.status_code`::String) = ?)",
+ },
+ {
+ name: "GreaterThan_NumericString",
+ key: attrKey("http.status_code", telemetrytypes.FieldDataTypeNumber, evo),
+ operator: qbtypes.FilterOperatorGreaterThan, value: "200",
+ expected: "toFloat64OrNull(attributes.`http.status_code`::String) > ?",
+ },
+ {
+ name: "Equal_NonNumericString_CastsToString",
+ key: attrKey("http.status_code", telemetrytypes.FieldDataTypeNumber, evo),
+ operator: qbtypes.FilterOperatorEqual, value: "ok",
+ expected: "toString(toFloat64OrNull(attributes.`http.status_code`::String)) = ?",
+ },
+ {
+ name: "Equal_Bool_CastsToString",
+ key: attrKey("http.status_code", telemetrytypes.FieldDataTypeNumber, evo),
+ operator: qbtypes.FilterOperatorEqual, value: true,
+ expected: "toString(toFloat64OrNull(attributes.`http.status_code`::String)) = ?",
},
}
@@ -271,7 +295,7 @@ func TestColumnExpressionForAttributeJSON(t *testing.T) {
key := attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, telemetrytypes.FieldDataTypeFloat64, nil)
require.NoError(t, err)
- assert.Equal(t, "multiIf(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL)), mapContains(attributes_number, 'attribute.latency'), toFloat64(attributes_number['attribute.latency']), NULL)", got)
+ assert.Equal(t, "multiIf(toFloat64OrNull(attributes.`latency`::String) IS NOT NULL, toFloat64OrNull(attributes.`latency`::String), mapContains(attributes_number, 'attribute.latency'), toFloat64(attributes_number['attribute.latency']), NULL)", got)
})
}
@@ -314,7 +338,7 @@ func TestConditionForAttributeJSONTypeCollision(t *testing.T) {
sb.Where(sb.Or(conds...))
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, "toFloat64OrNull(attributes.`http.status_code`::String) = ?")
- assert.Contains(t, sql, "toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)) = ?")
+ assert.Contains(t, sql, "toFloat64OrNull(attributes.`http.status_code`::String) = ?")
assert.Contains(t, sql, "attributes.`http.status_code` IS NOT NULL")
assert.NotEmpty(t, warnings, "a colliding name must surface the ambiguity warning")
}
@@ -339,7 +363,7 @@ func TestColumnExpressionForAttributeJSONTypeCollision(t *testing.T) {
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeString, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
- "multiIf(attributes.`http.status_code` IS NOT NULL, attributes.`http.status_code`::String, if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toString(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), NULL)",
+ "multiIf(attributes.`http.status_code` IS NOT NULL, attributes.`http.status_code`::String, toFloat64OrNull(attributes.`http.status_code`::String) IS NOT NULL, toString(toFloat64OrNull(attributes.`http.status_code`::String)), NULL)",
got)
}
@@ -363,7 +387,7 @@ func TestColumnExpressionForAttributeJSONTypeCollisionNumericAgg(t *testing.T) {
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeFloat64, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
- "multiIf(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), attributes.`http.status_code` IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), NULL)",
+ "multiIf(toFloat64OrNull(attributes.`http.status_code`::String) IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), attributes.`http.status_code` IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), NULL)",
got)
}
@@ -409,7 +433,7 @@ func TestColumnForUnspecifiedAttributeNoBranchFlip(t *testing.T) {
}
// TestConditionForAttributeJSONNegativeOperatorParity pins Map parity for numeric/bool attributes.
-// The value reads an absent key as NULL (accurateCastOrNull, or the straddle multiIf else); a
+// The value reads an absent key as NULL (toFloat64OrNull, or the straddle multiIf else); a
// positive operator excludes such a row via the exists guard, but a negative operator has no guard,
// so the condition builder folds the NULL to the Map's type zero (ifNull) for negatives only.
// String needs no fold — ::String already reads absent as ”. The fold rides the attributes
@@ -432,7 +456,7 @@ func TestConditionForAttributeJSONNegativeOperatorParity(t *testing.T) {
t.Run("not equal number after -> NULL folded to 0", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
sql := build(t, key, attrWindowAfter, qbtypes.FilterOperatorNotEqual, float64(200))
- assert.Contains(t, sql, "toFloat64(ifNull(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), 0)) <> ?")
+ assert.Contains(t, sql, "toFloat64(ifNull(toFloat64OrNull(attributes.`http.status_code`::String), 0)) <> ?")
})
t.Run("not equal bool after -> NULL folded to false", func(t *testing.T) {
@@ -447,27 +471,27 @@ func TestConditionForAttributeJSONNegativeOperatorParity(t *testing.T) {
t.Run("not equal number after, string value -> string cast over folded read, no type mismatch", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
sql := build(t, key, attrWindowAfter, qbtypes.FilterOperatorNotEqual, "teapot")
- assert.Contains(t, sql, "toString(ifNull(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), 0)) <> ?")
+ assert.Contains(t, sql, "toString(ifNull(toFloat64OrNull(attributes.`http.status_code`::String), 0)) <> ?")
assert.NotContains(t, sql, "ifNull(toString(")
})
t.Run("equal number after -> not folded, exists guard excludes absent", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
sql := build(t, key, attrWindowAfter, qbtypes.FilterOperatorEqual, float64(0))
- assert.Contains(t, sql, "toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)) = ?")
+ assert.Contains(t, sql, "toFloat64OrNull(attributes.`http.status_code`::String) = ?")
assert.NotContains(t, sql, "ifNull")
})
t.Run("not in number after -> each operand folded to 0", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
sql := build(t, key, attrWindowAfter, qbtypes.FilterOperatorNotIn, []any{float64(200), float64(404)})
- assert.Contains(t, sql, "(toFloat64(ifNull(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), 0)) <> ? AND toFloat64(ifNull(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), 0)) <> ?)")
+ assert.Contains(t, sql, "(toFloat64(ifNull(toFloat64OrNull(attributes.`http.status_code`::String), 0)) <> ? AND toFloat64(ifNull(toFloat64OrNull(attributes.`http.status_code`::String), 0)) <> ?)")
})
t.Run("not equal number straddle -> whole multiIf folded", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
sql := build(t, key, attrWindowStraddle, qbtypes.FilterOperatorNotEqual, float64(200))
- assert.Contains(t, sql, "toFloat64(ifNull(multiIf(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), mapContains(attributes_number, 'http.status_code'), attributes_number['http.status_code'], NULL), 0)) <> ?")
+ assert.Contains(t, sql, "toFloat64(ifNull(multiIf(toFloat64OrNull(attributes.`http.status_code`::String) IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), mapContains(attributes_number, 'http.status_code'), attributes_number['http.status_code'], NULL), 0)) <> ?")
})
t.Run("not equal number before -> harmless fold over the map read", func(t *testing.T) {
@@ -509,7 +533,7 @@ func TestConditionForAttributeJSONStraddleAbsentKeyExclusion(t *testing.T) {
return sql
}
- guard := "multiIf(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL), mapContains(attributes_number, 'http.status_code'), attributes_number['http.status_code'], NULL) IS NOT NULL"
+ guard := "multiIf(toFloat64OrNull(attributes.`http.status_code`::String) IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), mapContains(attributes_number, 'http.status_code'), attributes_number['http.status_code'], NULL) IS NOT NULL"
t.Run("equal zero takes no guard: the absent read is NULL", func(t *testing.T) {
key := attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo)
diff --git a/pkg/types/promotetypes/materialized.go b/pkg/types/promotetypes/materialized.go
index fe0388f7e3..6970f1f668 100644
--- a/pkg/types/promotetypes/materialized.go
+++ b/pkg/types/promotetypes/materialized.go
@@ -42,7 +42,7 @@ func NewMaterializedPromotePath(target Target, key *telemetrytypes.TelemetryFiel
Promote: false,
Indexes: indexes,
}
- if err := path.validateAndSetIndexDefaults(); err != nil {
+ if err := path.validateAndSetIndexDefaults(target); err != nil {
return nil, err
}
return path, nil
diff --git a/pkg/types/promotetypes/target.go b/pkg/types/promotetypes/target.go
index 5359731265..1749c4572a 100644
--- a/pkg/types/promotetypes/target.go
+++ b/pkg/types/promotetypes/target.go
@@ -27,14 +27,24 @@ func (t Target) BaseColumnPrefix() string { return t.BaseColumn + "." }
func (t Target) PromotedColumnPrefix() string { return t.PromotedColumn() + "." }
// IndexExpression folds logs strings to lower case over assumeNotNull for
-// case-insensitive LIKE searches; traces indexes are a bare type cast.
+// case-insensitive LIKE searches; traces indexes are the querier's read, which
+// an index matches only as that exact text.
func (t Target) IndexExpression(column, path, jsonDataType string) string {
- switch t.Entry.Signal {
- case telemetrytypes.SignalLogs:
+ if t.Entry.Signal == telemetrytypes.SignalLogs {
return schemamigrator.JSONSubColumnIndexExpr(column, path, jsonDataType)
- default:
- return simpleJSONSubColumnIndexExpr(column, path, jsonDataType)
}
+ subColumn := fmt.Sprintf("%s.%s", column, clickhousesql.Identifier(path))
+ expr, _ := tracestelemetryschema.AttributeJSONValueExpr(subColumn, telemetrytypes.MappingJSONDataTypeToFieldDataType[jsonDataType])
+ return expr
+}
+
+// IndexDataType folds traces Int64 into Float64, the one type the querier
+// reads every traces number as.
+func (t Target) IndexDataType(jsonDataType telemetrytypes.JSONDataType) telemetrytypes.JSONDataType {
+ if t.Entry.Signal != telemetrytypes.SignalLogs && jsonDataType == telemetrytypes.Int64 {
+ return telemetrytypes.Float64
+ }
+ return jsonDataType
}
func (t Target) JSONIndexLookup() telemetrytypes.JSONIndexLookup {
@@ -121,14 +131,6 @@ func TargetFor(signal telemetrytypes.Signal, context telemetrytypes.FieldContext
return Target{}, false
}
-// simpleJSONSubColumnIndexExpr renders column.`path`::Type with the path
-// quoted as one identifier, the form trace queries read the sub-column with;
-// the cast unwraps the Nullable the sub-column access returns, which bloom
-// filter indexes reject.
-func simpleJSONSubColumnIndexExpr(column, path, jsonDataType string) string {
- return fmt.Sprintf("%s.%s::%s", column, clickhousesql.Identifier(path), jsonDataType)
-}
-
// reservedPathPrefix returns the target prefix path carries, if any: the base
// or promoted column prefix, or the logs body search alias, which is stripped
// from queries before metadata lookup and can never match.
diff --git a/pkg/types/promotetypes/target_test.go b/pkg/types/promotetypes/target_test.go
index 3fa7631d4e..66d32b161d 100644
--- a/pkg/types/promotetypes/target_test.go
+++ b/pkg/types/promotetypes/target_test.go
@@ -3,6 +3,7 @@ package promotetypes
import (
"testing"
+ "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
)
@@ -47,6 +48,22 @@ func TestIndexExpression(t *testing.T) {
jsonDataType: "String",
want: "attributes.`user-name`::String",
},
+ {
+ name: "TracesFloat64_QuerierNumberRead",
+ target: NewTracesAttributesTarget(),
+ column: "attributes",
+ path: "http.status_code",
+ jsonDataType: "Float64",
+ want: "toFloat64OrNull(attributes.`http.status_code`::String)",
+ },
+ {
+ name: "TracesInt64_QuerierNumberRead",
+ target: NewTracesAttributesTarget(),
+ column: "attributes_promoted",
+ path: "retry.count",
+ jsonDataType: "Int64",
+ want: "toFloat64OrNull(attributes_promoted.`retry.count`::String)",
+ },
}
for _, testCase := range testCases {
@@ -55,3 +72,22 @@ func TestIndexExpression(t *testing.T) {
})
}
}
+
+func TestIndexDataType(t *testing.T) {
+ testCases := []struct {
+ name string
+ target Target
+ jsonDataType telemetrytypes.JSONDataType
+ want telemetrytypes.JSONDataType
+ }{
+ {name: "TracesInt64_Float64", target: NewTracesAttributesTarget(), jsonDataType: telemetrytypes.Int64, want: telemetrytypes.Float64},
+ {name: "TracesString_Unchanged", target: NewTracesAttributesTarget(), jsonDataType: telemetrytypes.String, want: telemetrytypes.String},
+ {name: "LogsInt64_Unchanged", target: NewLogsBodyTarget(), jsonDataType: telemetrytypes.Int64, want: telemetrytypes.Int64},
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ assert.Equal(t, testCase.want, testCase.target.IndexDataType(testCase.jsonDataType))
+ })
+ }
+}
diff --git a/pkg/types/promotetypes/types.go b/pkg/types/promotetypes/types.go
index 6f2b4193dc..ad64e31094 100644
--- a/pkg/types/promotetypes/types.go
+++ b/pkg/types/promotetypes/types.go
@@ -114,7 +114,7 @@ func (i *PromotePath) ValidateAndSetDefaults(target Target) error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cardinal paths can not be promoted or indexed")
}
- return i.validateAndSetIndexDefaults()
+ return i.validateAndSetIndexDefaults(target)
}
func (index WrappedIndex) SkipIndexType() (schemamigrator.IndexType, error) {
@@ -201,7 +201,7 @@ func validateIndexType(indexType string) error {
return nil
}
-func (i *PromotePath) validateAndSetIndexDefaults() error {
+func (i *PromotePath) validateAndSetIndexDefaults(target Target) error {
for idx, index := range i.Indexes {
if index.Type == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index type is required")
@@ -221,7 +221,7 @@ func (i *PromotePath) validateAndSetIndexDefaults() error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.FieldDataType)
}
- i.Indexes[idx].JSONDataType = jsonDataType
+ i.Indexes[idx].JSONDataType = target.IndexDataType(jsonDataType)
}
return nil
diff --git a/tests/fixtures/materialized.py b/tests/fixtures/materialized.py
index 45e6caec26..5fb7c9c32e 100644
--- a/tests/fixtures/materialized.py
+++ b/tests/fixtures/materialized.py
@@ -9,19 +9,26 @@ MATERIALIZED_TABLES = {
"traces": ("signoz_traces", "signoz_index_v3", "distributed_signoz_index_v3"),
}
+# data type -> (column type, attribute map)
+MATERIALIZED_COLUMN_TYPES = {
+ "string": ("LowCardinality(String)", "attributes_string"),
+ "number": ("Float64", "attributes_number"),
+}
+
@pytest.fixture(name="materialize_attribute", scope="function")
-def materialize_attribute(signoz: types.SigNoz) -> Generator[Callable[[str, str, bool], str]]:
+def materialize_attribute(signoz: types.SigNoz) -> Generator[Callable[..., str]]:
"""Teardown drops the columns and every index whose expression mentions a materialized key."""
cluster = signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
created: list[tuple[str, str, str]] = []
- def materialize(signal: str, key: str, indexed: bool) -> str:
+ def materialize(signal: str, key: str, indexed: bool, data_type: str = "string") -> str:
database, local_table, distributed_table = MATERIALIZED_TABLES[signal]
- column = f"attribute_string_{key.replace('.', '$$')}"
+ column_type, attribute_map = MATERIALIZED_COLUMN_TYPES[data_type]
+ column = f"attribute_{data_type}_{key.replace('.', '$$')}"
created.append((signal, key, column))
for table in (local_table, distributed_table):
- signoz.telemetrystore.conn.query(f"ALTER TABLE {database}.{table} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS `{column}` LowCardinality(String) DEFAULT attributes_string['{key}']")
+ signoz.telemetrystore.conn.query(f"ALTER TABLE {database}.{table} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS `{column}` {column_type} DEFAULT {attribute_map}['{key}']")
if indexed:
signoz.telemetrystore.conn.query(f"ALTER TABLE {database}.{local_table} ON CLUSTER '{cluster}' ADD INDEX IF NOT EXISTS `{column}_idx` `{column}` TYPE bloom_filter(0.01) GRANULARITY 64")
return column
diff --git a/tests/integration/tests/promote/02_materialized.py b/tests/integration/tests/promote/02_materialized.py
index fac8f33acf..ff3048655b 100644
--- a/tests/integration/tests/promote/02_materialized.py
+++ b/tests/integration/tests/promote/02_materialized.py
@@ -5,7 +5,9 @@ import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.traces import Traces
+PROMOTED_PATHS = "/api/v1/promoted_paths"
MATERIALIZED_PATH = "/api/v1/promoted_paths/materialized"
PROBE_INDEX_EXPRS_QUERY = "SELECT expr FROM system.data_skipping_indices WHERE database = 'signoz_traces' AND table = 'signoz_index_v3' AND expr LIKE 'CAST(%materialized.%probe%'"
@@ -14,7 +16,7 @@ def test_index_materialized_paths(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
- materialize_attribute: Callable[[str, str, bool], str],
+ materialize_attribute: Callable[..., str],
) -> None:
materialize_attribute("traces", "materialized.probe", True)
materialize_attribute("traces", "materialized.unindexed_probe", False)
@@ -60,3 +62,40 @@ def test_index_materialized_paths(
assert resp.status_code == HTTPStatus.OK, resp.text
assert "materialized.probe" not in {path["path"] for path in resp.json()["data"]["indexed"]}
assert "materialized.probe" in {path["path"] for path in resp.json()["data"]["skipped"]}
+
+
+def test_index_materialized_number_path(
+ signoz: types.SigNoz,
+ create_user_admin: types.Operation, # pylint: disable=unused-argument
+ get_token: Callable[[str, str], str],
+ materialize_attribute: Callable[..., str],
+ insert_traces: Callable[[list[Traces]], None],
+) -> None:
+ materialize_attribute("traces", "materialized.number_probe", True, "number")
+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+ resp = requests.post(
+ signoz.self.host_configs["8080"].get(MATERIALIZED_PATH),
+ json={"signal": "traces"},
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=30,
+ )
+ assert resp.status_code == HTTPStatus.OK, resp.text
+ indexed = {path["path"]: path["indexes"] for path in resp.json()["data"]["indexed"]}
+ assert indexed["materialized.number_probe"] == [{"fieldDataType": "number", "type": "bloom_filter(0.01)", "granularity": 64}]
+ exprs = signoz.telemetrystore.conn.query("SELECT expr FROM system.data_skipping_indices WHERE database = 'signoz_traces' AND table = 'signoz_index_v3' AND expr LIKE '%materialized.number_probe%'").result_rows
+ assert [row[0] for row in exprs] == ["toFloat64OrNull(CAST(attributes.`materialized.number_probe`, 'String'))"]
+
+ resp = requests.get(
+ signoz.self.host_configs["8080"].get(PROMOTED_PATHS),
+ params={"signal": "traces", "context": "attribute"},
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=30,
+ )
+ assert resp.status_code == HTTPStatus.OK, resp.text
+ listed = {path["path"]: path.get("indexes") for path in resp.json()["data"]}
+ assert listed["materialized.number_probe"] == [{"fieldDataType": "float64", "type": "bloom_filter(0.01)", "granularity": 64}]
+
+ # a non-numeric value must not fail the insert on the numeric index
+ insert_traces([Traces(attributes={"materialized.number_probe": 200}), Traces(attributes={"materialized.number_probe": "not-a-number"})])
+ assert signoz.telemetrystore.conn.query("SELECT count() FROM signoz_traces.distributed_signoz_index_v3 WHERE attributes.`materialized.number_probe` IS NOT NULL").result_rows == [(2,)]
diff --git a/tests/integration/tests/queriertraces/13_attributes_json_evolution.py b/tests/integration/tests/queriertraces/13_attributes_json_evolution.py
index 8b5dc59dc9..ce714ca7cc 100644
--- a/tests/integration/tests/queriertraces/13_attributes_json_evolution.py
+++ b/tests/integration/tests/queriertraces/13_attributes_json_evolution.py
@@ -335,6 +335,63 @@ def test_traces_attributes_json_collision_and_map_parity(
assert set(series.keys()) == expected, label
+def test_traces_attributes_json_number_reads(
+ signoz: types.SigNoz,
+ create_user_admin: None, # pylint: disable=unused-argument
+ get_token: Callable[[str, str], str],
+ insert_traces: Callable[[list[Traces]], None],
+ seed_attribute_evolution: Callable[[str, datetime], None],
+) -> None:
+ """A JSON number reads like the attributes_number map: a numeric string as its number, an
+ integer above 2^53 rounded, and a non-numeric string as no number."""
+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+ evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+ seed_attribute_evolution("traces", evolution_time)
+
+ hit = evolution_time + timedelta(minutes=5)
+ insert_traces(
+ [
+ Traces(
+ timestamp=hit,
+ trace_id=TraceIdGenerator.trace_id(),
+ span_id=TraceIdGenerator.span_id(),
+ name=route,
+ attributes={"http.route": route, "app.code": code},
+ attribute_write_mode="dual_write",
+ )
+ for route, code in [("/int", 404), ("/numeric_string", "404"), ("/big", 2**53 + 1), ("/text", "abc")]
+ ]
+ )
+
+ start_ms = int((hit - timedelta(minutes=1)).timestamp() * 1000)
+ end_ms = int((hit + timedelta(minutes=1)).timestamp() * 1000)
+
+ for label, filter_expression, expected in [
+ ("numeric_string_eq", "app.code = 404", {"/int", "/numeric_string"}),
+ ("big_int_rounded", f"app.code = {2**53}", {"/big"}),
+ ("text_not_a_number", "app.code > 0", {"/int", "/numeric_string", "/big"}),
+ ]:
+ response = make_query_request(
+ signoz,
+ token,
+ start_ms=start_ms,
+ end_ms=end_ms,
+ request_type=RequestType.TIME_SERIES,
+ queries=[
+ build_traces_scalar_query(
+ aggregations=[build_aggregation("count()")],
+ group_by=[build_group_by_field("http.route", field_context="attribute")],
+ filter_expression=filter_expression,
+ )
+ ],
+ )
+ assert response.status_code == HTTPStatus.OK, label
+ aggregations = (response.json()["data"]["data"]["results"][0].get("aggregations")) or []
+ series = index_series_by_label(aggregations[0]["series"], "http.route") if aggregations else {}
+ assert set(series.keys()) == expected, label
+
+
def test_traces_attributes_json_list_view(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
diff --git a/tests/integration/tests/queriertraces/14_skip_indexes.py b/tests/integration/tests/queriertraces/14_skip_indexes.py
new file mode 100644
index 0000000000..7e1f104369
--- /dev/null
+++ b/tests/integration/tests/queriertraces/14_skip_indexes.py
@@ -0,0 +1,80 @@
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+import pytest
+import requests
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.querier import RequestType, build_aggregation, build_traces_scalar_query, get_preview_statements, make_preview_query_request
+from fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, Traces
+
+PROMOTED_PATHS = "/api/v1/promoted_paths"
+
+
+def test_traces_filters_use_skip_indexes(
+ signoz: types.SigNoz,
+ create_user_admin: types.Operation, # pylint: disable=unused-argument
+ get_token: Callable[[str, str], str],
+ insert_traces: Callable[[list[Traces]], None],
+ seed_attribute_evolution: Callable[[str, datetime], None],
+ request: pytest.FixtureRequest,
+) -> None:
+ seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+ resp = requests.post(
+ signoz.self.host_configs["8080"].get(PROMOTED_PATHS),
+ json=[
+ {"signal": "traces", "context": "attribute", "path": "index.number_probe", "indexes": [{"fieldDataType": "number", "type": "bloom_filter(0.01)", "granularity": 64}]},
+ {"signal": "traces", "context": "attribute", "path": "index.string_probe", "indexes": [{"fieldDataType": "string", "type": "bloom_filter(0.01)", "granularity": 64}]},
+ ],
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=30,
+ )
+ assert resp.status_code == HTTPStatus.CREATED, resp.text
+ cluster = signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
+ for index_name in ("attributes.index.number_probe_float64_bloom_filter", "attributes.index.string_probe_string_bloom_filter"):
+ request.addfinalizer(lambda index_name=index_name: signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_traces.signoz_index_v3 ON CLUSTER '{cluster}' DROP INDEX IF EXISTS `{index_name}`"))
+
+ resp = requests.post(
+ signoz.self.host_configs["8080"].get(PROMOTED_PATHS),
+ json=[{"signal": "traces", "context": "attribute", "path": "index.bool_probe", "indexes": [{"fieldDataType": "bool", "type": "bloom_filter(0.01)", "granularity": 64}]}],
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=30,
+ )
+ assert resp.status_code == HTTPStatus.BAD_REQUEST, resp.text
+
+ now = datetime.now(tz=UTC)
+ insert_traces([Traces(timestamp=now - timedelta(seconds=10), attributes={"index.number_probe": 200, "index.string_probe": "probe"})])
+
+ for filter_expression, index_name in [
+ ("index.number_probe = 200", "attributes.index.number_probe_float64_bloom_filter"),
+ ("index.string_probe = 'probe'", "attributes.index.string_probe_string_bloom_filter"),
+ ("name = 'probe'", "idx_name"),
+ ("duration_nano > 100", "idx_duration"),
+ ("kind = 2", "idx_kind"),
+ ("kind_string = 'Server'", "idx_kind_string"),
+ ("status_code_string = 'Ok'", "idx_status_code_string"),
+ ("response_status_code = '200'", "idx_response_statusCode"),
+ ("http_method = 'GET'", "idx_http_method"),
+ ("http_url = 'probe'", "idx_http_url"),
+ ("http_host = 'probe'", "idx_http_host"),
+ ("trace_id = '0af7651916cd43dd8448eb211c80319c'", "idx_trace_id"),
+ ("span_id = 'b7ad6b7169203331'", "idx_span_id"),
+ ]:
+ statements = get_preview_statements(
+ make_preview_query_request(
+ signoz,
+ token,
+ start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
+ end_ms=int(now.timestamp() * 1000),
+ request_type=RequestType.SCALAR,
+ # count() over primary-key filters is answered by the exact count projection, which reads no index
+ queries=[build_traces_scalar_query(aggregations=[build_aggregation("sum(duration_nano)")], filter_expression=filter_expression)],
+ ),
+ "A",
+ )
+ used = {step["name"] for statement in statements for read in (statement["granules"] or {}).get("reads", []) for step in read["steps"]}
+ assert index_name in used, f"{filter_expression}: {used}"