SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
package logstelemetryschema
import (
"context"
"fmt"
"reflect"
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/clickhousesql"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
func parseStrValue(valueStr string, operator qbtypes.FilterOperator) (telemetrytypes.FieldDataType, any) {
valueType := telemetrytypes.FieldDataTypeString
// return the value as is for the following operators
// as they are always string
if operator == qbtypes.FilterOperatorContains || operator == qbtypes.FilterOperatorNotContains ||
operator == qbtypes.FilterOperatorRegexp || operator == qbtypes.FilterOperatorNotRegexp ||
operator == qbtypes.FilterOperatorLike || operator == qbtypes.FilterOperatorNotLike ||
operator == qbtypes.FilterOperatorILike || operator == qbtypes.FilterOperatorNotILike {
return valueType, valueStr
}
var err error
var parsedValue any
if parsedValue, err = strconv.ParseInt(valueStr, 10, 64); err == nil {
valueType = telemetrytypes.FieldDataTypeInt64
} else if parsedValue, err = strconv.ParseFloat(valueStr, 64); err == nil {
valueType = telemetrytypes.FieldDataTypeFloat64
} else if parsedValue, err = strconv.ParseBool(valueStr); err == nil {
valueType = telemetrytypes.FieldDataTypeBool
} else {
parsedValue = valueStr
valueType = telemetrytypes.FieldDataTypeString
}
return valueType, parsedValue
}
func InferDataType(value any, operator qbtypes.FilterOperator, key *telemetrytypes.TelemetryFieldKey) (telemetrytypes.FieldDataType, any) {
if operator.IsArrayOperator() && reflect.ValueOf(value).Kind() != reflect.Slice {
value = []any{value}
}
// closure to calculate the data type of the value
var closure func(value any, key *telemetrytypes.TelemetryFieldKey) (telemetrytypes.FieldDataType, any)
closure = func(value any, key *telemetrytypes.TelemetryFieldKey) (telemetrytypes.FieldDataType, any) {
// check if the value is a int, float, string, bool
valueType := telemetrytypes.FieldDataTypeUnspecified
switch v := value.(type) {
case []any:
// take the first element and infer the type
var scalerType telemetrytypes.FieldDataType
if len(v) > 0 {
// Note: [[...]] Slices inside Slices are not handled yet
if reflect.ValueOf(v[0]).Kind() == reflect.Slice {
return telemetrytypes.FieldDataTypeUnspecified, value
}
scalerType, _ = closure(v[0], key)
}
arrayType := telemetrytypes.ScalerFieldTypeToArrayFieldType[scalerType]
switch {
// decide on the field data type based on the key
case key.FieldDataType.IsArray():
return arrayType, v
default:
// TODO(Piyush): backward compatibility for the old String based JSON QB queries
if strings.HasSuffix(key.Name, telemetrytypes.ArrayAnyIndexSuffix) {
return arrayType, v
}
return scalerType, v
}
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
valueType = telemetrytypes.FieldDataTypeInt64
case float32, float64:
valueType = telemetrytypes.FieldDataTypeFloat64
case string:
valueType, value = parseStrValue(v, operator)
case bool:
valueType = telemetrytypes.FieldDataTypeBool
}
return valueType, value
}
// calculate the data type of the value
return closure(value, key)
}
// ClickHouse's JSONPath parser accepts only \\ and \" inside a quoted member.
func jsonPathMember(segment string) string {
escaped := strings.ReplaceAll(segment, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `"`, `\"`)
return `"` + escaped + `"`
}
// bodyJSONPathLiteral escapes each segment for JSONPath, then the whole path as a
// ClickHouse literal; the fixed "$." root is kept out of the literal escaping.
func bodyJSONPathLiteral(key *telemetrytypes.TelemetryFieldKey) string {
parts := strings.Split(key.Name, ".")
newParts := make([]string, 0, len(parts))
for _, part := range parts {
if strings.HasSuffix(part, "[*]") {
newParts = append(newParts, jsonPathMember(strings.TrimSuffix(part, "[*]"))+"[*]")
} else if strings.HasSuffix(part, "[]") {
newParts = append(newParts, jsonPathMember(strings.TrimSuffix(part, "[]"))+"[*]")
} else {
newParts = append(newParts, jsonPathMember(part))
}
}
members := clickhousesql.StringLiteral(strings.Join(newParts, "."))
return "'$." + members[1:]
}
func GetBodyJSONKey(_ context.Context, key *telemetrytypes.TelemetryFieldKey, operator qbtypes.FilterOperator, value any) (string, any) {
dataType, value := InferDataType(value, operator, key)
// for array types, we need to extract the value from the JSON_QUERY
if dataType == telemetrytypes.FieldDataTypeArrayInt64 ||
dataType == telemetrytypes.FieldDataTypeArrayFloat64 ||
dataType == telemetrytypes.FieldDataTypeArrayString ||
dataType == telemetrytypes.FieldDataTypeArrayBool ||
dataType == telemetrytypes.FieldDataTypeArrayNumber {
return fmt.Sprintf("JSONExtract(JSON_QUERY(body, %s), '%s')", bodyJSONPathLiteral(key), dataType.CHDataType()), value
}
if dataType != telemetrytypes.FieldDataTypeString {
// for all types except strings, we need to extract the value from the JSON_VALUE
return fmt.Sprintf("JSONExtract(JSON_VALUE(body, %s), '%s')", bodyJSONPathLiteral(key), dataType.CHDataType()), value
}
// JSON_VALUE returns a String; stringify list operands so a numeric element in a mixed
// set (e.g. IN ['alpha', 42]) doesn't hit a String-vs-number supertype error (CH 386).
if list, ok := value.([]any); ok {
strs := make([]any, len(list))
for i, e := range list {
strs[i] = fmt.Sprintf("%v", e)
}
value = strs
}
return fmt.Sprintf("JSON_VALUE(body, %s)", bodyJSONPathLiteral(key)), value
}
func GetBodyJSONKeyForExists(_ context.Context, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, _ any) string {
return fmt.Sprintf("JSON_EXISTS(body, %s)", bodyJSONPathLiteral(key))
}
// legacyElemType infers the has-family element type from the needle (legacy has no schema). It
// scans EVERY value so the chosen array type and all coerced needles agree — else ClickHouse
// raises "no supertype ... String" (code 386). Int64 stays distinct from Float64 so a quoted
// integer is exact past 2^53 (unquoted literals already arrive as float64, parsed upstream).
func legacyElemType(needle any) telemetrytypes.FieldDataType {
list, ok := needle.([]any)
if !ok {
list = []any{needle}
}
if len(list) == 0 {
return telemetrytypes.FieldDataTypeString
}
allInt, allNumeric := true, true
for _, v := range list {
switch t := v.(type) {
case float32, float64:
allInt = false
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
// integer Go types stay int-exact
case string:
if _, err := strconv.ParseInt(t, 10, 64); err != nil {
allInt = false
}
if _, err := strconv.ParseFloat(t, 64); err != nil {
allNumeric = false
}
default:
// booleans (and anything else) -> String; a bool renders to 'true'/'false', so a
// bool needle only matches genuine JSON booleans, not truthy numbers/strings.
allInt, allNumeric = false, false
}
}
switch {
case allInt:
return telemetrytypes.FieldDataTypeInt64
case allNumeric:
return telemetrytypes.FieldDataTypeFloat64
default:
return telemetrytypes.FieldDataTypeString
}
}
// legacyCoerceNeedle coerces a needle to elem type dt so its bound-arg type matches the
// extracted column (legacyElemType guarantees it's coercible).
func legacyCoerceNeedle(v any, dt telemetrytypes.FieldDataType) any {
switch dt {
case telemetrytypes.FieldDataTypeInt64:
if s, ok := v.(string); ok {
if i, err := strconv.ParseInt(s, 10, 64); err == nil {
return i
}
}
return v
case telemetrytypes.FieldDataTypeFloat64:
if s, ok := v.(string); ok {
f, _ := strconv.ParseFloat(s, 64)
return f
}
return v
default:
return bodyArrayNeedleString(v)
}
}
// getBodyJSONArrayKey extracts the leaf as Array(Nullable(<dt>)) — Nullable so a value of a
// different JSON type maps to NULL instead of corrupting (e.g. a non-numeric string → 0).
func getBodyJSONArrayKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) string {
name := key.Name
if !strings.HasSuffix(name, "[*]") && !strings.HasSuffix(name, "[]") {
name += "[*]"
}
arrKey := telemetrytypes.NewTelemetryFieldKey(name, key.FieldContext, key.FieldDataType)
return fmt.Sprintf("JSONExtract(JSON_QUERY(body, %s), 'Array(Nullable(%s))')", bodyJSONPathLiteral(arrKey), dt.CHDataType())
}
// getBodyJSONScalarKey builds the single-element-set fallback for a scalar body value: the leaf
// extracted as a scalar of type dt, plus a guard restricting it to a genuinely scalar body. The
// guard is required because JSON_VALUE returns '' for an array/object/missing value, which would
// otherwise zero-value match (has(x,0) / has(x,false) / has(x,'') on any array). ok=false when
// the path still traverses an array ([*]/[]).
func getBodyJSONScalarKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) (expr string, guard string, ok bool) {
name := strings.TrimSuffix(strings.TrimSuffix(key.Name, "[*]"), "[]")
if strings.Contains(name, "[") {
return "", "", false
}
scalarKey := telemetrytypes.NewTelemetryFieldKey(name, key.FieldContext, key.FieldDataType)
path := bodyJSONPathLiteral(scalarKey)
if dt == telemetrytypes.FieldDataTypeString {
expr = fmt.Sprintf("JSON_VALUE(body, %s)", path)
} else {
// Nullable so a scalar of a different type (e.g. a bool/string where a number is
// searched) extracts to NULL rather than the type's default (0/false), which would
// otherwise zero-value match has(x, 0).
expr = fmt.Sprintf("JSONExtract(JSON_VALUE(body, %s), 'Nullable(%s)')", path, dt.CHDataType())
}
keys := strings.Split(name, ".")
for i, k := range keys {
keys[i] = clickhousesql.StringLiteral(k)
}
guard = fmt.Sprintf("JSONType(body, %s) NOT IN ('Array', 'Object', 'Null')", strings.Join(keys, ", "))
return expr, guard, true
}
func bodyArrayNeedleString(v any) string {
switch t := v.(type) {
case string:
return t
case bool:
return strconv.FormatBool(t)
case float64:
return strconv.FormatFloat(t, 'f', -1, 64)
case float32:
return strconv.FormatFloat(float64(t), 'f', -1, 64)
default:
return fmt.Sprintf("%v", t)
}
}