SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
fix(clickhouseprometheus): merge fingerprints sharing a labelset instead of injecting a fingerprint label (#12152)
Srikanth Chekuri05b35dd4eeac
diff --git a/pkg/prometheus/clickhouseprometheus/client.go b/pkg/prometheus/clickhouseprometheus/client.goindex b911ee7de5..821509e792 100644--- a/pkg/prometheus/clickhouseprometheus/client.go+++ b/pkg/prometheus/clickhouseprometheus/client.go@@ -14,6 +14,7 @@ import ("github.com/SigNoz/signoz/pkg/types/ctxtypes""github.com/SigNoz/signoz/pkg/types/instrumentationtypes""github.com/SigNoz/signoz/pkg/types/telemetrytypes"+ "github.com/cespare/xxhash/v2"promValue "github.com/prometheus/prometheus/model/value""github.com/prometheus/prometheus/prompb""github.com/prometheus/prometheus/storage"@@ -184,7 +185,7 @@ func (client *client) getFingerprintsFromClickhouseQuery(ctx context.Context, qureturn nil, err}- labels, _, err := unmarshalLabels(labelString, fingerprint)+ labels, _, err := unmarshalLabels(labelString)if err != nil {return nil, err}@@ -281,7 +282,138 @@ func (client *client) querySamples(ctx context.Context, start int64, end int64,return nil, err}- return res, nil+ return mergeSeriesWithIdenticalLabels(res), nil+}++// mergeSeriesWithIdenticalLabels collapses series sharing one labelset into+// one series each. Distinct fingerprints can map to one labelset: a label+// value goes empty over a series' lifetime (#8563), the fingerprint+// algorithm changes across exporter versions, or the env changes. The+// engine treats the labelset as series identity — duplicates raise+// "duplicate series", and #8563's workaround of injecting a synthetic+// fingerprint label silently broke without() and vector matching. Merging+// at the last point before hand-off keeps any future input-side label+// normalization collision-safe. Grouping is by an order-insensitive 64-bit+// hash so the common no-collision case costs one hash and one map insert+// per series; hash-equal groups are confirmed by exact labelset equality+// before any merge. Regression:+// TestClient_QuerySamplesMergesIdenticalLabelSets and+// tests/integration/tests/promqlconformance/02_fingerprint_probe.py.+func mergeSeriesWithIdenticalLabels(series []*prompb.TimeSeries) []*prompb.TimeSeries {+ if len(series) < 2 {+ return series+ }++ groups := make(map[uint64][]*prompb.TimeSeries, len(series))+ order := make([]uint64, 0, len(series))+ for _, ts := range series {+ key := labelsHash(ts.Labels)+ if _, ok := groups[key]; !ok {+ order = append(order, key)+ }+ groups[key] = append(groups[key], ts)+ }+ if len(order) == len(series) {+ return series+ }++ res := make([]*prompb.TimeSeries, 0, len(order))+ for _, key := range order {+ group := groups[key]+ if len(group) == 1 {+ res = append(res, group[0])+ continue+ }+ for _, sub := range splitByLabelSet(group) {+ if len(sub) == 1 {+ res = append(res, sub[0])+ continue+ }+ res = append(res, mergeSamples(sub))+ }+ }+ return res+}++var labelHashSep = []byte{0xff}++// labelsHash combines per-label hashes commutatively, so the stored JSON's+// key order (not canonical across fingerprints) needs no sort.+func labelsHash(lbls []prompb.Label) uint64 {+ var h uint64+ var d xxhash.Digest+ for _, l := range lbls {+ d.Reset()+ _, _ = d.WriteString(l.Name)+ _, _ = d.Write(labelHashSep)+ _, _ = d.WriteString(l.Value)+ h += d.Sum64()+ }+ return h+}++// splitByLabelSet partitions a hash-equal group into sub-groups of exactly+// equal labelsets, preserving input order; series that merely collide on the+// 64-bit hash must not be merged.+func splitByLabelSet(group []*prompb.TimeSeries) [][]*prompb.TimeSeries {+ var out [][]*prompb.TimeSeries+outer:+ for _, ts := range group {+ for i, sub := range out {+ if labelSetsEqual(sub[0].Labels, ts.Labels) {+ out[i] = append(out[i], ts)+ continue outer+ }+ }+ out = append(out, []*prompb.TimeSeries{ts})+ }+ return out+}++func labelSetsEqual(a, b []prompb.Label) bool {+ if len(a) != len(b) {+ return false+ }+ for _, la := range a {+ found := false+ for _, lb := range b {+ if la.Name == lb.Name {+ found = la.Value == lb.Value+ break+ }+ }+ if !found {+ return false+ }+ }+ return true+}++// mergeSamples k-way merges sample streams that share one labelset. On+// equal timestamps the highest fingerprint wins: the input is in ascending+// fingerprint order (samples SQL), keeping the choice deterministic.+func mergeSamples(group []*prompb.TimeSeries) *prompb.TimeSeries {+ merged := &prompb.TimeSeries{Labels: group[0].Labels}+ idx := make([]int, len(group))+ for {+ minTs := int64(math.MaxInt64)+ for i, ts := range group {+ if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp < minTs {+ minTs = ts.Samples[idx[i]].Timestamp+ }+ }+ if minTs == math.MaxInt64 {+ return merged+ }+ var chosen prompb.Sample+ for i, ts := range group {+ if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp == minTs {+ chosen = ts.Samples[idx[i]]+ idx[i]+++ }+ }+ merged.Samples = append(merged.Samples, chosen)+ }}func (client *client) queryRaw(ctx context.Context, query string, ts int64) (*prompb.QueryResult, error) {diff --git a/pkg/prometheus/clickhouseprometheus/client_query_test.go b/pkg/prometheus/clickhouseprometheus/client_query_test.goindex 38e96b6f37..5417f575af 100644--- a/pkg/prometheus/clickhouseprometheus/client_query_test.go+++ b/pkg/prometheus/clickhouseprometheus/client_query_test.go@@ -152,21 +152,22 @@ func TestClient_getFingerprintsFromClickhouseQuery(t *testing.T) {setupMock: func(m cmock.ClickConnMockCommon, args ...any) {rows := [][]any{{uint64(123), `{"t1":"s1","t2":"s2"}`},- {uint64(234), `{"t1":"s1","t2":"s2"}`},+ {uint64(234), `{"t1":"s1","t2":"s2","empty":""}`},}m.ExpectQuery(`SELECT fingerprint,labels`).WithArgs(args...).WillReturnRows(cmock.NewRows(cols, rows),)},+ // No synthetic fingerprint label (#8563), empty-valued labels+ // dropped: both fingerprints present one labelset for+ // querySamples to merge.want: map[uint64][]prompb.Label{123: {- {Name: "fingerprint", Value: "123"},{Name: "t1", Value: "s1"},{Name: "t2", Value: "s2"},},234: {- {Name: "fingerprint", Value: "234"},{Name: "t1", Value: "s1"},{Name: "t2", Value: "s2"},},@@ -208,3 +209,86 @@ func TestClient_getFingerprintsFromClickhouseQuery(t *testing.T) {})}}++// Regression for the duplicate-series class behind #8563: fingerprints+// sharing one labelset must come back as one merged series, the higher+// fingerprint winning equal timestamps.+func TestClient_QuerySamplesMergesIdenticalLabelSets(t *testing.T) {+ ctx := context.Background()+ cols := []cmock.ColumnType{+ {Name: "metric_name", Type: "String"},+ {Name: "fingerprint", Type: "UInt64"},+ {Name: "unix_milli", Type: "Int64"},+ {Name: "value", Type: "Float64"},+ {Name: "flags", Type: "UInt32"},+ }++ canary := []prompb.Label{+ {Name: "__name__", Value: "requests"},+ {Name: "group", Value: "canary"},+ }+ production := []prompb.Label{+ {Name: "__name__", Value: "requests"},+ {Name: "group", Value: "production"},+ }+ fingerprints := map[uint64][]prompb.Label{+ 100: canary,+ 200: canary,+ 300: production,+ }++ telemetryStore := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)+ // Rows arrive ordered by (fingerprint, unix_milli), matching the SQL.+ values := [][]any{+ {"requests", uint64(100), int64(1000), float64(1.0), uint32(0)},+ {"requests", uint64(100), int64(2000), float64(2.0), uint32(0)},+ {"requests", uint64(200), int64(2000), float64(20.0), uint32(0)},+ {"requests", uint64(200), int64(3000), float64(30.0), uint32(0)},+ {"requests", uint64(300), int64(1500), float64(5.0), uint32(0)},+ }+ telemetryStore.Mock().ExpectQuery("SELECT metric_name, fingerprint, unix_milli, value, flags").+ WithArgs("requests", int64(1000), int64(3000)).+ WillReturnRows(cmock.NewRows(cols, values))++ readClient := client{telemetryStore: telemetryStore}+ result, err := readClient.querySamples(ctx, 1000, 3000, fingerprints, "requests", "SELECT metric_name, fingerprint, unix_milli, value, flags", nil)+ require.NoError(t, err)++ assert.Equal(t, []*prompb.TimeSeries{+ {+ Labels: canary,+ Samples: []prompb.Sample{+ {Timestamp: 1000, Value: 1.0},+ {Timestamp: 2000, Value: 20.0},+ {Timestamp: 3000, Value: 30.0},+ },+ },+ {+ Labels: production,+ Samples: []prompb.Sample{+ {Timestamp: 1500, Value: 5.0},+ },+ },+ }, result)+}++// Hash grouping must stay order-insensitive (stored JSON key order is not+// canonical across fingerprints), and a 64-bit hash collision between+// distinct labelsets must not merge them — splitByLabelSet is that guard.+func TestLabelsHashAndCollisionSplit(t *testing.T) {+ lbls := []prompb.Label{+ {Name: "__name__", Value: "requests"},+ {Name: "job", Value: "api"},+ {Name: "instance", Value: "0"},+ }+ reversed := []prompb.Label{lbls[2], lbls[1], lbls[0]}+ assert.Equal(t, labelsHash(lbls), labelsHash(reversed))++ a := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}}+ b := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "y"}}}+ c := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}}+ got := splitByLabelSet([]*prompb.TimeSeries{a, b, c})+ require.Len(t, got, 2)+ assert.Equal(t, []*prompb.TimeSeries{a, c}, got[0])+ assert.Equal(t, []*prompb.TimeSeries{b}, got[1])+}diff --git a/pkg/prometheus/clickhouseprometheus/json.go b/pkg/prometheus/clickhouseprometheus/json.goindex ec478223ed..c9859c6be3 100644--- a/pkg/prometheus/clickhouseprometheus/json.go+++ b/pkg/prometheus/clickhouseprometheus/json.go@@ -2,14 +2,15 @@ package clickhouseprometheusimport ("encoding/json"- "strconv"- "github.com/SigNoz/signoz/pkg/prometheus""github.com/prometheus/prometheus/prompb")// Unmarshals JSON into Prometheus labels. It does not preserve order.-func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, error) {+// Empty-valued labels are dropped: Prometheus treats them as absent, and+// keeping them lets two fingerprints present duplicate labelsets to the+// engine (the incident behind #8563).+func unmarshalLabels(s string) ([]prompb.Label, string, error) {var metricName stringm := make(map[string]string)if err := json.Unmarshal([]byte(s), &m); err != nil {@@ -17,6 +18,9 @@ func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, erro}res := make([]prompb.Label, 0, len(m))for n, v := range m {+ if v == "" {+ continue+ }if n == "__name__" {metricName = v}@@ -26,9 +30,5 @@ func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, erroValue: v,})}- res = append(res, prompb.Label{- Name: prometheus.FingerprintAsPromLabelName,- Value: strconv.FormatUint(fingerprint, 10),- })return res, metricName, nil}diff --git a/pkg/prometheus/clickhouseprometheus/merge_test.go b/pkg/prometheus/clickhouseprometheus/merge_test.gonew file mode 100644index 0000000000..e56f9ebdfa--- /dev/null+++ b/pkg/prometheus/clickhouseprometheus/merge_test.go@@ -0,0 +1,82 @@+package clickhouseprometheus++import (+ "testing"++ "github.com/prometheus/prometheus/prompb"+ "github.com/stretchr/testify/assert"+)++func mkSeries(value string, samples ...prompb.Sample) *prompb.TimeSeries {+ return &prompb.TimeSeries{+ Labels: []prompb.Label{{Name: "job", Value: value}},+ Samples: samples,+ }+}++func TestMergeSeriesWithIdenticalLabels(t *testing.T) {+ s := func(ts int64, v float64) prompb.Sample { return prompb.Sample{Timestamp: ts, Value: v} }++ t.Run("empty and single series pass through untouched", func(t *testing.T) {+ assert.Nil(t, mergeSeriesWithIdenticalLabels(nil))+ one := []*prompb.TimeSeries{mkSeries("a", s(1, 1))}+ got := mergeSeriesWithIdenticalLabels(one)+ assert.Equal(t, one, got)+ })++ t.Run("no collisions returns the input slice as is", func(t *testing.T) {+ in := []*prompb.TimeSeries{mkSeries("a", s(1, 1)), mkSeries("b", s(1, 2))}+ got := mergeSeriesWithIdenticalLabels(in)+ // same backing slice: the fast path must not rebuild anything+ assert.Equal(t, &in[0], &got[0])+ })++ t.Run("disjoint streams concatenate in timestamp order", func(t *testing.T) {+ // the #8563 shape: the series transitioned fingerprints at a point+ // in time, so the streams do not overlap at all+ got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{+ mkSeries("a", s(1, 1), s(2, 2)),+ mkSeries("a", s(3, 3), s(4, 4)),+ })+ assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples)+ })++ t.Run("three fingerprints one labelset", func(t *testing.T) {+ got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{+ mkSeries("a", s(1, 1), s(4, 4)),+ mkSeries("a", s(2, 2)),+ mkSeries("a", s(3, 3)),+ })+ assert.Len(t, got, 1)+ assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples)+ })++ t.Run("equal timestamps everywhere keep the last stream's value", func(t *testing.T) {+ // input is in ascending fingerprint order; the highest wins each tie+ got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{+ mkSeries("a", s(1, 1), s(2, 1)),+ mkSeries("a", s(1, 9), s(2, 9)),+ })+ assert.Equal(t, []prompb.Sample{s(1, 9), s(2, 9)}, got[0].Samples)+ })++ t.Run("zero-sample series in a group is harmless", func(t *testing.T) {+ got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{+ mkSeries("a"),+ mkSeries("a", s(1, 1)),+ })+ assert.Len(t, got, 1)+ assert.Equal(t, []prompb.Sample{s(1, 1)}, got[0].Samples)+ })++ t.Run("colliding and distinct series interleave without cross-talk", func(t *testing.T) {+ got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{+ mkSeries("a", s(1, 1)),+ mkSeries("b", s(1, 2)),+ mkSeries("a", s(2, 3)),+ })+ assert.Len(t, got, 2)+ assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 3)}, got[0].Samples)+ assert.Equal(t, []prompb.Sample{s(1, 2)}, got[1].Samples)+ })+}diff --git a/pkg/prometheus/label.go b/pkg/prometheus/label.godeleted file mode 100644index c0fb8899ae..0000000000--- a/pkg/prometheus/label.go+++ /dev/null@@ -1,39 +0,0 @@-package prometheus--import (- "github.com/SigNoz/signoz/pkg/errors"- "github.com/prometheus/prometheus/model/labels"- "github.com/prometheus/prometheus/promql"-)--const FingerprintAsPromLabelName string = "fingerprint"--func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error {- if len(labelsToRemove) == 0 || res == nil {- return nil- }-- switch res.Value.(type) {- case promql.Vector:- value := res.Value.(promql.Vector)- for i := range value {- b := labels.NewBuilder(value[i].Metric)- b.Del(labelsToRemove...)- newLabels := b.Labels()- value[i].Metric = newLabels- }- case promql.Matrix:- value := res.Value.(promql.Matrix)- for i := range value {- b := labels.NewBuilder(value[i].Metric)- b.Del(labelsToRemove...)- newLabels := b.Labels()- value[i].Metric = newLabels- }- case promql.Scalar:- return nil- default:- return errors.NewInternalf(errors.CodeInternal, "rule result is not a vector or scalar or matrix")- }- return nil-}diff --git a/pkg/prometheus/prometheustest/utils_test.go b/pkg/prometheus/prometheustest/utils_test.godeleted file mode 100644index 48d71bce9a..0000000000--- a/pkg/prometheus/prometheustest/utils_test.go+++ /dev/null@@ -1,107 +0,0 @@-package prometheustest--import (- "testing"-- "github.com/SigNoz/signoz/pkg/prometheus"- "github.com/prometheus/prometheus/model/labels"- "github.com/prometheus/prometheus/promql"-)--func TestRemoveExtraLabels(t *testing.T) {-- tests := []struct {- name string- res *promql.Result- remove []string- wantErr bool- verify func(t *testing.T, result *promql.Result, removed []string)- }{- {- name: "vector – label removed",- res: &promql.Result{- Value: promql.Vector{- promql.Sample{- Metric: labels.FromStrings(- "__name__", "http_requests_total",- "job", "demo",- "drop_me", "dropped",- ),- },- },- },- remove: []string{"drop_me"},- verify: func(t *testing.T, result *promql.Result, removed []string) {- k := result.Value.(promql.Vector)- for _, str := range removed {- get := k[0].Metric.Get(str)- if get != "" {- t.Fatalf("label not removed")- }- }- },- },- {- name: "scalar – nothing to strip",- res: &promql.Result{- Value: promql.Scalar{V: 99, T: 1},- },- remove: []string{"irrelevant"},- verify: func(t *testing.T, result *promql.Result, removed []string) {- sc := result.Value.(promql.Scalar)- if sc.V != 99 || sc.T != 1 {- t.Fatalf("scalar unexpectedly modified: got %+v", sc)- }- },- },- {- name: "matrix – label removed",- res: &promql.Result{- Value: promql.Matrix{- promql.Series{- Metric: labels.FromStrings(- "__name__", "http_requests_total",- "pod", "p0",- "drop_me", "dropped",- ),- Floats: []promql.FPoint{{T: 0, F: 1}, {T: 1, F: 2}},- },- promql.Series{- Metric: labels.FromStrings(- "__name__", "http_requests_total",- "pod", "p0",- "drop_me", "dropped",- ),- Floats: []promql.FPoint{{T: 0, F: 1}, {T: 1, F: 2}},- },- },- },- remove: []string{"drop_me"},- verify: func(t *testing.T, result *promql.Result, removed []string) {- mat := result.Value.(promql.Matrix)- for _, str := range removed {- for _, k := range mat {- if k.Metric.Get(str) != "" {- t.Fatalf("label not removed")- }- }- }- },- },- }-- for _, tc := range tests {- t.Run(tc.name, func(t *testing.T) {- err := prometheus.RemoveExtraLabels(tc.res, tc.remove...)- if tc.wantErr && err == nil {- t.Fatalf("expected error, got nil")- }- if !tc.wantErr && err != nil {- t.Fatalf("unexpected error: %v", err)- }- if tc.verify != nil {- tc.verify(t, tc.res, tc.remove)- }- })- }-}diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.goindex aceed4323f..21558c4250 100644--- a/pkg/query-service/app/clickhouseReader/reader.go+++ b/pkg/query-service/app/clickhouseReader/reader.go@@ -247,10 +247,6 @@ func (r *ClickHouseReader) GetInstantQueryMetricsResult(ctx context.Context, que}qry.Close()- err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName)- if err != nil {- return nil, nil, &model.ApiError{Typ: model.ErrorInternal, Err: err}- }return res, &qs, nil}@@ -271,10 +267,6 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model}qry.Close()- err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName)- if err != nil {- return nil, nil, &model.ApiError{Typ: model.ErrorInternal, Err: err}- }return res, &qs, nil}diff --git a/pkg/query-service/rules/prom_rule.go b/pkg/query-service/rules/prom_rule.goindex 81c802b814..f68bf43a14 100644--- a/pkg/query-service/rules/prom_rule.go+++ b/pkg/query-service/rules/prom_rule.go@@ -387,11 +387,6 @@ func (r *PromRule) RunAlertQuery(ctx context.Context, qs string, start, end timereturn nil, res.Err}- err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName)- if err != nil {- return nil, err- }-switch typ := res.Value.(type) {case promql.Vector:series := make([]promql.Series, 0, len(typ))