SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
package implinframonitoring
import (
"context"
"fmt"
"slices"
"github.com/SigNoz/signoz/pkg/clickhousesql"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/huandu/go-sqlbuilder"
"golang.org/x/sync/errgroup"
)
// buildNodeRecords assembles the page records. Condition counts come from
// conditionCounts in both modes. In list mode (isNodeNameInGroupBy=true) each
// group is one node, so exactly one count is 1; Condition is derived from
// which one. In grouped_list mode Condition stays NodeConditionNoData.
func buildNodeRecords(
isNodeNameInGroupBy bool,
resp *qbtypes.QueryRangeResponse,
pageGroups []map[string]string,
groupBy []qbtypes.GroupByKey,
metadataMap map[string]map[string]string,
nodeConditionCounts map[string]nodeConditionCounts,
podStatusCounts map[string]podStatusCounts,
) []inframonitoringtypes.NodeRecord {
metricsMap := parseFullQueryResponse(resp, groupBy)
records := make([]inframonitoringtypes.NodeRecord, 0, len(pageGroups))
for _, labels := range pageGroups {
compositeKey := compositeKeyFromLabels(labels, groupBy)
nodeName := labels[inframonitoringtypes.NodeNameAttrKey]
record := inframonitoringtypes.NodeRecord{ // initialize with default values
NodeName: nodeName,
Condition: inframonitoringtypes.NodeConditionNoData,
NodeCPU: -1,
NodeCPUAllocatable: -1,
NodeMemory: -1,
NodeMemoryAllocatable: -1,
Meta: map[string]string{},
}
if metrics, ok := metricsMap[compositeKey]; ok {
if v, exists := metrics["A"]; exists {
record.NodeCPU = v
}
if v, exists := metrics["B"]; exists {
record.NodeCPUAllocatable = v
}
if v, exists := metrics["C"]; exists {
record.NodeMemory = v
}
if v, exists := metrics["D"]; exists {
record.NodeMemoryAllocatable = v
}
}
if nodeConditionCountsForGroup, ok := nodeConditionCounts[compositeKey]; ok {
record.NodeCountsByReadiness = inframonitoringtypes.NodeCountsByReadiness{
Ready: nodeConditionCountsForGroup.Ready,
NotReady: nodeConditionCountsForGroup.NotReady,
}
// In list mode each group is one node; the count==1 bucket identifies the condition.
if isNodeNameInGroupBy {
switch {
case nodeConditionCountsForGroup.Ready == 1:
record.Condition = inframonitoringtypes.NodeConditionReady
case nodeConditionCountsForGroup.NotReady == 1:
record.Condition = inframonitoringtypes.NodeConditionNotReady
}
}
}
if podStatusCountsForGroup, ok := podStatusCounts[compositeKey]; ok {
record.PodCountsByStatus = podStatusCountsToResponse(podStatusCountsForGroup)
}
if attrs, ok := metadataMap[compositeKey]; ok {
for k, v := range attrs {
record.Meta[k] = v
}
}
records = append(records, record)
}
return records
}
// getTopNodeGroupsAndMetadata concurrently fetches metadata + the ordering-metric
// ranking (plus the full-scope pod-status / node-readiness keysets when filtering,
// to intersect all).
func (m *module) getTopNodeGroupsAndMetadata(
ctx context.Context,
orgID valuer.UUID,
req *inframonitoringtypes.PostableNodes,
) ([]map[string]string, map[string]map[string]string, map[string]podStatusCounts, *qbtypes.QueryWarnData, map[string]nodeConditionCounts, error) {
var (
orderByKey string
metadataMap map[string]map[string]string
allMetricGroups []rankedGroup
statusCounts map[string]podStatusCounts
statusWarning *qbtypes.QueryWarnData
nodeConditionCounts map[string]nodeConditionCounts
filter *qbtypes.Filter
filterByPodStatus []inframonitoringtypes.PodStatus
filterByNodeReadiness []inframonitoringtypes.NodeCondition
)
orderByKey = req.OrderBy.Key.Name
// When filtering by pod status / node readiness, resolve the full-scope
// keyset(s) concurrently (pageGroups=nil spans all groups under the user
// filter) to intersect metadata + ranked groups below. Filters compose as AND.
if req.Filter != nil {
filter = &req.Filter.Filter
filterByPodStatus = req.Filter.FilterByPodStatus
filterByNodeReadiness = req.Filter.FilterByNodeReadiness
}
g, gCtx := errgroup.WithContext(ctx)
g.Go(func() error {
var err error
metadataMap, err = m.getNodesTableMetadata(gCtx, orgID, req)
return err
})
if len(filterByPodStatus) != 0 {
g.Go(func() error {
var err error
statusCounts, statusWarning, err = m.getPerGroupPodStatusCountsWithReqMetricChecks(gCtx, orgID, req.Start, req.End, filter, req.GroupBy, nil, filterByPodStatus)
return err
})
}
if len(filterByNodeReadiness) != 0 {
g.Go(func() error {
var err error
nodeConditionCounts, err = m.getPerGroupNodeConditionCounts(gCtx, orgID, req.Start, req.End, filter, req.GroupBy, nil, filterByNodeReadiness)
return err
})
}
if orderByKey == inframonitoringtypes.NodeNameAttrKey {
if err := g.Wait(); err != nil {
return nil, nil, nil, nil, nil, err
}
// Secondary filter: keep only status/readiness-matching groups. A missing
// metric yields an empty statusCounts, so this correctly empties the result
// (the caller also surfaces the warning). Filters compose as AND.
if len(filterByPodStatus) != 0 {
metadataMap = intersectMap(metadataMap, statusCounts)
}
if len(filterByNodeReadiness) != 0 {
metadataMap = intersectMap(metadataMap, nodeConditionCounts)
}
pageGroups := inframonitoringtypes.PaginateMetadataByName(metadataMap, req.GroupBy, req.OrderBy.Direction, req.Offset, req.Limit, inframonitoringtypes.NodeNameAttrKey)
return pageGroups, metadataMap, statusCounts, statusWarning, nodeConditionCounts, nil
}
queryNamesForOrderBy := orderByToNodesQueryNames[orderByKey]
rankingQueryName := queryNamesForOrderBy[len(queryNamesForOrderBy)-1]
topReq := &qbtypes.QueryRangeRequest{
Start: uint64(req.Start),
End: uint64(req.End),
RequestType: qbtypes.RequestTypeScalar,
CompositeQuery: qbtypes.CompositeQuery{
Queries: make([]qbtypes.QueryEnvelope, 0, len(queryNamesForOrderBy)),
},
}
for _, envelope := range m.newNodesTableListQuery().CompositeQuery.Queries {
if !slices.Contains(queryNamesForOrderBy, envelope.GetQueryName()) {
continue
}
copied := envelope
if copied.Type == qbtypes.QueryTypeBuilder {
existingExpr := ""
if f := copied.GetFilter(); f != nil {
existingExpr = f.Expression
}
reqFilterExpr := ""
if req.Filter != nil {
reqFilterExpr = req.Filter.Expression
}
merged := mergeFilterExpressions(existingExpr, reqFilterExpr)
copied.SetFilter(&qbtypes.Filter{Expression: merged})
copied.SetGroupBy(req.GroupBy)
}
topReq.CompositeQuery.Queries = append(topReq.CompositeQuery.Queries, copied)
}
g.Go(func() error {
resp, err := m.querier.QueryRange(gCtx, orgID, topReq)
if err != nil {
return err
}
allMetricGroups = parseAndSortGroups(resp, rankingQueryName, req.GroupBy, req.OrderBy.Direction)
return nil
})
if err := g.Wait(); err != nil {
return nil, nil, nil, nil, nil, err
}
// Secondary filter: intersect ranked groups + metadata with the status/readiness
// keyset. A missing metric yields an empty keyset, correctly emptying the result
// (the caller also surfaces the warning). Filters compose as AND.
if len(filterByPodStatus) != 0 {
allMetricGroups = intersectRankedGroups(allMetricGroups, statusCounts)
metadataMap = intersectMap(metadataMap, statusCounts)
}
if len(filterByNodeReadiness) != 0 {
allMetricGroups = intersectRankedGroups(allMetricGroups, nodeConditionCounts)
metadataMap = intersectMap(metadataMap, nodeConditionCounts)
}
pageGroups := paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit)
return pageGroups, metadataMap, statusCounts, statusWarning, nodeConditionCounts, nil
}
func (m *module) getNodesTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNodes) (map[string]map[string]string, error) {
var nonGroupByAttrs []string
for _, key := range nodeAttrKeysForMetadata {
if !isKeyInGroupByAttrs(req.GroupBy, key) {
nonGroupByAttrs = append(nonGroupByAttrs, key)
}
}
var filter *qbtypes.Filter
if req.Filter != nil {
filter = &req.Filter.Filter
}
return m.getMetadata(ctx, orgID, nodesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, filter, req.Start, req.End)
}
// getPerGroupNodeConditionCounts computes per-group node counts bucketed by each
// node's latest condition_ready value (0 / 1) in the requested window.
// Pipeline:
//
// timeSeriesFPs: fp ↔ (node_name, groupBy cols) from the time_series table.
// User filter + page-groups filter applied here.
// latestConditionPerNode: INNER JOIN samples × timeSeriesFPs, collapsed to
// the latest condition value per node via argMax(value, unix_milli).
// countNodesPerCondition: per-group uniqExactIf into ready/not_ready buckets.
//
// Groups absent from the result map have implicit zero counts (caller default).
// applyNodeReadinessFilter adds the readiness push-down (condition_value IN (...))
// to the outer count builder. condition_value is numeric (1=Ready, 0=NotReady), so
// we map each requested enum to its int. No-op when the requested set is empty.
func applyNodeReadinessFilter(cb *sqlbuilder.SelectBuilder, filterByNodeReadiness []inframonitoringtypes.NodeCondition) {
if len(filterByNodeReadiness) == 0 {
return
}
nums := make([]int, len(filterByNodeReadiness))
for i, c := range filterByNodeReadiness {
v := inframonitoringtypes.NodeConditionNumNotReady
if c == inframonitoringtypes.NodeConditionReady {
v = inframonitoringtypes.NodeConditionNumReady
}
nums[i] = v
}
cb.Where(cb.In("condition_value", sqlbuilder.List(nums)))
}
func (m *module) getPerGroupNodeConditionCounts(
ctx context.Context,
orgID valuer.UUID,
start, end int64,
filter *qbtypes.Filter,
groupBy []qbtypes.GroupByKey,
pageGroups []map[string]string,
filterByNodeReadiness []inframonitoringtypes.NodeCondition,
) (map[string]nodeConditionCounts, error) {
// Empty pageGroups means "span all under user filter", allowed only in
// full-scope mode (filtering by readiness). Otherwise it's an empty page.
if len(groupBy) == 0 || (len(pageGroups) == 0 && len(filterByNodeReadiness) == 0) {
return map[string]nodeConditionCounts{}, nil
}
// Merge user filter with page-groups IN clauses.
userFilterExpr := ""
if filter != nil {
userFilterExpr = filter.Expression
}
pageGroupsFilterExpr := buildPageGroupsFilterExpr(pageGroups)
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, pageGroupsFilterExpr)
// Step-floor bounds + resolve tables in one shot to match QB v5 querier.
samplesStartMs, flooredEndMs, tsAdjustedStartMs, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
// ----- timeSeriesFPs -----
timeSeriesFPs := sqlbuilder.NewSelectBuilder()
timeSeriesFPsSelectCols := []string{
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS node_name", timeSeriesFPs.Var(inframonitoringtypes.NodeNameAttrKey)),
}
for _, key := range groupBy {
timeSeriesFPsSelectCols = append(timeSeriesFPsSelectCols,
fmt.Sprintf("JSONExtractString(labels, %s) AS %s", timeSeriesFPs.Var(key.Name), sqlbuilder.Escape(clickhousesql.Identifier(key.Name))),
)
}
timeSeriesFPs.Select(timeSeriesFPsSelectCols...)
timeSeriesFPs.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
timeSeriesFPs.Where(
timeSeriesFPs.E("metric_name", nodeConditionMetricName),
timeSeriesFPs.GE("unix_milli", tsAdjustedStartMs),
timeSeriesFPs.LE("unix_milli", flooredEndMs),
)
if mergedFilterExpr != "" {
filterClause, err := m.buildFilterClause(ctx, orgID, &qbtypes.Filter{Expression: mergedFilterExpr}, start, end)
if err != nil {
return nil, err
}
if filterClause != nil {
timeSeriesFPs.AddWhereClause(filterClause)
}
}
timeSeriesFPsGroupBy := []string{"fingerprint", "node_name"}
for _, key := range groupBy {
timeSeriesFPsGroupBy = append(timeSeriesFPsGroupBy, sqlbuilder.Escape(clickhousesql.Identifier(key.Name)))
}
timeSeriesFPs.GroupBy(timeSeriesFPsGroupBy...)
timeSeriesFPsSQL, timeSeriesFPsArgs := timeSeriesFPs.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- latestConditionPerNode -----
latestConditionPerNode := sqlbuilder.NewSelectBuilder()
latestConditionPerNodeSelectCols := []string{"tsfp.node_name AS node_name"}
latestConditionPerNodeGroupBy := []string{"node_name"}
for _, key := range groupBy {
col := sqlbuilder.Escape(clickhousesql.Identifier(key.Name))
latestConditionPerNodeSelectCols = append(latestConditionPerNodeSelectCols, fmt.Sprintf("tsfp.%s AS %s", col, col))
latestConditionPerNodeGroupBy = append(latestConditionPerNodeGroupBy, col)
}
latestConditionPerNodeSelectCols = append(latestConditionPerNodeSelectCols,
fmt.Sprintf("argMax(samples.%s, samples.unix_milli) AS condition_value", valueCol),
)
latestConditionPerNode.Select(latestConditionPerNodeSelectCols...)
latestConditionPerNode.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN time_series_fps AS tsfp ON samples.fingerprint = tsfp.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
))
latestConditionPerNode.Where(
latestConditionPerNode.E("samples.metric_name", nodeConditionMetricName),
latestConditionPerNode.GE("samples.unix_milli", samplesStartMs),
latestConditionPerNode.L("samples.unix_milli", flooredEndMs),
"tsfp.node_name != ''",
)
latestConditionPerNode.GroupBy(latestConditionPerNodeGroupBy...)
latestConditionPerNodeSQL, latestConditionPerNodeArgs := latestConditionPerNode.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- countNodesPerCondition (outer SELECT) -----
countNodesPerConditionSelectCols := make([]string, 0, len(groupBy)+2)
countNodesPerConditionGroupBy := make([]string, 0, len(groupBy))
for _, key := range groupBy {
col := sqlbuilder.Escape(clickhousesql.Identifier(key.Name))
countNodesPerConditionSelectCols = append(countNodesPerConditionSelectCols, col)
countNodesPerConditionGroupBy = append(countNodesPerConditionGroupBy, col)
}
countNodesPerConditionSelectCols = append(countNodesPerConditionSelectCols,
fmt.Sprintf("uniqExactIf(node_name, condition_value = %d) AS ready_count", inframonitoringtypes.NodeConditionNumReady),
fmt.Sprintf("uniqExactIf(node_name, condition_value = %d) AS not_ready_count", inframonitoringtypes.NodeConditionNumNotReady),
)
// Outer count query. Built with sqlbuilder so the readiness push-down uses a
// proper IN (keep only nodes whose readiness is in the requested set).
countBuilder := sqlbuilder.NewSelectBuilder()
countBuilder.Select(countNodesPerConditionSelectCols...)
countBuilder.From("latest_condition_per_node")
applyNodeReadinessFilter(countBuilder, filterByNodeReadiness)
countBuilder.GroupBy(countNodesPerConditionGroupBy...)
countNodesPerConditionSQL, countArgs := countBuilder.BuildWithFlavor(sqlbuilder.ClickHouse)
// Combine CTEs + outer.
cteFragments := []string{
fmt.Sprintf("time_series_fps AS (%s)", timeSeriesFPsSQL),
fmt.Sprintf("latest_condition_per_node AS (%s)", latestConditionPerNodeSQL),
}
finalSQL := querybuilder.CombineCTEs(cteFragments) + countNodesPerConditionSQL
finalArgs := querybuilder.PrependArgs([][]any{timeSeriesFPsArgs, latestConditionPerNodeArgs}, countArgs)
rows, err := m.telemetryStore.ClickhouseDB().Query(ctx, finalSQL, finalArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
result := make(map[string]nodeConditionCounts)
for rows.Next() {
groupVals := make([]string, len(groupBy))
scanPtrs := make([]any, 0, len(groupBy)+2)
for i := range groupVals {
scanPtrs = append(scanPtrs, &groupVals[i])
}
var ready, notReady uint64
scanPtrs = append(scanPtrs, &ready, ¬Ready)
if err := rows.Scan(scanPtrs...); err != nil {
return nil, err
}
result[compositeKeyFromList(groupVals)] = nodeConditionCounts{
Ready: int(ready),
NotReady: int(notReady),
}
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
}