Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

16.1 KB
Raw Blame
package implinframonitoring

import (
	"context"
	"fmt"
	"slices"

	"github.com/SigNoz/signoz/pkg/clickhousesql"
	"github.com/SigNoz/signoz/pkg/querybuilder"
	"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
	"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
	"github.com/SigNoz/signoz/pkg/valuer"
	"github.com/huandu/go-sqlbuilder"
	"golang.org/x/sync/errgroup"
)

// buildNodeRecords assembles the page records. Condition counts come from
// conditionCounts in both modes. In list mode (isNodeNameInGroupBy=true) each
// group is one node, so exactly one count is 1; Condition is derived from
// which one. In grouped_list mode Condition stays NodeConditionNoData.
func buildNodeRecords(
	isNodeNameInGroupBy bool,
	resp *qbtypes.QueryRangeResponse,
	pageGroups []map[string]string,
	groupBy []qbtypes.GroupByKey,
	metadataMap map[string]map[string]string,
	nodeConditionCounts map[string]nodeConditionCounts,
	podStatusCounts map[string]podStatusCounts,
) []inframonitoringtypes.NodeRecord {
	metricsMap := parseFullQueryResponse(resp, groupBy)

	records := make([]inframonitoringtypes.NodeRecord, 0, len(pageGroups))
	for _, labels := range pageGroups {
		compositeKey := compositeKeyFromLabels(labels, groupBy)
		nodeName := labels[inframonitoringtypes.NodeNameAttrKey]

		record := inframonitoringtypes.NodeRecord{ // initialize with default values
			NodeName:              nodeName,
			Condition:             inframonitoringtypes.NodeConditionNoData,
			NodeCPU:               -1,
			NodeCPUAllocatable:    -1,
			NodeMemory:            -1,
			NodeMemoryAllocatable: -1,
			Meta:                  map[string]string{},
		}

		if metrics, ok := metricsMap[compositeKey]; ok {
			if v, exists := metrics["A"]; exists {
				record.NodeCPU = v
			}
			if v, exists := metrics["B"]; exists {
				record.NodeCPUAllocatable = v
			}
			if v, exists := metrics["C"]; exists {
				record.NodeMemory = v
			}
			if v, exists := metrics["D"]; exists {
				record.NodeMemoryAllocatable = v
			}
		}

		if nodeConditionCountsForGroup, ok := nodeConditionCounts[compositeKey]; ok {
			record.NodeCountsByReadiness = inframonitoringtypes.NodeCountsByReadiness{
				Ready:    nodeConditionCountsForGroup.Ready,
				NotReady: nodeConditionCountsForGroup.NotReady,
			}

			// In list mode each group is one node; the count==1 bucket identifies the condition.
			if isNodeNameInGroupBy {
				switch {
				case nodeConditionCountsForGroup.Ready == 1:
					record.Condition = inframonitoringtypes.NodeConditionReady
				case nodeConditionCountsForGroup.NotReady == 1:
					record.Condition = inframonitoringtypes.NodeConditionNotReady
				}
			}
		}

		if podStatusCountsForGroup, ok := podStatusCounts[compositeKey]; ok {
			record.PodCountsByStatus = podStatusCountsToResponse(podStatusCountsForGroup)
		}

		if attrs, ok := metadataMap[compositeKey]; ok {
			for k, v := range attrs {
				record.Meta[k] = v
			}
		}

		records = append(records, record)
	}
	return records
}

// getTopNodeGroupsAndMetadata concurrently fetches metadata + the ordering-metric
// ranking (plus the full-scope pod-status / node-readiness keysets when filtering,
// to intersect all).
func (m *module) getTopNodeGroupsAndMetadata(
	ctx context.Context,
	orgID valuer.UUID,
	req *inframonitoringtypes.PostableNodes,
) ([]map[string]string, map[string]map[string]string, map[string]podStatusCounts, *qbtypes.QueryWarnData, map[string]nodeConditionCounts, error) {

	var (
		orderByKey            string
		metadataMap           map[string]map[string]string
		allMetricGroups       []rankedGroup
		statusCounts          map[string]podStatusCounts
		statusWarning         *qbtypes.QueryWarnData
		nodeConditionCounts   map[string]nodeConditionCounts
		filter                *qbtypes.Filter
		filterByPodStatus     []inframonitoringtypes.PodStatus
		filterByNodeReadiness []inframonitoringtypes.NodeCondition
	)

	orderByKey = req.OrderBy.Key.Name

	// When filtering by pod status / node readiness, resolve the full-scope
	// keyset(s) concurrently (pageGroups=nil spans all groups under the user
	// filter) to intersect metadata + ranked groups below. Filters compose as AND.
	if req.Filter != nil {
		filter = &req.Filter.Filter
		filterByPodStatus = req.Filter.FilterByPodStatus
		filterByNodeReadiness = req.Filter.FilterByNodeReadiness
	}

	g, gCtx := errgroup.WithContext(ctx)

	g.Go(func() error {
		var err error
		metadataMap, err = m.getNodesTableMetadata(gCtx, orgID, req)
		return err
	})

	if len(filterByPodStatus) != 0 {
		g.Go(func() error {
			var err error
			statusCounts, statusWarning, err = m.getPerGroupPodStatusCountsWithReqMetricChecks(gCtx, orgID, req.Start, req.End, filter, req.GroupBy, nil, filterByPodStatus)
			return err
		})
	}

	if len(filterByNodeReadiness) != 0 {
		g.Go(func() error {
			var err error
			nodeConditionCounts, err = m.getPerGroupNodeConditionCounts(gCtx, orgID, req.Start, req.End, filter, req.GroupBy, nil, filterByNodeReadiness)
			return err
		})
	}

	if orderByKey == inframonitoringtypes.NodeNameAttrKey {
		if err := g.Wait(); err != nil {
			return nil, nil, nil, nil, nil, err
		}
		// Secondary filter: keep only status/readiness-matching groups. A missing
		// metric yields an empty statusCounts, so this correctly empties the result
		// (the caller also surfaces the warning). Filters compose as AND.
		if len(filterByPodStatus) != 0 {
			metadataMap = intersectMap(metadataMap, statusCounts)
		}
		if len(filterByNodeReadiness) != 0 {
			metadataMap = intersectMap(metadataMap, nodeConditionCounts)
		}
		pageGroups := inframonitoringtypes.PaginateMetadataByName(metadataMap, req.GroupBy, req.OrderBy.Direction, req.Offset, req.Limit, inframonitoringtypes.NodeNameAttrKey)
		return pageGroups, metadataMap, statusCounts, statusWarning, nodeConditionCounts, nil
	}

	queryNamesForOrderBy := orderByToNodesQueryNames[orderByKey]
	rankingQueryName := queryNamesForOrderBy[len(queryNamesForOrderBy)-1]

	topReq := &qbtypes.QueryRangeRequest{
		Start:       uint64(req.Start),
		End:         uint64(req.End),
		RequestType: qbtypes.RequestTypeScalar,
		CompositeQuery: qbtypes.CompositeQuery{
			Queries: make([]qbtypes.QueryEnvelope, 0, len(queryNamesForOrderBy)),
		},
	}

	for _, envelope := range m.newNodesTableListQuery().CompositeQuery.Queries {
		if !slices.Contains(queryNamesForOrderBy, envelope.GetQueryName()) {
			continue
		}
		copied := envelope
		if copied.Type == qbtypes.QueryTypeBuilder {
			existingExpr := ""
			if f := copied.GetFilter(); f != nil {
				existingExpr = f.Expression
			}
			reqFilterExpr := ""
			if req.Filter != nil {
				reqFilterExpr = req.Filter.Expression
			}
			merged := mergeFilterExpressions(existingExpr, reqFilterExpr)
			copied.SetFilter(&qbtypes.Filter{Expression: merged})
			copied.SetGroupBy(req.GroupBy)
		}
		topReq.CompositeQuery.Queries = append(topReq.CompositeQuery.Queries, copied)
	}

	g.Go(func() error {
		resp, err := m.querier.QueryRange(gCtx, orgID, topReq)
		if err != nil {
			return err
		}
		allMetricGroups = parseAndSortGroups(resp, rankingQueryName, req.GroupBy, req.OrderBy.Direction)
		return nil
	})

	if err := g.Wait(); err != nil {
		return nil, nil, nil, nil, nil, err
	}

	// Secondary filter: intersect ranked groups + metadata with the status/readiness
	// keyset. A missing metric yields an empty keyset, correctly emptying the result
	// (the caller also surfaces the warning). Filters compose as AND.
	if len(filterByPodStatus) != 0 {
		allMetricGroups = intersectRankedGroups(allMetricGroups, statusCounts)
		metadataMap = intersectMap(metadataMap, statusCounts)
	}
	if len(filterByNodeReadiness) != 0 {
		allMetricGroups = intersectRankedGroups(allMetricGroups, nodeConditionCounts)
		metadataMap = intersectMap(metadataMap, nodeConditionCounts)
	}

	pageGroups := paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit)
	return pageGroups, metadataMap, statusCounts, statusWarning, nodeConditionCounts, nil
}

func (m *module) getNodesTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNodes) (map[string]map[string]string, error) {
	var nonGroupByAttrs []string
	for _, key := range nodeAttrKeysForMetadata {
		if !isKeyInGroupByAttrs(req.GroupBy, key) {
			nonGroupByAttrs = append(nonGroupByAttrs, key)
		}
	}
	var filter *qbtypes.Filter
	if req.Filter != nil {
		filter = &req.Filter.Filter
	}
	return m.getMetadata(ctx, orgID, nodesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, filter, req.Start, req.End)
}

// getPerGroupNodeConditionCounts computes per-group node counts bucketed by each
// node's latest condition_ready value (0 / 1) in the requested window.
// Pipeline:
//
//	timeSeriesFPs:           fp ↔ (node_name, groupBy cols) from the time_series table.
//	                         User filter + page-groups filter applied here.
//	latestConditionPerNode:  INNER JOIN samples × timeSeriesFPs, collapsed to
//	                         the latest condition value per node via argMax(value, unix_milli).
//	countNodesPerCondition:  per-group uniqExactIf into ready/not_ready buckets.
//
// Groups absent from the result map have implicit zero counts (caller default).
// applyNodeReadinessFilter adds the readiness push-down (condition_value IN (...))
// to the outer count builder. condition_value is numeric (1=Ready, 0=NotReady), so
// we map each requested enum to its int. No-op when the requested set is empty.
func applyNodeReadinessFilter(cb *sqlbuilder.SelectBuilder, filterByNodeReadiness []inframonitoringtypes.NodeCondition) {
	if len(filterByNodeReadiness) == 0 {
		return
	}
	nums := make([]int, len(filterByNodeReadiness))
	for i, c := range filterByNodeReadiness {
		v := inframonitoringtypes.NodeConditionNumNotReady
		if c == inframonitoringtypes.NodeConditionReady {
			v = inframonitoringtypes.NodeConditionNumReady
		}
		nums[i] = v
	}
	cb.Where(cb.In("condition_value", sqlbuilder.List(nums)))
}

func (m *module) getPerGroupNodeConditionCounts(
	ctx context.Context,
	orgID valuer.UUID,
	start, end int64,
	filter *qbtypes.Filter,
	groupBy []qbtypes.GroupByKey,
	pageGroups []map[string]string,
	filterByNodeReadiness []inframonitoringtypes.NodeCondition,
) (map[string]nodeConditionCounts, error) {
	// Empty pageGroups means "span all under user filter", allowed only in
	// full-scope mode (filtering by readiness). Otherwise it's an empty page.
	if len(groupBy) == 0 || (len(pageGroups) == 0 && len(filterByNodeReadiness) == 0) {
		return map[string]nodeConditionCounts{}, nil
	}

	// Merge user filter with page-groups IN clauses.
	userFilterExpr := ""
	if filter != nil {
		userFilterExpr = filter.Expression
	}
	pageGroupsFilterExpr := buildPageGroupsFilterExpr(pageGroups)
	mergedFilterExpr := mergeFilterExpressions(userFilterExpr, pageGroupsFilterExpr)

	// Step-floor bounds + resolve tables in one shot to match QB v5 querier.
	samplesStartMs, flooredEndMs, tsAdjustedStartMs, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
	valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)

	// ----- timeSeriesFPs -----
	timeSeriesFPs := sqlbuilder.NewSelectBuilder()
	timeSeriesFPsSelectCols := []string{
		"fingerprint",
		fmt.Sprintf("JSONExtractString(labels, %s) AS node_name", timeSeriesFPs.Var(inframonitoringtypes.NodeNameAttrKey)),
	}
	for _, key := range groupBy {
		timeSeriesFPsSelectCols = append(timeSeriesFPsSelectCols,
			fmt.Sprintf("JSONExtractString(labels, %s) AS %s", timeSeriesFPs.Var(key.Name), sqlbuilder.Escape(clickhousesql.Identifier(key.Name))),
		)
	}
	timeSeriesFPs.Select(timeSeriesFPsSelectCols...)
	timeSeriesFPs.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
	timeSeriesFPs.Where(
		timeSeriesFPs.E("metric_name", nodeConditionMetricName),
		timeSeriesFPs.GE("unix_milli", tsAdjustedStartMs),
		timeSeriesFPs.LE("unix_milli", flooredEndMs),
	)
	if mergedFilterExpr != "" {
		filterClause, err := m.buildFilterClause(ctx, orgID, &qbtypes.Filter{Expression: mergedFilterExpr}, start, end)
		if err != nil {
			return nil, err
		}
		if filterClause != nil {
			timeSeriesFPs.AddWhereClause(filterClause)
		}
	}
	timeSeriesFPsGroupBy := []string{"fingerprint", "node_name"}
	for _, key := range groupBy {
		timeSeriesFPsGroupBy = append(timeSeriesFPsGroupBy, sqlbuilder.Escape(clickhousesql.Identifier(key.Name)))
	}
	timeSeriesFPs.GroupBy(timeSeriesFPsGroupBy...)
	timeSeriesFPsSQL, timeSeriesFPsArgs := timeSeriesFPs.BuildWithFlavor(sqlbuilder.ClickHouse)

	// ----- latestConditionPerNode -----
	latestConditionPerNode := sqlbuilder.NewSelectBuilder()
	latestConditionPerNodeSelectCols := []string{"tsfp.node_name AS node_name"}
	latestConditionPerNodeGroupBy := []string{"node_name"}
	for _, key := range groupBy {
		col := sqlbuilder.Escape(clickhousesql.Identifier(key.Name))
		latestConditionPerNodeSelectCols = append(latestConditionPerNodeSelectCols, fmt.Sprintf("tsfp.%s AS %s", col, col))
		latestConditionPerNodeGroupBy = append(latestConditionPerNodeGroupBy, col)
	}
	latestConditionPerNodeSelectCols = append(latestConditionPerNodeSelectCols,
		fmt.Sprintf("argMax(samples.%s, samples.unix_milli) AS condition_value", valueCol),
	)
	latestConditionPerNode.Select(latestConditionPerNodeSelectCols...)
	latestConditionPerNode.From(fmt.Sprintf(
		"%s.%s AS samples INNER JOIN time_series_fps AS tsfp ON samples.fingerprint = tsfp.fingerprint",
		metricstelemetryschema.DBName, distributedSamplesTable,
	))
	latestConditionPerNode.Where(
		latestConditionPerNode.E("samples.metric_name", nodeConditionMetricName),
		latestConditionPerNode.GE("samples.unix_milli", samplesStartMs),
		latestConditionPerNode.L("samples.unix_milli", flooredEndMs),
		"tsfp.node_name != ''",
	)
	latestConditionPerNode.GroupBy(latestConditionPerNodeGroupBy...)
	latestConditionPerNodeSQL, latestConditionPerNodeArgs := latestConditionPerNode.BuildWithFlavor(sqlbuilder.ClickHouse)

	// ----- countNodesPerCondition (outer SELECT) -----
	countNodesPerConditionSelectCols := make([]string, 0, len(groupBy)+2)
	countNodesPerConditionGroupBy := make([]string, 0, len(groupBy))
	for _, key := range groupBy {
		col := sqlbuilder.Escape(clickhousesql.Identifier(key.Name))
		countNodesPerConditionSelectCols = append(countNodesPerConditionSelectCols, col)
		countNodesPerConditionGroupBy = append(countNodesPerConditionGroupBy, col)
	}
	countNodesPerConditionSelectCols = append(countNodesPerConditionSelectCols,
		fmt.Sprintf("uniqExactIf(node_name, condition_value = %d) AS ready_count", inframonitoringtypes.NodeConditionNumReady),
		fmt.Sprintf("uniqExactIf(node_name, condition_value = %d) AS not_ready_count", inframonitoringtypes.NodeConditionNumNotReady),
	)
	// Outer count query. Built with sqlbuilder so the readiness push-down uses a
	// proper IN (keep only nodes whose readiness is in the requested set).
	countBuilder := sqlbuilder.NewSelectBuilder()
	countBuilder.Select(countNodesPerConditionSelectCols...)
	countBuilder.From("latest_condition_per_node")
	applyNodeReadinessFilter(countBuilder, filterByNodeReadiness)
	countBuilder.GroupBy(countNodesPerConditionGroupBy...)
	countNodesPerConditionSQL, countArgs := countBuilder.BuildWithFlavor(sqlbuilder.ClickHouse)

	// Combine CTEs + outer.
	cteFragments := []string{
		fmt.Sprintf("time_series_fps AS (%s)", timeSeriesFPsSQL),
		fmt.Sprintf("latest_condition_per_node AS (%s)", latestConditionPerNodeSQL),
	}
	finalSQL := querybuilder.CombineCTEs(cteFragments) + countNodesPerConditionSQL
	finalArgs := querybuilder.PrependArgs([][]any{timeSeriesFPsArgs, latestConditionPerNodeArgs}, countArgs)

	rows, err := m.telemetryStore.ClickhouseDB().Query(ctx, finalSQL, finalArgs...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	result := make(map[string]nodeConditionCounts)
	for rows.Next() {
		groupVals := make([]string, len(groupBy))
		scanPtrs := make([]any, 0, len(groupBy)+2)
		for i := range groupVals {
			scanPtrs = append(scanPtrs, &groupVals[i])
		}
		var ready, notReady uint64
		scanPtrs = append(scanPtrs, &ready, &notReady)

		if err := rows.Scan(scanPtrs...); err != nil {
			return nil, err
		}
		result[compositeKeyFromList(groupVals)] = nodeConditionCounts{
			Ready:    int(ready),
			NotReady: int(notReady),
		}
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return result, nil
}