Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

8.8 KB
Raw Blame
package implpromote

import (
	"context"
	"log/slog"
	"strings"

	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
	"github.com/SigNoz/signoz/pkg/errors"
	"github.com/SigNoz/signoz/pkg/factory"
	"github.com/SigNoz/signoz/pkg/modules/promote"
	"github.com/SigNoz/signoz/pkg/telemetrystore"
	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
	"github.com/SigNoz/signoz/pkg/types/promotetypes"
	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)

var (
	CodeFailedToCreateIndex        = errors.MustNewCode("failed_to_create_index_promoted_paths")
	CodeFailedToQueryPromotedPaths = errors.MustNewCode("failed_to_query_promoted_paths")
)

type module struct {
	metadataStore  telemetrytypes.MetadataStore
	telemetryStore telemetrystore.TelemetryStore
	settings       factory.ScopedProviderSettings
}

func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore telemetrystore.TelemetryStore, providerSettings factory.ProviderSettings) promote.Module {
	return &module{
		metadataStore:  metadataStore,
		telemetryStore: telemetrystore,
		settings:       factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/promote/implpromote"),
	}
}

func (m *module) ListPromotedPaths(ctx context.Context, filters promotetypes.ListPromotedPathsFilters) ([]promotetypes.PromotePath, error) {
	response := make([]promotetypes.PromotePath, 0)
	for _, target := range promotetypes.Targets() {
		if !filters.MatchesTarget(target) {
			continue
		}
		paths, err := m.listPromotedPaths(ctx, target)
		if err != nil {
			return nil, err
		}
		for _, path := range paths {
			if filters.MatchesPath(path) {
				response = append(response, path)
			}
		}
	}
	return response, nil
}

func (m *module) listPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error) {
	promotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry)
	if err != nil {
		return nil, err
	}

	response := make([]promotetypes.PromotePath, 0, len(promotedPaths))
	for path := range promotedPaths {
		response = append(response, promotetypes.PromotePath{
			Signal:  target.Entry.Signal.StringValue(),
			Context: target.Entry.FieldContext.StringValue(),
			Path:    path,
			Promote: true,
		})
	}

	indexes, err := m.metadataStore.ListJSONIndexes(ctx, target.JSONIndexLookup())
	if err != nil {
		return nil, err
	}

	// aggr keys are full sub-column paths: index.BaseColumn carries the
	// column prefix and index.Name the bare path.
	aggr := map[string][]promotetypes.WrappedIndex{}
	for _, index := range indexes {
		fullPath := index.BaseColumn + index.Name
		aggr[fullPath] = append(aggr[fullPath], promotetypes.WrappedIndex{
			FieldDataType: index.FieldDataType,
			Type:          index.IndexType,
			Granularity:   index.Granularity,
		})
	}

	for i := range response {
		fullPath := target.PromotedColumnPrefix() + response[i].Path
		if indexes, ok := aggr[fullPath]; ok {
			response[i].Indexes = indexes
			delete(aggr, fullPath)
		}
	}

	for fullPath, indexes := range aggr {
		path := strings.TrimPrefix(fullPath, target.BaseColumnPrefix())
		path = strings.TrimPrefix(path, target.PromotedColumnPrefix())
		response = append(response, promotetypes.PromotePath{
			Signal:  target.Entry.Signal.StringValue(),
			Context: target.Entry.FieldContext.StringValue(),
			Path:    path,
			Indexes: indexes,
		})
	}
	return response, nil
}

func (m *module) IndexMaterializedPaths(ctx context.Context, req promotetypes.PostableIndexMaterializedPaths) (*promotetypes.IndexMaterializedPathsResult, error) {
	target, err := req.Target()
	if err != nil {
		return nil, err
	}

	keys, err := m.metadataStore.GetMaterializedKeys(ctx, target.Entry.Signal)
	if err != nil {
		return nil, err
	}

	paths := []*promotetypes.PromotePath{}
	names := []string{}
	for _, key := range keys {
		if key.FieldContext != target.Entry.FieldContext || len(key.Indexes) == 0 {
			continue
		}
		path, err := promotetypes.NewMaterializedPromotePath(target, key)
		if err != nil {
			m.settings.Logger().WarnContext(ctx, "skipping materialized path that cannot be indexed", slog.String("path", key.Name), errors.Attr(err))
			continue
		}
		paths = append(paths, path)
		names = append(names, path.Path)
	}

	result := &promotetypes.IndexMaterializedPathsResult{Indexed: []*promotetypes.PromotePath{}, Skipped: []*promotetypes.PromotePath{}}
	if len(paths) > 0 {
		existing, err := m.metadataStore.ListJSONIndexes(ctx, target.JSONIndexLookup(), names...)
		if err != nil {
			return nil, err
		}
		existingByPath := map[string][]promotetypes.WrappedIndex{}
		for _, index := range existing {
			existingByPath[index.Name] = append(existingByPath[index.Name], promotetypes.WrappedIndex{FieldDataType: index.FieldDataType, Type: index.IndexType, Granularity: index.Granularity})
		}
		for _, path := range paths {
			if indexes, ok := existingByPath[path.Path]; ok {
				path.Indexes = indexes
				result.Skipped = append(result.Skipped, path)
				continue
			}
			result.Indexed = append(result.Indexed, path)
		}
	}

	if !req.DryRun && len(result.Indexed) > 0 {
		if err := m.promotePaths(ctx, target, result.Indexed...); err != nil {
			return nil, err
		}
	}

	indexed := make([]string, 0, len(result.Indexed))
	for _, path := range result.Indexed {
		indexed = append(indexed, path.Path)
	}
	skipped := make([]string, 0, len(result.Skipped))
	for _, path := range result.Skipped {
		skipped = append(skipped, path.Path)
	}
	m.settings.Logger().InfoContext(ctx, "indexed materialized paths", slog.String("signal", req.Signal), slog.Bool("dry_run", req.DryRun), slog.Int("indexed_count", len(indexed)), slog.Any("indexed", indexed), slog.Int("skipped_count", len(skipped)), slog.Any("skipped", skipped))

	return result, nil
}

func (m *module) PromotePaths(ctx context.Context, paths ...*promotetypes.PromotePath) error {
	byTarget := map[promotetypes.Target][]*promotetypes.PromotePath{}
	targets := []promotetypes.Target{}
	for _, path := range paths {
		target, err := path.Target()
		if err != nil {
			return err
		}
		if _, ok := byTarget[target]; !ok {
			targets = append(targets, target)
		}
		byTarget[target] = append(byTarget[target], path)
	}

	for _, target := range targets {
		if err := m.promotePaths(ctx, target, byTarget[target]...); err != nil {
			return err
		}
	}
	return nil
}

// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, target promotetypes.Target, indexes []schemamigrator.Index) error {
	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
		instrumentationtypes.TelemetrySignal:  target.Entry.Signal.StringValue(),
		instrumentationtypes.CodeNamespace:    "promote",
		instrumentationtypes.CodeFunctionName: "createIndexes",
	})
	if len(indexes) == 0 {
		return nil
	}

	for _, index := range indexes {
		alterStmt := schemamigrator.AlterTableAddIndex{
			Database: target.DBName,
			Table:    target.LocalTableName,
			Index:    index,
		}
		op := alterStmt.OnCluster(m.telemetryStore.Cluster())
		if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
			return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
		}
	}

	return nil
}

func (m *module) promotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error {
	pathsStr := []string{}
	for _, path := range paths {
		pathsStr = append(pathsStr, path.Path)
	}

	existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry, pathsStr...)
	if err != nil {
		return err
	}

	var toInsert []string
	indexes := []schemamigrator.Index{}
	for _, it := range paths {
		if it.Promote {
			if _, promoted := existingPromotedPaths[it.Path]; !promoted {
				toInsert = append(toInsert, it.Path)
			}
		}
		if len(it.Indexes) > 0 {
			parentColumn := target.BaseColumn
			// if the path is already promoted or is being promoted, add it to the promoted column
			if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
				parentColumn = target.PromotedColumn()
			}

			for _, index := range it.Indexes {
				typeIndex, err := index.SkipIndexType()
				if err != nil {
					return err
				}
				indexes = append(indexes, schemamigrator.Index{
					Name:        schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, strings.ToLower(index.JSONDataType.StringValue()), typeIndex),
					Expression:  target.IndexExpression(parentColumn, it.Path, index.JSONDataType.StringValue()),
					Type:        index.Type,
					Granularity: index.Granularity,
				})
			}
		}
	}

	if len(toInsert) > 0 {
		err := m.metadataStore.PromotePaths(ctx, target.Entry, toInsert...)
		if err != nil {
			return err
		}
	}

	if len(indexes) > 0 {
		if err := m.createIndexes(ctx, target, indexes); err != nil {
			return err
		}
	}

	return nil
}