SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
package implpromote
import (
"context"
"log/slog"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
var (
CodeFailedToCreateIndex = errors.MustNewCode("failed_to_create_index_promoted_paths")
CodeFailedToQueryPromotedPaths = errors.MustNewCode("failed_to_query_promoted_paths")
)
type module struct {
metadataStore telemetrytypes.MetadataStore
telemetryStore telemetrystore.TelemetryStore
settings factory.ScopedProviderSettings
}
func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore telemetrystore.TelemetryStore, providerSettings factory.ProviderSettings) promote.Module {
return &module{
metadataStore: metadataStore,
telemetryStore: telemetrystore,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/promote/implpromote"),
}
}
func (m *module) ListPromotedPaths(ctx context.Context, filters promotetypes.ListPromotedPathsFilters) ([]promotetypes.PromotePath, error) {
response := make([]promotetypes.PromotePath, 0)
for _, target := range promotetypes.Targets() {
if !filters.MatchesTarget(target) {
continue
}
paths, err := m.listPromotedPaths(ctx, target)
if err != nil {
return nil, err
}
for _, path := range paths {
if filters.MatchesPath(path) {
response = append(response, path)
}
}
}
return response, nil
}
func (m *module) listPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error) {
promotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry)
if err != nil {
return nil, err
}
response := make([]promotetypes.PromotePath, 0, len(promotedPaths))
for path := range promotedPaths {
response = append(response, promotetypes.PromotePath{
Signal: target.Entry.Signal.StringValue(),
Context: target.Entry.FieldContext.StringValue(),
Path: path,
Promote: true,
})
}
indexes, err := m.metadataStore.ListJSONIndexes(ctx, target.JSONIndexLookup())
if err != nil {
return nil, err
}
// aggr keys are full sub-column paths: index.BaseColumn carries the
// column prefix and index.Name the bare path.
aggr := map[string][]promotetypes.WrappedIndex{}
for _, index := range indexes {
fullPath := index.BaseColumn + index.Name
aggr[fullPath] = append(aggr[fullPath], promotetypes.WrappedIndex{
FieldDataType: index.FieldDataType,
Type: index.IndexType,
Granularity: index.Granularity,
})
}
for i := range response {
fullPath := target.PromotedColumnPrefix() + response[i].Path
if indexes, ok := aggr[fullPath]; ok {
response[i].Indexes = indexes
delete(aggr, fullPath)
}
}
for fullPath, indexes := range aggr {
path := strings.TrimPrefix(fullPath, target.BaseColumnPrefix())
path = strings.TrimPrefix(path, target.PromotedColumnPrefix())
response = append(response, promotetypes.PromotePath{
Signal: target.Entry.Signal.StringValue(),
Context: target.Entry.FieldContext.StringValue(),
Path: path,
Indexes: indexes,
})
}
return response, nil
}
func (m *module) IndexMaterializedPaths(ctx context.Context, req promotetypes.PostableIndexMaterializedPaths) (*promotetypes.IndexMaterializedPathsResult, error) {
target, err := req.Target()
if err != nil {
return nil, err
}
keys, err := m.metadataStore.GetMaterializedKeys(ctx, target.Entry.Signal)
if err != nil {
return nil, err
}
paths := []*promotetypes.PromotePath{}
names := []string{}
for _, key := range keys {
if key.FieldContext != target.Entry.FieldContext || len(key.Indexes) == 0 {
continue
}
path, err := promotetypes.NewMaterializedPromotePath(target, key)
if err != nil {
m.settings.Logger().WarnContext(ctx, "skipping materialized path that cannot be indexed", slog.String("path", key.Name), errors.Attr(err))
continue
}
paths = append(paths, path)
names = append(names, path.Path)
}
result := &promotetypes.IndexMaterializedPathsResult{Indexed: []*promotetypes.PromotePath{}, Skipped: []*promotetypes.PromotePath{}}
if len(paths) > 0 {
existing, err := m.metadataStore.ListJSONIndexes(ctx, target.JSONIndexLookup(), names...)
if err != nil {
return nil, err
}
existingByPath := map[string][]promotetypes.WrappedIndex{}
for _, index := range existing {
existingByPath[index.Name] = append(existingByPath[index.Name], promotetypes.WrappedIndex{FieldDataType: index.FieldDataType, Type: index.IndexType, Granularity: index.Granularity})
}
for _, path := range paths {
if indexes, ok := existingByPath[path.Path]; ok {
path.Indexes = indexes
result.Skipped = append(result.Skipped, path)
continue
}
result.Indexed = append(result.Indexed, path)
}
}
if !req.DryRun && len(result.Indexed) > 0 {
if err := m.promotePaths(ctx, target, result.Indexed...); err != nil {
return nil, err
}
}
indexed := make([]string, 0, len(result.Indexed))
for _, path := range result.Indexed {
indexed = append(indexed, path.Path)
}
skipped := make([]string, 0, len(result.Skipped))
for _, path := range result.Skipped {
skipped = append(skipped, path.Path)
}
m.settings.Logger().InfoContext(ctx, "indexed materialized paths", slog.String("signal", req.Signal), slog.Bool("dry_run", req.DryRun), slog.Int("indexed_count", len(indexed)), slog.Any("indexed", indexed), slog.Int("skipped_count", len(skipped)), slog.Any("skipped", skipped))
return result, nil
}
func (m *module) PromotePaths(ctx context.Context, paths ...*promotetypes.PromotePath) error {
byTarget := map[promotetypes.Target][]*promotetypes.PromotePath{}
targets := []promotetypes.Target{}
for _, path := range paths {
target, err := path.Target()
if err != nil {
return err
}
if _, ok := byTarget[target]; !ok {
targets = append(targets, target)
}
byTarget[target] = append(byTarget[target], path)
}
for _, target := range targets {
if err := m.promotePaths(ctx, target, byTarget[target]...); err != nil {
return err
}
}
return nil
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, target promotetypes.Target, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: target.Entry.Signal.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
if len(indexes) == 0 {
return nil
}
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: target.DBName,
Table: target.LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
}
}
return nil
}
func (m *module) promotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error {
pathsStr := []string{}
for _, path := range paths {
pathsStr = append(pathsStr, path.Path)
}
existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry, pathsStr...)
if err != nil {
return err
}
var toInsert []string
indexes := []schemamigrator.Index{}
for _, it := range paths {
if it.Promote {
if _, promoted := existingPromotedPaths[it.Path]; !promoted {
toInsert = append(toInsert, it.Path)
}
}
if len(it.Indexes) > 0 {
parentColumn := target.BaseColumn
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = target.PromotedColumn()
}
for _, index := range it.Indexes {
typeIndex, err := index.SkipIndexType()
if err != nil {
return err
}
indexes = append(indexes, schemamigrator.Index{
Name: schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, strings.ToLower(index.JSONDataType.StringValue()), typeIndex),
Expression: target.IndexExpression(parentColumn, it.Path, index.JSONDataType.StringValue()),
Type: index.Type,
Granularity: index.Granularity,
})
}
}
}
if len(toInsert) > 0 {
err := m.metadataStore.PromotePaths(ctx, target.Entry, toInsert...)
if err != nil {
return err
}
}
if len(indexes) > 0 {
if err := m.createIndexes(ctx, target, indexes); err != nil {
return err
}
}
return nil
}