SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
package usage
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"time"
"log/slog"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/go-co-op/gocron"
"github.com/google/uuid"
"github.com/SigNoz/signoz/ee/query-service/model"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/query-service/utils/encryption"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/zeus"
)
const (
MaxRetries = 3
RetryInterval = 5 * time.Second
stateUnlocked uint32 = 0
stateLocked uint32 = 1
)
var (
locker = stateUnlocked
)
type Manager struct {
clickhouseConn clickhouse.Conn
licenseService licensing.Licensing
scheduler *gocron.Scheduler
zeus zeus.Zeus
orgGetter organization.Getter
flagger flagger.Flagger
}
func New(licenseService licensing.Licensing, clickhouseConn clickhouse.Conn, zeus zeus.Zeus, orgGetter organization.Getter, flagger flagger.Flagger) (*Manager, error) {
m := &Manager{
clickhouseConn: clickhouseConn,
licenseService: licenseService,
scheduler: gocron.NewScheduler(time.UTC).Every(1).Day().At("00:00"), // send usage every at 00:00 UTC
zeus: zeus,
orgGetter: orgGetter,
flagger: flagger,
}
return m, nil
}
// start loads collects and exports any exported snapshot and starts the exporter
func (lm *Manager) Start(ctx context.Context) error {
// compares the locker and stateUnlocked if both are same lock is applied else returns error
if !atomic.CompareAndSwapUint32(&locker, stateUnlocked, stateLocked) {
return fmt.Errorf("usage exporter is locked")
}
// upload usage once when starting the service
_, err := lm.scheduler.Do(func() { lm.UploadUsage(ctx) })
if err != nil {
return err
}
lm.UploadUsage(ctx)
lm.scheduler.StartAsync()
return nil
}
func (lm *Manager) UploadUsage(ctx context.Context) {
organizations, err := lm.orgGetter.ListByOwnedKeyRange(ctx)
if err != nil {
slog.ErrorContext(ctx, "failed to get organizations", errors.Attr(err))
return
}
for _, organization := range organizations {
// check if license is present or not
license, err := lm.licenseService.GetActive(ctx, organization.ID)
if err != nil {
slog.ErrorContext(ctx, "failed to get active license", errors.Attr(err))
return
}
if license == nil {
// we will not start the usage reporting if license is not present.
slog.InfoContext(ctx, "no license present, skipping usage reporting")
return
}
usages := []model.UsageDB{}
// get usage from clickhouse
dbs := []string{"signoz_logs", "signoz_traces", "signoz_metrics"}
query := `
SELECT tenant, collector_id, exporter_id, timestamp, data
FROM %s.distributed_usage as u1
GLOBAL INNER JOIN
(SELECT
tenant, collector_id, exporter_id, MAX(timestamp) as ts
FROM %s.distributed_usage as u2
where timestamp >= $1
GROUP BY tenant, collector_id, exporter_id
) as t1
ON
u1.tenant = t1.tenant AND u1.collector_id = t1.collector_id AND u1.exporter_id = t1.exporter_id and u1.timestamp = t1.ts
order by timestamp
`
for _, db := range dbs {
dbusages := []model.UsageDB{}
err := lm.clickhouseConn.Select(ctx, &dbusages, fmt.Sprintf(query, db, db), time.Now().Add(-(24 * time.Hour)))
if err != nil && !strings.Contains(err.Error(), "doesn't exist") {
slog.ErrorContext(ctx, "failed to get usage from clickhouse", errors.Attr(err))
return
}
for _, u := range dbusages {
u.Type = db
usages = append(usages, u)
}
}
if len(usages) <= 0 {
slog.InfoContext(ctx, "no snapshots to upload, skipping")
return
}
slog.InfoContext(ctx, "uploading usage data")
usagesPayload := []model.Usage{}
for _, usage := range usages {
usageDataBytes, err := encryption.Decrypt([]byte(usage.ExporterID[:32]), []byte(usage.Data))
if err != nil {
slog.ErrorContext(ctx, "error while decrypting usage data", errors.Attr(err))
return
}
usageData := model.Usage{}
err = json.Unmarshal(usageDataBytes, &usageData)
if err != nil {
slog.ErrorContext(ctx, "error while unmarshalling usage data", errors.Attr(err))
return
}
usageData.CollectorID = usage.CollectorID
usageData.ExporterID = usage.ExporterID
usageData.Type = usage.Type
usageData.Tenant = "default"
usageData.OrgName = "default"
usageData.TenantId = "default"
usagesPayload = append(usagesPayload, usageData)
}
key, _ := uuid.Parse(license.Key)
payload := model.UsagePayload{
LicenseKey: key,
Usage: usagesPayload,
}
body, errv2 := json.Marshal(payload)
if errv2 != nil {
slog.ErrorContext(ctx, "error while marshalling usage payload", errors.Attr(errv2))
return
}
evalCtx := featuretypes.NewFlaggerEvaluationContext(organization.ID)
useZeus := lm.flagger.BooleanOrEmpty(ctx, flagger.FeaturePutMetersInZeus, evalCtx)
if useZeus {
errv2 = lm.zeus.PutMetersV2(ctx, payload.LicenseKey.String(), body)
} else {
errv2 = lm.zeus.PutMeters(ctx, payload.LicenseKey.String(), body)
}
if errv2 != nil {
slog.ErrorContext(ctx, "failed to upload usage", errors.Attr(errv2))
// not returning error here since it is captured in the failed count
return
}
}
}
func (lm *Manager) Stop(ctx context.Context) {
lm.scheduler.Stop()
slog.InfoContext(ctx, "sending usage data before shutting down")
// send usage before shutting down
lm.UploadUsage(ctx)
atomic.StoreUint32(&locker, stateUnlocked)
}