Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(metrics): expose product health metrics (#21211)
p0ny4748a7fe44c5
diff --git a/backend/component/productmetrics/README.md b/backend/component/productmetrics/README.mdnew file mode 100644index 0000000000..87b56da879--- /dev/null+++ b/backend/component/productmetrics/README.md@@ -0,0 +1,45 @@+# Product metrics++Bytebase exposes product-health metrics from the server's `/metrics` endpoint.+They complement Go runtime and HTTP metrics with installation state and+process-local background-work health.++## Installation-state metrics++Installation-state metrics are read from shared metadata on every scrape so all+replicas report equivalent values. If the metadata or configured license cannot+be read and verified, the scrape fails instead of returning a partial state.++| Metric | Type | Description |+| --- | --- | --- |+| `bytebase_license_expiry_timestamp_seconds` | Gauge | License expiry as a Unix timestamp. `+Inf` means the Free plan or a perpetual license. A valid expired license retains its expiry timestamp while using Free-plan limits. |+| `bytebase_license_seats_used` | Gauge | Distinct end users currently occupying license seats. |+| `bytebase_license_seats_limit` | Gauge | Effective seat limit. `+Inf` means unlimited. |+| `bytebase_license_instances_used` | Gauge | Non-deleted registered instances. |+| `bytebase_license_instances_limit` | Gauge | Effective registered-instance limit. `+Inf` means unlimited. |++## Process-local metrics++Process metrics record work performed by the serving Bytebase process. In a+multi-replica deployment, query each replica or aggregate across replicas as+appropriate. Canceled shutdown work is not recorded as a failure.++| Metric | Type | Labels | Description |+| --- | --- | --- | --- |+| `bytebase_instance_sync_duration_seconds` | Native histogram | `result`, `bytebase_instance_id`, resource labels | Duration of completed instance synchronization attempts. |+| `bytebase_database_sync_total` | Counter | `result`, `bytebase_instance_id`, `database`, resource labels | Completed database schema synchronization attempts. |+| `bytebase_runner_run_duration_seconds` | Native histogram | `runner`, `result` | Duration of synchronous background runner cycles. |++`result` is `success`, `failure`, or `skipped`. Runner values are+`plan_check`, `task_pending`, `task_dispatch`, `instance_sync`, and+`database_sync`.++Resource-label names are exposed with a `label_` prefix. Characters outside+ASCII letters, digits, and underscores are replaced with underscores, and name+collisions receive deterministic `_conflictN` suffixes. The label schema is the+union of labels observed by that process; a resource without a union label emits+an empty value for it.++The duration metrics use the Prometheus client library's native histogram+implementation. Their bucket factor is at most 1.1, populated buckets are capped+at 100, and bucket state is not reset more often than once per hour.diff --git a/backend/component/productmetrics/productmetrics.go b/backend/component/productmetrics/productmetrics.gonew file mode 100644index 0000000000..f04f5a01f3--- /dev/null+++ b/backend/component/productmetrics/productmetrics.go@@ -0,0 +1,385 @@+// Package productmetrics collects Bytebase product-health metrics.+package productmetrics++import (+ "context"+ "math"+ "slices"+ "strconv"+ "strings"+ "sync"+ "time"++ "github.com/pkg/errors"+ "github.com/prometheus/client_golang/prometheus"+ metricpb "github.com/prometheus/client_model/go"++ "github.com/bytebase/bytebase/backend/enterprise"+ "github.com/bytebase/bytebase/backend/store"+)++const metricsCollectionTimeout = 5 * time.Second++// Runner identifies a synchronous background control-loop cycle.+type Runner string++const (+ RunnerPlanCheck Runner = "plan_check"+ RunnerTaskPending Runner = "task_pending"+ RunnerTaskDispatch Runner = "task_dispatch"+ RunnerInstanceSync Runner = "instance_sync"+ RunnerDatabaseSync Runner = "database_sync"+)++// RunnerResult is the terminal result of a runner cycle.+type RunnerResult string++const (+ ResultSuccess RunnerResult = "success"+ ResultFailure RunnerResult = "failure"+ ResultSkipped RunnerResult = "skipped"+)++type databaseIdentity struct {+ instanceID string+ databaseName string+}++type nativeHistogram struct {+ prometheus.Histogram+}++type dynamicHistogramMetric struct {+ desc *prometheus.Desc+ histogram prometheus.Histogram+ labelValues []string+}++type stateStore interface {+ GetWorkspaceID(context.Context) (string, error)+ CountSeatOccupyingUsers(context.Context, string) (int, error)+ CountActiveInstances(context.Context, string) (int, error)+}++type licenseStateProvider interface {+ GetVerifiedStateUncached(context.Context, string) (*enterprise.VerifiedState, error)+}++// ProductMetrics owns process-local event metrics and scrape-time product+// state. It deliberately is an unchecked Collector: dynamic resource label+// names are derived at collection time from the latest observed resource+// labels, which Prometheus's static descriptor registration cannot express.+type ProductMetrics struct {+ stores stateStore+ licenseService licenseStateProvider++ mu sync.RWMutex++ instanceLabels map[string]map[string]string+ databaseLabels map[string]map[string]string+ instanceHistograms map[string]map[RunnerResult]*nativeHistogram+ databaseCounters map[string]map[RunnerResult]uint64+ databases map[string]databaseIdentity+ runnerEvents map[Runner]map[RunnerResult]*nativeHistogram+}++// New creates a process-local product metrics collector.+func New(stores *store.Store, licenseService *enterprise.LicenseService) *ProductMetrics {+ metrics := &ProductMetrics{+ instanceLabels: make(map[string]map[string]string),+ databaseLabels: make(map[string]map[string]string),+ instanceHistograms: make(map[string]map[RunnerResult]*nativeHistogram),+ databaseCounters: make(map[string]map[RunnerResult]uint64),+ databases: make(map[string]databaseIdentity),+ runnerEvents: make(map[Runner]map[RunnerResult]*nativeHistogram),+ }+ if stores != nil {+ metrics.stores = stores+ }+ if licenseService != nil {+ metrics.licenseService = licenseService+ }+ return metrics+}++// Describe intentionally sends no descriptors. Resource labels can change+// between observations, so their descriptor is defined only at collection.+func (*ProductMetrics) Describe(_ chan<- *prometheus.Desc) {+ // Dynamic descriptors are emitted only from Collect.+}++// RecordInstanceSync records one complete instance-sync attempt. Cancellation+// is a shutdown path rather than a failed synchronization.+func (m *ProductMetrics) RecordInstanceSync(instance *store.InstanceMessage, duration time.Duration, err error) {+ if errors.Is(err, context.Canceled) {+ return+ }+ identity := ""+ labels := map[string]string(nil)+ if instance != nil {+ identity = instance.ResourceID+ labels = instance.Metadata.GetLabels()+ }+ m.mu.Lock()+ defer m.mu.Unlock()+ m.instanceLabels[identity] = cloneLabels(labels)+ result := ResultSuccess+ if err != nil {+ result = ResultFailure+ }+ byResult := m.instanceHistograms[identity]+ if byResult == nil {+ byResult = make(map[RunnerResult]*nativeHistogram)+ m.instanceHistograms[identity] = byResult+ }+ h := byResult[result]+ if h == nil {+ h = newNativeHistogram()+ byResult[result] = h+ }+ h.observe(duration.Seconds())+}++// RecordDatabaseSync records one database-schema synchronization attempt.+func (m *ProductMetrics) RecordDatabaseSync(database *store.DatabaseMessage, err error) {+ if errors.Is(err, context.Canceled) {+ return+ }+ identity := ""+ labels := map[string]string(nil)+ if database != nil {+ identity = database.InstanceID + "\x00" + database.DatabaseName+ labels = database.Metadata.GetLabels()+ }+ m.mu.Lock()+ defer m.mu.Unlock()+ m.databaseLabels[identity] = cloneLabels(labels)+ result := ResultSuccess+ if err != nil {+ result = ResultFailure+ }+ instanceID, databaseName := "", ""+ if database != nil {+ instanceID, databaseName = database.InstanceID, database.DatabaseName+ }+ m.databases[identity] = databaseIdentity{instanceID: instanceID, databaseName: databaseName}+ byResult := m.databaseCounters[identity]+ if byResult == nil {+ byResult = make(map[RunnerResult]uint64)+ m.databaseCounters[identity] = byResult+ }+ byResult[result]+++}++// RecordRunnerRun records a completed synchronous runner cycle.+func (m *ProductMetrics) RecordRunnerRun(runner Runner, result RunnerResult, duration time.Duration) {+ m.mu.Lock()+ defer m.mu.Unlock()+ byResult := m.runnerEvents[runner]+ if byResult == nil {+ byResult = make(map[RunnerResult]*nativeHistogram)+ m.runnerEvents[runner] = byResult+ }+ h := byResult[result]+ if h == nil {+ h = newNativeHistogram()+ byResult[result] = h+ }+ h.observe(duration.Seconds())+}++// Collect emits the process-local event state and current shared installation+// state. A state error invalidates the scrape rather than publishing a partial+// state snapshot.+func (m *ProductMetrics) Collect(ch chan<- prometheus.Metric) {+ m.collectEvents(ch)++ ctx, cancel := context.WithTimeout(context.Background(), metricsCollectionTimeout)+ defer cancel()+ if m.stores == nil || m.licenseService == nil {+ fail(ch, errors.New("product metrics dependencies are not configured"))+ return+ }+ workspaceID, err := m.stores.GetWorkspaceID(ctx)+ if err != nil {+ fail(ch, errors.Wrap(err, "failed to get workspace ID"))+ return+ }+ usedSeats, usedInstances := 0, 0+ if workspaceID != "" {+ usedSeats, err = m.stores.CountSeatOccupyingUsers(ctx, workspaceID)+ if err != nil {+ fail(ch, errors.Wrapf(err, "failed to count seat-occupying users for workspace %q", workspaceID))+ return+ }+ usedInstances, err = m.stores.CountActiveInstances(ctx, workspaceID)+ if err != nil {+ fail(ch, errors.Wrapf(err, "failed to count active instances for workspace %q", workspaceID))+ return+ }+ }+ state, err := m.licenseService.GetVerifiedStateUncached(ctx, workspaceID)+ if err != nil {+ fail(ch, errors.Wrapf(err, "failed to read verified license state for workspace %q", workspaceID))+ return+ }+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_seats_used", "Number of distinct end users occupying license seats.", nil, nil), prometheus.GaugeValue, float64(usedSeats))+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_seats_limit", "Effective seat limit. +Inf means unlimited.", nil, nil), prometheus.GaugeValue, prometheusValue(state.UserLimit))+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_instances_used", "Number of non-deleted registered instances.", nil, nil), prometheus.GaugeValue, float64(usedInstances))+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_instances_limit", "Effective registered instance limit. +Inf means unlimited.", nil, nil), prometheus.GaugeValue, prometheusValue(state.InstanceLimit))+ expiry := math.Inf(1)+ if state.ExpiresAt != nil {+ expiry = float64(state.ExpiresAt.Unix())+ }+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_expiry_timestamp_seconds", "License expiry Unix timestamp. +Inf means free or perpetual.", nil, nil), prometheus.GaugeValue, expiry)+}++func (m *ProductMetrics) collectEvents(ch chan<- prometheus.Metric) {+ m.mu.RLock()+ defer m.mu.RUnlock()++ instanceSchema := dynamicSchema(m.instanceLabels)+ instanceDesc := prometheus.NewDesc("bytebase_instance_sync_duration_seconds", "Duration of instance synchronization attempts.", append([]string{"result", "bytebase_instance_id"}, instanceSchema.keys...), nil)+ for identity, byResult := range m.instanceHistograms {+ for result, histogram := range byResult {+ labels := append([]string{string(result), identity}, instanceSchema.values(m.instanceLabels[identity])...)+ ch <- histogram.metric(instanceDesc, labels...)+ }+ }++ databaseSchema := dynamicSchema(m.databaseLabels)+ databaseDesc := prometheus.NewDesc("bytebase_database_sync_total", "Number of database schema synchronization attempts.", append([]string{"result", "bytebase_instance_id", "database"}, databaseSchema.keys...), nil)+ for identity, byResult := range m.databaseCounters {+ database := m.databases[identity]+ for result, count := range byResult {+ labels := append([]string{string(result), database.instanceID, database.databaseName}, databaseSchema.values(m.databaseLabels[identity])...)+ ch <- prometheus.MustNewConstMetric(databaseDesc, prometheus.CounterValue, float64(count), labels...)+ }+ }++ runnerDesc := prometheus.NewDesc("bytebase_runner_run_duration_seconds", "Duration of synchronous runner control-loop cycles.", []string{"runner", "result"}, nil)+ for runner, byResult := range m.runnerEvents {+ for result, histogram := range byResult {+ ch <- histogram.metric(runnerDesc, string(runner), string(result))+ }+ }+}++func newNativeHistogram() *nativeHistogram {+ return &nativeHistogram{Histogram: prometheus.NewHistogram(prometheus.HistogramOpts{+ Name: "bytebase_internal_product_duration_seconds",+ Help: "Internal accumulator for dynamically labeled product duration metrics.",+ NativeHistogramBucketFactor: 1.1,+ NativeHistogramMaxBucketNumber: 100,+ NativeHistogramMinResetDuration: time.Hour,+ })}+}++func (h *nativeHistogram) observe(value float64) {+ h.Observe(value)+}++func (h *nativeHistogram) metric(desc *prometheus.Desc, labels ...string) prometheus.Metric {+ return &dynamicHistogramMetric{desc: desc, histogram: h.Histogram, labelValues: labels}+}++func (m *dynamicHistogramMetric) Desc() *prometheus.Desc {+ return m.desc+}++func (m *dynamicHistogramMetric) Write(out *metricpb.Metric) error {+ if err := m.histogram.Write(out); err != nil {+ return errors.Wrap(err, "failed to write native histogram")+ }+ out.Label = prometheus.MakeLabelPairs(m.desc, m.labelValues)+ return nil+}++type labelSchema struct {+ keys []string+ mapped map[string]string+}++func dynamicSchema(labelsByIdentity map[string]map[string]string) labelSchema {+ original := make(map[string]struct{})+ for _, labels := range labelsByIdentity {+ for key := range labels {+ original[key] = struct{}{}+ }+ }+ originalKeys := make([]string, 0, len(original))+ for key := range original {+ originalKeys = append(originalKeys, key)+ }+ slices.Sort(originalKeys)+ schema := labelSchema{keys: make([]string, 0, len(originalKeys)), mapped: make(map[string]string, len(originalKeys))}+ used := make(map[string]struct{})+ for _, key := range originalKeys {+ baseName := "label_" + sanitizeLabelKey(key)+ name := baseName+ for suffix := 1; ; suffix++ {+ if _, exists := used[name]; !exists {+ break+ }+ name = baseName + "_conflict" + strconv.Itoa(suffix)+ }+ used[name] = struct{}{}+ schema.keys = append(schema.keys, name)+ schema.mapped[key] = name+ }+ return schema+}++func (s labelSchema) values(labels map[string]string) []string {+ values := make([]string, len(s.keys))+ if len(labels) == 0 {+ return values+ }+ for key, value := range labels {+ name := s.mapped[key]+ for j, metricKey := range s.keys {+ if metricKey == name {+ values[j] = value+ break+ }+ }+ }+ return values+}++func sanitizeLabelKey(key string) string {+ var b strings.Builder+ for _, r := range key {+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' {+ _, _ = b.WriteRune(r)+ } else {+ _ = b.WriteByte('_')+ }+ }+ if b.Len() == 0 {+ return "_"+ }+ return b.String()+}++func cloneLabels(labels map[string]string) map[string]string {+ if len(labels) == 0 {+ return nil+ }+ clone := make(map[string]string, len(labels))+ for key, value := range labels {+ clone[key] = value+ }+ return clone+}++func prometheusValue(value int) float64 {+ if value == math.MaxInt {+ return math.Inf(1)+ }+ return float64(value)+}++func fail(ch chan<- prometheus.Metric, err error) {+ ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)+}diff --git a/backend/component/productmetrics/productmetrics_test.go b/backend/component/productmetrics/productmetrics_test.gonew file mode 100644index 0000000000..bba9d82830--- /dev/null+++ b/backend/component/productmetrics/productmetrics_test.go@@ -0,0 +1,282 @@+package productmetrics++import (+ "context"+ "errors"+ "math"+ "sync"+ "testing"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ metricpb "github.com/prometheus/client_model/go"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/enterprise"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++type fakeStateStore struct {+ workspace string+ seats int+ instances int+ err error+}++func (s *fakeStateStore) GetWorkspaceID(context.Context) (string, error) {+ return s.workspace, s.err+}++func (s *fakeStateStore) CountSeatOccupyingUsers(context.Context, string) (int, error) {+ return s.seats, s.err+}++func (s *fakeStateStore) CountActiveInstances(context.Context, string) (int, error) {+ return s.instances, s.err+}++type fakeLicenseStateProvider struct {+ state *enterprise.VerifiedState+ err error+}++func (p *fakeLicenseStateProvider) GetVerifiedStateUncached(context.Context, string) (*enterprise.VerifiedState, error) {+ return p.state, p.err+}++func TestRegistryInstallationStateVariants(t *testing.T) {+ finiteExpiry := time.Now().Add(24 * time.Hour).Truncate(time.Second)+ expiredAt := time.Now().Add(-time.Hour).Truncate(time.Second)+ tests := []struct {+ name string+ state *enterprise.VerifiedState+ wantExpiry float64+ wantLimit float64+ }{+ {name: "free", state: &enterprise.VerifiedState{UserLimit: 20, InstanceLimit: 10}, wantExpiry: math.Inf(1), wantLimit: 10},+ {name: "perpetual", state: &enterprise.VerifiedState{UserLimit: math.MaxInt, InstanceLimit: math.MaxInt}, wantExpiry: math.Inf(1), wantLimit: math.Inf(1)},+ {name: "finite", state: &enterprise.VerifiedState{ExpiresAt: &finiteExpiry, UserLimit: 100, InstanceLimit: 25}, wantExpiry: float64(finiteExpiry.Unix()), wantLimit: 25},+ {name: "expired", state: &enterprise.VerifiedState{ExpiresAt: &expiredAt, UserLimit: 20, InstanceLimit: 10}, wantExpiry: float64(expiredAt.Unix()), wantLimit: 10},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ metrics := New(nil, nil)+ metrics.stores = &fakeStateStore{workspace: "ws", seats: 3, instances: 2}+ metrics.licenseService = &fakeLicenseStateProvider{state: test.state}+ registry := prometheus.NewRegistry()+ registry.MustRegister(metrics)+ families, err := registry.Gather()+ require.NoError(t, err)+ require.Equal(t, 2.0, familyGaugeValue(t, families, "bytebase_license_instances_used"))+ require.Equal(t, test.wantLimit, familyGaugeValue(t, families, "bytebase_license_instances_limit"))+ require.Equal(t, test.wantExpiry, familyGaugeValue(t, families, "bytebase_license_expiry_timestamp_seconds"))+ })+ }+}++func TestRegistryStateFailureAndEventLocality(t *testing.T) {+ newCollector := func() (*ProductMetrics, *prometheus.Registry) {+ metrics := New(nil, nil)+ metrics.stores = &fakeStateStore{workspace: "ws"}+ metrics.licenseService = &fakeLicenseStateProvider{state: &enterprise.VerifiedState{UserLimit: 20, InstanceLimit: 10}}+ registry := prometheus.NewRegistry()+ registry.MustRegister(metrics)+ return metrics, registry+ }+ metrics1, registry1 := newCollector()+ _, registry2 := newCollector()+ metrics1.RecordRunnerRun(RunnerPlanCheck, ResultSuccess, time.Second)+ families1, err := registry1.Gather()+ require.NoError(t, err)+ families2, err := registry2.Gather()+ require.NoError(t, err)+ require.Equal(t, 1.0, familyHistogramCount(families1, "bytebase_runner_run_duration_seconds"))+ require.Zero(t, familyHistogramCount(families2, "bytebase_runner_run_duration_seconds"))++ metrics1.licenseService = &fakeLicenseStateProvider{err: errors.New("malformed license")}+ _, err = registry1.Gather()+ require.ErrorContains(t, err, "malformed license")+}++func TestResourceMetricLabelsAndCumulativeMutation(t *testing.T) {+ m := New(nil, nil)+ instance := &store.InstanceMessage{+ ResourceID: "prod",+ Metadata: &storepb.Instance{Labels: map[string]string{+ "env.prod": "before",+ "env-prod": "dash",+ "env_prod_conflict1": "reserved",+ "\u4e2d\u6587": "unicode",+ }},+ }+ m.RecordInstanceSync(instance, time.Second, nil)+ collidingMetric := findMetric(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds")+ require.Equal(t, "before", metricLabel(collidingMetric, "label_env_prod_conflict1"))+ require.Equal(t, "dash", metricLabel(collidingMetric, "label_env_prod"))+ require.Equal(t, "reserved", metricLabel(collidingMetric, "label_env_prod_conflict1_conflict1"))+ require.Equal(t, "unicode", metricLabel(collidingMetric, "label___"))+ m.RecordInstanceSync(&store.InstanceMessage{ResourceID: "staging", Metadata: &storepb.Instance{Labels: map[string]string{"owner": "platform"}}}, time.Second, nil)+ unionMetric := findMetricWithLabels(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds", map[string]string{"bytebase_instance_id": "staging"})+ require.Equal(t, "", metricLabel(unionMetric, "label_env_prod_conflict1"))+ require.Equal(t, "platform", metricLabel(unionMetric, "label_owner"))+ instance.Metadata.Labels = map[string]string{"env.prod": "after"}+ m.RecordInstanceSync(instance, 2*time.Second, nil)++ metrics := eventMetrics(m)+ metric := findMetricWithLabels(t, metrics, "bytebase_instance_sync_duration_seconds", map[string]string{"bytebase_instance_id": "prod"})+ require.Equal(t, uint64(2), metric.GetHistogram().GetSampleCount())+ require.Equal(t, 3.0, metric.GetHistogram().GetSampleSum())+ require.Equal(t, int32(3), metric.GetHistogram().GetSchema())+ require.Empty(t, metric.GetHistogram().GetBucket())+ require.Equal(t, "prod", metricLabel(metric, "bytebase_instance_id"))+ require.Equal(t, "after", metricLabel(metric, "label_env_prod"))+}++func TestDatabaseAndRunnerMetrics(t *testing.T) {+ m := New(nil, nil)+ database := &store.DatabaseMessage{+ InstanceID: "prod",+ DatabaseName: "app",+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{"region": "us"}},+ }+ m.RecordDatabaseSync(database, nil)+ m.RecordDatabaseSync(database, context.DeadlineExceeded)+ m.RecordDatabaseSync(database, context.Canceled)+ m.RecordRunnerRun(RunnerPlanCheck, ResultSuccess, time.Second)++ metrics := eventMetrics(m)+ databaseMetric := findMetricWithLabels(t, metrics, "bytebase_database_sync_total", map[string]string{"result": "success"})+ require.Equal(t, "prod", metricLabel(databaseMetric, "bytebase_instance_id"))+ require.Equal(t, "app", metricLabel(databaseMetric, "database"))+ require.Equal(t, "success", metricLabel(databaseMetric, "result"))+ require.Equal(t, 1.0, databaseMetric.GetCounter().GetValue())+ runnerMetric := findMetric(t, metrics, "bytebase_runner_run_duration_seconds")+ require.Equal(t, int32(3), runnerMetric.GetHistogram().GetSchema())+ require.Empty(t, runnerMetric.GetHistogram().GetBucket())+}++func TestNativeHistogramBucketCap(t *testing.T) {+ histogram := newNativeHistogram()+ for i := range 150 {+ histogram.observe(math.Pow(2, float64(i)))+ }+ metric := &metricpb.Metric{}+ require.NoError(t, histogram.Write(metric))+ nativeBucketCount := 0+ for _, span := range metric.GetHistogram().GetPositiveSpan() {+ nativeBucketCount += int(span.GetLength())+ }+ require.LessOrEqual(t, nativeBucketCount, 100)+ require.Less(t, metric.GetHistogram().GetSchema(), int32(3))+}++func TestNativeHistogramBucketAssignment(t *testing.T) {+ tests := []struct {+ value float64+ wantIndex int32+ }{+ {value: 0.95, wantIndex: 0},+ {value: 1, wantIndex: 0},+ {value: 1.05, wantIndex: 1},+ {value: 2, wantIndex: 8},+ }+ for _, test := range tests {+ histogram := newNativeHistogram()+ histogram.observe(test.value)+ metric := &metricpb.Metric{}+ require.NoError(t, histogram.Write(metric))+ require.Equal(t, int32(3), metric.GetHistogram().GetSchema())+ require.Len(t, metric.GetHistogram().GetPositiveSpan(), 1)+ require.Equal(t, test.wantIndex, metric.GetHistogram().GetPositiveSpan()[0].GetOffset())+ }+}++func TestConcurrentObserveAndCollect(t *testing.T) {+ m := New(nil, nil)+ instance := &store.InstanceMessage{ResourceID: "prod", Metadata: &storepb.Instance{}}+ var wg sync.WaitGroup+ for range 10 {+ wg.Go(func() {+ for range 100 {+ m.RecordInstanceSync(instance, time.Millisecond, nil)+ eventMetrics(m)+ }+ })+ }+ wg.Wait()+ metric := findMetric(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds")+ require.Equal(t, uint64(1000), metric.GetHistogram().GetSampleCount())+}++func eventMetrics(m *ProductMetrics) []*metricpb.Metric {+ ch := make(chan prometheus.Metric, 1000)+ m.collectEvents(ch)+ metrics := make([]*metricpb.Metric, 0, len(ch))+ for range len(ch) {+ metric := <-ch+ written := &metricpb.Metric{}+ if metric.Write(written) == nil {+ metrics = append(metrics, written)+ }+ }+ return metrics+}++func findMetric(t *testing.T, metrics []*metricpb.Metric, name string) *metricpb.Metric {+ return findMetricWithLabels(t, metrics, name, nil)+}++func findMetricWithLabels(t *testing.T, metrics []*metricpb.Metric, name string, labels map[string]string) *metricpb.Metric {+ t.Helper()+ for _, metric := range metrics {+ matches := true+ for name, value := range labels {+ matches = matches && metricLabel(metric, name) == value+ }+ if !matches {+ continue+ }+ if metric.GetHistogram() != nil && name == "bytebase_instance_sync_duration_seconds" && metricLabel(metric, "bytebase_instance_id") != "" {+ return metric+ }+ if metric.GetCounter() != nil && name == "bytebase_database_sync_total" {+ return metric+ }+ if metric.GetHistogram() != nil && name == "bytebase_runner_run_duration_seconds" && metricLabel(metric, "runner") != "" {+ return metric+ }+ }+ t.Fatalf("metric %q not found", name)+ return nil+}++func metricLabel(metric *metricpb.Metric, name string) string {+ for _, pair := range metric.GetLabel() {+ if pair.GetName() == name {+ return pair.GetValue()+ }+ }+ return ""+}++func familyGaugeValue(t *testing.T, families []*metricpb.MetricFamily, name string) float64 {+ t.Helper()+ for _, family := range families {+ if family.GetName() == name {+ require.Len(t, family.Metric, 1)+ return family.Metric[0].GetGauge().GetValue()+ }+ }+ t.Fatalf("metric family %q not found", name)+ return 0+}++func familyHistogramCount(families []*metricpb.MetricFamily, name string) float64 {+ for _, family := range families {+ if family.GetName() == name && len(family.Metric) > 0 {+ return float64(family.Metric[0].GetHistogram().GetSampleCount())+ }+ }+ return 0+}diff --git a/backend/enterprise/license.go b/backend/enterprise/license.goindex f3f643faa4..985106eb35 100644--- a/backend/enterprise/license.go+++ b/backend/enterprise/license.go@@ -328,27 +328,11 @@ func (s *LicenseService) GetUserLimit(ctx context.Context, workspaceID string) i// metadata. A metadata read failure is returned as an error instead of silently// falling back to the Free plan.func (s *LicenseService) GetUserLimitUncached(ctx context.Context, workspaceID string) (int, error) {- setting, err := s.store.GetSystemSettingUncached(ctx, workspaceID)+ state, err := s.GetVerifiedStateUncached(ctx, workspaceID)if err != nil {- return 0, errors.Wrap(err, "failed to get system setting")- }- if setting == nil {- return userLimitFromSubscription(defaultFreeSubscription), nil- }- if setting.License == "" {- return userLimitFromSubscription(defaultFreeSubscription), nil- }- subscription := defaultFreeSubscription- parsedSubscription, parseErr := s.parseLicense(setting.License, workspaceID)- if parseErr == nil {- subscription = parsedSubscription+ return 0, err}- // An expired or malformed license has the effective Free plan, mirroring- // LoadSubscription.- if isExpired(subscription) {- subscription = &v1pb.Subscription{Plan: v1pb.PlanType_FREE}- }- return userLimitFromSubscription(subscription), nil+ return state.UserLimit, nil}func userLimitFromSubscription(subscription *v1pb.Subscription) int {@@ -383,6 +367,62 @@ func (s *LicenseService) GetInstanceLimit(ctx context.Context, workspaceID strinreturn limit}+// VerifiedState is the effective license state read directly from shared+// metadata. Expired, correctly signed licenses use Free limits while retaining+// their expiration timestamp; a malformed configured license is an error.+type VerifiedState struct {+ ExpiresAt *time.Time+ UserLimit int+ InstanceLimit int+}++// GetVerifiedStateUncached bypasses all setting and subscription caches. It is+// used by state-metric collection so every replica observes the same verified+// license state for a scrape.+func (s *LicenseService) GetVerifiedStateUncached(ctx context.Context, workspaceID string) (*VerifiedState, error) {+ setting, err := s.store.GetSystemSettingUncached(ctx, workspaceID)+ if err != nil {+ return nil, errors.Wrap(err, "failed to get system setting")+ }+ if setting == nil || setting.License == "" {+ return verifiedState(defaultFreeSubscription), nil+ }+ subscription, err := s.parseLicenseUncheckedExpiry(setting.License, workspaceID)+ if err != nil {+ return nil, errors.Wrap(err, "failed to verify configured license")+ }+ state := verifiedState(subscription)+ if isExpired(subscription) {+ free := verifiedState(defaultFreeSubscription)+ free.ExpiresAt = state.ExpiresAt+ return free, nil+ }+ return state, nil+}++func verifiedState(subscription *v1pb.Subscription) *VerifiedState {+ state := &VerifiedState{+ UserLimit: userLimitFromSubscription(subscription),+ InstanceLimit: instanceLimitFromSubscription(subscription),+ }+ if subscription.GetExpiresTime() != nil {+ expiresAt := subscription.GetExpiresTime().AsTime()+ state.ExpiresAt = &expiresAt+ }+ return state+}++func instanceLimitFromSubscription(subscription *v1pb.Subscription) int {+ if subscription.Instances > 0 {+ return int(subscription.Instances)+ }+ limit := instanceLimitValues[subscription.Plan]+ if limit == -1 {+ return math.MaxInt+ }+ return limit+}+// GetActivatedInstanceLimit returns the activated instance limit for the current subscription.func (s *LicenseService) GetActivatedInstanceLimit(ctx context.Context, workspaceID string) int {limit := s.LoadSubscription(ctx, workspaceID).ActiveInstances@@ -557,8 +597,22 @@ func (s *LicenseService) CheckReplicaLimit(ctx context.Context) error {}func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscription, error) {+ subscription, err := s.parseLicenseUncheckedExpiry(license, workspaceID)+ if err != nil {+ return nil, err+ }+ if isExpired(subscription) {+ return nil, common.Errorf(common.Invalid, "license has expired at %v", subscription.ExpiresTime.AsTime())+ }+ return subscription, nil+}++// parseLicenseUncheckedExpiry verifies a license's signature and all static+// claims while deliberately retaining an otherwise valid expired token. The+// caller chooses whether expiry is an error or an effective Free fallback.+func (s *LicenseService) parseLicenseUncheckedExpiry(license, workspaceID string) (*v1pb.Subscription, error) {claim := &Claims{}- token, err := jwt.ParseWithClaims(license, claim, func(token *jwt.Token) (any, error) {+ token, err := jwt.NewParser(jwt.WithoutClaimsValidation()).ParseWithClaims(license, claim, func(token *jwt.Token) (any, error) {if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {return nil, common.Errorf(common.Invalid, "unexpected signing method: %v", token.Header["alg"])}@@ -577,6 +631,9 @@ func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscrif !token.Valid {return nil, common.Errorf(common.Invalid, "invalid token")}+ if claim.NotBefore != nil && time.Now().Before(claim.NotBefore.Time) {+ return nil, common.Errorf(common.Invalid, "license is not valid before %v", claim.NotBefore.Time)+ }if s.config.Issuer != claim.Issuer {return nil, common.Errorf(common.Invalid, "iss is not valid, expect %s but found '%v'", s.config.Issuer, claim.Issuer)@@ -611,10 +668,6 @@ func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscrif claim.ExpiresAt != nil && !claim.ExpiresAt.IsZero() {expiresTime = timestamppb.New(claim.ExpiresAt.Time)}- if expiresTime != nil && expiresTime.AsTime().Before(time.Now()) {- return nil, errors.Errorf("license has expired at %v", expiresTime.AsTime())- }-return &v1pb.Subscription{ActiveInstances: int32(claim.ActiveInstances),Instances: int32(claim.Instances),diff --git a/backend/enterprise/license_test.go b/backend/enterprise/license_test.goindex 2c95abdf41..c37c3e4095 100644--- a/backend/enterprise/license_test.go+++ b/backend/enterprise/license_test.go@@ -137,6 +137,28 @@ func TestCreateLicenseUsesEqualInstanceClaims(t *testing.T) {}}+func TestParseLicenseExpiredIsInvalid(t *testing.T) {+ privateKey, err := rsa.GenerateKey(rand.Reader, 2048)+ require.NoError(t, err)+ service := &LicenseService{+ config: &Config{+ PublicKey: &privateKey.PublicKey,+ PrivateKey: privateKey,+ Version: keyID,+ Issuer: issuer,+ Audience: audience,+ },+ }+ license, err := service.CreateLicense(&LicenseParams{+ Plan: v1pb.PlanType_TEAM.String(),+ ExpiresAt: time.Now().Add(-time.Hour),+ })+ require.NoError(t, err)++ _, err = service.parseLicense(license, "test-workspace")+ require.Equal(t, common.Invalid, common.ErrorCode(err))+}+func TestGetUserLimitUncached(t *testing.T) {ctx := context.Background()container := testcontainer.GetTestPgContainer(ctx, t)@@ -198,6 +220,11 @@ func TestGetUserLimitUncached(t *testing.T) {limit, err = licenseService.GetUserLimitUncached(ctx, "ws-a")require.NoError(t, err)require.Equal(t, 100, limit)+ state, err := licenseService.GetVerifiedStateUncached(ctx, "ws-a")+ require.NoError(t, err)+ require.Nil(t, state.ExpiresAt)+ require.Equal(t, 100, state.UserLimit)+ require.Equal(t, math.MaxInt, state.InstanceLimit)// Legacy Enterprise license without a seat claim: unlimited.storeLicense(t, &LicenseParams{@@ -208,11 +235,21 @@ func TestGetUserLimitUncached(t *testing.T) {require.Equal(t, math.MaxInt, limit)// Expired Enterprise license: falls back to the Free plan limit.+ expiresAt := time.Now().Add(-time.Hour)storeLicense(t, &LicenseParams{Plan: v1pb.PlanType_ENTERPRISE.String(), Seats: 100, WorkspaceID: "ws-a",- ExpiresAt: time.Now().Add(-time.Hour),+ ExpiresAt: expiresAt,})limit, err = licenseService.GetUserLimitUncached(ctx, "ws-a")require.NoError(t, err)require.Equal(t, userLimitValues[v1pb.PlanType_FREE], limit)+ state, err = licenseService.GetVerifiedStateUncached(ctx, "ws-a")+ require.NoError(t, err)+ require.WithinDuration(t, expiresAt, *state.ExpiresAt, time.Second)+ require.Equal(t, userLimitValues[v1pb.PlanType_FREE], state.UserLimit)+ require.Equal(t, instanceLimitValues[v1pb.PlanType_FREE], state.InstanceLimit)++ require.NoError(t, s.UpdateLicense(ctx, "ws-a", "not-a-license"))+ _, err = licenseService.GetVerifiedStateUncached(ctx, "ws-a")+ require.Error(t, err)}diff --git a/backend/runner/plancheck/scheduler.go b/backend/runner/plancheck/scheduler.goindex 7ec9009c5f..34313c49a0 100644--- a/backend/runner/plancheck/scheduler.go+++ b/backend/runner/plancheck/scheduler.go@@ -12,6 +12,7 @@ import ("github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/bus"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store"@@ -22,12 +23,13 @@ const ()// NewScheduler creates a new plan check scheduler.-func NewScheduler(s *store.Store, bus *bus.Bus, executor *CombinedExecutor, licenseService *enterprise.LicenseService) *Scheduler {+func NewScheduler(s *store.Store, bus *bus.Bus, executor *CombinedExecutor, licenseService *enterprise.LicenseService, productMetrics *productmetrics.ProductMetrics) *Scheduler {return &Scheduler{store: s,bus: bus,executor: executor,licenseService: licenseService,+ productMetrics: productMetrics,}}@@ -37,6 +39,7 @@ type Scheduler struct {bus *bus.Busexecutor *CombinedExecutorlicenseService *enterprise.LicenseService+ productMetrics *productmetrics.ProductMetrics}// Run runs the scheduler.@@ -66,6 +69,8 @@ func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {}func (s *Scheduler) runOnce(ctx context.Context) {+ startedAt := time.Now()+ result := productmetrics.ResultFailuredefer func() {if r := recover(); r != nil {err, ok := r.(error)@@ -74,6 +79,9 @@ func (s *Scheduler) runOnce(ctx context.Context) {}slog.Error("Plan check scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))}+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerPlanCheck, result, time.Since(startedAt))+ }}()claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)@@ -85,6 +93,7 @@ func (s *Scheduler) runOnce(ctx context.Context) {for _, c := range claimed {go s.runPlanCheckRun(ctx, c.ProjectID, c.UID, c.PlanUID, c.ApprovalInputVersion)}+ result = productmetrics.ResultSuccess}func (s *Scheduler) runPlanCheckRun(ctx context.Context, projectID string, uid int64, planUID int64, approvalInputVersion int64) {diff --git a/backend/runner/plancheck/scheduler_test.go b/backend/runner/plancheck/scheduler_test.goindex ee37da775f..26067b336b 100644--- a/backend/runner/plancheck/scheduler_test.go+++ b/backend/runner/plancheck/scheduler_test.go@@ -4,13 +4,55 @@ import ("context""testing"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/component/bus"+ "github.com/bytebase/bytebase/backend/component/productmetrics"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store")+func TestRunOnceRecordsEmptySuccess(t *testing.T) {+ ctx := context.Background()+ stores := setupPlancheckStore(ctx, t)+ b, err := bus.New()+ require.NoError(t, err)+ metrics := productmetrics.New(nil, nil)++ NewScheduler(stores, b, nil, nil, metrics).runOnce(ctx)+ NewScheduler(nil, nil, nil, nil, metrics).runOnce(ctx)++ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerPlanCheck, productmetrics.ResultFailure))+}++func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {+ t.Helper()+ ch := make(chan prometheus.Metric, 16)+ go func() {+ metrics.Collect(ch)+ close(ch)+ }()++ var count uint64+ for metric := range ch {+ var dtoMetric dto.Metric+ if metric.Write(&dtoMetric) != nil {+ continue+ }+ labels := map[string]string{}+ for _, label := range dtoMetric.GetLabel() {+ labels[label.GetName()] = label.GetValue()+ }+ if labels["runner"] == string(runner) && labels["result"] == string(result) {+ count += dtoMetric.GetHistogram().GetSampleCount()+ }+ }+ return count+}+func TestMarkPlanCheckRunDoneSkipsDraftIssue(t *testing.T) {ctx := context.Background()stores := setupPlancheckStore(ctx, t)@@ -43,7 +85,7 @@ func TestMarkPlanCheckRunDoneSkipsDraftIssue(t *testing.T) {require.NoError(t, err)require.Len(t, claimed, 1)- scheduler := NewScheduler(stores, b, nil, nil)+ scheduler := NewScheduler(stores, b, nil, nil, nil)scheduler.markPlanCheckRunDone(ctx, "project-a", claimed[0].UID, plan.UID, 1, nil)run, err := stores.GetPlanCheckRun(ctx, "project-a", plan.UID)@@ -81,7 +123,7 @@ func TestRunPlanCheckRunStopsAfterProjectArchive(t *testing.T) {Workspace: "default",Delete: &archived,}))- scheduler := NewScheduler(stores, b, nil, nil)+ scheduler := NewScheduler(stores, b, nil, nil, nil)scheduler.runPlanCheckRun(ctx, "project-a", claimed[0].UID, plan.UID, 1)run, err := stores.GetPlanCheckRun(ctx, "project-a", plan.UID)diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.goindex 94ee4ddd79..982cbb5ba1 100644--- a/backend/runner/schemasync/syncer.go+++ b/backend/runner/schemasync/syncer.go@@ -9,6 +9,7 @@ import ("slices""strings""sync"+ "sync/atomic""time""github.com/pkg/errors"@@ -20,6 +21,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/dbfactory"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db"@@ -37,11 +39,12 @@ const ()// NewSyncer creates a schema syncer.-func NewSyncer(stores *store.Store, dbFactory *dbfactory.DBFactory, licenseService *enterprise.LicenseService) *Syncer {+func NewSyncer(stores *store.Store, dbFactory *dbfactory.DBFactory, licenseService *enterprise.LicenseService, productMetrics *productmetrics.ProductMetrics) *Syncer {return &Syncer{store: stores,dbFactory: dbFactory,licenseService: licenseService,+ productMetrics: productMetrics,}}@@ -52,6 +55,7 @@ type Syncer struct {store *store.StoredbFactory *dbfactory.DBFactorylicenseService *enterprise.LicenseService+ productMetrics *productmetrics.ProductMetricsdatabaseSyncMap sync.Map // map[string]*store.DatabaseMessage}@@ -89,56 +93,10 @@ func (s *Syncer) Run(ctx context.Context, wg *sync.WaitGroup) {slog.Warn("Database sync checker skipped due to HA license restriction", log.BBError(err))continue}- instances, err := s.store.ListAllInstances(ctx, false)- if err != nil {- slog.Error("Failed to list instance", log.BBError(err))- return- }- instanceMap := make(map[string]*store.InstanceMessage)- for _, instance := range instances {- instanceMap[instance.ResourceID] = instance+ if err := s.syncQueuedDatabases(ctx); err != nil {+ slog.Error("Failed to run database sync cycle", log.BBError(err))+ continue}- dbwp := pool.New().WithMaxGoroutines(MaximumOutstanding)- s.databaseSyncMap.Range(func(key, value any) bool {- database, ok := value.(*store.DatabaseMessage)- if !ok {- return true- }-- s.databaseSyncMap.Delete(key)- instance, ok := instanceMap[database.InstanceID]- if !ok || !s.canScheduleDatabaseSync(ctx, instance, database) {- return true- }- dbwp.Go(func() {- slog.Debug("Sync database schema", slog.String("instance", database.InstanceID), slog.String("database", database.DatabaseName))- if err := s.SyncDatabaseSchema(ctx, database); err != nil {- slog.Debug("Failed to sync database schema",- slog.String("instance", database.InstanceID),- slog.String("databaseName", database.DatabaseName),- log.BBError(err))- // Save sync error to database metadata- if _, updateErr := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- MetadataUpdates: []func(*storepb.DatabaseMetadata){- func(md *storepb.DatabaseMetadata) {- md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_FAILED- md.SyncError = err.Error()- md.LastSyncTime = timestamppb.Now()- },- },- }); updateErr != nil {- slog.Error("Failed to update database sync error",- slog.String("instance", database.InstanceID),- slog.String("database", database.DatabaseName),- log.BBError(updateErr))- }- }- })- return true- })- dbwp.Wait()case <-ctx.Done(): // if cancel() executereturn}@@ -148,6 +106,8 @@ func (s *Syncer) Run(ctx context.Context, wg *sync.WaitGroup) {}func (s *Syncer) trySyncAll(ctx context.Context) {+ startedAt := time.Now()+ result := productmetrics.ResultFailuredefer func() {if r := recover(); r != nil {err, ok := r.(error)@@ -156,6 +116,9 @@ func (s *Syncer) trySyncAll(ctx context.Context) {}slog.Error("Instance syncer PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))}+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerInstanceSync, result, time.Since(startedAt))+ }}()lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeySchemaSyncer)@@ -165,15 +128,18 @@ func (s *Syncer) trySyncAll(ctx context.Context) {}if !acquired {slog.Debug("Schema syncer advisory lock held by another replica, skipping")+ result = productmetrics.ResultSkippedreturn}defer func() {if err := lock.Release(); err != nil {+ result = productmetrics.ResultFailureslog.Error("Failed to release schema syncer advisory lock", log.BBError(err))}}()wp := pool.New().WithMaxGoroutines(MaximumOutstanding)+ var syncFailed atomic.Boolinstances, err := s.store.ListAllInstances(ctx, false)if err != nil {slog.Error("Failed to retrieve instances", log.BBError(err))@@ -182,7 +148,15 @@ func (s *Syncer) trySyncAll(ctx context.Context) {now := time.Now()for _, instance := range instances {instance := instance- if !s.canScheduleInstanceSync(ctx, instance) {+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)+ if err != nil {+ syncFailed.Store(true)+ slog.Error("Failed to determine whether instance sync can be scheduled",+ slog.String("instance", instance.ResourceID),+ log.BBError(err))+ continue+ }+ if !canSchedule {continue}interval := s.getOrDefaultSyncInterval(ctx, instance)@@ -200,6 +174,7 @@ func (s *Syncer) trySyncAll(ctx context.Context) {wp.Go(func() {slog.Debug("Sync instance schema", slog.String("instance", instance.ResourceID))if _, _, _, err := s.SyncInstance(ctx, instance); err != nil {+ syncFailed.Store(true)slog.Debug("Failed to sync instance",slog.String("instance", instance.ResourceID),slog.String("error", err.Error()))@@ -227,7 +202,16 @@ func (s *Syncer) trySyncAll(ctx context.Context) {if !ok {continue}- if !s.canScheduleDatabaseSync(ctx, instance, database) {+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)+ if err != nil {+ syncFailed.Store(true)+ slog.Error("Failed to determine whether database sync can be scheduled",+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ log.BBError(err))+ continue+ }+ if !canSchedule {continue}// The database inherits the sync interval from the instance.@@ -245,33 +229,144 @@ func (s *Syncer) trySyncAll(ctx context.Context) {s.databaseSyncMap.Store(database.String(), database)}+ if !syncFailed.Load() {+ result = productmetrics.ResultSuccess+ }}-func (s *Syncer) canScheduleInstanceSync(ctx context.Context, instance *store.InstanceMessage) bool {- return instance.ProjectID == nil || s.isProjectActive(ctx, *instance.ProjectID)+func (s *Syncer) syncQueuedDatabases(ctx context.Context) (retErr error) {+ startedAt := time.Now()+ result := productmetrics.ResultFailure+ defer func() {+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerDatabaseSync, result, time.Since(startedAt))+ }+ }()+ defer func() {+ if r := recover(); r != nil {+ panicErr, ok := r.(error)+ if !ok {+ panicErr = errors.Errorf("%v", r)+ }+ retErr = errors.Wrap(panicErr, "database sync checker panic")+ slog.Error("Database sync checker PANIC RECOVER", log.BBError(retErr), log.BBStack("panic-stack"))+ }+ }()++ instances, err := s.store.ListAllInstances(ctx, false)+ if err != nil {+ return errors.Wrap(err, "failed to list instances")+ }+ instanceMap := make(map[string]*store.InstanceMessage)+ for _, instance := range instances {+ instanceMap[instance.ResourceID] = instance+ }+ dbwp := pool.New().WithMaxGoroutines(MaximumOutstanding)+ var syncFailed atomic.Bool+ s.databaseSyncMap.Range(func(key, value any) bool {+ database, ok := value.(*store.DatabaseMessage)+ if !ok {+ return true+ }++ instance, ok := instanceMap[database.InstanceID]+ if !ok {+ s.databaseSyncMap.Delete(key)+ return true+ }+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)+ if err != nil {+ syncFailed.Store(true)+ slog.Error("Failed to determine whether queued database sync can be scheduled",+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ log.BBError(err))+ return true+ }+ s.databaseSyncMap.Delete(key)+ if !canSchedule {+ return true+ }+ dbwp.Go(func() {+ slog.Debug("Sync database schema", slog.String("instance", database.InstanceID), slog.String("database", database.DatabaseName))+ if err := s.SyncDatabaseSchema(ctx, database); err != nil {+ syncFailed.Store(true)+ slog.Debug("Failed to sync database schema",+ slog.String("instance", database.InstanceID),+ slog.String("databaseName", database.DatabaseName),+ log.BBError(err))+ // Save sync error to database metadata+ if _, updateErr := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ MetadataUpdates: []func(*storepb.DatabaseMetadata){+ func(md *storepb.DatabaseMetadata) {+ md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_FAILED+ md.SyncError = err.Error()+ md.LastSyncTime = timestamppb.Now()+ },+ },+ }); updateErr != nil {+ syncFailed.Store(true)+ slog.Error("Failed to update database sync error",+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ log.BBError(updateErr))+ }+ }+ })+ return true+ })+ dbwp.Wait()++ if syncFailed.Load() {+ return errors.New("one or more queued database synchronizations failed")+ }+ result = productmetrics.ResultSuccess+ return nil}-func (s *Syncer) canScheduleDatabaseSync(ctx context.Context, instance *store.InstanceMessage, database *store.DatabaseMessage) bool {- return !database.Deleted && s.canScheduleInstanceSync(ctx, instance) && s.isProjectActive(ctx, database.ProjectID)+func (s *Syncer) canScheduleInstanceSync(ctx context.Context, instance *store.InstanceMessage) (bool, error) {+ if instance.ProjectID == nil {+ return true, nil+ }+ return s.isProjectActive(ctx, *instance.ProjectID)}-func (s *Syncer) isProjectActive(ctx context.Context, projectID string) bool {+func (s *Syncer) canScheduleDatabaseSync(ctx context.Context, instance *store.InstanceMessage, database *store.DatabaseMessage) (bool, error) {+ if database.Deleted {+ return false, nil+ }+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)+ if err != nil || !canSchedule {+ return canSchedule, err+ }+ return s.isProjectActive(ctx, database.ProjectID)+}++func (s *Syncer) isProjectActive(ctx context.Context, projectID string) (bool, error) {project, err := s.store.GetProjectByResourceID(ctx, projectID)if err != nil {- slog.Error("failed to get project for schema sync", slog.String("project", projectID), log.BBError(err))- return false+ return false, errors.Wrapf(err, "failed to get project %q for schema sync", projectID)}if project == nil {slog.Warn("project not found for schema sync", slog.String("project", projectID))- return false+ return false, nil}- return !project.Deleted+ return !project.Deleted, nil}func (s *Syncer) SyncAllDatabases(ctx context.Context, instance *store.InstanceMessage) {find := &store.FindDatabaseMessage{}if instance != nil {- if !s.canScheduleInstanceSync(ctx, instance) {+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)+ if err != nil {+ slog.Error("Failed to determine whether instance sync can be scheduled",+ slog.String("instance", instance.ResourceID),+ log.BBError(err))+ return+ }+ if !canSchedule {return}find.InstanceID = &instance.ResourceID@@ -284,9 +379,22 @@ func (s *Syncer) SyncAllDatabases(ctx context.Context, instance *store.InstanceM}for _, database := range databases {- if database.Deleted || instance != nil && !s.canScheduleDatabaseSync(ctx, instance, database) {+ if database.Deleted {continue}+ if instance != nil {+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)+ if err != nil {+ slog.Error("Failed to determine whether database sync can be scheduled",+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ log.BBError(err))+ continue+ }+ if !canSchedule {+ continue+ }+ }s.databaseSyncMap.Store(database.String(), database)}}@@ -329,7 +437,20 @@ func (s *Syncer) GetInstanceMeta(ctx context.Context, instance *store.InstanceMe}// SyncInstance syncs the schema for all databases in an instance.-func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessage) (*store.InstanceMessage, []*storepb.DatabaseSchemaMetadata, []*store.DatabaseMessage, error) {+func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessage) (updatedInstance *store.InstanceMessage, allDatabases []*storepb.DatabaseSchemaMetadata, newDatabases []*store.DatabaseMessage, retErr error) {+ startedAt := time.Now()+ defer func() {+ panicValue := recover()+ if panicValue != nil {+ retErr = errorFromPanic(panicValue, "instance sync panic")+ }+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordInstanceSync(instance, time.Since(startedAt), retErr)+ }+ if panicValue != nil {+ panic(panicValue)+ }+ }()instanceMeta, err := s.GetInstanceMeta(ctx, instance)if err != nil {return nil, nil, nil, err@@ -347,7 +468,7 @@ func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessaif instanceMeta.Version != instance.Metadata.GetVersion() {metadata.Version = instanceMeta.Version}- updatedInstance, err := s.store.UpdateInstance(ctx, updateInstance)+ updatedInstance, err = s.store.UpdateInstance(ctx, updateInstance)if err != nil {return nil, nil, nil, err}@@ -355,7 +476,6 @@ func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessaif err != nil {return nil, nil, nil, errors.Wrapf(err, "failed to sync database for instance: %s. Failed to find database list", instance.ResourceID)}- var newDatabases []*store.DatabaseMessagevar filteredDatabaseMetadatas []*storepb.DatabaseSchemaMetadatavar databaseProjectID stringif instance.ProjectID != nil {@@ -410,6 +530,18 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databif database == nil {return "", errors.New("cannot sync nil database")}+ defer func() {+ panicValue := recover()+ if panicValue != nil {+ retErr = errorFromPanic(panicValue, "database schema sync panic")+ }+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordDatabaseSync(database, retErr)+ }+ if panicValue != nil {+ panic(panicValue)+ }+ }()instance, err := s.store.GetInstanceByResourceID(ctx, database.InstanceID)if err != nil {return "", errors.Wrapf(err, "failed to get instance %q", database.InstanceID)@@ -500,6 +632,14 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databreturn "", nil}+func errorFromPanic(value any, message string) error {+ err, ok := value.(error)+ if !ok {+ err = errors.Errorf("%v", value)+ }+ return errors.Wrap(err, message)+}+// SyncDatabaseSchemaToHistory will sync the schema for a database and create a sync history record.func (s *Syncer) SyncDatabaseSchemaToHistory(ctx context.Context, database *store.DatabaseMessage) (string, error) {return s.doSyncDatabaseSchema(ctx, database, true)diff --git a/backend/runner/schemasync/syncer_test.go b/backend/runner/schemasync/syncer_test.goindex ae3dcd7543..a0712e0800 100644--- a/backend/runner/schemasync/syncer_test.go+++ b/backend/runner/schemasync/syncer_test.go@@ -6,16 +6,141 @@ import ("testing""time"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go""github.com/stretchr/testify/require""google.golang.org/protobuf/types/known/durationpb""google.golang.org/protobuf/types/known/timestamppb""github.com/bytebase/bytebase/backend/common/testcontainer"+ "github.com/bytebase/bytebase/backend/component/productmetrics"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/migrator""github.com/bytebase/bytebase/backend/store")+func TestSyncCyclesRecordEmptySuccess(t *testing.T) {+ ctx := context.Background()+ stores := setupSyncerStore(ctx, t)+ metrics := productmetrics.New(nil, nil)+ syncer := NewSyncer(stores, nil, nil, metrics)++ syncer.trySyncAll(ctx)+ require.NoError(t, syncer.syncQueuedDatabases(ctx))+ lock, acquired, err := store.TryAdvisoryLock(ctx, stores.GetDB(), store.AdvisoryLockKeySchemaSyncer)+ require.NoError(t, err)+ require.True(t, acquired)+ syncer.trySyncAll(ctx)+ require.NoError(t, lock.Release())+ panicSyncer := &Syncer{productMetrics: metrics}+ panicSyncer.trySyncAll(ctx)+ require.Error(t, panicSyncer.syncQueuedDatabases(ctx))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSuccess))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultSuccess))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultFailure))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultFailure))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSkipped))+}++func TestSyncInstancePanicRecordsFailure(t *testing.T) {+ metrics := productmetrics.New(nil, nil)+ instance := &store.InstanceMessage{+ ResourceID: "instance-a",+ Metadata: &storepb.Instance{+ DataSources: []*storepb.DataSource{{Type: storepb.DataSourceType_ADMIN}},+ },+ }+ syncer := &Syncer{productMetrics: metrics}++ require.Panics(t, func() {+ _, _, _, _ = syncer.SyncInstance(context.Background(), instance)+ })+ require.Equal(t, uint64(1), syncMetricCount(t, metrics, map[string]string{+ "result": string(productmetrics.ResultFailure),+ "bytebase_instance_id": instance.ResourceID,+ }))+}++func TestSyncDatabaseSchemaPanicRecordsFailure(t *testing.T) {+ metrics := productmetrics.New(nil, nil)+ database := &store.DatabaseMessage{InstanceID: "instance-a", DatabaseName: "db-a"}+ syncer := &Syncer{productMetrics: metrics}++ require.Panics(t, func() {+ _ = syncer.SyncDatabaseSchema(context.Background(), database)+ })+ require.Equal(t, uint64(1), syncMetricCount(t, metrics, map[string]string{+ "result": string(productmetrics.ResultFailure),+ "bytebase_instance_id": database.InstanceID,+ "database": database.DatabaseName,+ }))+}++func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {+ t.Helper()+ ch := make(chan prometheus.Metric, 16)+ go func() {+ metrics.Collect(ch)+ close(ch)+ }()++ var count uint64+ for metric := range ch {+ var dtoMetric dto.Metric+ if metric.Write(&dtoMetric) != nil {+ continue+ }+ labels := map[string]string{}+ for _, label := range dtoMetric.GetLabel() {+ labels[label.GetName()] = label.GetValue()+ }+ if labels["runner"] == string(runner) && labels["result"] == string(result) {+ count += dtoMetric.GetHistogram().GetSampleCount()+ }+ }+ return count+}++func syncMetricCount(t *testing.T, metrics *productmetrics.ProductMetrics, expectedLabels map[string]string) uint64 {+ t.Helper()+ ch := make(chan prometheus.Metric, 16)+ go func() {+ metrics.Collect(ch)+ close(ch)+ }()++ for metric := range ch {+ var dtoMetric dto.Metric+ if metric.Write(&dtoMetric) != nil {+ continue+ }+ labels := map[string]string{}+ for _, label := range dtoMetric.GetLabel() {+ labels[label.GetName()] = label.GetValue()+ }+ if len(labels) != len(expectedLabels) {+ continue+ }+ matches := true+ for name, value := range expectedLabels {+ if labels[name] != value {+ matches = false+ break+ }+ }+ if !matches {+ continue+ }+ if dtoMetric.GetHistogram() != nil {+ return dtoMetric.GetHistogram().GetSampleCount()+ }+ if dtoMetric.GetCounter() != nil {+ return uint64(dtoMetric.GetCounter().GetValue())+ }+ }+ return 0+}+// TestSyncDatabaseSchemaNilDatabase pins the guard that prevents a nil// *store.DatabaseMessage from panicking the syncer. Callers may receive// (nil, nil) from store.GetDatabase when the referenced database is not@@ -85,7 +210,7 @@ func TestTrySyncAllSkipsDatabasesInArchivedProjects(t *testing.T) {})require.NoError(t, err)- syncer := NewSyncer(s, nil, nil)+ syncer := NewSyncer(s, nil, nil, nil)syncer.trySyncAll(ctx)queued := 0@@ -113,7 +238,7 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {ctx := context.Background()s := setupSyncerStore(ctx, t)projectID := "project-a"- syncer := NewSyncer(s, nil, nil)+ syncer := NewSyncer(s, nil, nil, nil)instance := &store.InstanceMessage{ProjectID: &projectID}archived := true@@ -122,7 +247,9 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {Workspace: "default",Delete: &archived,}))- require.False(t, syncer.canScheduleInstanceSync(ctx, instance))+ canSchedule, err := syncer.canScheduleInstanceSync(ctx, instance)+ require.NoError(t, err)+ require.False(t, canSchedule)archived = falserequire.NoError(t, s.UpdateProjects(ctx, &store.UpdateProjectMessage{@@ -130,7 +257,73 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {Workspace: "default",Delete: &archived,}))- require.True(t, syncer.canScheduleInstanceSync(ctx, instance))+ canSchedule, err = syncer.canScheduleInstanceSync(ctx, instance)+ require.NoError(t, err)+ require.True(t, canSchedule)+}++func TestTrySyncAllProjectLookupFailureRecordsFailure(t *testing.T) {+ ctx := context.Background()+ stores := setupSyncerStore(ctx, t)+ projectID := "project-a"+ _, err := stores.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: "instance-a",+ Workspace: "default",+ ProjectID: &projectID,+ Metadata: &storepb.Instance{+ Activation: true,+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)+ _, err = stores.GetDB().ExecContext(ctx, "ALTER TABLE project RENAME TO unavailable_project")+ require.NoError(t, err)++ metrics := productmetrics.New(nil, nil)+ syncer := NewSyncer(stores, nil, nil, metrics)+ syncer.trySyncAll(ctx)++ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultFailure))+ require.Zero(t, runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSuccess))+}++func TestSyncQueuedDatabasesProjectLookupFailureRecordsFailure(t *testing.T) {+ ctx := context.Background()+ stores := setupSyncerStore(ctx, t)+ _, err := stores.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: "instance-a",+ Workspace: "default",+ Metadata: &storepb.Instance{+ Activation: true,+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)+ database, err := stores.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "project-a",+ InstanceID: "instance-a",+ DatabaseName: "app",+ Metadata: &storepb.DatabaseMetadata{},+ })+ require.NoError(t, err)++ metrics := productmetrics.New(nil, nil)+ syncer := NewSyncer(stores, nil, nil, metrics)+ syncer.SyncDatabaseAsync(database)+ _, err = stores.GetDB().ExecContext(ctx, "ALTER TABLE project RENAME TO unavailable_project")+ require.NoError(t, err)++ require.Error(t, syncer.syncQueuedDatabases(ctx))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultFailure))+ require.Zero(t, runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultSuccess))+ queued := 0+ syncer.databaseSyncMap.Range(func(_, _ any) bool {+ queued+++ return true+ })+ require.Equal(t, 1, queued)}func setupSyncerStore(ctx context.Context, t *testing.T) *store.Store {diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex deeb30841a..3b476a49a0 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -13,6 +13,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/productmetrics"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store")@@ -56,6 +57,20 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wafunc (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {startedAt := time.Now()+ result := productmetrics.ResultFailure+ defer func() {+ if r := recover(); r != nil {+ panicErr, ok := r.(error)+ if !ok {+ panicErr = errors.Errorf("%v", r)+ }+ err = errors.Wrap(panicErr, "pending task runs scheduler panic")+ slog.Error("Pending task runs scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))+ }+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerTaskPending, result, time.Since(startedAt))+ }+ }()tx, err := s.store.GetDB().BeginTx(ctx, nil)if err != nil {return errors.Wrapf(err, "failed to begin pending scheduler transaction")@@ -75,6 +90,7 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {slog.Debug("Pending scheduler advisory lock held by another replica, skipping",slog.Duration("duration", time.Since(startedAt)),)+ result = productmetrics.ResultSkippedreturn nil}@@ -91,8 +107,12 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {return errors.Wrapf(err, "failed to create scheduling context")}+ var processingErr errorfor _, taskRun := range taskRuns {if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {+ if processingErr == nil {+ processingErr = err+ }slog.Error("failed to schedule pending task run",slog.Int64("taskRunID", taskRun.ID),log.BBError(err),@@ -103,7 +123,11 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {if err := tx.Commit(); err != nil {return errors.Wrapf(err, "failed to commit pending scheduler transaction")}+ if processingErr != nil {+ return errors.Wrap(processingErr, "failed to process one or more pending task runs")+ }+ result = productmetrics.ResultSuccessreturn nil}diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex d92d6b81d4..d49b3b893f 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -12,6 +12,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/bus"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/component/webhook"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store"@@ -50,20 +51,43 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa}}-func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {+func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) (retErr error) {+ startedAt := time.Now()+ result := productmetrics.ResultFailure+ defer func() {+ if r := recover(); r != nil {+ panicErr, ok := r.(error)+ if !ok {+ panicErr = errors.Errorf("%v", r)+ }+ retErr = errors.Wrap(panicErr, "running task runs scheduler panic")+ slog.Error("Running task runs scheduler PANIC RECOVER", log.BBError(retErr), log.BBStack("panic-stack"))+ }+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerTaskDispatch, result, time.Since(startedAt))+ }+ }()// Atomically claim all AVAILABLE task runsclaimed, err := s.store.ClaimAvailableTaskRuns(ctx, s.profile.ReplicaID)if err != nil {return errors.Wrapf(err, "failed to claim available task runs")}+ var processingErr errorfor _, c := range claimed {taskRunCtx := taskRunLogContext(ctx, c.ProjectID, c.TaskRunUID)if err := s.executeTaskRun(taskRunCtx, c.ProjectID, c.TaskRunUID, c.TaskUID); err != nil {+ if processingErr == nil {+ processingErr = err+ }slog.ErrorContext(taskRunCtx, "failed to execute task run", log.BBError(err))}}+ if processingErr != nil {+ return errors.Wrap(processingErr, "failed to process one or more available task runs")+ }+ result = productmetrics.ResultSuccessreturn nil}diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex d7aa088cff..d4996237b0 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -12,6 +12,7 @@ import ("github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/bus""github.com/bytebase/bytebase/backend/component/config"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/component/webhook""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -33,6 +34,7 @@ type Scheduler struct {bus *bus.BuswebhookManager *webhook.ManagerlicenseService *enterprise.LicenseService+ productMetrics *productmetrics.ProductMetricsexecutorMap map[storepb.Task_Type]Executorprofile *config.Profile// haFailSince is when CheckReplicaLimit first started failing.@@ -47,6 +49,7 @@ func NewScheduler(webhookManager *webhook.Manager,licenseService *enterprise.LicenseService,profile *config.Profile,+ productMetrics *productmetrics.ProductMetrics,) *Scheduler {return &Scheduler{store: store,@@ -54,6 +57,7 @@ func NewScheduler(webhookManager: webhookManager,licenseService: licenseService,profile: profile,+ productMetrics: productMetrics,executorMap: map[storepb.Task_Type]Executor{},}}diff --git a/backend/runner/taskrun/scheduler_test.go b/backend/runner/taskrun/scheduler_test.goindex 0ef1592dab..78cd9fbc74 100644--- a/backend/runner/taskrun/scheduler_test.go+++ b/backend/runner/taskrun/scheduler_test.go@@ -4,14 +4,84 @@ import ("context""testing"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/component/bus"+ "github.com/bytebase/bytebase/backend/component/config"+ "github.com/bytebase/bytebase/backend/component/productmetrics"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store")+func TestTaskCyclesRecordEmptySuccess(t *testing.T) {+ ctx := context.Background()+ stores := setupRolloutCreatorStore(ctx, t)+ b, err := bus.New()+ require.NoError(t, err)+ metrics := productmetrics.New(nil, nil)+ scheduler := &Scheduler{+ store: stores,+ bus: b,+ profile: &config.Profile{ReplicaID: "replica-a"},+ productMetrics: metrics,+ }++ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))+ require.NoError(t, scheduler.scheduleRunningTaskRuns(ctx))+ tx, err := stores.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ acquired, err := store.TryAdvisoryXactLock(ctx, tx, store.AdvisoryLockKeyPendingScheduler)+ require.NoError(t, err)+ require.True(t, acquired)+ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))+ require.NoError(t, tx.Rollback())+ panicScheduler := &Scheduler{productMetrics: metrics}+ require.Error(t, panicScheduler.schedulePendingTaskRuns(ctx))+ require.Error(t, panicScheduler.scheduleRunningTaskRuns(ctx))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultSuccess))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultSuccess))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultFailure))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultFailure))+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultSkipped))++ canceledMetrics := productmetrics.New(nil, nil)+ scheduler.productMetrics = canceledMetrics+ canceledContext, cancel := context.WithCancel(ctx)+ cancel()+ require.Error(t, scheduler.schedulePendingTaskRuns(canceledContext))+ require.Error(t, scheduler.scheduleRunningTaskRuns(canceledContext))+ require.Zero(t, runnerRunCount(t, canceledMetrics, productmetrics.RunnerTaskPending, productmetrics.ResultFailure))+ require.Zero(t, runnerRunCount(t, canceledMetrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultFailure))+}++func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {+ t.Helper()+ ch := make(chan prometheus.Metric, 16)+ go func() {+ metrics.Collect(ch)+ close(ch)+ }()++ var count uint64+ for metric := range ch {+ var dtoMetric dto.Metric+ if metric.Write(&dtoMetric) != nil {+ continue+ }+ labels := map[string]string{}+ for _, label := range dtoMetric.GetLabel() {+ labels[label.GetName()] = label.GetValue()+ }+ if labels["runner"] == string(runner) && labels["result"] == string(result) {+ count += dtoMetric.GetHistogram().GetSampleCount()+ }+ }+ return count+}+func TestCompletionWebhookEnvironmentUsesLastEnvironmentOrder(t *testing.T) {tasks := []*store.TaskMessage{{ID: 1, Environment: "dev"},diff --git a/backend/server/echo_routes.go b/backend/server/echo_routes.goindex 84779cda0c..14c9094da8 100644--- a/backend/server/echo_routes.go+++ b/backend/server/echo_routes.go@@ -10,7 +10,6 @@ import ("github.com/labstack/echo/v5/middleware""github.com/pkg/errors""github.com/prometheus/client_golang/prometheus"- "github.com/prometheus/client_golang/prometheus/promhttp"connectcors "connectrpc.com/cors"@@ -22,9 +21,8 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/config"- "github.com/bytebase/bytebase/backend/enterprise"+ "github.com/bytebase/bytebase/backend/component/productmetrics"stripeplugin "github.com/bytebase/bytebase/backend/plugin/stripe"- "github.com/bytebase/bytebase/backend/store")func configureEchoRouters(@@ -34,8 +32,7 @@ func configureEchoRouters(oauth2Service *oauth2.Service,mcpServer *mcp.Server,stripeWebhookHandler *stripeapi.WebhookHandler,- stores *store.Store,- licenseService *enterprise.LicenseService,+ productMetrics *productmetrics.ProductMetrics,profile *config.Profile,) {e.Use(recoverMiddleware)@@ -67,7 +64,7 @@ func configureEchoRouters(registerPprof(e, &profile.RuntimeDebug)- registerMetricsRoute(e, profile, stores, licenseService)+ registerMetricsRoute(e, profile, productMetrics)e.GET("/healthz", func(c *echo.Context) error {return c.String(http.StatusOK, "OK")@@ -97,7 +94,7 @@ func configureEchoRouters(embedFrontend(e)}-func registerMetricsRoute(e *echo.Echo, profile *config.Profile, stores *store.Store, licenseService *enterprise.LicenseService) {+func registerMetricsRoute(e *echo.Echo, profile *config.Profile, productMetrics *productmetrics.ProductMetrics) {if profile.SaaS {return}@@ -125,18 +122,8 @@ func registerMetricsRoute(e *echo.Echo, profile *config.Profile, stores *store.S// Use promhttp directly: pass the local registry as the Registerer// for self-instrumentation; pass the Gatherers fold as the gather// source. Both observability surfaces preserved.- // Product-level license gauges are registered on the local registry only;- // they are computed synchronously on every scrape from fresh shared- // metadata. A collection failure fails the whole scrape (HTTP 500 via- // HTTPErrorOnError) instead of emitting zero, stale, or missing gauges.- registry.MustRegister(newLicenseSeatsCollector(stores, licenseService))- e.GET("/metrics", echo.WrapHandler(promhttp.InstrumentMetricHandler(- registry,- promhttp.HandlerFor(- prometheus.Gatherers{registry, prometheus.DefaultGatherer},- promhttp.HandlerOpts{ErrorHandling: promhttp.HTTPErrorOnError},- ),- )))+ registry.MustRegister(productMetrics)+ e.GET("/metrics", echo.WrapHandler(newMetricsHandler(registry)))}func recoverMiddleware(next echo.HandlerFunc) echo.HandlerFunc {diff --git a/backend/server/echo_routes_test.go b/backend/server/echo_routes_test.goindex b7bba6047b..e38fd37ba9 100644--- a/backend/server/echo_routes_test.go+++ b/backend/server/echo_routes_test.go@@ -9,6 +9,7 @@ import ("github.com/labstack/echo/v5""github.com/bytebase/bytebase/backend/component/config"+ "github.com/bytebase/bytebase/backend/component/productmetrics")func TestSecurityHeadersMiddleware_GA4Sources(t *testing.T) {@@ -93,12 +94,12 @@ func TestMetricsRouteDisabledInSaaS(t *testing.T) {t.Run(tt.name, func(t *testing.T) {e := echo.New()if tt.saas {- registerMetricsRoute(e, &config.Profile{SaaS: true}, nil, nil)+ registerMetricsRoute(e, &config.Profile{SaaS: true}, nil)} else {// Self-host: use a real store and license service so the// product-level license gauges are registered and collected._, st, licenseService := newMetricsTestEcho(t)- registerMetricsRoute(e, &config.Profile{SaaS: false}, st, licenseService)+ registerMetricsRoute(e, &config.Profile{SaaS: false}, productmetrics.New(st, licenseService))}req := httptest.NewRequest(http.MethodGet, "/metrics", nil)diff --git a/backend/server/metrics.go b/backend/server/metrics.goindex 5f363596f7..c356731feb 100644--- a/backend/server/metrics.go+++ b/backend/server/metrics.go@@ -1,93 +1,20 @@package serverimport (- "context"- "math"- "time"+ "net/http"- "github.com/pkg/errors""github.com/prometheus/client_golang/prometheus"-- "github.com/bytebase/bytebase/backend/enterprise"- "github.com/bytebase/bytebase/backend/store"+ "github.com/prometheus/client_golang/prometheus/promhttp")-// metricsCollectionTimeout bounds each collection so a slow metadata read cannot-// hang a scrape indefinitely. Collectors receive no request context.-const metricsCollectionTimeout = 5 * time.Second--// licenseSeatsCollector exposes product-level license seat gauges at /metrics.-// Both values are computed synchronously on every scrape from fresh shared-// metadata (no per-replica caches), so replicas expose equivalent values.-// A failed metadata read fails the whole scrape instead of emitting zero,-// stale, or missing gauges.-type licenseSeatsCollector struct {- stores *store.Store- licenseService *enterprise.LicenseService-- usedSeats *prometheus.Desc- seatLimit *prometheus.Desc-}--func newLicenseSeatsCollector(stores *store.Store, licenseService *enterprise.LicenseService) *licenseSeatsCollector {- return &licenseSeatsCollector{- stores: stores,- licenseService: licenseService,- usedSeats: prometheus.NewDesc(- "bytebase_license_seats_used",- "Number of distinct end users occupying license seats in this Bytebase installation, including pending invites and group members, excluding deleted users and non-user identities.",- nil, nil,+// newMetricsHandler keeps the server-local registry isolated while folding in+// package-global instrumentation that is registered elsewhere in Bytebase.+func newMetricsHandler(registry *prometheus.Registry) http.Handler {+ return promhttp.InstrumentMetricHandler(+ registry,+ promhttp.HandlerFor(+ prometheus.Gatherers{registry, prometheus.DefaultGatherer},+ promhttp.HandlerOpts{ErrorHandling: promhttp.HTTPErrorOnError},),- seatLimit: prometheus.NewDesc(- "bytebase_license_seats_limit",- "Effective seat limit enforced for this Bytebase installation. +Inf means the license has no seat limit.",- nil, nil,- ),- }-}--func (c *licenseSeatsCollector) Describe(ch chan<- *prometheus.Desc) {- ch <- c.usedSeats- ch <- c.seatLimit-}--func (c *licenseSeatsCollector) Collect(ch chan<- prometheus.Metric) {- ctx, cancel := context.WithTimeout(context.Background(), metricsCollectionTimeout)- defer cancel()-- workspaceID, err := c.stores.GetWorkspaceID(ctx)- if err != nil {- failMetricCollection(ch, errors.Wrap(err, "failed to get workspace ID"))- return- }-- used := 0- if workspaceID != "" {- used, err = c.stores.CountSeatOccupyingUsers(ctx, workspaceID)- if err != nil {- failMetricCollection(ch, errors.Wrapf(err, "failed to count seat-occupying users for workspace %q", workspaceID))- return- }- }-- limit, err := c.licenseService.GetUserLimitUncached(ctx, workspaceID)- if err != nil {- failMetricCollection(ch, errors.Wrapf(err, "failed to read the user limit for workspace %q", workspaceID))- return- }-- limitValue := float64(limit)- if limit == math.MaxInt {- limitValue = math.Inf(1)- }-- ch <- prometheus.MustNewConstMetric(c.usedSeats, prometheus.GaugeValue, float64(used))- ch <- prometheus.MustNewConstMetric(c.seatLimit, prometheus.GaugeValue, limitValue)-}--// failMetricCollection emits an invalid metric, which makes Registry.Gather-// fail and therefore fails the whole scrape (HTTP 500 via HTTPErrorOnError)-// instead of reporting zero, stale, or partial values.-func failMetricCollection(ch chan<- prometheus.Metric, err error) {- ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)+ )}diff --git a/backend/server/metrics_test.go b/backend/server/metrics_test.goindex b38ab50b03..59b32dc5fd 100644--- a/backend/server/metrics_test.go+++ b/backend/server/metrics_test.go@@ -3,17 +3,24 @@ package serverimport ("context""fmt"+ "io""net/http""net/http/httptest"+ "strings"+ "sync""testing"+ "time""github.com/labstack/echo/v5"+ metricpb "github.com/prometheus/client_model/go"+ "github.com/prometheus/common/expfmt""github.com/stretchr/testify/require""google.golang.org/protobuf/encoding/protojson""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/testcontainer""github.com/bytebase/bytebase/backend/component/config"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/migrator"@@ -28,6 +35,11 @@ const devEnterpriseLicense = "eyJhbGciOiJSUzI1NiIsImtpZCI6InYxIiwidHlwIjoiSldUIn// newMetricsTestEcho returns an echo server with the self-host /metrics route// wired to a real store and license service backed by a fresh PostgreSQL.func newMetricsTestEcho(t *testing.T) (*echo.Echo, *store.Store, *enterprise.LicenseService) {+ e, st, licenseService, _ := newMetricsTestEchoWithCollector(t)+ return e, st, licenseService+}++func newMetricsTestEchoWithCollector(t *testing.T) (*echo.Echo, *store.Store, *enterprise.LicenseService, *productmetrics.ProductMetrics) {t.Helper()ctx := context.Background()container := testcontainer.GetTestPgContainer(ctx, t)@@ -46,8 +58,9 @@ func newMetricsTestEcho(t *testing.T) (*echo.Echo, *store.Store, *enterprise.Licrequire.NoError(t, err)e := echo.New()- registerMetricsRoute(e, &config.Profile{SaaS: false}, st, licenseService)- return e, st, licenseService+ metrics := productmetrics.New(st, licenseService)+ registerMetricsRoute(e, &config.Profile{SaaS: false}, metrics)+ return e, st, licenseService, metrics}func scrapeMetrics(t *testing.T, e *echo.Echo, wantStatus int) string {@@ -67,10 +80,18 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {body := scrapeMetrics(t, e, http.StatusOK)require.Contains(t, body, "bytebase_license_seats_used 0")require.Contains(t, body, "bytebase_license_seats_limit 20")+ require.Contains(t, body, "bytebase_license_instances_used 0")+ require.Contains(t, body, "bytebase_license_instances_limit 10")+ require.Contains(t, body, "bytebase_license_expiry_timestamp_seconds +Inf")_, err := st.GetDB().ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('ws-test');INSERT INTO setting (name, workspace, value) VALUES ('SYSTEM', 'ws-test', '{}'::jsonb);+ INSERT INTO project (resource_id, workspace, name) VALUES ('project-a', 'ws-test', 'Project A');+ INSERT INTO instance (resource_id, workspace, project, deleted) VALUES+ ('workspace-instance', 'ws-test', NULL, FALSE),+ ('project-instance', 'ws-test', 'project-a', FALSE),+ ('deleted-instance', 'ws-test', NULL, TRUE);INSERT INTO principal (name, email, password_hash, deleted) VALUES('alice', '[email protected]', 'x', FALSE),('bob', '[email protected]', 'x', FALSE),@@ -112,6 +133,7 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {body = scrapeMetrics(t, e, http.StatusOK)require.Contains(t, body, "bytebase_license_seats_used 4")require.Contains(t, body, "bytebase_license_seats_limit 20")+ require.Contains(t, body, "bytebase_license_instances_used 2")// A colliding group email in another workspace must not leak into this// workspace's count. Keep the second workspace deleted so GetWorkspaceID@@ -139,6 +161,12 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {body = scrapeMetrics(t, e, http.StatusOK)require.Contains(t, body, "bytebase_license_seats_used 4")require.Contains(t, body, "bytebase_license_seats_limit +Inf")+ require.Contains(t, body, "bytebase_license_instances_limit +Inf")++ // A configured but malformed license invalidates the complete scrape.+ require.NoError(t, st.UpdateLicense(ctx, "ws-test", "not-a-license"))+ scrapeMetrics(t, e, http.StatusInternalServerError)+ require.NoError(t, licenseService.StoreLicense(ctx, "ws-test", devEnterpriseLicense))// A failing metadata read must fail the scrape instead of emitting// zero, stale, or missing gauges.@@ -146,6 +174,74 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {scrapeMetrics(t, e, http.StatusInternalServerError)}+func TestMetricsEventsAreServerLocal(t *testing.T) {+ e1, _, _, metrics1 := newMetricsTestEchoWithCollector(t)+ e2, _, _, _ := newMetricsTestEchoWithCollector(t)+ metrics1.RecordRunnerRun(productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess, time.Second)++ body1 := scrapeMetrics(t, e1, http.StatusOK)+ require.Contains(t, body1, `bytebase_runner_run_duration_seconds_count{result="success",runner="plan_check"} 1`)+ body2 := scrapeMetrics(t, e2, http.StatusOK)+ require.NotContains(t, body2, "bytebase_runner_run_duration_seconds")+ for _, name := range []string{+ "bytebase_license_expiry_timestamp_seconds",+ "bytebase_license_instances_used",+ "bytebase_license_instances_limit",+ } {+ require.Equal(t, metricTextLine(body1, name), metricTextLine(body2, name))+ }++ var wg sync.WaitGroup+ scrapeErr := make(chan string, 8)+ for range 8 {+ wg.Go(func() {+ metrics1.RecordRunnerRun(productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess, time.Millisecond)+ req := httptest.NewRequest(http.MethodGet, "/metrics", nil)+ rec := httptest.NewRecorder()+ e1.ServeHTTP(rec, req)+ if rec.Code != http.StatusOK {+ scrapeErr <- rec.Body.String()+ }+ })+ }+ wg.Wait()+ close(scrapeErr)+ for err := range scrapeErr {+ t.Errorf("concurrent scrape failed: %s", err)+ }++ format := expfmt.NewFormat(expfmt.TypeProtoDelim)+ req := httptest.NewRequest(http.MethodGet, "/metrics", nil)+ req.Header.Set("Accept", string(format))+ rec := httptest.NewRecorder()+ e1.ServeHTTP(rec, req)+ require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())+ decoder := expfmt.NewDecoder(rec.Body, format)+ foundNative := false+ for {+ family := &metricpb.MetricFamily{}+ if err := decoder.Decode(family); err != nil {+ require.ErrorIs(t, err, io.EOF)+ break+ }+ if family.GetName() == "bytebase_runner_run_duration_seconds" {+ require.Len(t, family.Metric, 1)+ require.NotNil(t, family.Metric[0].GetHistogram().Schema)+ foundNative = true+ }+ }+ require.True(t, foundNative)+}++func metricTextLine(body, name string) string {+ for line := range strings.SplitSeq(body, "\n") {+ if strings.HasPrefix(line, name+" ") {+ return line+ }+ }+ return ""+}+func TestMetricsLicenseSeatsAllUsers(t *testing.T) {ctx := context.Background()e, st, _ := newMetricsTestEcho(t)diff --git a/backend/server/server.go b/backend/server/server.goindex f5b745b5b4..f098e1ae00 100644--- a/backend/server/server.go+++ b/backend/server/server.go@@ -25,6 +25,7 @@ import ("github.com/bytebase/bytebase/backend/component/config""github.com/bytebase/bytebase/backend/component/dbfactory""github.com/bytebase/bytebase/backend/component/iam"+ "github.com/bytebase/bytebase/backend/component/productmetrics""github.com/bytebase/bytebase/backend/component/review""github.com/bytebase/bytebase/backend/component/sampleinstance""github.com/bytebase/bytebase/backend/component/sheet"@@ -213,15 +214,19 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {// Configure echo server.s.echoServer = echo.New()- s.schemaSyncer = schemasync.NewSyncer(stores, s.dbFactory, s.licenseService)+ var productMetrics *productmetrics.ProductMetrics+ if !profile.SaaS {+ productMetrics = productmetrics.New(stores, s.licenseService)+ }+ s.schemaSyncer = schemasync.NewSyncer(stores, s.dbFactory, s.licenseService, productMetrics)s.approvalRunner = review.NewRunner(stores, s.bus, s.webhookManager, s.licenseService)- s.taskScheduler = taskrun.NewScheduler(stores, s.bus, s.webhookManager, s.licenseService, profile)+ s.taskScheduler = taskrun.NewScheduler(stores, s.bus, s.webhookManager, s.licenseService, profile, productMetrics)s.taskScheduler.Register(storepb.Task_DATABASE_CREATE, taskrun.NewDatabaseCreateExecutor(stores, s.dbFactory, s.schemaSyncer))s.taskScheduler.Register(storepb.Task_DATABASE_MIGRATE, taskrun.NewDatabaseMigrateExecutor(stores, s.dbFactory, s.bus, s.schemaSyncer, profile))combinedExecutor := plancheck.NewCombinedExecutor(stores, sheetManager, s.dbFactory)- s.planCheckScheduler = plancheck.NewScheduler(stores, s.bus, combinedExecutor, s.licenseService)+ s.planCheckScheduler = plancheck.NewScheduler(stores, s.bus, combinedExecutor, s.licenseService, productMetrics)s.notifyListener = notifylistener.NewListener(stores.GetDB(), s.bus)// Data cleaner@@ -246,7 +251,7 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {if err != nil {return nil, errors.Wrapf(err, "failed to create MCP server")}- configureEchoRouters(s.echoServer, s.lspServer, directorySyncServer, oauth2Service, mcpServer, stripeWebhookHandler, s.store, s.licenseService, profile)+ configureEchoRouters(s.echoServer, s.lspServer, directorySyncServer, oauth2Service, mcpServer, stripeWebhookHandler, productMetrics, profile)serverStarted = truereturn s, nildiff --git a/go.mod b/go.modindex f883ce576a..d7f68cee3c 100644--- a/go.mod+++ b/go.mod@@ -337,8 +337,8 @@ require (github.com/pquerna/cachecontrol v0.2.0 // indirectgithub.com/pquerna/otp v1.5.0github.com/prometheus/client_golang v1.24.1- github.com/prometheus/client_model v0.6.2 // indirect- github.com/prometheus/common v0.70.1 // indirect+ github.com/prometheus/client_model v0.6.2+ github.com/prometheus/common v0.70.1github.com/prometheus/procfs v0.21.1 // indirectgithub.com/segmentio/asm v1.2.1 // indirectgithub.com/shirou/gopsutil/v3 v3.24.5 // indirect