Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(metrics): expose product health metrics (#21211)

p0ny4748a7fe44c5
diff --git a/backend/component/productmetrics/README.md b/backend/component/productmetrics/README.md
new file mode 100644
index 0000000000..87b56da879
--- /dev/null
+++ b/backend/component/productmetrics/README.md
@@ -0,0 +1,45 @@
+# Product metrics
+
+Bytebase exposes product-health metrics from the server's `/metrics` endpoint.
+They complement Go runtime and HTTP metrics with installation state and
+process-local background-work health.
+
+## Installation-state metrics
+
+Installation-state metrics are read from shared metadata on every scrape so all
+replicas report equivalent values. If the metadata or configured license cannot
+be read and verified, the scrape fails instead of returning a partial state.
+
+| Metric | Type | Description |
+| --- | --- | --- |
+| `bytebase_license_expiry_timestamp_seconds` | Gauge | License expiry as a Unix timestamp. `+Inf` means the Free plan or a perpetual license. A valid expired license retains its expiry timestamp while using Free-plan limits. |
+| `bytebase_license_seats_used` | Gauge | Distinct end users currently occupying license seats. |
+| `bytebase_license_seats_limit` | Gauge | Effective seat limit. `+Inf` means unlimited. |
+| `bytebase_license_instances_used` | Gauge | Non-deleted registered instances. |
+| `bytebase_license_instances_limit` | Gauge | Effective registered-instance limit. `+Inf` means unlimited. |
+
+## Process-local metrics
+
+Process metrics record work performed by the serving Bytebase process. In a
+multi-replica deployment, query each replica or aggregate across replicas as
+appropriate. Canceled shutdown work is not recorded as a failure.
+
+| Metric | Type | Labels | Description |
+| --- | --- | --- | --- |
+| `bytebase_instance_sync_duration_seconds` | Native histogram | `result`, `bytebase_instance_id`, resource labels | Duration of completed instance synchronization attempts. |
+| `bytebase_database_sync_total` | Counter | `result`, `bytebase_instance_id`, `database`, resource labels | Completed database schema synchronization attempts. |
+| `bytebase_runner_run_duration_seconds` | Native histogram | `runner`, `result` | Duration of synchronous background runner cycles. |
+
+`result` is `success`, `failure`, or `skipped`. Runner values are
+`plan_check`, `task_pending`, `task_dispatch`, `instance_sync`, and
+`database_sync`.
+
+Resource-label names are exposed with a `label_` prefix. Characters outside
+ASCII letters, digits, and underscores are replaced with underscores, and name
+collisions receive deterministic `_conflictN` suffixes. The label schema is the
+union of labels observed by that process; a resource without a union label emits
+an empty value for it.
+
+The duration metrics use the Prometheus client library's native histogram
+implementation. Their bucket factor is at most 1.1, populated buckets are capped
+at 100, and bucket state is not reset more often than once per hour.
diff --git a/backend/component/productmetrics/productmetrics.go b/backend/component/productmetrics/productmetrics.go
new file mode 100644
index 0000000000..f04f5a01f3
--- /dev/null
+++ b/backend/component/productmetrics/productmetrics.go
@@ -0,0 +1,385 @@
+// Package productmetrics collects Bytebase product-health metrics.
+package productmetrics
+
+import (
+ "context"
+ "math"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/pkg/errors"
+ "github.com/prometheus/client_golang/prometheus"
+ metricpb "github.com/prometheus/client_model/go"
+
+ "github.com/bytebase/bytebase/backend/enterprise"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+const metricsCollectionTimeout = 5 * time.Second
+
+// Runner identifies a synchronous background control-loop cycle.
+type Runner string
+
+const (
+ RunnerPlanCheck Runner = "plan_check"
+ RunnerTaskPending Runner = "task_pending"
+ RunnerTaskDispatch Runner = "task_dispatch"
+ RunnerInstanceSync Runner = "instance_sync"
+ RunnerDatabaseSync Runner = "database_sync"
+)
+
+// RunnerResult is the terminal result of a runner cycle.
+type RunnerResult string
+
+const (
+ ResultSuccess RunnerResult = "success"
+ ResultFailure RunnerResult = "failure"
+ ResultSkipped RunnerResult = "skipped"
+)
+
+type databaseIdentity struct {
+ instanceID string
+ databaseName string
+}
+
+type nativeHistogram struct {
+ prometheus.Histogram
+}
+
+type dynamicHistogramMetric struct {
+ desc *prometheus.Desc
+ histogram prometheus.Histogram
+ labelValues []string
+}
+
+type stateStore interface {
+ GetWorkspaceID(context.Context) (string, error)
+ CountSeatOccupyingUsers(context.Context, string) (int, error)
+ CountActiveInstances(context.Context, string) (int, error)
+}
+
+type licenseStateProvider interface {
+ GetVerifiedStateUncached(context.Context, string) (*enterprise.VerifiedState, error)
+}
+
+// ProductMetrics owns process-local event metrics and scrape-time product
+// state. It deliberately is an unchecked Collector: dynamic resource label
+// names are derived at collection time from the latest observed resource
+// labels, which Prometheus's static descriptor registration cannot express.
+type ProductMetrics struct {
+ stores stateStore
+ licenseService licenseStateProvider
+
+ mu sync.RWMutex
+
+ instanceLabels map[string]map[string]string
+ databaseLabels map[string]map[string]string
+ instanceHistograms map[string]map[RunnerResult]*nativeHistogram
+ databaseCounters map[string]map[RunnerResult]uint64
+ databases map[string]databaseIdentity
+ runnerEvents map[Runner]map[RunnerResult]*nativeHistogram
+}
+
+// New creates a process-local product metrics collector.
+func New(stores *store.Store, licenseService *enterprise.LicenseService) *ProductMetrics {
+ metrics := &ProductMetrics{
+ instanceLabels: make(map[string]map[string]string),
+ databaseLabels: make(map[string]map[string]string),
+ instanceHistograms: make(map[string]map[RunnerResult]*nativeHistogram),
+ databaseCounters: make(map[string]map[RunnerResult]uint64),
+ databases: make(map[string]databaseIdentity),
+ runnerEvents: make(map[Runner]map[RunnerResult]*nativeHistogram),
+ }
+ if stores != nil {
+ metrics.stores = stores
+ }
+ if licenseService != nil {
+ metrics.licenseService = licenseService
+ }
+ return metrics
+}
+
+// Describe intentionally sends no descriptors. Resource labels can change
+// between observations, so their descriptor is defined only at collection.
+func (*ProductMetrics) Describe(_ chan<- *prometheus.Desc) {
+ // Dynamic descriptors are emitted only from Collect.
+}
+
+// RecordInstanceSync records one complete instance-sync attempt. Cancellation
+// is a shutdown path rather than a failed synchronization.
+func (m *ProductMetrics) RecordInstanceSync(instance *store.InstanceMessage, duration time.Duration, err error) {
+ if errors.Is(err, context.Canceled) {
+ return
+ }
+ identity := ""
+ labels := map[string]string(nil)
+ if instance != nil {
+ identity = instance.ResourceID
+ labels = instance.Metadata.GetLabels()
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.instanceLabels[identity] = cloneLabels(labels)
+ result := ResultSuccess
+ if err != nil {
+ result = ResultFailure
+ }
+ byResult := m.instanceHistograms[identity]
+ if byResult == nil {
+ byResult = make(map[RunnerResult]*nativeHistogram)
+ m.instanceHistograms[identity] = byResult
+ }
+ h := byResult[result]
+ if h == nil {
+ h = newNativeHistogram()
+ byResult[result] = h
+ }
+ h.observe(duration.Seconds())
+}
+
+// RecordDatabaseSync records one database-schema synchronization attempt.
+func (m *ProductMetrics) RecordDatabaseSync(database *store.DatabaseMessage, err error) {
+ if errors.Is(err, context.Canceled) {
+ return
+ }
+ identity := ""
+ labels := map[string]string(nil)
+ if database != nil {
+ identity = database.InstanceID + "\x00" + database.DatabaseName
+ labels = database.Metadata.GetLabels()
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.databaseLabels[identity] = cloneLabels(labels)
+ result := ResultSuccess
+ if err != nil {
+ result = ResultFailure
+ }
+ instanceID, databaseName := "", ""
+ if database != nil {
+ instanceID, databaseName = database.InstanceID, database.DatabaseName
+ }
+ m.databases[identity] = databaseIdentity{instanceID: instanceID, databaseName: databaseName}
+ byResult := m.databaseCounters[identity]
+ if byResult == nil {
+ byResult = make(map[RunnerResult]uint64)
+ m.databaseCounters[identity] = byResult
+ }
+ byResult[result]++
+}
+
+// RecordRunnerRun records a completed synchronous runner cycle.
+func (m *ProductMetrics) RecordRunnerRun(runner Runner, result RunnerResult, duration time.Duration) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ byResult := m.runnerEvents[runner]
+ if byResult == nil {
+ byResult = make(map[RunnerResult]*nativeHistogram)
+ m.runnerEvents[runner] = byResult
+ }
+ h := byResult[result]
+ if h == nil {
+ h = newNativeHistogram()
+ byResult[result] = h
+ }
+ h.observe(duration.Seconds())
+}
+
+// Collect emits the process-local event state and current shared installation
+// state. A state error invalidates the scrape rather than publishing a partial
+// state snapshot.
+func (m *ProductMetrics) Collect(ch chan<- prometheus.Metric) {
+ m.collectEvents(ch)
+
+ ctx, cancel := context.WithTimeout(context.Background(), metricsCollectionTimeout)
+ defer cancel()
+ if m.stores == nil || m.licenseService == nil {
+ fail(ch, errors.New("product metrics dependencies are not configured"))
+ return
+ }
+ workspaceID, err := m.stores.GetWorkspaceID(ctx)
+ if err != nil {
+ fail(ch, errors.Wrap(err, "failed to get workspace ID"))
+ return
+ }
+ usedSeats, usedInstances := 0, 0
+ if workspaceID != "" {
+ usedSeats, err = m.stores.CountSeatOccupyingUsers(ctx, workspaceID)
+ if err != nil {
+ fail(ch, errors.Wrapf(err, "failed to count seat-occupying users for workspace %q", workspaceID))
+ return
+ }
+ usedInstances, err = m.stores.CountActiveInstances(ctx, workspaceID)
+ if err != nil {
+ fail(ch, errors.Wrapf(err, "failed to count active instances for workspace %q", workspaceID))
+ return
+ }
+ }
+ state, err := m.licenseService.GetVerifiedStateUncached(ctx, workspaceID)
+ if err != nil {
+ fail(ch, errors.Wrapf(err, "failed to read verified license state for workspace %q", workspaceID))
+ return
+ }
+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_seats_used", "Number of distinct end users occupying license seats.", nil, nil), prometheus.GaugeValue, float64(usedSeats))
+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_seats_limit", "Effective seat limit. +Inf means unlimited.", nil, nil), prometheus.GaugeValue, prometheusValue(state.UserLimit))
+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_instances_used", "Number of non-deleted registered instances.", nil, nil), prometheus.GaugeValue, float64(usedInstances))
+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_instances_limit", "Effective registered instance limit. +Inf means unlimited.", nil, nil), prometheus.GaugeValue, prometheusValue(state.InstanceLimit))
+ expiry := math.Inf(1)
+ if state.ExpiresAt != nil {
+ expiry = float64(state.ExpiresAt.Unix())
+ }
+ ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("bytebase_license_expiry_timestamp_seconds", "License expiry Unix timestamp. +Inf means free or perpetual.", nil, nil), prometheus.GaugeValue, expiry)
+}
+
+func (m *ProductMetrics) collectEvents(ch chan<- prometheus.Metric) {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+
+ instanceSchema := dynamicSchema(m.instanceLabels)
+ instanceDesc := prometheus.NewDesc("bytebase_instance_sync_duration_seconds", "Duration of instance synchronization attempts.", append([]string{"result", "bytebase_instance_id"}, instanceSchema.keys...), nil)
+ for identity, byResult := range m.instanceHistograms {
+ for result, histogram := range byResult {
+ labels := append([]string{string(result), identity}, instanceSchema.values(m.instanceLabels[identity])...)
+ ch <- histogram.metric(instanceDesc, labels...)
+ }
+ }
+
+ databaseSchema := dynamicSchema(m.databaseLabels)
+ databaseDesc := prometheus.NewDesc("bytebase_database_sync_total", "Number of database schema synchronization attempts.", append([]string{"result", "bytebase_instance_id", "database"}, databaseSchema.keys...), nil)
+ for identity, byResult := range m.databaseCounters {
+ database := m.databases[identity]
+ for result, count := range byResult {
+ labels := append([]string{string(result), database.instanceID, database.databaseName}, databaseSchema.values(m.databaseLabels[identity])...)
+ ch <- prometheus.MustNewConstMetric(databaseDesc, prometheus.CounterValue, float64(count), labels...)
+ }
+ }
+
+ runnerDesc := prometheus.NewDesc("bytebase_runner_run_duration_seconds", "Duration of synchronous runner control-loop cycles.", []string{"runner", "result"}, nil)
+ for runner, byResult := range m.runnerEvents {
+ for result, histogram := range byResult {
+ ch <- histogram.metric(runnerDesc, string(runner), string(result))
+ }
+ }
+}
+
+func newNativeHistogram() *nativeHistogram {
+ return &nativeHistogram{Histogram: prometheus.NewHistogram(prometheus.HistogramOpts{
+ Name: "bytebase_internal_product_duration_seconds",
+ Help: "Internal accumulator for dynamically labeled product duration metrics.",
+ NativeHistogramBucketFactor: 1.1,
+ NativeHistogramMaxBucketNumber: 100,
+ NativeHistogramMinResetDuration: time.Hour,
+ })}
+}
+
+func (h *nativeHistogram) observe(value float64) {
+ h.Observe(value)
+}
+
+func (h *nativeHistogram) metric(desc *prometheus.Desc, labels ...string) prometheus.Metric {
+ return &dynamicHistogramMetric{desc: desc, histogram: h.Histogram, labelValues: labels}
+}
+
+func (m *dynamicHistogramMetric) Desc() *prometheus.Desc {
+ return m.desc
+}
+
+func (m *dynamicHistogramMetric) Write(out *metricpb.Metric) error {
+ if err := m.histogram.Write(out); err != nil {
+ return errors.Wrap(err, "failed to write native histogram")
+ }
+ out.Label = prometheus.MakeLabelPairs(m.desc, m.labelValues)
+ return nil
+}
+
+type labelSchema struct {
+ keys []string
+ mapped map[string]string
+}
+
+func dynamicSchema(labelsByIdentity map[string]map[string]string) labelSchema {
+ original := make(map[string]struct{})
+ for _, labels := range labelsByIdentity {
+ for key := range labels {
+ original[key] = struct{}{}
+ }
+ }
+ originalKeys := make([]string, 0, len(original))
+ for key := range original {
+ originalKeys = append(originalKeys, key)
+ }
+ slices.Sort(originalKeys)
+ schema := labelSchema{keys: make([]string, 0, len(originalKeys)), mapped: make(map[string]string, len(originalKeys))}
+ used := make(map[string]struct{})
+ for _, key := range originalKeys {
+ baseName := "label_" + sanitizeLabelKey(key)
+ name := baseName
+ for suffix := 1; ; suffix++ {
+ if _, exists := used[name]; !exists {
+ break
+ }
+ name = baseName + "_conflict" + strconv.Itoa(suffix)
+ }
+ used[name] = struct{}{}
+ schema.keys = append(schema.keys, name)
+ schema.mapped[key] = name
+ }
+ return schema
+}
+
+func (s labelSchema) values(labels map[string]string) []string {
+ values := make([]string, len(s.keys))
+ if len(labels) == 0 {
+ return values
+ }
+ for key, value := range labels {
+ name := s.mapped[key]
+ for j, metricKey := range s.keys {
+ if metricKey == name {
+ values[j] = value
+ break
+ }
+ }
+ }
+ return values
+}
+
+func sanitizeLabelKey(key string) string {
+ var b strings.Builder
+ for _, r := range key {
+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' {
+ _, _ = b.WriteRune(r)
+ } else {
+ _ = b.WriteByte('_')
+ }
+ }
+ if b.Len() == 0 {
+ return "_"
+ }
+ return b.String()
+}
+
+func cloneLabels(labels map[string]string) map[string]string {
+ if len(labels) == 0 {
+ return nil
+ }
+ clone := make(map[string]string, len(labels))
+ for key, value := range labels {
+ clone[key] = value
+ }
+ return clone
+}
+
+func prometheusValue(value int) float64 {
+ if value == math.MaxInt {
+ return math.Inf(1)
+ }
+ return float64(value)
+}
+
+func fail(ch chan<- prometheus.Metric, err error) {
+ ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
+}
diff --git a/backend/component/productmetrics/productmetrics_test.go b/backend/component/productmetrics/productmetrics_test.go
new file mode 100644
index 0000000000..bba9d82830
--- /dev/null
+++ b/backend/component/productmetrics/productmetrics_test.go
@@ -0,0 +1,282 @@
+package productmetrics
+
+import (
+ "context"
+ "errors"
+ "math"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ metricpb "github.com/prometheus/client_model/go"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/enterprise"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+type fakeStateStore struct {
+ workspace string
+ seats int
+ instances int
+ err error
+}
+
+func (s *fakeStateStore) GetWorkspaceID(context.Context) (string, error) {
+ return s.workspace, s.err
+}
+
+func (s *fakeStateStore) CountSeatOccupyingUsers(context.Context, string) (int, error) {
+ return s.seats, s.err
+}
+
+func (s *fakeStateStore) CountActiveInstances(context.Context, string) (int, error) {
+ return s.instances, s.err
+}
+
+type fakeLicenseStateProvider struct {
+ state *enterprise.VerifiedState
+ err error
+}
+
+func (p *fakeLicenseStateProvider) GetVerifiedStateUncached(context.Context, string) (*enterprise.VerifiedState, error) {
+ return p.state, p.err
+}
+
+func TestRegistryInstallationStateVariants(t *testing.T) {
+ finiteExpiry := time.Now().Add(24 * time.Hour).Truncate(time.Second)
+ expiredAt := time.Now().Add(-time.Hour).Truncate(time.Second)
+ tests := []struct {
+ name string
+ state *enterprise.VerifiedState
+ wantExpiry float64
+ wantLimit float64
+ }{
+ {name: "free", state: &enterprise.VerifiedState{UserLimit: 20, InstanceLimit: 10}, wantExpiry: math.Inf(1), wantLimit: 10},
+ {name: "perpetual", state: &enterprise.VerifiedState{UserLimit: math.MaxInt, InstanceLimit: math.MaxInt}, wantExpiry: math.Inf(1), wantLimit: math.Inf(1)},
+ {name: "finite", state: &enterprise.VerifiedState{ExpiresAt: &finiteExpiry, UserLimit: 100, InstanceLimit: 25}, wantExpiry: float64(finiteExpiry.Unix()), wantLimit: 25},
+ {name: "expired", state: &enterprise.VerifiedState{ExpiresAt: &expiredAt, UserLimit: 20, InstanceLimit: 10}, wantExpiry: float64(expiredAt.Unix()), wantLimit: 10},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ metrics := New(nil, nil)
+ metrics.stores = &fakeStateStore{workspace: "ws", seats: 3, instances: 2}
+ metrics.licenseService = &fakeLicenseStateProvider{state: test.state}
+ registry := prometheus.NewRegistry()
+ registry.MustRegister(metrics)
+ families, err := registry.Gather()
+ require.NoError(t, err)
+ require.Equal(t, 2.0, familyGaugeValue(t, families, "bytebase_license_instances_used"))
+ require.Equal(t, test.wantLimit, familyGaugeValue(t, families, "bytebase_license_instances_limit"))
+ require.Equal(t, test.wantExpiry, familyGaugeValue(t, families, "bytebase_license_expiry_timestamp_seconds"))
+ })
+ }
+}
+
+func TestRegistryStateFailureAndEventLocality(t *testing.T) {
+ newCollector := func() (*ProductMetrics, *prometheus.Registry) {
+ metrics := New(nil, nil)
+ metrics.stores = &fakeStateStore{workspace: "ws"}
+ metrics.licenseService = &fakeLicenseStateProvider{state: &enterprise.VerifiedState{UserLimit: 20, InstanceLimit: 10}}
+ registry := prometheus.NewRegistry()
+ registry.MustRegister(metrics)
+ return metrics, registry
+ }
+ metrics1, registry1 := newCollector()
+ _, registry2 := newCollector()
+ metrics1.RecordRunnerRun(RunnerPlanCheck, ResultSuccess, time.Second)
+ families1, err := registry1.Gather()
+ require.NoError(t, err)
+ families2, err := registry2.Gather()
+ require.NoError(t, err)
+ require.Equal(t, 1.0, familyHistogramCount(families1, "bytebase_runner_run_duration_seconds"))
+ require.Zero(t, familyHistogramCount(families2, "bytebase_runner_run_duration_seconds"))
+
+ metrics1.licenseService = &fakeLicenseStateProvider{err: errors.New("malformed license")}
+ _, err = registry1.Gather()
+ require.ErrorContains(t, err, "malformed license")
+}
+
+func TestResourceMetricLabelsAndCumulativeMutation(t *testing.T) {
+ m := New(nil, nil)
+ instance := &store.InstanceMessage{
+ ResourceID: "prod",
+ Metadata: &storepb.Instance{Labels: map[string]string{
+ "env.prod": "before",
+ "env-prod": "dash",
+ "env_prod_conflict1": "reserved",
+ "\u4e2d\u6587": "unicode",
+ }},
+ }
+ m.RecordInstanceSync(instance, time.Second, nil)
+ collidingMetric := findMetric(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds")
+ require.Equal(t, "before", metricLabel(collidingMetric, "label_env_prod_conflict1"))
+ require.Equal(t, "dash", metricLabel(collidingMetric, "label_env_prod"))
+ require.Equal(t, "reserved", metricLabel(collidingMetric, "label_env_prod_conflict1_conflict1"))
+ require.Equal(t, "unicode", metricLabel(collidingMetric, "label___"))
+ m.RecordInstanceSync(&store.InstanceMessage{ResourceID: "staging", Metadata: &storepb.Instance{Labels: map[string]string{"owner": "platform"}}}, time.Second, nil)
+ unionMetric := findMetricWithLabels(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds", map[string]string{"bytebase_instance_id": "staging"})
+ require.Equal(t, "", metricLabel(unionMetric, "label_env_prod_conflict1"))
+ require.Equal(t, "platform", metricLabel(unionMetric, "label_owner"))
+ instance.Metadata.Labels = map[string]string{"env.prod": "after"}
+ m.RecordInstanceSync(instance, 2*time.Second, nil)
+
+ metrics := eventMetrics(m)
+ metric := findMetricWithLabels(t, metrics, "bytebase_instance_sync_duration_seconds", map[string]string{"bytebase_instance_id": "prod"})
+ require.Equal(t, uint64(2), metric.GetHistogram().GetSampleCount())
+ require.Equal(t, 3.0, metric.GetHistogram().GetSampleSum())
+ require.Equal(t, int32(3), metric.GetHistogram().GetSchema())
+ require.Empty(t, metric.GetHistogram().GetBucket())
+ require.Equal(t, "prod", metricLabel(metric, "bytebase_instance_id"))
+ require.Equal(t, "after", metricLabel(metric, "label_env_prod"))
+}
+
+func TestDatabaseAndRunnerMetrics(t *testing.T) {
+ m := New(nil, nil)
+ database := &store.DatabaseMessage{
+ InstanceID: "prod",
+ DatabaseName: "app",
+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{"region": "us"}},
+ }
+ m.RecordDatabaseSync(database, nil)
+ m.RecordDatabaseSync(database, context.DeadlineExceeded)
+ m.RecordDatabaseSync(database, context.Canceled)
+ m.RecordRunnerRun(RunnerPlanCheck, ResultSuccess, time.Second)
+
+ metrics := eventMetrics(m)
+ databaseMetric := findMetricWithLabels(t, metrics, "bytebase_database_sync_total", map[string]string{"result": "success"})
+ require.Equal(t, "prod", metricLabel(databaseMetric, "bytebase_instance_id"))
+ require.Equal(t, "app", metricLabel(databaseMetric, "database"))
+ require.Equal(t, "success", metricLabel(databaseMetric, "result"))
+ require.Equal(t, 1.0, databaseMetric.GetCounter().GetValue())
+ runnerMetric := findMetric(t, metrics, "bytebase_runner_run_duration_seconds")
+ require.Equal(t, int32(3), runnerMetric.GetHistogram().GetSchema())
+ require.Empty(t, runnerMetric.GetHistogram().GetBucket())
+}
+
+func TestNativeHistogramBucketCap(t *testing.T) {
+ histogram := newNativeHistogram()
+ for i := range 150 {
+ histogram.observe(math.Pow(2, float64(i)))
+ }
+ metric := &metricpb.Metric{}
+ require.NoError(t, histogram.Write(metric))
+ nativeBucketCount := 0
+ for _, span := range metric.GetHistogram().GetPositiveSpan() {
+ nativeBucketCount += int(span.GetLength())
+ }
+ require.LessOrEqual(t, nativeBucketCount, 100)
+ require.Less(t, metric.GetHistogram().GetSchema(), int32(3))
+}
+
+func TestNativeHistogramBucketAssignment(t *testing.T) {
+ tests := []struct {
+ value float64
+ wantIndex int32
+ }{
+ {value: 0.95, wantIndex: 0},
+ {value: 1, wantIndex: 0},
+ {value: 1.05, wantIndex: 1},
+ {value: 2, wantIndex: 8},
+ }
+ for _, test := range tests {
+ histogram := newNativeHistogram()
+ histogram.observe(test.value)
+ metric := &metricpb.Metric{}
+ require.NoError(t, histogram.Write(metric))
+ require.Equal(t, int32(3), metric.GetHistogram().GetSchema())
+ require.Len(t, metric.GetHistogram().GetPositiveSpan(), 1)
+ require.Equal(t, test.wantIndex, metric.GetHistogram().GetPositiveSpan()[0].GetOffset())
+ }
+}
+
+func TestConcurrentObserveAndCollect(t *testing.T) {
+ m := New(nil, nil)
+ instance := &store.InstanceMessage{ResourceID: "prod", Metadata: &storepb.Instance{}}
+ var wg sync.WaitGroup
+ for range 10 {
+ wg.Go(func() {
+ for range 100 {
+ m.RecordInstanceSync(instance, time.Millisecond, nil)
+ eventMetrics(m)
+ }
+ })
+ }
+ wg.Wait()
+ metric := findMetric(t, eventMetrics(m), "bytebase_instance_sync_duration_seconds")
+ require.Equal(t, uint64(1000), metric.GetHistogram().GetSampleCount())
+}
+
+func eventMetrics(m *ProductMetrics) []*metricpb.Metric {
+ ch := make(chan prometheus.Metric, 1000)
+ m.collectEvents(ch)
+ metrics := make([]*metricpb.Metric, 0, len(ch))
+ for range len(ch) {
+ metric := <-ch
+ written := &metricpb.Metric{}
+ if metric.Write(written) == nil {
+ metrics = append(metrics, written)
+ }
+ }
+ return metrics
+}
+
+func findMetric(t *testing.T, metrics []*metricpb.Metric, name string) *metricpb.Metric {
+ return findMetricWithLabels(t, metrics, name, nil)
+}
+
+func findMetricWithLabels(t *testing.T, metrics []*metricpb.Metric, name string, labels map[string]string) *metricpb.Metric {
+ t.Helper()
+ for _, metric := range metrics {
+ matches := true
+ for name, value := range labels {
+ matches = matches && metricLabel(metric, name) == value
+ }
+ if !matches {
+ continue
+ }
+ if metric.GetHistogram() != nil && name == "bytebase_instance_sync_duration_seconds" && metricLabel(metric, "bytebase_instance_id") != "" {
+ return metric
+ }
+ if metric.GetCounter() != nil && name == "bytebase_database_sync_total" {
+ return metric
+ }
+ if metric.GetHistogram() != nil && name == "bytebase_runner_run_duration_seconds" && metricLabel(metric, "runner") != "" {
+ return metric
+ }
+ }
+ t.Fatalf("metric %q not found", name)
+ return nil
+}
+
+func metricLabel(metric *metricpb.Metric, name string) string {
+ for _, pair := range metric.GetLabel() {
+ if pair.GetName() == name {
+ return pair.GetValue()
+ }
+ }
+ return ""
+}
+
+func familyGaugeValue(t *testing.T, families []*metricpb.MetricFamily, name string) float64 {
+ t.Helper()
+ for _, family := range families {
+ if family.GetName() == name {
+ require.Len(t, family.Metric, 1)
+ return family.Metric[0].GetGauge().GetValue()
+ }
+ }
+ t.Fatalf("metric family %q not found", name)
+ return 0
+}
+
+func familyHistogramCount(families []*metricpb.MetricFamily, name string) float64 {
+ for _, family := range families {
+ if family.GetName() == name && len(family.Metric) > 0 {
+ return float64(family.Metric[0].GetHistogram().GetSampleCount())
+ }
+ }
+ return 0
+}
diff --git a/backend/enterprise/license.go b/backend/enterprise/license.go
index f3f643faa4..985106eb35 100644
--- a/backend/enterprise/license.go
+++ b/backend/enterprise/license.go
@@ -328,27 +328,11 @@ func (s *LicenseService) GetUserLimit(ctx context.Context, workspaceID string) i
// metadata. A metadata read failure is returned as an error instead of silently
// falling back to the Free plan.
func (s *LicenseService) GetUserLimitUncached(ctx context.Context, workspaceID string) (int, error) {
- setting, err := s.store.GetSystemSettingUncached(ctx, workspaceID)
+ state, err := s.GetVerifiedStateUncached(ctx, workspaceID)
if err != nil {
- return 0, errors.Wrap(err, "failed to get system setting")
- }
- if setting == nil {
- return userLimitFromSubscription(defaultFreeSubscription), nil
- }
- if setting.License == "" {
- return userLimitFromSubscription(defaultFreeSubscription), nil
- }
- subscription := defaultFreeSubscription
- parsedSubscription, parseErr := s.parseLicense(setting.License, workspaceID)
- if parseErr == nil {
- subscription = parsedSubscription
+ return 0, err
}
- // An expired or malformed license has the effective Free plan, mirroring
- // LoadSubscription.
- if isExpired(subscription) {
- subscription = &v1pb.Subscription{Plan: v1pb.PlanType_FREE}
- }
- return userLimitFromSubscription(subscription), nil
+ return state.UserLimit, nil
}
func userLimitFromSubscription(subscription *v1pb.Subscription) int {
@@ -383,6 +367,62 @@ func (s *LicenseService) GetInstanceLimit(ctx context.Context, workspaceID strin
return limit
}
+// VerifiedState is the effective license state read directly from shared
+// metadata. Expired, correctly signed licenses use Free limits while retaining
+// their expiration timestamp; a malformed configured license is an error.
+type VerifiedState struct {
+ ExpiresAt *time.Time
+ UserLimit int
+ InstanceLimit int
+}
+
+// GetVerifiedStateUncached bypasses all setting and subscription caches. It is
+// used by state-metric collection so every replica observes the same verified
+// license state for a scrape.
+func (s *LicenseService) GetVerifiedStateUncached(ctx context.Context, workspaceID string) (*VerifiedState, error) {
+ setting, err := s.store.GetSystemSettingUncached(ctx, workspaceID)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to get system setting")
+ }
+ if setting == nil || setting.License == "" {
+ return verifiedState(defaultFreeSubscription), nil
+ }
+ subscription, err := s.parseLicenseUncheckedExpiry(setting.License, workspaceID)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to verify configured license")
+ }
+ state := verifiedState(subscription)
+ if isExpired(subscription) {
+ free := verifiedState(defaultFreeSubscription)
+ free.ExpiresAt = state.ExpiresAt
+ return free, nil
+ }
+ return state, nil
+}
+
+func verifiedState(subscription *v1pb.Subscription) *VerifiedState {
+ state := &VerifiedState{
+ UserLimit: userLimitFromSubscription(subscription),
+ InstanceLimit: instanceLimitFromSubscription(subscription),
+ }
+ if subscription.GetExpiresTime() != nil {
+ expiresAt := subscription.GetExpiresTime().AsTime()
+ state.ExpiresAt = &expiresAt
+ }
+ return state
+}
+
+func instanceLimitFromSubscription(subscription *v1pb.Subscription) int {
+ if subscription.Instances > 0 {
+ return int(subscription.Instances)
+ }
+ limit := instanceLimitValues[subscription.Plan]
+ if limit == -1 {
+ return math.MaxInt
+ }
+ return limit
+}
+
// GetActivatedInstanceLimit returns the activated instance limit for the current subscription.
func (s *LicenseService) GetActivatedInstanceLimit(ctx context.Context, workspaceID string) int {
limit := s.LoadSubscription(ctx, workspaceID).ActiveInstances
@@ -557,8 +597,22 @@ func (s *LicenseService) CheckReplicaLimit(ctx context.Context) error {
}
func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscription, error) {
+ subscription, err := s.parseLicenseUncheckedExpiry(license, workspaceID)
+ if err != nil {
+ return nil, err
+ }
+ if isExpired(subscription) {
+ return nil, common.Errorf(common.Invalid, "license has expired at %v", subscription.ExpiresTime.AsTime())
+ }
+ return subscription, nil
+}
+
+// parseLicenseUncheckedExpiry verifies a license's signature and all static
+// claims while deliberately retaining an otherwise valid expired token. The
+// caller chooses whether expiry is an error or an effective Free fallback.
+func (s *LicenseService) parseLicenseUncheckedExpiry(license, workspaceID string) (*v1pb.Subscription, error) {
claim := &Claims{}
- token, err := jwt.ParseWithClaims(license, claim, func(token *jwt.Token) (any, error) {
+ token, err := jwt.NewParser(jwt.WithoutClaimsValidation()).ParseWithClaims(license, claim, func(token *jwt.Token) (any, error) {
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
return nil, common.Errorf(common.Invalid, "unexpected signing method: %v", token.Header["alg"])
}
@@ -577,6 +631,9 @@ func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscr
if !token.Valid {
return nil, common.Errorf(common.Invalid, "invalid token")
}
+ if claim.NotBefore != nil && time.Now().Before(claim.NotBefore.Time) {
+ return nil, common.Errorf(common.Invalid, "license is not valid before %v", claim.NotBefore.Time)
+ }
if s.config.Issuer != claim.Issuer {
return nil, common.Errorf(common.Invalid, "iss is not valid, expect %s but found '%v'", s.config.Issuer, claim.Issuer)
@@ -611,10 +668,6 @@ func (s *LicenseService) parseLicense(license, workspaceID string) (*v1pb.Subscr
if claim.ExpiresAt != nil && !claim.ExpiresAt.IsZero() {
expiresTime = timestamppb.New(claim.ExpiresAt.Time)
}
- if expiresTime != nil && expiresTime.AsTime().Before(time.Now()) {
- return nil, errors.Errorf("license has expired at %v", expiresTime.AsTime())
- }
-
return &v1pb.Subscription{
ActiveInstances: int32(claim.ActiveInstances),
Instances: int32(claim.Instances),
diff --git a/backend/enterprise/license_test.go b/backend/enterprise/license_test.go
index 2c95abdf41..c37c3e4095 100644
--- a/backend/enterprise/license_test.go
+++ b/backend/enterprise/license_test.go
@@ -137,6 +137,28 @@ func TestCreateLicenseUsesEqualInstanceClaims(t *testing.T) {
}
}
+func TestParseLicenseExpiredIsInvalid(t *testing.T) {
+ privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+ require.NoError(t, err)
+ service := &LicenseService{
+ config: &Config{
+ PublicKey: &privateKey.PublicKey,
+ PrivateKey: privateKey,
+ Version: keyID,
+ Issuer: issuer,
+ Audience: audience,
+ },
+ }
+ license, err := service.CreateLicense(&LicenseParams{
+ Plan: v1pb.PlanType_TEAM.String(),
+ ExpiresAt: time.Now().Add(-time.Hour),
+ })
+ require.NoError(t, err)
+
+ _, err = service.parseLicense(license, "test-workspace")
+ require.Equal(t, common.Invalid, common.ErrorCode(err))
+}
+
func TestGetUserLimitUncached(t *testing.T) {
ctx := context.Background()
container := testcontainer.GetTestPgContainer(ctx, t)
@@ -198,6 +220,11 @@ func TestGetUserLimitUncached(t *testing.T) {
limit, err = licenseService.GetUserLimitUncached(ctx, "ws-a")
require.NoError(t, err)
require.Equal(t, 100, limit)
+ state, err := licenseService.GetVerifiedStateUncached(ctx, "ws-a")
+ require.NoError(t, err)
+ require.Nil(t, state.ExpiresAt)
+ require.Equal(t, 100, state.UserLimit)
+ require.Equal(t, math.MaxInt, state.InstanceLimit)
// Legacy Enterprise license without a seat claim: unlimited.
storeLicense(t, &LicenseParams{
@@ -208,11 +235,21 @@ func TestGetUserLimitUncached(t *testing.T) {
require.Equal(t, math.MaxInt, limit)
// Expired Enterprise license: falls back to the Free plan limit.
+ expiresAt := time.Now().Add(-time.Hour)
storeLicense(t, &LicenseParams{
Plan: v1pb.PlanType_ENTERPRISE.String(), Seats: 100, WorkspaceID: "ws-a",
- ExpiresAt: time.Now().Add(-time.Hour),
+ ExpiresAt: expiresAt,
})
limit, err = licenseService.GetUserLimitUncached(ctx, "ws-a")
require.NoError(t, err)
require.Equal(t, userLimitValues[v1pb.PlanType_FREE], limit)
+ state, err = licenseService.GetVerifiedStateUncached(ctx, "ws-a")
+ require.NoError(t, err)
+ require.WithinDuration(t, expiresAt, *state.ExpiresAt, time.Second)
+ require.Equal(t, userLimitValues[v1pb.PlanType_FREE], state.UserLimit)
+ require.Equal(t, instanceLimitValues[v1pb.PlanType_FREE], state.InstanceLimit)
+
+ require.NoError(t, s.UpdateLicense(ctx, "ws-a", "not-a-license"))
+ _, err = licenseService.GetVerifiedStateUncached(ctx, "ws-a")
+ require.Error(t, err)
}
diff --git a/backend/runner/plancheck/scheduler.go b/backend/runner/plancheck/scheduler.go
index 7ec9009c5f..34313c49a0 100644
--- a/backend/runner/plancheck/scheduler.go
+++ b/backend/runner/plancheck/scheduler.go
@@ -12,6 +12,7 @@ import (
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/bus"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
@@ -22,12 +23,13 @@ const (
)
// NewScheduler creates a new plan check scheduler.
-func NewScheduler(s *store.Store, bus *bus.Bus, executor *CombinedExecutor, licenseService *enterprise.LicenseService) *Scheduler {
+func NewScheduler(s *store.Store, bus *bus.Bus, executor *CombinedExecutor, licenseService *enterprise.LicenseService, productMetrics *productmetrics.ProductMetrics) *Scheduler {
return &Scheduler{
store: s,
bus: bus,
executor: executor,
licenseService: licenseService,
+ productMetrics: productMetrics,
}
}
@@ -37,6 +39,7 @@ type Scheduler struct {
bus *bus.Bus
executor *CombinedExecutor
licenseService *enterprise.LicenseService
+ productMetrics *productmetrics.ProductMetrics
}
// Run runs the scheduler.
@@ -66,6 +69,8 @@ func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {
}
func (s *Scheduler) runOnce(ctx context.Context) {
+ startedAt := time.Now()
+ result := productmetrics.ResultFailure
defer func() {
if r := recover(); r != nil {
err, ok := r.(error)
@@ -74,6 +79,9 @@ func (s *Scheduler) runOnce(ctx context.Context) {
}
slog.Error("Plan check scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
}
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerPlanCheck, result, time.Since(startedAt))
+ }
}()
claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)
@@ -85,6 +93,7 @@ func (s *Scheduler) runOnce(ctx context.Context) {
for _, c := range claimed {
go s.runPlanCheckRun(ctx, c.ProjectID, c.UID, c.PlanUID, c.ApprovalInputVersion)
}
+ result = productmetrics.ResultSuccess
}
func (s *Scheduler) runPlanCheckRun(ctx context.Context, projectID string, uid int64, planUID int64, approvalInputVersion int64) {
diff --git a/backend/runner/plancheck/scheduler_test.go b/backend/runner/plancheck/scheduler_test.go
index ee37da775f..26067b336b 100644
--- a/backend/runner/plancheck/scheduler_test.go
+++ b/backend/runner/plancheck/scheduler_test.go
@@ -4,13 +4,55 @@ import (
"context"
"testing"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/component/bus"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
)
+func TestRunOnceRecordsEmptySuccess(t *testing.T) {
+ ctx := context.Background()
+ stores := setupPlancheckStore(ctx, t)
+ b, err := bus.New()
+ require.NoError(t, err)
+ metrics := productmetrics.New(nil, nil)
+
+ NewScheduler(stores, b, nil, nil, metrics).runOnce(ctx)
+ NewScheduler(nil, nil, nil, nil, metrics).runOnce(ctx)
+
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerPlanCheck, productmetrics.ResultFailure))
+}
+
+func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {
+ t.Helper()
+ ch := make(chan prometheus.Metric, 16)
+ go func() {
+ metrics.Collect(ch)
+ close(ch)
+ }()
+
+ var count uint64
+ for metric := range ch {
+ var dtoMetric dto.Metric
+ if metric.Write(&dtoMetric) != nil {
+ continue
+ }
+ labels := map[string]string{}
+ for _, label := range dtoMetric.GetLabel() {
+ labels[label.GetName()] = label.GetValue()
+ }
+ if labels["runner"] == string(runner) && labels["result"] == string(result) {
+ count += dtoMetric.GetHistogram().GetSampleCount()
+ }
+ }
+ return count
+}
+
func TestMarkPlanCheckRunDoneSkipsDraftIssue(t *testing.T) {
ctx := context.Background()
stores := setupPlancheckStore(ctx, t)
@@ -43,7 +85,7 @@ func TestMarkPlanCheckRunDoneSkipsDraftIssue(t *testing.T) {
require.NoError(t, err)
require.Len(t, claimed, 1)
- scheduler := NewScheduler(stores, b, nil, nil)
+ scheduler := NewScheduler(stores, b, nil, nil, nil)
scheduler.markPlanCheckRunDone(ctx, "project-a", claimed[0].UID, plan.UID, 1, nil)
run, err := stores.GetPlanCheckRun(ctx, "project-a", plan.UID)
@@ -81,7 +123,7 @@ func TestRunPlanCheckRunStopsAfterProjectArchive(t *testing.T) {
Workspace: "default",
Delete: &archived,
}))
- scheduler := NewScheduler(stores, b, nil, nil)
+ scheduler := NewScheduler(stores, b, nil, nil, nil)
scheduler.runPlanCheckRun(ctx, "project-a", claimed[0].UID, plan.UID, 1)
run, err := stores.GetPlanCheckRun(ctx, "project-a", plan.UID)
diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.go
index 94ee4ddd79..982cbb5ba1 100644
--- a/backend/runner/schemasync/syncer.go
+++ b/backend/runner/schemasync/syncer.go
@@ -9,6 +9,7 @@ import (
"slices"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/pkg/errors"
@@ -20,6 +21,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/dbfactory"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
@@ -37,11 +39,12 @@ const (
)
// NewSyncer creates a schema syncer.
-func NewSyncer(stores *store.Store, dbFactory *dbfactory.DBFactory, licenseService *enterprise.LicenseService) *Syncer {
+func NewSyncer(stores *store.Store, dbFactory *dbfactory.DBFactory, licenseService *enterprise.LicenseService, productMetrics *productmetrics.ProductMetrics) *Syncer {
return &Syncer{
store: stores,
dbFactory: dbFactory,
licenseService: licenseService,
+ productMetrics: productMetrics,
}
}
@@ -52,6 +55,7 @@ type Syncer struct {
store *store.Store
dbFactory *dbfactory.DBFactory
licenseService *enterprise.LicenseService
+ productMetrics *productmetrics.ProductMetrics
databaseSyncMap sync.Map // map[string]*store.DatabaseMessage
}
@@ -89,56 +93,10 @@ func (s *Syncer) Run(ctx context.Context, wg *sync.WaitGroup) {
slog.Warn("Database sync checker skipped due to HA license restriction", log.BBError(err))
continue
}
- instances, err := s.store.ListAllInstances(ctx, false)
- if err != nil {
- slog.Error("Failed to list instance", log.BBError(err))
- return
- }
- instanceMap := make(map[string]*store.InstanceMessage)
- for _, instance := range instances {
- instanceMap[instance.ResourceID] = instance
+ if err := s.syncQueuedDatabases(ctx); err != nil {
+ slog.Error("Failed to run database sync cycle", log.BBError(err))
+ continue
}
- dbwp := pool.New().WithMaxGoroutines(MaximumOutstanding)
- s.databaseSyncMap.Range(func(key, value any) bool {
- database, ok := value.(*store.DatabaseMessage)
- if !ok {
- return true
- }
-
- s.databaseSyncMap.Delete(key)
- instance, ok := instanceMap[database.InstanceID]
- if !ok || !s.canScheduleDatabaseSync(ctx, instance, database) {
- return true
- }
- dbwp.Go(func() {
- slog.Debug("Sync database schema", slog.String("instance", database.InstanceID), slog.String("database", database.DatabaseName))
- if err := s.SyncDatabaseSchema(ctx, database); err != nil {
- slog.Debug("Failed to sync database schema",
- slog.String("instance", database.InstanceID),
- slog.String("databaseName", database.DatabaseName),
- log.BBError(err))
- // Save sync error to database metadata
- if _, updateErr := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- MetadataUpdates: []func(*storepb.DatabaseMetadata){
- func(md *storepb.DatabaseMetadata) {
- md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_FAILED
- md.SyncError = err.Error()
- md.LastSyncTime = timestamppb.Now()
- },
- },
- }); updateErr != nil {
- slog.Error("Failed to update database sync error",
- slog.String("instance", database.InstanceID),
- slog.String("database", database.DatabaseName),
- log.BBError(updateErr))
- }
- }
- })
- return true
- })
- dbwp.Wait()
case <-ctx.Done(): // if cancel() execute
return
}
@@ -148,6 +106,8 @@ func (s *Syncer) Run(ctx context.Context, wg *sync.WaitGroup) {
}
func (s *Syncer) trySyncAll(ctx context.Context) {
+ startedAt := time.Now()
+ result := productmetrics.ResultFailure
defer func() {
if r := recover(); r != nil {
err, ok := r.(error)
@@ -156,6 +116,9 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
}
slog.Error("Instance syncer PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
}
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerInstanceSync, result, time.Since(startedAt))
+ }
}()
lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeySchemaSyncer)
@@ -165,15 +128,18 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
}
if !acquired {
slog.Debug("Schema syncer advisory lock held by another replica, skipping")
+ result = productmetrics.ResultSkipped
return
}
defer func() {
if err := lock.Release(); err != nil {
+ result = productmetrics.ResultFailure
slog.Error("Failed to release schema syncer advisory lock", log.BBError(err))
}
}()
wp := pool.New().WithMaxGoroutines(MaximumOutstanding)
+ var syncFailed atomic.Bool
instances, err := s.store.ListAllInstances(ctx, false)
if err != nil {
slog.Error("Failed to retrieve instances", log.BBError(err))
@@ -182,7 +148,15 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
now := time.Now()
for _, instance := range instances {
instance := instance
- if !s.canScheduleInstanceSync(ctx, instance) {
+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
+ if err != nil {
+ syncFailed.Store(true)
+ slog.Error("Failed to determine whether instance sync can be scheduled",
+ slog.String("instance", instance.ResourceID),
+ log.BBError(err))
+ continue
+ }
+ if !canSchedule {
continue
}
interval := s.getOrDefaultSyncInterval(ctx, instance)
@@ -200,6 +174,7 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
wp.Go(func() {
slog.Debug("Sync instance schema", slog.String("instance", instance.ResourceID))
if _, _, _, err := s.SyncInstance(ctx, instance); err != nil {
+ syncFailed.Store(true)
slog.Debug("Failed to sync instance",
slog.String("instance", instance.ResourceID),
slog.String("error", err.Error()))
@@ -227,7 +202,16 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
if !ok {
continue
}
- if !s.canScheduleDatabaseSync(ctx, instance, database) {
+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
+ if err != nil {
+ syncFailed.Store(true)
+ slog.Error("Failed to determine whether database sync can be scheduled",
+ slog.String("instance", database.InstanceID),
+ slog.String("database", database.DatabaseName),
+ log.BBError(err))
+ continue
+ }
+ if !canSchedule {
continue
}
// The database inherits the sync interval from the instance.
@@ -245,33 +229,144 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
s.databaseSyncMap.Store(database.String(), database)
}
+ if !syncFailed.Load() {
+ result = productmetrics.ResultSuccess
+ }
}
-func (s *Syncer) canScheduleInstanceSync(ctx context.Context, instance *store.InstanceMessage) bool {
- return instance.ProjectID == nil || s.isProjectActive(ctx, *instance.ProjectID)
+func (s *Syncer) syncQueuedDatabases(ctx context.Context) (retErr error) {
+ startedAt := time.Now()
+ result := productmetrics.ResultFailure
+ defer func() {
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerDatabaseSync, result, time.Since(startedAt))
+ }
+ }()
+ defer func() {
+ if r := recover(); r != nil {
+ panicErr, ok := r.(error)
+ if !ok {
+ panicErr = errors.Errorf("%v", r)
+ }
+ retErr = errors.Wrap(panicErr, "database sync checker panic")
+ slog.Error("Database sync checker PANIC RECOVER", log.BBError(retErr), log.BBStack("panic-stack"))
+ }
+ }()
+
+ instances, err := s.store.ListAllInstances(ctx, false)
+ if err != nil {
+ return errors.Wrap(err, "failed to list instances")
+ }
+ instanceMap := make(map[string]*store.InstanceMessage)
+ for _, instance := range instances {
+ instanceMap[instance.ResourceID] = instance
+ }
+ dbwp := pool.New().WithMaxGoroutines(MaximumOutstanding)
+ var syncFailed atomic.Bool
+ s.databaseSyncMap.Range(func(key, value any) bool {
+ database, ok := value.(*store.DatabaseMessage)
+ if !ok {
+ return true
+ }
+
+ instance, ok := instanceMap[database.InstanceID]
+ if !ok {
+ s.databaseSyncMap.Delete(key)
+ return true
+ }
+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
+ if err != nil {
+ syncFailed.Store(true)
+ slog.Error("Failed to determine whether queued database sync can be scheduled",
+ slog.String("instance", database.InstanceID),
+ slog.String("database", database.DatabaseName),
+ log.BBError(err))
+ return true
+ }
+ s.databaseSyncMap.Delete(key)
+ if !canSchedule {
+ return true
+ }
+ dbwp.Go(func() {
+ slog.Debug("Sync database schema", slog.String("instance", database.InstanceID), slog.String("database", database.DatabaseName))
+ if err := s.SyncDatabaseSchema(ctx, database); err != nil {
+ syncFailed.Store(true)
+ slog.Debug("Failed to sync database schema",
+ slog.String("instance", database.InstanceID),
+ slog.String("databaseName", database.DatabaseName),
+ log.BBError(err))
+ // Save sync error to database metadata
+ if _, updateErr := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{
+ InstanceID: database.InstanceID,
+ DatabaseName: database.DatabaseName,
+ MetadataUpdates: []func(*storepb.DatabaseMetadata){
+ func(md *storepb.DatabaseMetadata) {
+ md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_FAILED
+ md.SyncError = err.Error()
+ md.LastSyncTime = timestamppb.Now()
+ },
+ },
+ }); updateErr != nil {
+ syncFailed.Store(true)
+ slog.Error("Failed to update database sync error",
+ slog.String("instance", database.InstanceID),
+ slog.String("database", database.DatabaseName),
+ log.BBError(updateErr))
+ }
+ }
+ })
+ return true
+ })
+ dbwp.Wait()
+
+ if syncFailed.Load() {
+ return errors.New("one or more queued database synchronizations failed")
+ }
+ result = productmetrics.ResultSuccess
+ return nil
}
-func (s *Syncer) canScheduleDatabaseSync(ctx context.Context, instance *store.InstanceMessage, database *store.DatabaseMessage) bool {
- return !database.Deleted && s.canScheduleInstanceSync(ctx, instance) && s.isProjectActive(ctx, database.ProjectID)
+func (s *Syncer) canScheduleInstanceSync(ctx context.Context, instance *store.InstanceMessage) (bool, error) {
+ if instance.ProjectID == nil {
+ return true, nil
+ }
+ return s.isProjectActive(ctx, *instance.ProjectID)
}
-func (s *Syncer) isProjectActive(ctx context.Context, projectID string) bool {
+func (s *Syncer) canScheduleDatabaseSync(ctx context.Context, instance *store.InstanceMessage, database *store.DatabaseMessage) (bool, error) {
+ if database.Deleted {
+ return false, nil
+ }
+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
+ if err != nil || !canSchedule {
+ return canSchedule, err
+ }
+ return s.isProjectActive(ctx, database.ProjectID)
+}
+
+func (s *Syncer) isProjectActive(ctx context.Context, projectID string) (bool, error) {
project, err := s.store.GetProjectByResourceID(ctx, projectID)
if err != nil {
- slog.Error("failed to get project for schema sync", slog.String("project", projectID), log.BBError(err))
- return false
+ return false, errors.Wrapf(err, "failed to get project %q for schema sync", projectID)
}
if project == nil {
slog.Warn("project not found for schema sync", slog.String("project", projectID))
- return false
+ return false, nil
}
- return !project.Deleted
+ return !project.Deleted, nil
}
func (s *Syncer) SyncAllDatabases(ctx context.Context, instance *store.InstanceMessage) {
find := &store.FindDatabaseMessage{}
if instance != nil {
- if !s.canScheduleInstanceSync(ctx, instance) {
+ canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
+ if err != nil {
+ slog.Error("Failed to determine whether instance sync can be scheduled",
+ slog.String("instance", instance.ResourceID),
+ log.BBError(err))
+ return
+ }
+ if !canSchedule {
return
}
find.InstanceID = &instance.ResourceID
@@ -284,9 +379,22 @@ func (s *Syncer) SyncAllDatabases(ctx context.Context, instance *store.InstanceM
}
for _, database := range databases {
- if database.Deleted || instance != nil && !s.canScheduleDatabaseSync(ctx, instance, database) {
+ if database.Deleted {
continue
}
+ if instance != nil {
+ canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
+ if err != nil {
+ slog.Error("Failed to determine whether database sync can be scheduled",
+ slog.String("instance", database.InstanceID),
+ slog.String("database", database.DatabaseName),
+ log.BBError(err))
+ continue
+ }
+ if !canSchedule {
+ continue
+ }
+ }
s.databaseSyncMap.Store(database.String(), database)
}
}
@@ -329,7 +437,20 @@ func (s *Syncer) GetInstanceMeta(ctx context.Context, instance *store.InstanceMe
}
// SyncInstance syncs the schema for all databases in an instance.
-func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessage) (*store.InstanceMessage, []*storepb.DatabaseSchemaMetadata, []*store.DatabaseMessage, error) {
+func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessage) (updatedInstance *store.InstanceMessage, allDatabases []*storepb.DatabaseSchemaMetadata, newDatabases []*store.DatabaseMessage, retErr error) {
+ startedAt := time.Now()
+ defer func() {
+ panicValue := recover()
+ if panicValue != nil {
+ retErr = errorFromPanic(panicValue, "instance sync panic")
+ }
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordInstanceSync(instance, time.Since(startedAt), retErr)
+ }
+ if panicValue != nil {
+ panic(panicValue)
+ }
+ }()
instanceMeta, err := s.GetInstanceMeta(ctx, instance)
if err != nil {
return nil, nil, nil, err
@@ -347,7 +468,7 @@ func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessa
if instanceMeta.Version != instance.Metadata.GetVersion() {
metadata.Version = instanceMeta.Version
}
- updatedInstance, err := s.store.UpdateInstance(ctx, updateInstance)
+ updatedInstance, err = s.store.UpdateInstance(ctx, updateInstance)
if err != nil {
return nil, nil, nil, err
}
@@ -355,7 +476,6 @@ func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessa
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "failed to sync database for instance: %s. Failed to find database list", instance.ResourceID)
}
- var newDatabases []*store.DatabaseMessage
var filteredDatabaseMetadatas []*storepb.DatabaseSchemaMetadata
var databaseProjectID string
if instance.ProjectID != nil {
@@ -410,6 +530,18 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab
if database == nil {
return "", errors.New("cannot sync nil database")
}
+ defer func() {
+ panicValue := recover()
+ if panicValue != nil {
+ retErr = errorFromPanic(panicValue, "database schema sync panic")
+ }
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordDatabaseSync(database, retErr)
+ }
+ if panicValue != nil {
+ panic(panicValue)
+ }
+ }()
instance, err := s.store.GetInstanceByResourceID(ctx, database.InstanceID)
if err != nil {
return "", errors.Wrapf(err, "failed to get instance %q", database.InstanceID)
@@ -500,6 +632,14 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab
return "", nil
}
+func errorFromPanic(value any, message string) error {
+ err, ok := value.(error)
+ if !ok {
+ err = errors.Errorf("%v", value)
+ }
+ return errors.Wrap(err, message)
+}
+
// SyncDatabaseSchemaToHistory will sync the schema for a database and create a sync history record.
func (s *Syncer) SyncDatabaseSchemaToHistory(ctx context.Context, database *store.DatabaseMessage) (string, error) {
return s.doSyncDatabaseSchema(ctx, database, true)
diff --git a/backend/runner/schemasync/syncer_test.go b/backend/runner/schemasync/syncer_test.go
index ae3dcd7543..a0712e0800 100644
--- a/backend/runner/schemasync/syncer_test.go
+++ b/backend/runner/schemasync/syncer_test.go
@@ -6,16 +6,141 @@ import (
"testing"
"time"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/bytebase/bytebase/backend/common/testcontainer"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/migrator"
"github.com/bytebase/bytebase/backend/store"
)
+func TestSyncCyclesRecordEmptySuccess(t *testing.T) {
+ ctx := context.Background()
+ stores := setupSyncerStore(ctx, t)
+ metrics := productmetrics.New(nil, nil)
+ syncer := NewSyncer(stores, nil, nil, metrics)
+
+ syncer.trySyncAll(ctx)
+ require.NoError(t, syncer.syncQueuedDatabases(ctx))
+ lock, acquired, err := store.TryAdvisoryLock(ctx, stores.GetDB(), store.AdvisoryLockKeySchemaSyncer)
+ require.NoError(t, err)
+ require.True(t, acquired)
+ syncer.trySyncAll(ctx)
+ require.NoError(t, lock.Release())
+ panicSyncer := &Syncer{productMetrics: metrics}
+ panicSyncer.trySyncAll(ctx)
+ require.Error(t, panicSyncer.syncQueuedDatabases(ctx))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSuccess))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultSuccess))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultFailure))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultFailure))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSkipped))
+}
+
+func TestSyncInstancePanicRecordsFailure(t *testing.T) {
+ metrics := productmetrics.New(nil, nil)
+ instance := &store.InstanceMessage{
+ ResourceID: "instance-a",
+ Metadata: &storepb.Instance{
+ DataSources: []*storepb.DataSource{{Type: storepb.DataSourceType_ADMIN}},
+ },
+ }
+ syncer := &Syncer{productMetrics: metrics}
+
+ require.Panics(t, func() {
+ _, _, _, _ = syncer.SyncInstance(context.Background(), instance)
+ })
+ require.Equal(t, uint64(1), syncMetricCount(t, metrics, map[string]string{
+ "result": string(productmetrics.ResultFailure),
+ "bytebase_instance_id": instance.ResourceID,
+ }))
+}
+
+func TestSyncDatabaseSchemaPanicRecordsFailure(t *testing.T) {
+ metrics := productmetrics.New(nil, nil)
+ database := &store.DatabaseMessage{InstanceID: "instance-a", DatabaseName: "db-a"}
+ syncer := &Syncer{productMetrics: metrics}
+
+ require.Panics(t, func() {
+ _ = syncer.SyncDatabaseSchema(context.Background(), database)
+ })
+ require.Equal(t, uint64(1), syncMetricCount(t, metrics, map[string]string{
+ "result": string(productmetrics.ResultFailure),
+ "bytebase_instance_id": database.InstanceID,
+ "database": database.DatabaseName,
+ }))
+}
+
+func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {
+ t.Helper()
+ ch := make(chan prometheus.Metric, 16)
+ go func() {
+ metrics.Collect(ch)
+ close(ch)
+ }()
+
+ var count uint64
+ for metric := range ch {
+ var dtoMetric dto.Metric
+ if metric.Write(&dtoMetric) != nil {
+ continue
+ }
+ labels := map[string]string{}
+ for _, label := range dtoMetric.GetLabel() {
+ labels[label.GetName()] = label.GetValue()
+ }
+ if labels["runner"] == string(runner) && labels["result"] == string(result) {
+ count += dtoMetric.GetHistogram().GetSampleCount()
+ }
+ }
+ return count
+}
+
+func syncMetricCount(t *testing.T, metrics *productmetrics.ProductMetrics, expectedLabels map[string]string) uint64 {
+ t.Helper()
+ ch := make(chan prometheus.Metric, 16)
+ go func() {
+ metrics.Collect(ch)
+ close(ch)
+ }()
+
+ for metric := range ch {
+ var dtoMetric dto.Metric
+ if metric.Write(&dtoMetric) != nil {
+ continue
+ }
+ labels := map[string]string{}
+ for _, label := range dtoMetric.GetLabel() {
+ labels[label.GetName()] = label.GetValue()
+ }
+ if len(labels) != len(expectedLabels) {
+ continue
+ }
+ matches := true
+ for name, value := range expectedLabels {
+ if labels[name] != value {
+ matches = false
+ break
+ }
+ }
+ if !matches {
+ continue
+ }
+ if dtoMetric.GetHistogram() != nil {
+ return dtoMetric.GetHistogram().GetSampleCount()
+ }
+ if dtoMetric.GetCounter() != nil {
+ return uint64(dtoMetric.GetCounter().GetValue())
+ }
+ }
+ return 0
+}
+
// TestSyncDatabaseSchemaNilDatabase pins the guard that prevents a nil
// *store.DatabaseMessage from panicking the syncer. Callers may receive
// (nil, nil) from store.GetDatabase when the referenced database is not
@@ -85,7 +210,7 @@ func TestTrySyncAllSkipsDatabasesInArchivedProjects(t *testing.T) {
})
require.NoError(t, err)
- syncer := NewSyncer(s, nil, nil)
+ syncer := NewSyncer(s, nil, nil, nil)
syncer.trySyncAll(ctx)
queued := 0
@@ -113,7 +238,7 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {
ctx := context.Background()
s := setupSyncerStore(ctx, t)
projectID := "project-a"
- syncer := NewSyncer(s, nil, nil)
+ syncer := NewSyncer(s, nil, nil, nil)
instance := &store.InstanceMessage{ProjectID: &projectID}
archived := true
@@ -122,7 +247,9 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {
Workspace: "default",
Delete: &archived,
}))
- require.False(t, syncer.canScheduleInstanceSync(ctx, instance))
+ canSchedule, err := syncer.canScheduleInstanceSync(ctx, instance)
+ require.NoError(t, err)
+ require.False(t, canSchedule)
archived = false
require.NoError(t, s.UpdateProjects(ctx, &store.UpdateProjectMessage{
@@ -130,7 +257,73 @@ func TestCanScheduleInstanceSyncRequiresActiveProject(t *testing.T) {
Workspace: "default",
Delete: &archived,
}))
- require.True(t, syncer.canScheduleInstanceSync(ctx, instance))
+ canSchedule, err = syncer.canScheduleInstanceSync(ctx, instance)
+ require.NoError(t, err)
+ require.True(t, canSchedule)
+}
+
+func TestTrySyncAllProjectLookupFailureRecordsFailure(t *testing.T) {
+ ctx := context.Background()
+ stores := setupSyncerStore(ctx, t)
+ projectID := "project-a"
+ _, err := stores.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: "instance-a",
+ Workspace: "default",
+ ProjectID: &projectID,
+ Metadata: &storepb.Instance{
+ Activation: true,
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+ _, err = stores.GetDB().ExecContext(ctx, "ALTER TABLE project RENAME TO unavailable_project")
+ require.NoError(t, err)
+
+ metrics := productmetrics.New(nil, nil)
+ syncer := NewSyncer(stores, nil, nil, metrics)
+ syncer.trySyncAll(ctx)
+
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultFailure))
+ require.Zero(t, runnerRunCount(t, metrics, productmetrics.RunnerInstanceSync, productmetrics.ResultSuccess))
+}
+
+func TestSyncQueuedDatabasesProjectLookupFailureRecordsFailure(t *testing.T) {
+ ctx := context.Background()
+ stores := setupSyncerStore(ctx, t)
+ _, err := stores.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: "instance-a",
+ Workspace: "default",
+ Metadata: &storepb.Instance{
+ Activation: true,
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+ database, err := stores.UpsertDatabase(ctx, &store.DatabaseMessage{
+ ProjectID: "project-a",
+ InstanceID: "instance-a",
+ DatabaseName: "app",
+ Metadata: &storepb.DatabaseMetadata{},
+ })
+ require.NoError(t, err)
+
+ metrics := productmetrics.New(nil, nil)
+ syncer := NewSyncer(stores, nil, nil, metrics)
+ syncer.SyncDatabaseAsync(database)
+ _, err = stores.GetDB().ExecContext(ctx, "ALTER TABLE project RENAME TO unavailable_project")
+ require.NoError(t, err)
+
+ require.Error(t, syncer.syncQueuedDatabases(ctx))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultFailure))
+ require.Zero(t, runnerRunCount(t, metrics, productmetrics.RunnerDatabaseSync, productmetrics.ResultSuccess))
+ queued := 0
+ syncer.databaseSyncMap.Range(func(_, _ any) bool {
+ queued++
+ return true
+ })
+ require.Equal(t, 1, queued)
}
func setupSyncerStore(ctx context.Context, t *testing.T) *store.Store {
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index deeb30841a..3b476a49a0 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -13,6 +13,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
)
@@ -56,6 +57,20 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {
startedAt := time.Now()
+ result := productmetrics.ResultFailure
+ defer func() {
+ if r := recover(); r != nil {
+ panicErr, ok := r.(error)
+ if !ok {
+ panicErr = errors.Errorf("%v", r)
+ }
+ err = errors.Wrap(panicErr, "pending task runs scheduler panic")
+ slog.Error("Pending task runs scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
+ }
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerTaskPending, result, time.Since(startedAt))
+ }
+ }()
tx, err := s.store.GetDB().BeginTx(ctx, nil)
if err != nil {
return errors.Wrapf(err, "failed to begin pending scheduler transaction")
@@ -75,6 +90,7 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {
slog.Debug("Pending scheduler advisory lock held by another replica, skipping",
slog.Duration("duration", time.Since(startedAt)),
)
+ result = productmetrics.ResultSkipped
return nil
}
@@ -91,8 +107,12 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {
return errors.Wrapf(err, "failed to create scheduling context")
}
+ var processingErr error
for _, taskRun := range taskRuns {
if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {
+ if processingErr == nil {
+ processingErr = err
+ }
slog.Error("failed to schedule pending task run",
slog.Int64("taskRunID", taskRun.ID),
log.BBError(err),
@@ -103,7 +123,11 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {
if err := tx.Commit(); err != nil {
return errors.Wrapf(err, "failed to commit pending scheduler transaction")
}
+ if processingErr != nil {
+ return errors.Wrap(processingErr, "failed to process one or more pending task runs")
+ }
+ result = productmetrics.ResultSuccess
return nil
}
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index d92d6b81d4..d49b3b893f 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -12,6 +12,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/bus"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/component/webhook"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
@@ -50,20 +51,43 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
}
}
-func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {
+func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) (retErr error) {
+ startedAt := time.Now()
+ result := productmetrics.ResultFailure
+ defer func() {
+ if r := recover(); r != nil {
+ panicErr, ok := r.(error)
+ if !ok {
+ panicErr = errors.Errorf("%v", r)
+ }
+ retErr = errors.Wrap(panicErr, "running task runs scheduler panic")
+ slog.Error("Running task runs scheduler PANIC RECOVER", log.BBError(retErr), log.BBStack("panic-stack"))
+ }
+ if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
+ s.productMetrics.RecordRunnerRun(productmetrics.RunnerTaskDispatch, result, time.Since(startedAt))
+ }
+ }()
// Atomically claim all AVAILABLE task runs
claimed, err := s.store.ClaimAvailableTaskRuns(ctx, s.profile.ReplicaID)
if err != nil {
return errors.Wrapf(err, "failed to claim available task runs")
}
+ var processingErr error
for _, c := range claimed {
taskRunCtx := taskRunLogContext(ctx, c.ProjectID, c.TaskRunUID)
if err := s.executeTaskRun(taskRunCtx, c.ProjectID, c.TaskRunUID, c.TaskUID); err != nil {
+ if processingErr == nil {
+ processingErr = err
+ }
slog.ErrorContext(taskRunCtx, "failed to execute task run", log.BBError(err))
}
}
+ if processingErr != nil {
+ return errors.Wrap(processingErr, "failed to process one or more available task runs")
+ }
+ result = productmetrics.ResultSuccess
return nil
}
diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.go
index d7aa088cff..d4996237b0 100644
--- a/backend/runner/taskrun/scheduler.go
+++ b/backend/runner/taskrun/scheduler.go
@@ -12,6 +12,7 @@ import (
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/bus"
"github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/component/webhook"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -33,6 +34,7 @@ type Scheduler struct {
bus *bus.Bus
webhookManager *webhook.Manager
licenseService *enterprise.LicenseService
+ productMetrics *productmetrics.ProductMetrics
executorMap map[storepb.Task_Type]Executor
profile *config.Profile
// haFailSince is when CheckReplicaLimit first started failing.
@@ -47,6 +49,7 @@ func NewScheduler(
webhookManager *webhook.Manager,
licenseService *enterprise.LicenseService,
profile *config.Profile,
+ productMetrics *productmetrics.ProductMetrics,
) *Scheduler {
return &Scheduler{
store: store,
@@ -54,6 +57,7 @@ func NewScheduler(
webhookManager: webhookManager,
licenseService: licenseService,
profile: profile,
+ productMetrics: productMetrics,
executorMap: map[storepb.Task_Type]Executor{},
}
}
diff --git a/backend/runner/taskrun/scheduler_test.go b/backend/runner/taskrun/scheduler_test.go
index 0ef1592dab..78cd9fbc74 100644
--- a/backend/runner/taskrun/scheduler_test.go
+++ b/backend/runner/taskrun/scheduler_test.go
@@ -4,14 +4,84 @@ import (
"context"
"testing"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/component/bus"
+ "github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
)
+func TestTaskCyclesRecordEmptySuccess(t *testing.T) {
+ ctx := context.Background()
+ stores := setupRolloutCreatorStore(ctx, t)
+ b, err := bus.New()
+ require.NoError(t, err)
+ metrics := productmetrics.New(nil, nil)
+ scheduler := &Scheduler{
+ store: stores,
+ bus: b,
+ profile: &config.Profile{ReplicaID: "replica-a"},
+ productMetrics: metrics,
+ }
+
+ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))
+ require.NoError(t, scheduler.scheduleRunningTaskRuns(ctx))
+ tx, err := stores.GetDB().BeginTx(ctx, nil)
+ require.NoError(t, err)
+ acquired, err := store.TryAdvisoryXactLock(ctx, tx, store.AdvisoryLockKeyPendingScheduler)
+ require.NoError(t, err)
+ require.True(t, acquired)
+ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))
+ require.NoError(t, tx.Rollback())
+ panicScheduler := &Scheduler{productMetrics: metrics}
+ require.Error(t, panicScheduler.schedulePendingTaskRuns(ctx))
+ require.Error(t, panicScheduler.scheduleRunningTaskRuns(ctx))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultSuccess))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultSuccess))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultFailure))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultFailure))
+ require.Equal(t, uint64(1), runnerRunCount(t, metrics, productmetrics.RunnerTaskPending, productmetrics.ResultSkipped))
+
+ canceledMetrics := productmetrics.New(nil, nil)
+ scheduler.productMetrics = canceledMetrics
+ canceledContext, cancel := context.WithCancel(ctx)
+ cancel()
+ require.Error(t, scheduler.schedulePendingTaskRuns(canceledContext))
+ require.Error(t, scheduler.scheduleRunningTaskRuns(canceledContext))
+ require.Zero(t, runnerRunCount(t, canceledMetrics, productmetrics.RunnerTaskPending, productmetrics.ResultFailure))
+ require.Zero(t, runnerRunCount(t, canceledMetrics, productmetrics.RunnerTaskDispatch, productmetrics.ResultFailure))
+}
+
+func runnerRunCount(t *testing.T, metrics *productmetrics.ProductMetrics, runner productmetrics.Runner, result productmetrics.RunnerResult) uint64 {
+ t.Helper()
+ ch := make(chan prometheus.Metric, 16)
+ go func() {
+ metrics.Collect(ch)
+ close(ch)
+ }()
+
+ var count uint64
+ for metric := range ch {
+ var dtoMetric dto.Metric
+ if metric.Write(&dtoMetric) != nil {
+ continue
+ }
+ labels := map[string]string{}
+ for _, label := range dtoMetric.GetLabel() {
+ labels[label.GetName()] = label.GetValue()
+ }
+ if labels["runner"] == string(runner) && labels["result"] == string(result) {
+ count += dtoMetric.GetHistogram().GetSampleCount()
+ }
+ }
+ return count
+}
+
func TestCompletionWebhookEnvironmentUsesLastEnvironmentOrder(t *testing.T) {
tasks := []*store.TaskMessage{
{ID: 1, Environment: "dev"},
diff --git a/backend/server/echo_routes.go b/backend/server/echo_routes.go
index 84779cda0c..14c9094da8 100644
--- a/backend/server/echo_routes.go
+++ b/backend/server/echo_routes.go
@@ -10,7 +10,6 @@ import (
"github.com/labstack/echo/v5/middleware"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
connectcors "connectrpc.com/cors"
@@ -22,9 +21,8 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/config"
- "github.com/bytebase/bytebase/backend/enterprise"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
stripeplugin "github.com/bytebase/bytebase/backend/plugin/stripe"
- "github.com/bytebase/bytebase/backend/store"
)
func configureEchoRouters(
@@ -34,8 +32,7 @@ func configureEchoRouters(
oauth2Service *oauth2.Service,
mcpServer *mcp.Server,
stripeWebhookHandler *stripeapi.WebhookHandler,
- stores *store.Store,
- licenseService *enterprise.LicenseService,
+ productMetrics *productmetrics.ProductMetrics,
profile *config.Profile,
) {
e.Use(recoverMiddleware)
@@ -67,7 +64,7 @@ func configureEchoRouters(
registerPprof(e, &profile.RuntimeDebug)
- registerMetricsRoute(e, profile, stores, licenseService)
+ registerMetricsRoute(e, profile, productMetrics)
e.GET("/healthz", func(c *echo.Context) error {
return c.String(http.StatusOK, "OK")
@@ -97,7 +94,7 @@ func configureEchoRouters(
embedFrontend(e)
}
-func registerMetricsRoute(e *echo.Echo, profile *config.Profile, stores *store.Store, licenseService *enterprise.LicenseService) {
+func registerMetricsRoute(e *echo.Echo, profile *config.Profile, productMetrics *productmetrics.ProductMetrics) {
if profile.SaaS {
return
}
@@ -125,18 +122,8 @@ func registerMetricsRoute(e *echo.Echo, profile *config.Profile, stores *store.S
// Use promhttp directly: pass the local registry as the Registerer
// for self-instrumentation; pass the Gatherers fold as the gather
// source. Both observability surfaces preserved.
- // Product-level license gauges are registered on the local registry only;
- // they are computed synchronously on every scrape from fresh shared
- // metadata. A collection failure fails the whole scrape (HTTP 500 via
- // HTTPErrorOnError) instead of emitting zero, stale, or missing gauges.
- registry.MustRegister(newLicenseSeatsCollector(stores, licenseService))
- e.GET("/metrics", echo.WrapHandler(promhttp.InstrumentMetricHandler(
- registry,
- promhttp.HandlerFor(
- prometheus.Gatherers{registry, prometheus.DefaultGatherer},
- promhttp.HandlerOpts{ErrorHandling: promhttp.HTTPErrorOnError},
- ),
- )))
+ registry.MustRegister(productMetrics)
+ e.GET("/metrics", echo.WrapHandler(newMetricsHandler(registry)))
}
func recoverMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
diff --git a/backend/server/echo_routes_test.go b/backend/server/echo_routes_test.go
index b7bba6047b..e38fd37ba9 100644
--- a/backend/server/echo_routes_test.go
+++ b/backend/server/echo_routes_test.go
@@ -9,6 +9,7 @@ import (
"github.com/labstack/echo/v5"
"github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
)
func TestSecurityHeadersMiddleware_GA4Sources(t *testing.T) {
@@ -93,12 +94,12 @@ func TestMetricsRouteDisabledInSaaS(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
e := echo.New()
if tt.saas {
- registerMetricsRoute(e, &config.Profile{SaaS: true}, nil, nil)
+ registerMetricsRoute(e, &config.Profile{SaaS: true}, nil)
} else {
// Self-host: use a real store and license service so the
// product-level license gauges are registered and collected.
_, st, licenseService := newMetricsTestEcho(t)
- registerMetricsRoute(e, &config.Profile{SaaS: false}, st, licenseService)
+ registerMetricsRoute(e, &config.Profile{SaaS: false}, productmetrics.New(st, licenseService))
}
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
diff --git a/backend/server/metrics.go b/backend/server/metrics.go
index 5f363596f7..c356731feb 100644
--- a/backend/server/metrics.go
+++ b/backend/server/metrics.go
@@ -1,93 +1,20 @@
package server
import (
- "context"
- "math"
- "time"
+ "net/http"
- "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
-
- "github.com/bytebase/bytebase/backend/enterprise"
- "github.com/bytebase/bytebase/backend/store"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
)
-// metricsCollectionTimeout bounds each collection so a slow metadata read cannot
-// hang a scrape indefinitely. Collectors receive no request context.
-const metricsCollectionTimeout = 5 * time.Second
-
-// licenseSeatsCollector exposes product-level license seat gauges at /metrics.
-// Both values are computed synchronously on every scrape from fresh shared
-// metadata (no per-replica caches), so replicas expose equivalent values.
-// A failed metadata read fails the whole scrape instead of emitting zero,
-// stale, or missing gauges.
-type licenseSeatsCollector struct {
- stores *store.Store
- licenseService *enterprise.LicenseService
-
- usedSeats *prometheus.Desc
- seatLimit *prometheus.Desc
-}
-
-func newLicenseSeatsCollector(stores *store.Store, licenseService *enterprise.LicenseService) *licenseSeatsCollector {
- return &licenseSeatsCollector{
- stores: stores,
- licenseService: licenseService,
- usedSeats: prometheus.NewDesc(
- "bytebase_license_seats_used",
- "Number of distinct end users occupying license seats in this Bytebase installation, including pending invites and group members, excluding deleted users and non-user identities.",
- nil, nil,
+// newMetricsHandler keeps the server-local registry isolated while folding in
+// package-global instrumentation that is registered elsewhere in Bytebase.
+func newMetricsHandler(registry *prometheus.Registry) http.Handler {
+ return promhttp.InstrumentMetricHandler(
+ registry,
+ promhttp.HandlerFor(
+ prometheus.Gatherers{registry, prometheus.DefaultGatherer},
+ promhttp.HandlerOpts{ErrorHandling: promhttp.HTTPErrorOnError},
),
- seatLimit: prometheus.NewDesc(
- "bytebase_license_seats_limit",
- "Effective seat limit enforced for this Bytebase installation. +Inf means the license has no seat limit.",
- nil, nil,
- ),
- }
-}
-
-func (c *licenseSeatsCollector) Describe(ch chan<- *prometheus.Desc) {
- ch <- c.usedSeats
- ch <- c.seatLimit
-}
-
-func (c *licenseSeatsCollector) Collect(ch chan<- prometheus.Metric) {
- ctx, cancel := context.WithTimeout(context.Background(), metricsCollectionTimeout)
- defer cancel()
-
- workspaceID, err := c.stores.GetWorkspaceID(ctx)
- if err != nil {
- failMetricCollection(ch, errors.Wrap(err, "failed to get workspace ID"))
- return
- }
-
- used := 0
- if workspaceID != "" {
- used, err = c.stores.CountSeatOccupyingUsers(ctx, workspaceID)
- if err != nil {
- failMetricCollection(ch, errors.Wrapf(err, "failed to count seat-occupying users for workspace %q", workspaceID))
- return
- }
- }
-
- limit, err := c.licenseService.GetUserLimitUncached(ctx, workspaceID)
- if err != nil {
- failMetricCollection(ch, errors.Wrapf(err, "failed to read the user limit for workspace %q", workspaceID))
- return
- }
-
- limitValue := float64(limit)
- if limit == math.MaxInt {
- limitValue = math.Inf(1)
- }
-
- ch <- prometheus.MustNewConstMetric(c.usedSeats, prometheus.GaugeValue, float64(used))
- ch <- prometheus.MustNewConstMetric(c.seatLimit, prometheus.GaugeValue, limitValue)
-}
-
-// failMetricCollection emits an invalid metric, which makes Registry.Gather
-// fail and therefore fails the whole scrape (HTTP 500 via HTTPErrorOnError)
-// instead of reporting zero, stale, or partial values.
-func failMetricCollection(ch chan<- prometheus.Metric, err error) {
- ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
+ )
}
diff --git a/backend/server/metrics_test.go b/backend/server/metrics_test.go
index b38ab50b03..59b32dc5fd 100644
--- a/backend/server/metrics_test.go
+++ b/backend/server/metrics_test.go
@@ -3,17 +3,24 @@ package server
import (
"context"
"fmt"
+ "io"
"net/http"
"net/http/httptest"
+ "strings"
+ "sync"
"testing"
+ "time"
"github.com/labstack/echo/v5"
+ metricpb "github.com/prometheus/client_model/go"
+ "github.com/prometheus/common/expfmt"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/testcontainer"
"github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/migrator"
@@ -28,6 +35,11 @@ const devEnterpriseLicense = "eyJhbGciOiJSUzI1NiIsImtpZCI6InYxIiwidHlwIjoiSldUIn
// newMetricsTestEcho returns an echo server with the self-host /metrics route
// wired to a real store and license service backed by a fresh PostgreSQL.
func newMetricsTestEcho(t *testing.T) (*echo.Echo, *store.Store, *enterprise.LicenseService) {
+ e, st, licenseService, _ := newMetricsTestEchoWithCollector(t)
+ return e, st, licenseService
+}
+
+func newMetricsTestEchoWithCollector(t *testing.T) (*echo.Echo, *store.Store, *enterprise.LicenseService, *productmetrics.ProductMetrics) {
t.Helper()
ctx := context.Background()
container := testcontainer.GetTestPgContainer(ctx, t)
@@ -46,8 +58,9 @@ func newMetricsTestEcho(t *testing.T) (*echo.Echo, *store.Store, *enterprise.Lic
require.NoError(t, err)
e := echo.New()
- registerMetricsRoute(e, &config.Profile{SaaS: false}, st, licenseService)
- return e, st, licenseService
+ metrics := productmetrics.New(st, licenseService)
+ registerMetricsRoute(e, &config.Profile{SaaS: false}, metrics)
+ return e, st, licenseService, metrics
}
func scrapeMetrics(t *testing.T, e *echo.Echo, wantStatus int) string {
@@ -67,10 +80,18 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {
body := scrapeMetrics(t, e, http.StatusOK)
require.Contains(t, body, "bytebase_license_seats_used 0")
require.Contains(t, body, "bytebase_license_seats_limit 20")
+ require.Contains(t, body, "bytebase_license_instances_used 0")
+ require.Contains(t, body, "bytebase_license_instances_limit 10")
+ require.Contains(t, body, "bytebase_license_expiry_timestamp_seconds +Inf")
_, err := st.GetDB().ExecContext(ctx, `
INSERT INTO workspace (resource_id) VALUES ('ws-test');
INSERT INTO setting (name, workspace, value) VALUES ('SYSTEM', 'ws-test', '{}'::jsonb);
+ INSERT INTO project (resource_id, workspace, name) VALUES ('project-a', 'ws-test', 'Project A');
+ INSERT INTO instance (resource_id, workspace, project, deleted) VALUES
+ ('workspace-instance', 'ws-test', NULL, FALSE),
+ ('project-instance', 'ws-test', 'project-a', FALSE),
+ ('deleted-instance', 'ws-test', NULL, TRUE);
INSERT INTO principal (name, email, password_hash, deleted) VALUES
('alice', '[email protected]', 'x', FALSE),
('bob', '[email protected]', 'x', FALSE),
@@ -112,6 +133,7 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {
body = scrapeMetrics(t, e, http.StatusOK)
require.Contains(t, body, "bytebase_license_seats_used 4")
require.Contains(t, body, "bytebase_license_seats_limit 20")
+ require.Contains(t, body, "bytebase_license_instances_used 2")
// A colliding group email in another workspace must not leak into this
// workspace's count. Keep the second workspace deleted so GetWorkspaceID
@@ -139,6 +161,12 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {
body = scrapeMetrics(t, e, http.StatusOK)
require.Contains(t, body, "bytebase_license_seats_used 4")
require.Contains(t, body, "bytebase_license_seats_limit +Inf")
+ require.Contains(t, body, "bytebase_license_instances_limit +Inf")
+
+ // A configured but malformed license invalidates the complete scrape.
+ require.NoError(t, st.UpdateLicense(ctx, "ws-test", "not-a-license"))
+ scrapeMetrics(t, e, http.StatusInternalServerError)
+ require.NoError(t, licenseService.StoreLicense(ctx, "ws-test", devEnterpriseLicense))
// A failing metadata read must fail the scrape instead of emitting
// zero, stale, or missing gauges.
@@ -146,6 +174,74 @@ func TestCollisionMetricsLicenseSeats(t *testing.T) {
scrapeMetrics(t, e, http.StatusInternalServerError)
}
+func TestMetricsEventsAreServerLocal(t *testing.T) {
+ e1, _, _, metrics1 := newMetricsTestEchoWithCollector(t)
+ e2, _, _, _ := newMetricsTestEchoWithCollector(t)
+ metrics1.RecordRunnerRun(productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess, time.Second)
+
+ body1 := scrapeMetrics(t, e1, http.StatusOK)
+ require.Contains(t, body1, `bytebase_runner_run_duration_seconds_count{result="success",runner="plan_check"} 1`)
+ body2 := scrapeMetrics(t, e2, http.StatusOK)
+ require.NotContains(t, body2, "bytebase_runner_run_duration_seconds")
+ for _, name := range []string{
+ "bytebase_license_expiry_timestamp_seconds",
+ "bytebase_license_instances_used",
+ "bytebase_license_instances_limit",
+ } {
+ require.Equal(t, metricTextLine(body1, name), metricTextLine(body2, name))
+ }
+
+ var wg sync.WaitGroup
+ scrapeErr := make(chan string, 8)
+ for range 8 {
+ wg.Go(func() {
+ metrics1.RecordRunnerRun(productmetrics.RunnerPlanCheck, productmetrics.ResultSuccess, time.Millisecond)
+ req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+ rec := httptest.NewRecorder()
+ e1.ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ scrapeErr <- rec.Body.String()
+ }
+ })
+ }
+ wg.Wait()
+ close(scrapeErr)
+ for err := range scrapeErr {
+ t.Errorf("concurrent scrape failed: %s", err)
+ }
+
+ format := expfmt.NewFormat(expfmt.TypeProtoDelim)
+ req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+ req.Header.Set("Accept", string(format))
+ rec := httptest.NewRecorder()
+ e1.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
+ decoder := expfmt.NewDecoder(rec.Body, format)
+ foundNative := false
+ for {
+ family := &metricpb.MetricFamily{}
+ if err := decoder.Decode(family); err != nil {
+ require.ErrorIs(t, err, io.EOF)
+ break
+ }
+ if family.GetName() == "bytebase_runner_run_duration_seconds" {
+ require.Len(t, family.Metric, 1)
+ require.NotNil(t, family.Metric[0].GetHistogram().Schema)
+ foundNative = true
+ }
+ }
+ require.True(t, foundNative)
+}
+
+func metricTextLine(body, name string) string {
+ for line := range strings.SplitSeq(body, "\n") {
+ if strings.HasPrefix(line, name+" ") {
+ return line
+ }
+ }
+ return ""
+}
+
func TestMetricsLicenseSeatsAllUsers(t *testing.T) {
ctx := context.Background()
e, st, _ := newMetricsTestEcho(t)
diff --git a/backend/server/server.go b/backend/server/server.go
index f5b745b5b4..f098e1ae00 100644
--- a/backend/server/server.go
+++ b/backend/server/server.go
@@ -25,6 +25,7 @@ import (
"github.com/bytebase/bytebase/backend/component/config"
"github.com/bytebase/bytebase/backend/component/dbfactory"
"github.com/bytebase/bytebase/backend/component/iam"
+ "github.com/bytebase/bytebase/backend/component/productmetrics"
"github.com/bytebase/bytebase/backend/component/review"
"github.com/bytebase/bytebase/backend/component/sampleinstance"
"github.com/bytebase/bytebase/backend/component/sheet"
@@ -213,15 +214,19 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
// Configure echo server.
s.echoServer = echo.New()
- s.schemaSyncer = schemasync.NewSyncer(stores, s.dbFactory, s.licenseService)
+ var productMetrics *productmetrics.ProductMetrics
+ if !profile.SaaS {
+ productMetrics = productmetrics.New(stores, s.licenseService)
+ }
+ s.schemaSyncer = schemasync.NewSyncer(stores, s.dbFactory, s.licenseService, productMetrics)
s.approvalRunner = review.NewRunner(stores, s.bus, s.webhookManager, s.licenseService)
- s.taskScheduler = taskrun.NewScheduler(stores, s.bus, s.webhookManager, s.licenseService, profile)
+ s.taskScheduler = taskrun.NewScheduler(stores, s.bus, s.webhookManager, s.licenseService, profile, productMetrics)
s.taskScheduler.Register(storepb.Task_DATABASE_CREATE, taskrun.NewDatabaseCreateExecutor(stores, s.dbFactory, s.schemaSyncer))
s.taskScheduler.Register(storepb.Task_DATABASE_MIGRATE, taskrun.NewDatabaseMigrateExecutor(stores, s.dbFactory, s.bus, s.schemaSyncer, profile))
combinedExecutor := plancheck.NewCombinedExecutor(stores, sheetManager, s.dbFactory)
- s.planCheckScheduler = plancheck.NewScheduler(stores, s.bus, combinedExecutor, s.licenseService)
+ s.planCheckScheduler = plancheck.NewScheduler(stores, s.bus, combinedExecutor, s.licenseService, productMetrics)
s.notifyListener = notifylistener.NewListener(stores.GetDB(), s.bus)
// Data cleaner
@@ -246,7 +251,7 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to create MCP server")
}
- configureEchoRouters(s.echoServer, s.lspServer, directorySyncServer, oauth2Service, mcpServer, stripeWebhookHandler, s.store, s.licenseService, profile)
+ configureEchoRouters(s.echoServer, s.lspServer, directorySyncServer, oauth2Service, mcpServer, stripeWebhookHandler, productMetrics, profile)
serverStarted = true
return s, nil
diff --git a/go.mod b/go.mod
index f883ce576a..d7f68cee3c 100644
--- a/go.mod
+++ b/go.mod
@@ -337,8 +337,8 @@ require (
github.com/pquerna/cachecontrol v0.2.0 // indirect
github.com/pquerna/otp v1.5.0
github.com/prometheus/client_golang v1.24.1
- github.com/prometheus/client_model v0.6.2 // indirect
- github.com/prometheus/common v0.70.1 // indirect
+ github.com/prometheus/client_model v0.6.2
+ github.com/prometheus/common v0.70.1
github.com/prometheus/procfs v0.21.1 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/shirou/gopsutil/v3 v3.24.5 // indirect