Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: report SQL review config telemetry (#20626)

p0ny4b33d11ed267
diff --git a/backend/api/v1/review_config_service.go b/backend/api/v1/review_config_service.go
index db4c49fbc6..aab227c594 100644
--- a/backend/api/v1/review_config_service.go
+++ b/backend/api/v1/review_config_service.go
@@ -64,15 +64,11 @@ func (s *ReviewConfigService) ListReviewConfigs(ctx context.Context, _ *connect.
return nil, connect.NewError(connect.CodeInternal, err)
}
- response := &v1pb.ListReviewConfigsResponse{}
- for _, message := range messages {
- sqlReview, err := s.convertToV1ReviewConfig(ctx, message)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, err)
- }
- response.ReviewConfigs = append(response.ReviewConfigs, sqlReview)
+ reviewConfigs, err := s.convertToV1ReviewConfigs(ctx, messages)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
}
- return connect.NewResponse(response), nil
+ return connect.NewResponse(&v1pb.ListReviewConfigsResponse{ReviewConfigs: reviewConfigs}), nil
}
// GetReviewConfig gets the review config.
@@ -235,6 +231,34 @@ func convertToReviewConfigMessage(reviewConfig *v1pb.ReviewConfig) (*store.Revie
}
func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, reviewConfigMessage *store.ReviewConfigMessage) (*v1pb.ReviewConfig, error) {
+ reviewConfigs, err := s.convertToV1ReviewConfigs(ctx, []*store.ReviewConfigMessage{reviewConfigMessage})
+ if err != nil {
+ return nil, err
+ }
+ if len(reviewConfigs) == 0 {
+ return nil, errors.New("review config not found")
+ }
+ return reviewConfigs[0], nil
+}
+
+func (s *ReviewConfigService) convertToV1ReviewConfigs(ctx context.Context, reviewConfigMessages []*store.ReviewConfigMessage) ([]*v1pb.ReviewConfig, error) {
+ configs := make([]*v1pb.ReviewConfig, 0, len(reviewConfigMessages))
+ configByName := make(map[string]*v1pb.ReviewConfig, len(reviewConfigMessages))
+ for _, message := range reviewConfigMessages {
+ payload := message.Payload
+ if payload == nil {
+ payload = &storepb.ReviewConfigPayload{}
+ }
+ config := &v1pb.ReviewConfig{
+ Name: common.FormatReviewConfig(message.ID),
+ Title: message.Name,
+ Enabled: message.Enforce,
+ Rules: ConvertToV1PBSQLReviewRules(payload.SqlReviewRules),
+ }
+ configs = append(configs, config)
+ configByName[config.Name] = config
+ }
+
policyType := storepb.Policy_TAG
tagPolicies, err := s.store.ListPolicies(ctx, &store.FindPolicyMessage{
Workspace: common.GetWorkspaceIDFromContext(ctx),
@@ -245,19 +269,13 @@ func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, revie
return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to list tag policy"))
}
- config := &v1pb.ReviewConfig{
- Name: common.FormatReviewConfig(reviewConfigMessage.ID),
- Title: reviewConfigMessage.Name,
- Enabled: reviewConfigMessage.Enforce,
- Rules: ConvertToV1PBSQLReviewRules(reviewConfigMessage.Payload.SqlReviewRules),
- }
-
for _, policy := range tagPolicies {
p := &v1pb.TagPolicy{}
if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(policy.Payload), p); err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to unmarshal tag policy"))
}
- if p.Tags[common.ReservedTagReviewConfig] != config.Name {
+ config, ok := configByName[p.Tags[common.ReservedTagReviewConfig]]
+ if !ok {
continue
}
@@ -296,7 +314,7 @@ func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, revie
}
}
- return config, nil
+ return configs, nil
}
// validateSQLReviewRules validates the SQL review rule.
diff --git a/backend/component/telemetry/reporter.go b/backend/component/telemetry/reporter.go
new file mode 100644
index 0000000000..cbe809b5ea
--- /dev/null
+++ b/backend/component/telemetry/reporter.go
@@ -0,0 +1,108 @@
+// Package telemetry provides telemetry reporting to hub.bytebase.com.
+package telemetry
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/bytebase/bytebase/backend/common"
+)
+
+var hubEventURL = "https://hub.bytebase.com/v1/events"
+
+// Reporter sends telemetry events to hub.bytebase.com.
+type Reporter struct {
+ mu sync.RWMutex
+ version string
+ gitCommit string
+ releaseMode common.ReleaseMode
+ enabled bool
+
+ httpClient *http.Client
+}
+
+var (
+ globalReporter *Reporter
+ globalReporterOnce sync.Once
+)
+
+// InitGlobalReporter initializes the global telemetry reporter.
+func InitGlobalReporter(version, gitCommit string, releaseMode common.ReleaseMode, enabled bool) {
+ globalReporterOnce.Do(func() {
+ globalReporter = &Reporter{
+ version: version,
+ gitCommit: gitCommit,
+ releaseMode: releaseMode,
+ enabled: enabled,
+ httpClient: &http.Client{
+ Timeout: 120 * time.Second,
+ },
+ }
+ })
+}
+
+type sqlReviewConfigSnapshotPayload struct {
+ WorkspaceID string `json:"workspaceId"`
+ Email string `json:"email"`
+ Version string `json:"version"`
+ Commit string `json:"commit"`
+ SQLReviewConfigSnapshot struct {
+ Snapshot string `json:"snapshot"`
+ EmailDomains []string `json:"emailDomains"`
+ } `json:"sqlReviewConfigSnapshot"`
+}
+
+// ReportSQLReviewConfigSnapshot reports the current SQL review config snapshot.
+func ReportSQLReviewConfigSnapshot(ctx context.Context, workspaceID string, email string, snapshot string, emailDomains []string) {
+ if globalReporter == nil {
+ return
+ }
+
+ globalReporter.mu.RLock()
+ if !globalReporter.enabled {
+ globalReporter.mu.RUnlock()
+ return
+ }
+ version := globalReporter.version
+ gitCommit := globalReporter.gitCommit
+ releaseMode := globalReporter.releaseMode
+ globalReporter.mu.RUnlock()
+
+ if releaseMode != common.ReleaseModeProd || workspaceID == "" || email == "" || snapshot == "" {
+ return
+ }
+
+ payload := sqlReviewConfigSnapshotPayload{
+ WorkspaceID: workspaceID,
+ Email: email,
+ Version: version,
+ Commit: gitCommit,
+ }
+ payload.SQLReviewConfigSnapshot.Snapshot = snapshot
+ payload.SQLReviewConfigSnapshot.EmailDomains = emailDomains
+
+ go globalReporter.send(context.WithoutCancel(ctx), payload)
+}
+
+func (r *Reporter) send(ctx context.Context, payload any) {
+ body, err := json.Marshal(payload)
+ if err != nil {
+ return
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, hubEventURL, bytes.NewReader(body))
+ if err != nil {
+ return
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ return
+ }
+ resp.Body.Close()
+}
diff --git a/backend/component/telemetry/reporter_test.go b/backend/component/telemetry/reporter_test.go
new file mode 100644
index 0000000000..991dcf917a
--- /dev/null
+++ b/backend/component/telemetry/reporter_test.go
@@ -0,0 +1,130 @@
+package telemetry
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/bytebase/bytebase/backend/common"
+)
+
+func TestReportSQLReviewConfigSnapshot(t *testing.T) {
+ const snapshot = `{"reviewConfigs":[{"name":"reviewConfigs/basic","enabled":true}]}`
+
+ var (
+ mu sync.Mutex
+ payload map[string]any
+ done = make(chan struct{})
+ )
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer close(done)
+
+ if r.Method != http.MethodPost {
+ t.Errorf("method = %q, want %q", r.Method, http.MethodPost)
+ }
+ if got := r.Header.Get("Content-Type"); got != "application/json" {
+ t.Errorf("Content-Type = %q, want %q", got, "application/json")
+ }
+ var got map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
+ t.Errorf("failed to decode payload: %v", err)
+ return
+ }
+ mu.Lock()
+ payload = got
+ mu.Unlock()
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ oldHubEventURL := hubEventURL
+ hubEventURL = server.URL
+ defer func() {
+ hubEventURL = oldHubEventURL
+ }()
+
+ globalReporter = &Reporter{
+ version: "3.19.1",
+ gitCommit: "abcdef",
+ releaseMode: common.ReleaseModeProd,
+ enabled: true,
+ httpClient: server.Client(),
+ }
+ defer func() {
+ globalReporter = nil
+ }()
+
+ ReportSQLReviewConfigSnapshot(context.Background(), "workspace-id", "[email protected]", snapshot, []string{"example.com", "test.com"})
+
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Fatal("timed out waiting for telemetry request")
+ }
+
+ mu.Lock()
+ defer mu.Unlock()
+ if payload["workspaceId"] != "workspace-id" {
+ t.Fatalf("workspaceId = %v, want %q", payload["workspaceId"], "workspace-id")
+ }
+ if payload["email"] != "[email protected]" {
+ t.Fatalf("email = %v, want %q", payload["email"], "[email protected]")
+ }
+ if payload["version"] != "3.19.1" {
+ t.Fatalf("version = %v, want %q", payload["version"], "3.19.1")
+ }
+ if payload["commit"] != "abcdef" {
+ t.Fatalf("commit = %v, want %q", payload["commit"], "abcdef")
+ }
+ event, ok := payload["sqlReviewConfigSnapshot"].(map[string]any)
+ if !ok {
+ t.Fatalf("sqlReviewConfigSnapshot = %T, want object", payload["sqlReviewConfigSnapshot"])
+ }
+ if event["snapshot"] != snapshot {
+ t.Fatalf("snapshot = %v, want %q", event["snapshot"], snapshot)
+ }
+ domains, ok := event["emailDomains"].([]any)
+ if !ok {
+ t.Fatalf("emailDomains = %T, want array", event["emailDomains"])
+ }
+ if len(domains) != 2 || domains[0] != "example.com" || domains[1] != "test.com" {
+ t.Fatalf("emailDomains = %v, want %v", domains, []string{"example.com", "test.com"})
+ }
+}
+
+func TestReportSQLReviewConfigSnapshotSkipsNonProd(t *testing.T) {
+ requestReceived := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+ close(requestReceived)
+ }))
+ defer server.Close()
+
+ oldHubEventURL := hubEventURL
+ hubEventURL = server.URL
+ defer func() {
+ hubEventURL = oldHubEventURL
+ }()
+
+ globalReporter = &Reporter{
+ version: "3.19.1",
+ gitCommit: "abcdef",
+ releaseMode: common.ReleaseModeDev,
+ enabled: true,
+ httpClient: server.Client(),
+ }
+ defer func() {
+ globalReporter = nil
+ }()
+
+ ReportSQLReviewConfigSnapshot(context.Background(), "workspace-id", "[email protected]", "{}", []string{"example.com"})
+
+ select {
+ case <-requestReceived:
+ t.Fatal("telemetry request was sent in non-prod release mode")
+ case <-time.After(100 * time.Millisecond):
+ }
+}
diff --git a/backend/server/server.go b/backend/server/server.go
index 62c42eed5f..81aa0f3561 100644
--- a/backend/server/server.go
+++ b/backend/server/server.go
@@ -27,6 +27,7 @@ import (
"github.com/bytebase/bytebase/backend/component/iam"
"github.com/bytebase/bytebase/backend/component/sampleinstance"
"github.com/bytebase/bytebase/backend/component/sheet"
+ "github.com/bytebase/bytebase/backend/component/telemetry"
"github.com/bytebase/bytebase/backend/component/webhook"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -151,13 +152,14 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
EnableMetricCollection: true,
EnableDebug: profile.Debug,
}
+ var workspaceID string
if !s.profile.SaaS {
s.sampleInstanceManager = sampleinstance.NewManager(stores, profile)
// Load workspace-dependent settings if workspace exists.
// On first boot (no workspace yet), these remain at defaults and get
// initialized when the workspace is created and settings are updated via API.
- if workspaceID, _ := stores.GetWorkspaceID(ctx); workspaceID != "" {
+ if workspaceID, _ = stores.GetWorkspaceID(ctx); workspaceID != "" {
if err := s.sampleInstanceManager.StartIfExist(ctx, workspaceID); err != nil {
slog.Warn("failed to start sample instances", log.BBError(err))
}
@@ -175,6 +177,17 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
if logSetup.EnableDebug {
log.LogLevel.Set(slog.LevelDebug)
}
+ telemetry.InitGlobalReporter(
+ profile.Version,
+ profile.GitCommit,
+ profile.Mode,
+ logSetup.GetEnableMetricCollection(),
+ )
+ if !s.profile.SaaS && profile.Mode == common.ReleaseModeProd && logSetup.GetEnableMetricCollection() && workspaceID != "" {
+ go func() {
+ reportSQLReviewConfigSnapshot(context.WithoutCancel(ctx), stores, workspaceID)
+ }()
+ }
s.bus, err = bus.New()
if err != nil {
diff --git a/backend/server/sql_review_telemetry.go b/backend/server/sql_review_telemetry.go
new file mode 100644
index 0000000000..b704c94f92
--- /dev/null
+++ b/backend/server/sql_review_telemetry.go
@@ -0,0 +1,80 @@
+package server
+
+import (
+ "context"
+ "log/slog"
+ "slices"
+ "strings"
+
+ "connectrpc.com/connect"
+ "google.golang.org/protobuf/encoding/protojson"
+
+ apiv1 "github.com/bytebase/bytebase/backend/api/v1"
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/telemetry"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+func reportSQLReviewConfigSnapshot(ctx context.Context, stores *store.Store, workspaceID string) {
+ ctx = context.WithValue(ctx, common.WorkspaceIDContextKey, workspaceID)
+ users, err := stores.ListUsers(ctx, &store.FindUserMessage{})
+ if err != nil {
+ slog.Warn("failed to list users for SQL review config telemetry", log.BBError(err))
+ return
+ }
+ email, emailDomains := buildSQLReviewTelemetryIdentity(users)
+ if email == "" {
+ slog.Warn("skipping SQL review config telemetry because no active end user was found")
+ return
+ }
+
+ service := apiv1.NewReviewConfigService(stores)
+ response, err := service.ListReviewConfigs(ctx, connect.NewRequest(&v1pb.ListReviewConfigsRequest{}))
+ if err != nil {
+ slog.Warn("failed to list SQL review configs for telemetry", log.BBError(err))
+ return
+ }
+
+ snapshot, err := marshalSQLReviewConfigSnapshot(response.Msg)
+ if err != nil {
+ slog.Warn("failed to marshal SQL review config snapshot for telemetry", log.BBError(err))
+ return
+ }
+ telemetry.ReportSQLReviewConfigSnapshot(ctx, workspaceID, email, snapshot, emailDomains)
+}
+
+func marshalSQLReviewConfigSnapshot(response *v1pb.ListReviewConfigsResponse) (string, error) {
+ snapshot, err := protojson.MarshalOptions{EmitUnpopulated: true}.Marshal(response)
+ if err != nil {
+ return "", err
+ }
+ return string(snapshot), nil
+}
+
+func buildSQLReviewTelemetryIdentity(users []*store.UserMessage) (string, []string) {
+ email := ""
+ minID := 0
+ domainSet := make(map[string]bool)
+ for _, user := range users {
+ if user == nil || user.Email == "" {
+ continue
+ }
+ if email == "" || user.ID < minID {
+ email = user.Email
+ minID = user.ID
+ }
+
+ if i := strings.LastIndex(user.Email, "@"); i >= 0 && i+1 < len(user.Email) {
+ domainSet[strings.ToLower(user.Email[i+1:])] = true
+ }
+ }
+
+ domains := make([]string, 0, len(domainSet))
+ for domain := range domainSet {
+ domains = append(domains, domain)
+ }
+ slices.Sort(domains)
+ return email, domains
+}
diff --git a/backend/server/sql_review_telemetry_test.go b/backend/server/sql_review_telemetry_test.go
new file mode 100644
index 0000000000..cd18861d7b
--- /dev/null
+++ b/backend/server/sql_review_telemetry_test.go
@@ -0,0 +1,73 @@
+package server
+
+import (
+ "strings"
+ "testing"
+
+ "google.golang.org/protobuf/proto"
+
+ "github.com/bytebase/bytebase/backend/common"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+func TestMarshalSQLReviewConfigSnapshot(t *testing.T) {
+ response := &v1pb.ListReviewConfigsResponse{
+ ReviewConfigs: []*v1pb.ReviewConfig{
+ {
+ Name: "reviewConfigs/basic",
+ Title: "Basic",
+ Enabled: true,
+ Resources: []string{"environments/prod"},
+ Rules: []*v1pb.SQLReviewRule{
+ {
+ Type: v1pb.SQLReviewRule_NAMING_TABLE,
+ Level: v1pb.SQLReviewRule_ERROR,
+ Engine: v1pb.Engine_MYSQL,
+ Payload: &v1pb.SQLReviewRule_NamingPayload{
+ NamingPayload: &v1pb.SQLReviewRule_NamingRulePayload{
+ MaxLength: 64,
+ Format: "^[a-z]+$",
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ snapshot, err := marshalSQLReviewConfigSnapshot(response)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(snapshot, `"reviewConfigs"`) {
+ t.Fatalf("snapshot %q does not contain reviewConfigs", snapshot)
+ }
+ if !strings.Contains(snapshot, `"namingPayload"`) {
+ t.Fatalf("snapshot %q does not contain rule payload", snapshot)
+ }
+
+ got := &v1pb.ListReviewConfigsResponse{}
+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(snapshot), got); err != nil {
+ t.Fatal(err)
+ }
+ if !proto.Equal(response, got) {
+ t.Fatalf("roundtrip mismatch\n got: %v\nwant: %v", got, response)
+ }
+}
+
+func TestBuildSQLReviewTelemetryIdentity(t *testing.T) {
+ email, domains := buildSQLReviewTelemetryIdentity([]*store.UserMessage{
+ {ID: 30, Email: "[email protected]"},
+ {ID: 10, Email: "[email protected]"},
+ {ID: 20, Email: "[email protected]"},
+ {ID: 40, Email: "invalid-email"},
+ })
+
+ if email != "[email protected]" {
+ t.Fatalf("email = %q, want %q", email, "[email protected]")
+ }
+ if len(domains) != 2 || domains[0] != "example.com" || domains[1] != "test.com" {
+ t.Fatalf("domains = %v, want %v", domains, []string{"example.com", "test.com"})
+ }
+}