Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: report SQL review config telemetry (#20626)
p0ny4b33d11ed267
diff --git a/backend/api/v1/review_config_service.go b/backend/api/v1/review_config_service.goindex db4c49fbc6..aab227c594 100644--- a/backend/api/v1/review_config_service.go+++ b/backend/api/v1/review_config_service.go@@ -64,15 +64,11 @@ func (s *ReviewConfigService) ListReviewConfigs(ctx context.Context, _ *connect.return nil, connect.NewError(connect.CodeInternal, err)}- response := &v1pb.ListReviewConfigsResponse{}- for _, message := range messages {- sqlReview, err := s.convertToV1ReviewConfig(ctx, message)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, err)- }- response.ReviewConfigs = append(response.ReviewConfigs, sqlReview)+ reviewConfigs, err := s.convertToV1ReviewConfigs(ctx, messages)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, err)}- return connect.NewResponse(response), nil+ return connect.NewResponse(&v1pb.ListReviewConfigsResponse{ReviewConfigs: reviewConfigs}), nil}// GetReviewConfig gets the review config.@@ -235,6 +231,34 @@ func convertToReviewConfigMessage(reviewConfig *v1pb.ReviewConfig) (*store.Revie}func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, reviewConfigMessage *store.ReviewConfigMessage) (*v1pb.ReviewConfig, error) {+ reviewConfigs, err := s.convertToV1ReviewConfigs(ctx, []*store.ReviewConfigMessage{reviewConfigMessage})+ if err != nil {+ return nil, err+ }+ if len(reviewConfigs) == 0 {+ return nil, errors.New("review config not found")+ }+ return reviewConfigs[0], nil+}++func (s *ReviewConfigService) convertToV1ReviewConfigs(ctx context.Context, reviewConfigMessages []*store.ReviewConfigMessage) ([]*v1pb.ReviewConfig, error) {+ configs := make([]*v1pb.ReviewConfig, 0, len(reviewConfigMessages))+ configByName := make(map[string]*v1pb.ReviewConfig, len(reviewConfigMessages))+ for _, message := range reviewConfigMessages {+ payload := message.Payload+ if payload == nil {+ payload = &storepb.ReviewConfigPayload{}+ }+ config := &v1pb.ReviewConfig{+ Name: common.FormatReviewConfig(message.ID),+ Title: message.Name,+ Enabled: message.Enforce,+ Rules: ConvertToV1PBSQLReviewRules(payload.SqlReviewRules),+ }+ configs = append(configs, config)+ configByName[config.Name] = config+ }+policyType := storepb.Policy_TAGtagPolicies, err := s.store.ListPolicies(ctx, &store.FindPolicyMessage{Workspace: common.GetWorkspaceIDFromContext(ctx),@@ -245,19 +269,13 @@ func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, reviereturn nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to list tag policy"))}- config := &v1pb.ReviewConfig{- Name: common.FormatReviewConfig(reviewConfigMessage.ID),- Title: reviewConfigMessage.Name,- Enabled: reviewConfigMessage.Enforce,- Rules: ConvertToV1PBSQLReviewRules(reviewConfigMessage.Payload.SqlReviewRules),- }-for _, policy := range tagPolicies {p := &v1pb.TagPolicy{}if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(policy.Payload), p); err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to unmarshal tag policy"))}- if p.Tags[common.ReservedTagReviewConfig] != config.Name {+ config, ok := configByName[p.Tags[common.ReservedTagReviewConfig]]+ if !ok {continue}@@ -296,7 +314,7 @@ func (s *ReviewConfigService) convertToV1ReviewConfig(ctx context.Context, revie}}- return config, nil+ return configs, nil}// validateSQLReviewRules validates the SQL review rule.diff --git a/backend/component/telemetry/reporter.go b/backend/component/telemetry/reporter.gonew file mode 100644index 0000000000..cbe809b5ea--- /dev/null+++ b/backend/component/telemetry/reporter.go@@ -0,0 +1,108 @@+// Package telemetry provides telemetry reporting to hub.bytebase.com.+package telemetry++import (+ "bytes"+ "context"+ "encoding/json"+ "net/http"+ "sync"+ "time"++ "github.com/bytebase/bytebase/backend/common"+)++var hubEventURL = "https://hub.bytebase.com/v1/events"++// Reporter sends telemetry events to hub.bytebase.com.+type Reporter struct {+ mu sync.RWMutex+ version string+ gitCommit string+ releaseMode common.ReleaseMode+ enabled bool++ httpClient *http.Client+}++var (+ globalReporter *Reporter+ globalReporterOnce sync.Once+)++// InitGlobalReporter initializes the global telemetry reporter.+func InitGlobalReporter(version, gitCommit string, releaseMode common.ReleaseMode, enabled bool) {+ globalReporterOnce.Do(func() {+ globalReporter = &Reporter{+ version: version,+ gitCommit: gitCommit,+ releaseMode: releaseMode,+ enabled: enabled,+ httpClient: &http.Client{+ Timeout: 120 * time.Second,+ },+ }+ })+}++type sqlReviewConfigSnapshotPayload struct {+ WorkspaceID string `json:"workspaceId"`+ Email string `json:"email"`+ Version string `json:"version"`+ Commit string `json:"commit"`+ SQLReviewConfigSnapshot struct {+ Snapshot string `json:"snapshot"`+ EmailDomains []string `json:"emailDomains"`+ } `json:"sqlReviewConfigSnapshot"`+}++// ReportSQLReviewConfigSnapshot reports the current SQL review config snapshot.+func ReportSQLReviewConfigSnapshot(ctx context.Context, workspaceID string, email string, snapshot string, emailDomains []string) {+ if globalReporter == nil {+ return+ }++ globalReporter.mu.RLock()+ if !globalReporter.enabled {+ globalReporter.mu.RUnlock()+ return+ }+ version := globalReporter.version+ gitCommit := globalReporter.gitCommit+ releaseMode := globalReporter.releaseMode+ globalReporter.mu.RUnlock()++ if releaseMode != common.ReleaseModeProd || workspaceID == "" || email == "" || snapshot == "" {+ return+ }++ payload := sqlReviewConfigSnapshotPayload{+ WorkspaceID: workspaceID,+ Email: email,+ Version: version,+ Commit: gitCommit,+ }+ payload.SQLReviewConfigSnapshot.Snapshot = snapshot+ payload.SQLReviewConfigSnapshot.EmailDomains = emailDomains++ go globalReporter.send(context.WithoutCancel(ctx), payload)+}++func (r *Reporter) send(ctx context.Context, payload any) {+ body, err := json.Marshal(payload)+ if err != nil {+ return+ }++ req, err := http.NewRequestWithContext(ctx, http.MethodPost, hubEventURL, bytes.NewReader(body))+ if err != nil {+ return+ }+ req.Header.Set("Content-Type", "application/json")++ resp, err := r.httpClient.Do(req)+ if err != nil {+ return+ }+ resp.Body.Close()+}diff --git a/backend/component/telemetry/reporter_test.go b/backend/component/telemetry/reporter_test.gonew file mode 100644index 0000000000..991dcf917a--- /dev/null+++ b/backend/component/telemetry/reporter_test.go@@ -0,0 +1,130 @@+package telemetry++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "sync"+ "testing"+ "time"++ "github.com/bytebase/bytebase/backend/common"+)++func TestReportSQLReviewConfigSnapshot(t *testing.T) {+ const snapshot = `{"reviewConfigs":[{"name":"reviewConfigs/basic","enabled":true}]}`++ var (+ mu sync.Mutex+ payload map[string]any+ done = make(chan struct{})+ )+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ defer close(done)++ if r.Method != http.MethodPost {+ t.Errorf("method = %q, want %q", r.Method, http.MethodPost)+ }+ if got := r.Header.Get("Content-Type"); got != "application/json" {+ t.Errorf("Content-Type = %q, want %q", got, "application/json")+ }+ var got map[string]any+ if err := json.NewDecoder(r.Body).Decode(&got); err != nil {+ t.Errorf("failed to decode payload: %v", err)+ return+ }+ mu.Lock()+ payload = got+ mu.Unlock()+ w.WriteHeader(http.StatusOK)+ }))+ defer server.Close()++ oldHubEventURL := hubEventURL+ hubEventURL = server.URL+ defer func() {+ hubEventURL = oldHubEventURL+ }()++ globalReporter = &Reporter{+ version: "3.19.1",+ gitCommit: "abcdef",+ releaseMode: common.ReleaseModeProd,+ enabled: true,+ httpClient: server.Client(),+ }+ defer func() {+ globalReporter = nil+ }()++ ReportSQLReviewConfigSnapshot(context.Background(), "workspace-id", "[email protected]", snapshot, []string{"example.com", "test.com"})++ select {+ case <-done:+ case <-time.After(2 * time.Second):+ t.Fatal("timed out waiting for telemetry request")+ }++ mu.Lock()+ defer mu.Unlock()+ if payload["workspaceId"] != "workspace-id" {+ t.Fatalf("workspaceId = %v, want %q", payload["workspaceId"], "workspace-id")+ }+ if payload["email"] != "[email protected]" {+ t.Fatalf("email = %v, want %q", payload["email"], "[email protected]")+ }+ if payload["version"] != "3.19.1" {+ t.Fatalf("version = %v, want %q", payload["version"], "3.19.1")+ }+ if payload["commit"] != "abcdef" {+ t.Fatalf("commit = %v, want %q", payload["commit"], "abcdef")+ }+ event, ok := payload["sqlReviewConfigSnapshot"].(map[string]any)+ if !ok {+ t.Fatalf("sqlReviewConfigSnapshot = %T, want object", payload["sqlReviewConfigSnapshot"])+ }+ if event["snapshot"] != snapshot {+ t.Fatalf("snapshot = %v, want %q", event["snapshot"], snapshot)+ }+ domains, ok := event["emailDomains"].([]any)+ if !ok {+ t.Fatalf("emailDomains = %T, want array", event["emailDomains"])+ }+ if len(domains) != 2 || domains[0] != "example.com" || domains[1] != "test.com" {+ t.Fatalf("emailDomains = %v, want %v", domains, []string{"example.com", "test.com"})+ }+}++func TestReportSQLReviewConfigSnapshotSkipsNonProd(t *testing.T) {+ requestReceived := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {+ close(requestReceived)+ }))+ defer server.Close()++ oldHubEventURL := hubEventURL+ hubEventURL = server.URL+ defer func() {+ hubEventURL = oldHubEventURL+ }()++ globalReporter = &Reporter{+ version: "3.19.1",+ gitCommit: "abcdef",+ releaseMode: common.ReleaseModeDev,+ enabled: true,+ httpClient: server.Client(),+ }+ defer func() {+ globalReporter = nil+ }()++ ReportSQLReviewConfigSnapshot(context.Background(), "workspace-id", "[email protected]", "{}", []string{"example.com"})++ select {+ case <-requestReceived:+ t.Fatal("telemetry request was sent in non-prod release mode")+ case <-time.After(100 * time.Millisecond):+ }+}diff --git a/backend/server/server.go b/backend/server/server.goindex 62c42eed5f..81aa0f3561 100644--- a/backend/server/server.go+++ b/backend/server/server.go@@ -27,6 +27,7 @@ import ("github.com/bytebase/bytebase/backend/component/iam""github.com/bytebase/bytebase/backend/component/sampleinstance""github.com/bytebase/bytebase/backend/component/sheet"+ "github.com/bytebase/bytebase/backend/component/telemetry""github.com/bytebase/bytebase/backend/component/webhook""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -151,13 +152,14 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {EnableMetricCollection: true,EnableDebug: profile.Debug,}+ var workspaceID stringif !s.profile.SaaS {s.sampleInstanceManager = sampleinstance.NewManager(stores, profile)// Load workspace-dependent settings if workspace exists.// On first boot (no workspace yet), these remain at defaults and get// initialized when the workspace is created and settings are updated via API.- if workspaceID, _ := stores.GetWorkspaceID(ctx); workspaceID != "" {+ if workspaceID, _ = stores.GetWorkspaceID(ctx); workspaceID != "" {if err := s.sampleInstanceManager.StartIfExist(ctx, workspaceID); err != nil {slog.Warn("failed to start sample instances", log.BBError(err))}@@ -175,6 +177,17 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {if logSetup.EnableDebug {log.LogLevel.Set(slog.LevelDebug)}+ telemetry.InitGlobalReporter(+ profile.Version,+ profile.GitCommit,+ profile.Mode,+ logSetup.GetEnableMetricCollection(),+ )+ if !s.profile.SaaS && profile.Mode == common.ReleaseModeProd && logSetup.GetEnableMetricCollection() && workspaceID != "" {+ go func() {+ reportSQLReviewConfigSnapshot(context.WithoutCancel(ctx), stores, workspaceID)+ }()+ }s.bus, err = bus.New()if err != nil {diff --git a/backend/server/sql_review_telemetry.go b/backend/server/sql_review_telemetry.gonew file mode 100644index 0000000000..b704c94f92--- /dev/null+++ b/backend/server/sql_review_telemetry.go@@ -0,0 +1,80 @@+package server++import (+ "context"+ "log/slog"+ "slices"+ "strings"++ "connectrpc.com/connect"+ "google.golang.org/protobuf/encoding/protojson"++ apiv1 "github.com/bytebase/bytebase/backend/api/v1"+ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/telemetry"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/store"+)++func reportSQLReviewConfigSnapshot(ctx context.Context, stores *store.Store, workspaceID string) {+ ctx = context.WithValue(ctx, common.WorkspaceIDContextKey, workspaceID)+ users, err := stores.ListUsers(ctx, &store.FindUserMessage{})+ if err != nil {+ slog.Warn("failed to list users for SQL review config telemetry", log.BBError(err))+ return+ }+ email, emailDomains := buildSQLReviewTelemetryIdentity(users)+ if email == "" {+ slog.Warn("skipping SQL review config telemetry because no active end user was found")+ return+ }++ service := apiv1.NewReviewConfigService(stores)+ response, err := service.ListReviewConfigs(ctx, connect.NewRequest(&v1pb.ListReviewConfigsRequest{}))+ if err != nil {+ slog.Warn("failed to list SQL review configs for telemetry", log.BBError(err))+ return+ }++ snapshot, err := marshalSQLReviewConfigSnapshot(response.Msg)+ if err != nil {+ slog.Warn("failed to marshal SQL review config snapshot for telemetry", log.BBError(err))+ return+ }+ telemetry.ReportSQLReviewConfigSnapshot(ctx, workspaceID, email, snapshot, emailDomains)+}++func marshalSQLReviewConfigSnapshot(response *v1pb.ListReviewConfigsResponse) (string, error) {+ snapshot, err := protojson.MarshalOptions{EmitUnpopulated: true}.Marshal(response)+ if err != nil {+ return "", err+ }+ return string(snapshot), nil+}++func buildSQLReviewTelemetryIdentity(users []*store.UserMessage) (string, []string) {+ email := ""+ minID := 0+ domainSet := make(map[string]bool)+ for _, user := range users {+ if user == nil || user.Email == "" {+ continue+ }+ if email == "" || user.ID < minID {+ email = user.Email+ minID = user.ID+ }++ if i := strings.LastIndex(user.Email, "@"); i >= 0 && i+1 < len(user.Email) {+ domainSet[strings.ToLower(user.Email[i+1:])] = true+ }+ }++ domains := make([]string, 0, len(domainSet))+ for domain := range domainSet {+ domains = append(domains, domain)+ }+ slices.Sort(domains)+ return email, domains+}diff --git a/backend/server/sql_review_telemetry_test.go b/backend/server/sql_review_telemetry_test.gonew file mode 100644index 0000000000..cd18861d7b--- /dev/null+++ b/backend/server/sql_review_telemetry_test.go@@ -0,0 +1,73 @@+package server++import (+ "strings"+ "testing"++ "google.golang.org/protobuf/proto"++ "github.com/bytebase/bytebase/backend/common"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/store"+)++func TestMarshalSQLReviewConfigSnapshot(t *testing.T) {+ response := &v1pb.ListReviewConfigsResponse{+ ReviewConfigs: []*v1pb.ReviewConfig{+ {+ Name: "reviewConfigs/basic",+ Title: "Basic",+ Enabled: true,+ Resources: []string{"environments/prod"},+ Rules: []*v1pb.SQLReviewRule{+ {+ Type: v1pb.SQLReviewRule_NAMING_TABLE,+ Level: v1pb.SQLReviewRule_ERROR,+ Engine: v1pb.Engine_MYSQL,+ Payload: &v1pb.SQLReviewRule_NamingPayload{+ NamingPayload: &v1pb.SQLReviewRule_NamingRulePayload{+ MaxLength: 64,+ Format: "^[a-z]+$",+ },+ },+ },+ },+ },+ },+ }++ snapshot, err := marshalSQLReviewConfigSnapshot(response)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(snapshot, `"reviewConfigs"`) {+ t.Fatalf("snapshot %q does not contain reviewConfigs", snapshot)+ }+ if !strings.Contains(snapshot, `"namingPayload"`) {+ t.Fatalf("snapshot %q does not contain rule payload", snapshot)+ }++ got := &v1pb.ListReviewConfigsResponse{}+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(snapshot), got); err != nil {+ t.Fatal(err)+ }+ if !proto.Equal(response, got) {+ t.Fatalf("roundtrip mismatch\n got: %v\nwant: %v", got, response)+ }+}++func TestBuildSQLReviewTelemetryIdentity(t *testing.T) {+ email, domains := buildSQLReviewTelemetryIdentity([]*store.UserMessage{+ {ID: 30, Email: "[email protected]"},+ {ID: 10, Email: "[email protected]"},+ {ID: 20, Email: "[email protected]"},+ {ID: 40, Email: "invalid-email"},+ })++ if email != "[email protected]" {+ t.Fatalf("email = %q, want %q", email, "[email protected]")+ }+ if len(domains) != 2 || domains[0] != "example.com" || domains[1] != "test.com" {+ t.Fatalf("domains = %v, want %v", domains, []string{"example.com", "test.com"})+ }+}