Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: row-locking read-modify-write for workspace profile setting updates (#21060)
Vincent Huang8835311af741
diff --git a/backend/api/v1/setting_service.go b/backend/api/v1/setting_service.goindex 93d8f34fdd..1c59cb3972 100644--- a/backend/api/v1/setting_service.go+++ b/backend/api/v1/setting_service.go@@ -185,232 +185,11 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Req}var storeSettingValue proto.Message- var resetAuditLogStdout boolvar resetClassification boolswitch storeSettingName {case storepb.SettingName_WORKSPACE_PROFILE:- if request.Msg.UpdateMask == nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("update mask is required"))- }- payload := convertWorkspaceProfileSetting(request.Msg.Setting.Value.GetWorkspaceProfile())- // Merge onto the profile as stored in the database, not the setting- // cache: the whole profile is written back below, so a stale cached base- // would silently revert fields changed out-of-band (e.g. an emergency- // SQL flip of the MCP kill switch) when an admin saves an unrelated- // field.- freshSetting, err := s.store.GetSettingUncached(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find setting %s with error: %v", storeSettingName, err))- }- if freshSetting == nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("cannot find setting %v", storeSettingName))- }- profileValue, ok := freshSetting.Value.(*storepb.WorkspaceProfileSetting)- if !ok {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storeSettingName))- }- // The uncached read still lands in the setting cache (ListSettings- // caches what it returns), so mutate a clone: a validate-only request- // must never alter the served in-memory state.- oldSetting := proto.CloneOf(profileValue)-- for _, path := range request.Msg.UpdateMask.Paths {- switch path {- case "value.workspace_profile.enable_debug":- oldSetting.EnableDebug = payload.EnableDebug- level := slog.LevelInfo- if payload.EnableDebug {- level = slog.LevelDebug- }- log.LogLevel.Set(level)- case "value.workspace_profile.disallow_signup":- if s.profile.SaaS {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_signup cannot be changed in SaaS mode"))- }- if payload.DisallowSignup {- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_SELF_SERVICE_SIGNUP); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- }- oldSetting.DisallowSignup = payload.DisallowSignup- case "value.workspace_profile.external_url":- if s.profile.SaaS {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the external_url cannot be changed in SaaS mode"))- }- // Prevent changing external URL via UI when it's set via command-line flag- if s.profile.ExternalURL != "" {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))- }- if payload.ExternalUrl != "" {- externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)- if err != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))- }- payload.ExternalUrl = externalURL- }- oldSetting.ExternalUrl = payload.ExternalUrl- case "value.workspace_profile.require_mfa":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TWO_FA); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- oldSetting.Require_2Fa = payload.Require_2Fa- case "value.workspace_profile.access_token_duration":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- if payload.AccessTokenDuration != nil && payload.AccessTokenDuration.Seconds > 0 && payload.AccessTokenDuration.AsDuration() < time.Minute {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("access token duration should be at least one minute"))- }- oldSetting.AccessTokenDuration = payload.AccessTokenDuration- case "value.workspace_profile.refresh_token_duration":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- if payload.RefreshTokenDuration != nil && payload.RefreshTokenDuration.Seconds > 0 && payload.RefreshTokenDuration.AsDuration() < time.Hour {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("refresh token duration should be at least one hour"))- }- oldSetting.RefreshTokenDuration = payload.RefreshTokenDuration- case "value.workspace_profile.inactive_session_timeout":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- oldSetting.InactiveSessionTimeout = payload.InactiveSessionTimeout- case "value.workspace_profile.announcement":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DASHBOARD_ANNOUNCEMENT); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- if payload.Announcement != nil {- if err := validateAnnouncementTheme(payload.Announcement.Theme); err != nil {- return nil, err- }- }- oldSetting.Announcement = payload.Announcement- case "value.workspace_profile.maximum_request_expiration":- if payload.MaximumRequestExpiration != nil {- // If the value is less than or equal to 0, we will remove the setting. AKA no limit.- if payload.MaximumRequestExpiration.Seconds <= 0 {- payload.MaximumRequestExpiration = nil- }- }- oldSetting.MaximumRequestExpiration = payload.MaximumRequestExpiration- case "value.workspace_profile.maximum_role_expiration":- if payload.MaximumRoleExpiration != nil {- // If the value is less than or equal to 0, we will remove the setting. AKA no limit.- if payload.MaximumRoleExpiration.Seconds <= 0 {- payload.MaximumRoleExpiration = nil- }- }- oldSetting.MaximumRoleExpiration = payload.MaximumRoleExpiration- case "value.workspace_profile.domains":- if err := validateDomains(payload.Domains); err != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid domains, error %v", err))- }- oldSetting.Domains = payload.Domains- case "value.workspace_profile.enforce_identity_domain":- if payload.EnforceIdentityDomain {- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_USER_EMAIL_DOMAIN_RESTRICTION); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- }- oldSetting.EnforceIdentityDomain = payload.EnforceIdentityDomain- case "value.workspace_profile.database_change_mode":- oldSetting.DatabaseChangeMode = payload.DatabaseChangeMode- case "value.workspace_profile.mcp_capability":- if err := validateMCPCapability(payload.McpCapability); err != nil {- return nil, err- }- oldSetting.McpCapability = payload.McpCapability- case "value.workspace_profile.allow_email_code_signin":- if s.profile.SaaS {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("allow_email_code_signin cannot be changed in SaaS mode"))- }- if payload.AllowEmailCodeSignin {- emailSetting, err := s.store.GetSetting(ctx, workspaceID, storepb.SettingName_EMAIL)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to load email setting"))- }- if emailSetting == nil {- return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("cannot enable email code signin without an EMAIL setting"))- }- }- oldSetting.AllowEmailCodeSignin = payload.AllowEmailCodeSignin- case "value.workspace_profile.disallow_password_signin":- if s.profile.SaaS {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_password_signin cannot be changed in SaaS mode"))- }- if payload.DisallowPasswordSignin {- // We should still allow users to turn it off.- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_PASSWORD_SIGNIN); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }-- settingWsID := common.GetWorkspaceIDFromContext(ctx)- identityProviders, err := s.store.ListIdentityProviders(ctx, &store.FindIdentityProviderMessage{Workspace: &settingWsID})- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to list identity providers: %v", err))- }- if len(identityProviders) == 0 {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot disallow password signin when no identity provider is set"))- }- }- oldSetting.DisallowPasswordSignin = payload.DisallowPasswordSignin- case "value.workspace_profile.enable_metric_collection":- oldSetting.EnableMetricCollection = payload.EnableMetricCollection- case "value.workspace_profile.enable_audit_log_stdout":- if payload.EnableAuditLogStdout {- // Require TEAM or ENTERPRISE license- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- }- resetAuditLogStdout = true- oldSetting.EnableAuditLogStdout = payload.EnableAuditLogStdout- case "value.workspace_profile.watermark":- if payload.Watermark {- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_WATERMARK); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- }- oldSetting.Watermark = payload.Watermark- case "value.workspace_profile.directory_sync_token":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DIRECTORY_SYNC); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- // Generate a new token if the payload is empty.- // This handles both initial setup and token reset (when user explicitly sends empty string).- if payload.DirectorySyncToken == "" {- payload.DirectorySyncToken = uuid.New().String()- }- oldSetting.DirectorySyncToken = payload.DirectorySyncToken- case "value.workspace_profile.password_restriction":- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_PASSWORD_RESTRICTIONS); err != nil {- return nil, connect.NewError(connect.CodePermissionDenied, err)- }- if payload.PasswordRestriction != nil && payload.PasswordRestriction.MinLength < 8 {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid password minimum length, should no less than 8"))- }- oldSetting.PasswordRestriction = payload.PasswordRestriction- case "value.workspace_profile.sql_result_size":- oldSetting.SqlResultSize = payload.SqlResultSize- case "value.workspace_profile.query_timeout":- oldSetting.QueryTimeout = payload.QueryTimeout- case "value.workspace_profile.sql_editor_theme_id":- oldSetting.SqlEditorThemeId = payload.SqlEditorThemeId- case "value.workspace_profile.sql_editor_custom_theme":- if err := validateSQLEditorCustomTheme(payload.SqlEditorCustomTheme); err != nil {- return nil, err- }- oldSetting.SqlEditorCustomTheme = payload.SqlEditorCustomTheme- default:- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid update mask path %v", path))- }- }-- if len(oldSetting.Domains) == 0 && oldSetting.EnforceIdentityDomain {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("identity domain can be enforced only when workspace domains are set"))- }- storeSettingValue = oldSetting+ return s.updateWorkspaceProfileSetting(ctx, request, workspaceID)case storepb.SettingName_WORKSPACE_APPROVAL:if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_APPROVAL_WORKFLOW); err != nil {return nil, connect.NewError(connect.CodePermissionDenied, err)@@ -736,15 +515,6 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Reqreturn nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to set setting: %v", err))}- // Dynamically update audit logger runtime flag if enable_audit_log_stdout was changed- if resetAuditLogStdout {- workspaceProfile, err := s.store.GetWorkspaceProfileSetting(ctx, workspaceID)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get workspace setting message: %v", err))- }- s.profile.RuntimeEnableAuditLogStdout.Store(workspaceProfile.EnableAuditLogStdout)- }-// It's a temporary solution to map the classification to all projects before we support it in the UX.if resetClassification {classification, err := s.store.GetDataClassificationSetting(ctx, workspaceID)@@ -784,6 +554,368 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Reqreturn connect.NewResponse(settingMessage), nil}+// updateWorkspaceProfileSetting handles the WORKSPACE_PROFILE branch of+// UpdateSetting through the store's row-locking read-modify-write primitive:+// the update-mask merge and its validations run inside the transaction against+// the value of the locked row, so validation sees the true final state and a+// concurrent write (another admin, emergency SQL) can never be silently+// reverted by a merge based on a stale read. Validate-only requests run the+// same merge and validations against an uncached snapshot and write nothing.+func (s *SettingService) updateWorkspaceProfileSetting(ctx context.Context, request *connect.Request[v1pb.UpdateSettingRequest], workspaceID string) (*connect.Response[v1pb.Setting], error) {+ if request.Msg.UpdateMask == nil {+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("update mask is required"))+ }+ payload := convertWorkspaceProfileSetting(request.Msg.Setting.Value.GetWorkspaceProfile())+ // License, store, and profile checks run before the row-locking+ // transaction so apply stays free of database reads.+ if err := s.preflightWorkspaceProfilePaths(ctx, workspaceID, request, payload); err != nil {+ return nil, err+ }+ var lockedBefore *storepb.WorkspaceProfileSetting+ apply := func(current proto.Message) (proto.Message, error) {+ oldSetting, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storepb.SettingName_WORKSPACE_PROFILE))+ }+ // The audit before-image is the row this merge actually ran against,+ // not the possibly stale pre-lock snapshot captured by UpdateSetting.+ lockedBefore = proto.CloneOf(oldSetting)+ if err := mergeWorkspaceProfilePaths(request, payload, oldSetting); err != nil {+ return nil, err+ }+ if len(oldSetting.Domains) == 0 && oldSetting.EnforceIdentityDomain {+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("identity domain can be enforced only when workspace domains are set"))+ }+ return oldSetting, nil+ }++ if request.Msg.ValidateOnly {+ freshSetting, err := s.store.GetSettingUncached(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find setting %s with error: %v", storepb.SettingName_WORKSPACE_PROFILE, err))+ }+ if freshSetting == nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("cannot find setting %v", storepb.SettingName_WORKSPACE_PROFILE))+ }+ profileValue, ok := freshSetting.Value.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storepb.SettingName_WORKSPACE_PROFILE))+ }+ // GetSettingUncached returns a fresh object today (uncached reads no+ // longer populate the cache), but validate against a clone anyway so+ // a validate-only request can never mutate shared state should the+ // read path change.+ if _, err := apply(proto.CloneOf(profileValue)); err != nil {+ return nil, err+ }+ return connect.NewResponse(&v1pb.Setting{+ Name: request.Msg.Setting.Name,+ Value: request.Msg.Setting.Value,+ }), nil+ }++ // Runtime derivatives (audit stdout flag, debug log level, pprof gate)+ // reconcile from the freshly re-read committed state on EVERY successful+ // profile publication — not only when this request touched those fields —+ // so a publication skipped by an earlier failure is repaired by the next+ // profile write, mirroring startup derivation (server.go). Skipped in+ // SaaS, where workspace-scoped stored values must not drive process-global+ // state on a shared replica (the corresponding mask paths are also+ // rejected in preflight).+ //+ // The audit-log entitlement is computed here, outside the publish mutex:+ // runtime audit stdout requires the stored flag AND a valid license, same+ // as startup — and postCommit must not call IsFeatureEnabled itself,+ // because it runs under settingPublishMu and a setting cache miss there+ // would re-acquire that mutex.+ auditLogFeatureEnabled := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG) == nil+ postCommit := func(current *store.SettingMessage) {+ if s.profile.SaaS {+ return+ }+ profile, ok := current.Value.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return+ }+ s.profile.RuntimeEnableAuditLogStdout.Store(profile.EnableAuditLogStdout && auditLogFeatureEnabled)+ s.profile.RuntimeDebug.Store(profile.EnableDebug)+ level := slog.LevelInfo+ if profile.EnableDebug {+ level = slog.LevelDebug+ }+ log.LogLevel.Set(level)+ }+ setting, err := s.store.UpdateSettingAtomic(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE, apply, postCommit)+ if err != nil {+ var connectErr *connect.Error+ if errors.As(err, &connectErr) {+ return nil, err+ }+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to set setting: %v", err))+ }++ // Re-capture the audit before-image from the locked row the merge ran+ // against, overwriting UpdateSetting's earlier pre-lock snapshot.+ if setServiceData, ok := common.GetSetServiceDataFromContext(ctx); ok && lockedBefore != nil {+ v1pbSetting, err := convertToSettingMessage(&store.SettingMessage{+ Name: storepb.SettingName_WORKSPACE_PROFILE,+ Workspace: workspaceID,+ Value: lockedBefore,+ })+ if err != nil {+ slog.Warn("audit: failed to convert to v1.Setting", log.BBError(err))+ }+ p, err := anypb.New(v1pbSetting)+ if err != nil {+ slog.Warn("audit: failed to convert to anypb.Any", log.BBError(err))+ }+ setServiceData(p)+ }++ settingMessage, err := convertToSettingMessage(setting)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to convert setting message: %v", err))+ }+ return connect.NewResponse(settingMessage), nil+}++// preflightWorkspaceProfilePaths runs the update-mask validations that need+// license, store, or profile lookups. It runs BEFORE the row-locking+// transaction: apply must stay free of database reads, because it executes+// while holding the transaction's pooled connection and the row lock, and a+// nested read wanting a second connection is a bounded-pool starvation cycle.+// None of these checks depend on the locked row's state, so hoisting them is+// behavior-preserving. May normalize payload fields (e.g. external_url).+func (s *SettingService) preflightWorkspaceProfilePaths(ctx context.Context, workspaceID string, request *connect.Request[v1pb.UpdateSettingRequest], payload *storepb.WorkspaceProfileSetting) error {+ for _, path := range request.Msg.UpdateMask.Paths {+ switch path {+ case "value.workspace_profile.enable_debug":+ // Debug flips the process-global log level and pprof exposure; on+ // a shared SaaS replica that would affect other workspaces.+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the enable_debug cannot be changed in SaaS mode"))+ }+ case "value.workspace_profile.disallow_signup":+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_signup cannot be changed in SaaS mode"))+ }+ if payload.DisallowSignup {+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_SELF_SERVICE_SIGNUP); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ }+ case "value.workspace_profile.external_url":+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the external_url cannot be changed in SaaS mode"))+ }+ // Prevent changing external URL via UI when it's set via command-line flag+ if s.profile.ExternalURL != "" {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))+ }+ if payload.ExternalUrl != "" {+ externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)+ if err != nil {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))+ }+ payload.ExternalUrl = externalURL+ }+ case "value.workspace_profile.require_mfa":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TWO_FA); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ case "value.workspace_profile.access_token_duration":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ if payload.AccessTokenDuration != nil && payload.AccessTokenDuration.Seconds > 0 && payload.AccessTokenDuration.AsDuration() < time.Minute {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("access token duration should be at least one minute"))+ }+ case "value.workspace_profile.refresh_token_duration":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ if payload.RefreshTokenDuration != nil && payload.RefreshTokenDuration.Seconds > 0 && payload.RefreshTokenDuration.AsDuration() < time.Hour {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("refresh token duration should be at least one hour"))+ }+ case "value.workspace_profile.inactive_session_timeout":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ case "value.workspace_profile.announcement":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DASHBOARD_ANNOUNCEMENT); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ if payload.Announcement != nil {+ if err := validateAnnouncementTheme(payload.Announcement.Theme); err != nil {+ return err+ }+ }+ case "value.workspace_profile.enforce_identity_domain":+ if payload.EnforceIdentityDomain {+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_USER_EMAIL_DOMAIN_RESTRICTION); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ }+ case "value.workspace_profile.allow_email_code_signin":+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("allow_email_code_signin cannot be changed in SaaS mode"))+ }+ if payload.AllowEmailCodeSignin {+ emailSetting, err := s.store.GetSetting(ctx, workspaceID, storepb.SettingName_EMAIL)+ if err != nil {+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to load email setting"))+ }+ if emailSetting == nil {+ return connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("cannot enable email code signin without an EMAIL setting"))+ }+ }+ case "value.workspace_profile.disallow_password_signin":+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_password_signin cannot be changed in SaaS mode"))+ }+ if payload.DisallowPasswordSignin {+ // We should still allow users to turn it off.+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_PASSWORD_SIGNIN); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }++ settingWsID := common.GetWorkspaceIDFromContext(ctx)+ identityProviders, err := s.store.ListIdentityProviders(ctx, &store.FindIdentityProviderMessage{Workspace: &settingWsID})+ if err != nil {+ return connect.NewError(connect.CodeInternal, errors.Errorf("failed to list identity providers: %v", err))+ }+ if len(identityProviders) == 0 {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot disallow password signin when no identity provider is set"))+ }+ }+ case "value.workspace_profile.enable_audit_log_stdout":+ // Audit stdout output is process-global; on a shared SaaS replica+ // it would affect other workspaces.+ if s.profile.SaaS {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the enable_audit_log_stdout cannot be changed in SaaS mode"))+ }+ if payload.EnableAuditLogStdout {+ // Require TEAM or ENTERPRISE license+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ }+ case "value.workspace_profile.watermark":+ if payload.Watermark {+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_WATERMARK); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ }+ case "value.workspace_profile.directory_sync_token":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DIRECTORY_SYNC); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ case "value.workspace_profile.password_restriction":+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_PASSWORD_RESTRICTIONS); err != nil {+ return connect.NewError(connect.CodePermissionDenied, err)+ }+ if payload.PasswordRestriction != nil && payload.PasswordRestriction.MinLength < 8 {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid password minimum length, should no less than 8"))+ }+ default:+ // Unknown paths are rejected by the merge pass.+ }+ }+ return nil+}++// mergeWorkspaceProfilePaths applies the request's update-mask paths onto+// oldSetting. It is pure apart from directory-sync token generation: it+// performs no database or license reads — those ran in+// preflightWorkspaceProfilePaths — because it executes inside the row-locking+// transaction. Only validations that are payload-local or need the merged+// state live here.+func mergeWorkspaceProfilePaths(request *connect.Request[v1pb.UpdateSettingRequest], payload *storepb.WorkspaceProfileSetting, oldSetting *storepb.WorkspaceProfileSetting) error {+ for _, path := range request.Msg.UpdateMask.Paths {+ switch path {+ case "value.workspace_profile.enable_debug":+ oldSetting.EnableDebug = payload.EnableDebug+ case "value.workspace_profile.disallow_signup":+ oldSetting.DisallowSignup = payload.DisallowSignup+ case "value.workspace_profile.external_url":+ oldSetting.ExternalUrl = payload.ExternalUrl+ case "value.workspace_profile.require_mfa":+ oldSetting.Require_2Fa = payload.Require_2Fa+ case "value.workspace_profile.access_token_duration":+ oldSetting.AccessTokenDuration = payload.AccessTokenDuration+ case "value.workspace_profile.refresh_token_duration":+ oldSetting.RefreshTokenDuration = payload.RefreshTokenDuration+ case "value.workspace_profile.inactive_session_timeout":+ oldSetting.InactiveSessionTimeout = payload.InactiveSessionTimeout+ case "value.workspace_profile.announcement":+ oldSetting.Announcement = payload.Announcement+ case "value.workspace_profile.maximum_request_expiration":+ if payload.MaximumRequestExpiration != nil {+ // If the value is less than or equal to 0, we will remove the setting. AKA no limit.+ if payload.MaximumRequestExpiration.Seconds <= 0 {+ payload.MaximumRequestExpiration = nil+ }+ }+ oldSetting.MaximumRequestExpiration = payload.MaximumRequestExpiration+ case "value.workspace_profile.maximum_role_expiration":+ if payload.MaximumRoleExpiration != nil {+ // If the value is less than or equal to 0, we will remove the setting. AKA no limit.+ if payload.MaximumRoleExpiration.Seconds <= 0 {+ payload.MaximumRoleExpiration = nil+ }+ }+ oldSetting.MaximumRoleExpiration = payload.MaximumRoleExpiration+ case "value.workspace_profile.domains":+ if err := validateDomains(payload.Domains); err != nil {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid domains, error %v", err))+ }+ oldSetting.Domains = payload.Domains+ case "value.workspace_profile.enforce_identity_domain":+ oldSetting.EnforceIdentityDomain = payload.EnforceIdentityDomain+ case "value.workspace_profile.database_change_mode":+ oldSetting.DatabaseChangeMode = payload.DatabaseChangeMode+ case "value.workspace_profile.mcp_capability":+ if err := validateMCPCapability(payload.McpCapability); err != nil {+ return err+ }+ oldSetting.McpCapability = payload.McpCapability+ case "value.workspace_profile.allow_email_code_signin":+ oldSetting.AllowEmailCodeSignin = payload.AllowEmailCodeSignin+ case "value.workspace_profile.disallow_password_signin":+ oldSetting.DisallowPasswordSignin = payload.DisallowPasswordSignin+ case "value.workspace_profile.enable_metric_collection":+ oldSetting.EnableMetricCollection = payload.EnableMetricCollection+ case "value.workspace_profile.enable_audit_log_stdout":+ oldSetting.EnableAuditLogStdout = payload.EnableAuditLogStdout+ case "value.workspace_profile.watermark":+ oldSetting.Watermark = payload.Watermark+ case "value.workspace_profile.directory_sync_token":+ // Generate a new token if the payload is empty.+ // This handles both initial setup and token reset (when user explicitly sends empty string).+ if payload.DirectorySyncToken == "" {+ payload.DirectorySyncToken = uuid.New().String()+ }+ oldSetting.DirectorySyncToken = payload.DirectorySyncToken+ case "value.workspace_profile.password_restriction":+ oldSetting.PasswordRestriction = payload.PasswordRestriction+ case "value.workspace_profile.sql_result_size":+ oldSetting.SqlResultSize = payload.SqlResultSize+ case "value.workspace_profile.query_timeout":+ oldSetting.QueryTimeout = payload.QueryTimeout+ case "value.workspace_profile.sql_editor_theme_id":+ oldSetting.SqlEditorThemeId = payload.SqlEditorThemeId+ case "value.workspace_profile.sql_editor_custom_theme":+ if err := validateSQLEditorCustomTheme(payload.SqlEditorCustomTheme); err != nil {+ return err+ }+ oldSetting.SqlEditorCustomTheme = payload.SqlEditorCustomTheme+ default:+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid update mask path %v", path))+ }+ }+ return nil+}+// TestEmailSetting sends a test email using the provided config.func (s *SettingService) TestEmailSetting(ctx context.Context, req *connect.Request[v1pb.TestEmailSettingRequest]) (*connect.Response[v1pb.TestEmailSettingResponse], error) {emailSetting := convertEmailSetting(req.Msg.EmailSetting)diff --git a/backend/api/v1/setting_service_test.go b/backend/api/v1/setting_service_test.goindex 4269d1e06f..872a6c4489 100644--- a/backend/api/v1/setting_service_test.go+++ b/backend/api/v1/setting_service_test.go@@ -4,10 +4,13 @@ import ("context""testing"+ "connectrpc.com/connect""github.com/stretchr/testify/require"colorpb "google.golang.org/genproto/googleapis/type/color"+ "google.golang.org/protobuf/types/known/fieldmaskpb""google.golang.org/protobuf/types/known/wrapperspb"+ "github.com/bytebase/bytebase/backend/component/config"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")@@ -252,3 +255,30 @@ func TestValidateEnvironmentsColor(t *testing.T) {})}}++// TestPreflightWorkspaceProfileSaaSRestrictedPaths pins that the update-mask+// paths controlling process-global runtime behavior (debug/pprof, audit-log+// stdout) are rejected in SaaS mode: on a shared replica they would let one+// workspace admin change process-wide behavior affecting other workspaces.+func TestPreflightWorkspaceProfileSaaSRestrictedPaths(t *testing.T) {+ newRequest := func(path string) *connect.Request[v1pb.UpdateSettingRequest] {+ return connect.NewRequest(&v1pb.UpdateSettingRequest{+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{path}},+ })+ }+ saas := &SettingService{profile: &config.Profile{SaaS: true}}+ selfHosted := &SettingService{profile: &config.Profile{}}++ for _, path := range []string{+ "value.workspace_profile.enable_debug",+ "value.workspace_profile.enable_audit_log_stdout",+ } {+ err := saas.preflightWorkspaceProfilePaths(context.Background(), "ws", newRequest(path), &storepb.WorkspaceProfileSetting{})+ require.Error(t, err, path)+ require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), path)++ // Self-hosted keeps accepting them (payload zero-values skip license checks).+ err = selfHosted.preflightWorkspaceProfilePaths(context.Background(), "ws", newRequest(path), &storepb.WorkspaceProfileSetting{})+ require.NoError(t, err, path)+ }+}diff --git a/backend/store/export_test.go b/backend/store/export_test.gonew file mode 100644index 0000000000..ccee7c0ea2--- /dev/null+++ b/backend/store/export_test.go@@ -0,0 +1,8 @@+package store++// SetSettingPublishHookForTest installs a hook that runs between a setting+// update's commit and its cache publish — test-only, to deterministically+// expose the publish-ordering window.+func (s *Store) SetSettingPublishHookForTest(hook func()) {+ s.settingPublishHookForTest = hook+}diff --git a/backend/store/setting.go b/backend/store/setting.goindex 275bc58055..ae7fbe3cc1 100644--- a/backend/store/setting.go+++ b/backend/store/setting.go@@ -4,6 +4,7 @@ import ("context""database/sql""math"+ "time""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"@@ -237,10 +238,27 @@ type FindSettingMessage struct {// GetSetting returns the setting by name.func (s *Store) GetSetting(ctx context.Context, workspace string, name storepb.SettingName) (*SettingMessage, error) {- if v, ok := s.settingCache.Get(getSettingCacheKey(workspace, name)); ok && s.enableCache {+ // With caching disabled (HA), every read goes straight to the database:+ // there is no cache to fill, and taking the publish mutex here would+ // serialize all setting reads in the process behind a single lock.+ if !s.enableCache {+ return s.GetSettingUncached(ctx, workspace, name)+ }+ if v, ok := s.settingCache.Get(getSettingCacheKey(workspace, name)); ok {return v, nil}- return s.GetSettingUncached(ctx, workspace, name)+ // Fill the cache under the publish mutex so the fill's content is current+ // at publish time: an unordered fill could read a pre-commit snapshot and+ // cache it after a concurrent writer already published a newer value. No+ // row lock is held here, so the row-lock -> publish-lock order is kept.+ s.settingPublishMu.Lock()+ defer s.settingPublishMu.Unlock()+ setting, err := s.GetSettingUncached(ctx, workspace, name)+ if err != nil || setting == nil {+ return setting, err+ }+ s.settingCache.Add(getSettingCacheKey(workspace, name), setting)+ return setting, nil}// GetSettingUncached reads the setting directly from the database, bypassing@@ -320,12 +338,129 @@ func (s *Store) ListSettings(ctx context.Context, find *FindSettingMessage) ([]*return nil, err}- for _, setting := range settingMessages {- s.settingCache.Add(getSettingCacheKey(setting.Workspace, setting.Name), setting)- }+ // Deliberately no cache publication: this path runs outside the publish+ // mutex (GetSettingUncached callers hit it on every read), and an+ // unordered fill could pin a pre-commit snapshot over a newer published+ // value. Publication happens only in GetSetting's fill, UpsertSetting,+ // and UpdateSettingAtomic — all ordered by settingPublishMu.return settingMessages, nil}+// UpdateSettingAtomic performs a row-locking read-modify-write of a setting:+// the current value is read under SELECT ... FOR UPDATE, apply transforms it,+// and the result is written back in the same transaction — so a concurrent+// write can never be silently reverted by a merge based on a stale read (the+// lost-update hazard of read-then-UpsertSetting). apply receives the value+// freshly unmarshaled from the locked row and may mutate it in place; an error+// from apply aborts the transaction with no write and is returned unwrapped so+// callers keep typed errors. After commit, the served state is refreshed via+// publishSetting: the row is re-read under the publish mutex and the fresh+// value goes to the cache and to postCommit (optional, for derived state) —+// so publications always carry current truth regardless of the order in+// which updaters reach the mutex. apply must stay free of database reads:+// it runs while holding the transaction's pooled connection and the row+// lock, and a nested read wanting a second connection is the bounded-pool+// starvation cycle.+func (s *Store) UpdateSettingAtomic(ctx context.Context, workspace string, name storepb.SettingName, apply func(current proto.Message) (proto.Message, error), postCommit func(current *SettingMessage)) (*SettingMessage, error) {+ tx, err := s.GetDB().BeginTx(ctx, nil)+ if err != nil {+ return nil, errors.Wrap(err, "failed to begin transaction")+ }+ defer tx.Rollback()++ q := qb.Q().Space(`+ SELECT value FROM setting WHERE workspace = ? AND name = ? FOR UPDATE+ `, workspace, name.String())+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }+ var valueString string+ if err := tx.QueryRowContext(ctx, query, args...).Scan(&valueString); err != nil {+ if err == sql.ErrNoRows {+ return nil, errors.Errorf("setting %s not found", name)+ }+ return nil, errors.Wrapf(err, "failed to lock setting %s", name)+ }+ current, err := getSettingMessage(name)+ if err != nil {+ return nil, err+ }+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(valueString), current); err != nil {+ return nil, errors.Wrapf(err, "failed to unmarshal setting %s", name)+ }++ next, err := apply(current)+ if err != nil {+ return nil, err+ }++ nextBytes, err := protojson.Marshal(next)+ if err != nil {+ return nil, errors.Wrap(err, "failed to marshal setting value")+ }+ q = qb.Q().Space(`+ UPDATE setting SET value = ? WHERE workspace = ? AND name = ?+ `, string(nextBytes), workspace, name.String())+ query, args, err = q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return nil, errors.Wrapf(err, "failed to update setting %s", name)+ }+ // Commit BEFORE acquiring the publish mutex: commit returns the pooled+ // connection and releases the row lock, so waiting for the mutex never+ // holds pool resources (connection-holders never wait for the mutex, so+ // the mutex/pool wait graph stays acyclic). Ordering is preserved because+ // publishSetting re-reads the row under the mutex — whichever updater+ // publishes last still publishes current truth.+ if err := tx.Commit(); err != nil {+ return nil, errors.Wrap(err, "failed to commit transaction")+ }+ s.publishSetting(ctx, workspace, name, postCommit)+ return &SettingMessage{Name: name, Workspace: workspace, Value: next}, nil+}++// publishSetting refreshes the served state for a setting after a committed+// write: under settingPublishMu it re-reads the row and publishes the fresh+// value to the cache (when enabled) and to postCommit for derived state. It+// must be called with no transaction, row lock, or pooled connection held —+// the mutex-then-connection order here is what keeps the publish protocol+// deadlock-free against the bounded metadata pool. If the re-read fails, the+// cache entry is evicted rather than risk publishing stale state, and+// postCommit is skipped (derived state converges on the next write).+func (s *Store) publishSetting(ctx context.Context, workspace string, name storepb.SettingName, postCommit func(current *SettingMessage)) {+ if !s.enableCache && postCommit == nil {+ return+ }+ s.settingPublishMu.Lock()+ defer s.settingPublishMu.Unlock()+ if s.settingPublishHookForTest != nil {+ s.settingPublishHookForTest()+ }+ // The write is already committed, so publication must not depend on the+ // caller still listening: re-read on a bounded context detached from+ // request cancellation. If the read still fails (database unreachable),+ // evict and skip postCommit — the cache heals on the next fill, and+ // derived runtime state heals on the next successful publication of this+ // setting (postCommit callbacks must therefore reconcile from the fresh+ // value unconditionally, not only for fields their request touched).+ publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ defer cancel()+ fresh, err := s.GetSettingUncached(publishCtx, workspace, name)+ if err != nil || fresh == nil {+ s.settingCache.Remove(getSettingCacheKey(workspace, name))+ return+ }+ if s.enableCache {+ s.settingCache.Add(getSettingCacheKey(workspace, name), fresh)+ }+ if postCommit != nil {+ postCommit(fresh)+ }+}+// UpsertSetting upserts the setting by name.func (s *Store) UpsertSetting(ctx context.Context, update *SettingMessage) (*SettingMessage, error) {valueBytes, err := protojson.Marshal(update.Value)@@ -373,7 +508,7 @@ func (s *Store) UpsertSetting(ctx context.Context, update *SettingMessage) (*Set}setting.Value = msg- s.settingCache.Add(getSettingCacheKey(setting.Workspace, setting.Name), &setting)+ s.publishSetting(ctx, setting.Workspace, setting.Name, nil)return &setting, nil}@@ -389,6 +524,11 @@ func (s *Store) DeleteSetting(ctx context.Context, workspace string, name storepreturn err}+ // Invalidate under the ordering mutex: a cache-miss fill reads and+ // publishes atomically under the same mutex, so once serialized against+ // it, a fill can no longer resurrect the deleted row.+ s.settingPublishMu.Lock()+ defer s.settingPublishMu.Unlock()s.settingCache.Remove(getSettingCacheKey(workspace, name))return nil}diff --git a/backend/store/setting_atomic_test.go b/backend/store/setting_atomic_test.gonew file mode 100644index 0000000000..a3f27d6801--- /dev/null+++ b/backend/store/setting_atomic_test.go@@ -0,0 +1,460 @@+package store_test++import (+ "context"+ "database/sql"+ "fmt"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/proto"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/migrator"+ "github.com/bytebase/bytebase/backend/store"+)++// newSettingAtomicFixture boots a real PostgreSQL, migrates the schema, seeds+// the workspace and an empty WORKSPACE_PROFILE row, and returns a store with+// the setting cache ENABLED — the production non-HA shape, so the tests also+// pin that the cache reflects committed state.+func newSettingAtomicFixture(t *testing.T) (context.Context, *sql.DB, *store.Store) {+ t.Helper()+ return newSettingAtomicFixtureWithCache(t, true)+}++func newSettingAtomicFixtureWithCache(t *testing.T, enableCache bool) (context.Context, *sql.DB, *store.Store) {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ t.Cleanup(cancel)+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(context.Background()) })+ db := container.GetDB()+ require.NoError(t, migrator.MigrateSchema(ctx, db))++ _, err := db.ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('default');`)+ require.NoError(t, err)++ pgURL := fmt.Sprintf(+ "host=%s port=%s user=postgres password=root-password database=postgres",+ container.GetHost(), container.GetPort(),+ )+ s, err := store.New(ctx, pgURL, enableCache)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, s.Close()) })++ _, err = s.UpsertSetting(ctx, &store.SettingMessage{+ Name: storepb.SettingName_WORKSPACE_PROFILE,+ Workspace: "default",+ Value: &storepb.WorkspaceProfileSetting{},+ })+ require.NoError(t, err)+ return ctx, db, s+}++func mustProfile(t *testing.T, setting *store.SettingMessage) *storepb.WorkspaceProfileSetting {+ t.Helper()+ profile, ok := setting.Value.(*storepb.WorkspaceProfileSetting)+ require.True(t, ok, "unexpected setting value type %T", setting.Value)+ return profile+}++// TestUpdateSettingAtomicInterleaving pins the lost-update protection: while+// one update holds the row lock inside its apply callback, a concurrent update+// to a DIFFERENT field must wait behind it and then merge onto the committed+// result — both mutations survive in the final row and in the setting cache.+// Against a read-then-UpsertSetting implementation this fails: the second+// update merges onto a stale read and silently reverts the first's field.+func TestUpdateSettingAtomicInterleaving(t *testing.T) {+ ctx, db, s := newSettingAtomicFixture(t)++ entered := make(chan struct{})+ release := make(chan struct{})+ firstResult := make(chan error, 1)+ go func() {+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.EnableMetricCollection = true+ close(entered)+ <-release+ return profile, nil+ }, nil)+ firstResult <- err+ }()+ // The first update is now inside apply and holds the row lock.+ select {+ case <-entered:+ case <-time.After(15 * time.Second):+ t.Fatal("first update never reached its apply callback")+ }++ secondResult := make(chan error, 1)+ go func() {+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.McpCapability = storepb.WorkspaceProfileSetting_DISABLED+ return profile, nil+ }, nil)+ secondResult <- err+ }()++ // Deterministic barrier: the second update must be observed waiting behind+ // the first's row lock before the first is released.+ require.Eventually(t, func() bool {+ var waiting bool+ err := db.QueryRowContext(ctx, `+ SELECT EXISTS (+ SELECT 1+ FROM pg_stat_activity+ WHERE cardinality(pg_blocking_pids(pid)) > 0+ AND query LIKE '%setting%FOR UPDATE%'+ )+ `).Scan(&waiting)+ return err == nil && waiting+ }, 10*time.Second, 10*time.Millisecond,+ "the second update should wait behind the first's row lock")+ select {+ case err := <-secondResult:+ t.Fatalf("second update completed while the first held the row lock: %v", err)+ default:+ }++ close(release)+ for _, result := range []chan error{firstResult, secondResult} {+ select {+ case err := <-result:+ require.NoError(t, err)+ case <-time.After(15 * time.Second):+ t.Fatal("update did not complete; possible deadlock")+ }+ }++ // Both mutations survive in the database ...+ fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ freshProfile := mustProfile(t, fresh)+ require.True(t, freshProfile.EnableMetricCollection, "first update's field must survive")+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, freshProfile.McpCapability,+ "second update's field must survive")+ // ... and in the setting cache.+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ cachedProfile := mustProfile(t, cached)+ require.True(t, cachedProfile.EnableMetricCollection)+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, cachedProfile.McpCapability)+}++// TestUpdateSettingAtomicApplyAbort pins that an error from apply aborts the+// transaction with no write, leaves the cache untouched, and surfaces+// unwrapped so callers keep typed errors.+func TestUpdateSettingAtomicApplyAbort(t *testing.T) {+ ctx, _, s := newSettingAtomicFixture(t)++ sentinel := errors.New("validation failed")+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ // Mutate before failing: the mutation must not leak into the row+ // or the served cache.+ profile.EnableMetricCollection = true+ return nil, sentinel+ }, nil)+ require.ErrorIs(t, err, sentinel)++ fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ require.False(t, mustProfile(t, fresh).EnableMetricCollection)+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ require.False(t, mustProfile(t, cached).EnableMetricCollection)+}++// TestUpdateSettingAtomicMissingRow pins that a missing setting row is an+// error, not a silent create — the primitive updates existing state only.+func TestUpdateSettingAtomicMissingRow(t *testing.T) {+ ctx, _, s := newSettingAtomicFixture(t)++ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_AI,+ func(current proto.Message) (proto.Message, error) {+ return current, nil+ }, nil)+ require.Error(t, err)+ require.Contains(t, err.Error(), "not found")+}++// TestUpdateSettingAtomicPublishOrder pins that the setting cache is published+// in commit order: while one update is paused between its commit and its cache+// publish (test hook), a later update must not be able to commit and publish —+// otherwise the paused update's older value would overwrite the newer one in+// the cache, serving reverted state until the next refresh. Red against an+// implementation whose commit releases the row lock before an unordered cache+// write.+func TestUpdateSettingAtomicPublishOrder(t *testing.T) {+ ctx, _, s := newSettingAtomicFixture(t)++ pauseFirst := make(chan struct{})+ resumeFirst := make(chan struct{})+ // Only the first update pauses; later publishes pass through WITHOUT+ // blocking (sync.Once.Do would serialize concurrent callers and mask the+ // very ordering bug this test pins).+ var first atomic.Bool+ s.SetSettingPublishHookForTest(func() {+ if first.CompareAndSwap(false, true) {+ close(pauseFirst)+ <-resumeFirst+ }+ })++ // postCommit callbacks run inside the ordered publish window and receive+ // the freshly re-read value, so derived state converges on committed+ // truth regardless of which updater reaches the mutex first.+ var postCommitMu sync.Mutex+ var postCommitSeen []*storepb.WorkspaceProfileSetting+ recordPostCommit := func() func(current *store.SettingMessage) {+ return func(current *store.SettingMessage) {+ profile, ok := current.Value.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return+ }+ postCommitMu.Lock()+ defer postCommitMu.Unlock()+ postCommitSeen = append(postCommitSeen, profile)+ }+ }++ firstResult := make(chan error, 1)+ go func() {+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.EnableMetricCollection = true+ return profile, nil+ }, recordPostCommit())+ firstResult <- err+ }()+ // The first update has committed and is paused before publishing.+ select {+ case <-pauseFirst:+ case <-time.After(15 * time.Second):+ t.Fatal("first update never reached the publish window")+ }++ secondResult := make(chan error, 1)+ go func() {+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.McpCapability = storepb.WorkspaceProfileSetting_DISABLED+ return profile, nil+ }, recordPostCommit())+ secondResult <- err+ }()++ // Deterministic barrier: wait until the second update's write is visible+ // in the database. A writer must never retain its transaction (a+ // bounded-pool connection and the row lock) while waiting for the publish+ // mutex — that is the connection-pool deadlock cycle — so the commit must+ // land while the first still holds the publish window.+ require.Eventually(t, func() bool {+ committed, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ if err != nil {+ return false+ }+ profile, ok := committed.Value.(*storepb.WorkspaceProfileSetting)+ return ok && profile.McpCapability == storepb.WorkspaceProfileSetting_DISABLED+ }, 10*time.Second, 10*time.Millisecond,+ "second update must commit while its publish waits on the mutex")++ // Having committed, it must still not have completed: its publication is+ // blocked behind the first's paused window — completing here would be+ // exactly the out-of-order publish.+ select {+ case err := <-secondResult:+ t.Fatalf("second update published while the first held the publish window: %v", err)+ default:+ }++ close(resumeFirst)+ for _, result := range []chan error{firstResult, secondResult} {+ select {+ case err := <-result:+ require.NoError(t, err)+ case <-time.After(15 * time.Second):+ t.Fatal("update did not complete; possible deadlock")+ }+ }++ // The served cache must hold the final committed state: the second update+ // ran after the first's commit, so its merge carries BOTH fields.+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ cachedProfile := mustProfile(t, cached)+ require.True(t, cachedProfile.EnableMetricCollection, "cache must not serve pre-first-update state")+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, cachedProfile.McpCapability,+ "cache must not be overwritten by the earlier committer's stale publish")+ postCommitMu.Lock()+ defer postCommitMu.Unlock()+ require.Len(t, postCommitSeen, 2, "both postCommit callbacks must run")+ for _, seen := range postCommitSeen {+ require.True(t, seen.EnableMetricCollection,+ "postCommit must observe committed truth, not the caller's own write")+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, seen.McpCapability,+ "postCommit must observe committed truth, not the caller's own write")+ }+}++// TestSettingCacheNoFillFromUncachedReads pins that uncached reads do not+// populate the setting cache: GetSettingUncached (and ListSettings under it)+// runs outside the publish-ordering mutex, so a fill from that path could+// cache a pre-commit snapshot after a newer value was already published.+// Uncached readers must leave publication to the ordered paths.+func TestSettingCacheNoFillFromUncachedReads(t *testing.T) {+ ctx, db, s := newSettingAtomicFixture(t)+ // Drop the seed's cache entry so the next cached read must fill.+ s.DeleteCache()++ // An uncached read must not leave a cache entry behind ...+ _, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)++ // ... so after an out-of-band change, a cached read serves the new value.+ _, err = db.ExecContext(ctx, `+ UPDATE setting SET value = jsonb_set(value, '{enableMetricCollection}', 'true')+ WHERE workspace = 'default' AND name = 'WORKSPACE_PROFILE';+ `)+ require.NoError(t, err)++ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ require.True(t, mustProfile(t, cached).EnableMetricCollection,+ "GetSetting must not serve a snapshot cached by an uncached read")+}++// TestGetSettingCacheDisabledNotSerialized pins the HA read path: with the+// cache disabled (store.New(..., !profile.HA)), GetSetting must go straight to+// the database and never touch the publish-ordering mutex — otherwise every+// setting read in an HA process serializes behind a single lock, including+// while a writer sits in its commit-to-publish window.+func TestGetSettingCacheDisabledNotSerialized(t *testing.T) {+ ctx, _, s := newSettingAtomicFixtureWithCache(t, false)++ pause := make(chan struct{})+ resume := make(chan struct{})+ var first atomic.Bool+ s.SetSettingPublishHookForTest(func() {+ if first.CompareAndSwap(false, true) {+ close(pause)+ <-resume+ }+ })++ writerResult := make(chan error, 1)+ go func() {+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.EnableMetricCollection = true+ return profile, nil+ }, func(*store.SettingMessage) {})+ writerResult <- err+ }()+ select {+ case <-pause:+ case <-time.After(15 * time.Second):+ t.Fatal("writer never reached the publish window")+ }++ // The writer holds the publish mutex; a cache-disabled read must still+ // complete (it reads committed state and consults no cache).+ readResult := make(chan error, 1)+ go func() {+ setting, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ if err == nil && !mustProfile(t, setting).EnableMetricCollection {+ err = errors.New("read did not observe the committed value")+ }+ readResult <- err+ }()+ select {+ case err := <-readResult:+ require.NoError(t, err)+ case <-time.After(5 * time.Second):+ t.Fatal("cache-disabled GetSetting blocked behind the publish mutex")+ }++ close(resume)+ select {+ case err := <-writerResult:+ require.NoError(t, err)+ case <-time.After(15 * time.Second):+ t.Fatal("writer did not complete")+ }+}++// TestUpdateSettingAtomicPublishSurvivesCancellation pins that publication+// completes even when the request context dies right after commit: the write+// is already durable, so the cache refresh and postCommit must not depend on+// the caller still listening. Red against a publish path that re-reads on the+// request context — the canceled read was swallowed and postCommit silently+// skipped, leaving derived runtime state diverged from the database forever.+func TestUpdateSettingAtomicPublishSurvivesCancellation(t *testing.T) {+ ctx, _, s := newSettingAtomicFixture(t)++ updateCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ var once atomic.Bool+ s.SetSettingPublishHookForTest(func() {+ // Fires after commit, before the publish re-read.+ if once.CompareAndSwap(false, true) {+ cancel()+ }+ })++ var observed *storepb.WorkspaceProfileSetting+ _, err := s.UpdateSettingAtomic(updateCtx, "default", storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.EnableMetricCollection = true+ return profile, nil+ }, func(current *store.SettingMessage) {+ if profile, ok := current.Value.(*storepb.WorkspaceProfileSetting); ok {+ observed = profile+ }+ })+ require.NoError(t, err)++ require.NotNil(t, observed, "postCommit must run despite request cancellation after commit")+ require.True(t, observed.EnableMetricCollection, "postCommit must observe the committed state")++ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ require.True(t, mustProfile(t, cached).EnableMetricCollection,+ "the served cache must reflect the committed write despite request cancellation")+}diff --git a/backend/store/store.go b/backend/store/store.goindex d02eb6b14b..b6bc1e0267 100644--- a/backend/store/store.go+++ b/backend/store/store.go@@ -2,6 +2,8 @@package storeimport (+ "sync"+"context""database/sql""fmt"@@ -20,19 +22,30 @@ type Store struct {enableCache bool// Cache.- Secret string- userEmailCache *lru.Cache[string, *UserMessage]- instanceCache *lru.Cache[string, *InstanceMessage]- databaseCache *lru.Cache[string, *DatabaseMessage]- projectCache *lru.Cache[string, *ProjectMessage]- policyCache *lru.Cache[string, *PolicyMessage]- settingCache *lru.Cache[string, *SettingMessage]- rolesCache *expirable.LRU[string, *RoleMessage]- groupCache *expirable.LRU[string, *GroupMessage]- groupMembersCache *expirable.LRU[string, map[string]bool]- memberGroupsCache *expirable.LRU[string, []string]- dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]- iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]+ Secret string+ userEmailCache *lru.Cache[string, *UserMessage]+ instanceCache *lru.Cache[string, *InstanceMessage]+ databaseCache *lru.Cache[string, *DatabaseMessage]+ projectCache *lru.Cache[string, *ProjectMessage]+ policyCache *lru.Cache[string, *PolicyMessage]+ settingCache *lru.Cache[string, *SettingMessage]++ // settingPublishMu serializes all setting cache publications (writer+ // republishes, cache-miss fills, invalidations). Publishers re-read the+ // row under the mutex, so whichever publishes last publishes current+ // truth. It must be acquired with no transaction, row lock, or pooled+ // connection held — writers commit first, then publish — so the+ // mutex/pool wait graph stays acyclic against the bounded metadata pool.+ settingPublishMu sync.Mutex+ // settingPublishHookForTest, when set, runs between commit and cache+ // publish so tests can deterministically expose the ordering window.+ settingPublishHookForTest func()+ rolesCache *expirable.LRU[string, *RoleMessage]+ groupCache *expirable.LRU[string, *GroupMessage]+ groupMembersCache *expirable.LRU[string, map[string]bool]+ memberGroupsCache *expirable.LRU[string, []string]+ dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]+ iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]// Large objects.sheetFullCache *lru.Cache[string, *SheetMessage]@@ -116,7 +129,11 @@ func (s *Store) GetDB() *sql.DB {// DeleteCache deletes the cache.func (s *Store) DeleteCache() {+ // The setting cache purge participates in the publish-ordering invariant+ // so an in-flight fill cannot republish a just-purged snapshot.+ s.settingPublishMu.Lock()s.settingCache.Purge()+ s.settingPublishMu.Unlock()s.policyCache.Purge()s.userEmailCache.Purge()}