Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: row-locking read-modify-write for workspace profile setting updates (#21060)

Vincent Huang8835311af741
diff --git a/backend/api/v1/setting_service.go b/backend/api/v1/setting_service.go
index 93d8f34fdd..1c59cb3972 100644
--- a/backend/api/v1/setting_service.go
+++ b/backend/api/v1/setting_service.go
@@ -185,232 +185,11 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Req
}
var storeSettingValue proto.Message
- var resetAuditLogStdout bool
var resetClassification bool
switch storeSettingName {
case storepb.SettingName_WORKSPACE_PROFILE:
- if request.Msg.UpdateMask == nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("update mask is required"))
- }
- payload := convertWorkspaceProfileSetting(request.Msg.Setting.Value.GetWorkspaceProfile())
- // Merge onto the profile as stored in the database, not the setting
- // cache: the whole profile is written back below, so a stale cached base
- // would silently revert fields changed out-of-band (e.g. an emergency
- // SQL flip of the MCP kill switch) when an admin saves an unrelated
- // field.
- freshSetting, err := s.store.GetSettingUncached(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find setting %s with error: %v", storeSettingName, err))
- }
- if freshSetting == nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("cannot find setting %v", storeSettingName))
- }
- profileValue, ok := freshSetting.Value.(*storepb.WorkspaceProfileSetting)
- if !ok {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storeSettingName))
- }
- // The uncached read still lands in the setting cache (ListSettings
- // caches what it returns), so mutate a clone: a validate-only request
- // must never alter the served in-memory state.
- oldSetting := proto.CloneOf(profileValue)
-
- for _, path := range request.Msg.UpdateMask.Paths {
- switch path {
- case "value.workspace_profile.enable_debug":
- oldSetting.EnableDebug = payload.EnableDebug
- level := slog.LevelInfo
- if payload.EnableDebug {
- level = slog.LevelDebug
- }
- log.LogLevel.Set(level)
- case "value.workspace_profile.disallow_signup":
- if s.profile.SaaS {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_signup cannot be changed in SaaS mode"))
- }
- if payload.DisallowSignup {
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_SELF_SERVICE_SIGNUP); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- }
- oldSetting.DisallowSignup = payload.DisallowSignup
- case "value.workspace_profile.external_url":
- if s.profile.SaaS {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the external_url cannot be changed in SaaS mode"))
- }
- // Prevent changing external URL via UI when it's set via command-line flag
- if s.profile.ExternalURL != "" {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))
- }
- if payload.ExternalUrl != "" {
- externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)
- if err != nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))
- }
- payload.ExternalUrl = externalURL
- }
- oldSetting.ExternalUrl = payload.ExternalUrl
- case "value.workspace_profile.require_mfa":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TWO_FA); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- oldSetting.Require_2Fa = payload.Require_2Fa
- case "value.workspace_profile.access_token_duration":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- if payload.AccessTokenDuration != nil && payload.AccessTokenDuration.Seconds > 0 && payload.AccessTokenDuration.AsDuration() < time.Minute {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("access token duration should be at least one minute"))
- }
- oldSetting.AccessTokenDuration = payload.AccessTokenDuration
- case "value.workspace_profile.refresh_token_duration":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- if payload.RefreshTokenDuration != nil && payload.RefreshTokenDuration.Seconds > 0 && payload.RefreshTokenDuration.AsDuration() < time.Hour {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("refresh token duration should be at least one hour"))
- }
- oldSetting.RefreshTokenDuration = payload.RefreshTokenDuration
- case "value.workspace_profile.inactive_session_timeout":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- oldSetting.InactiveSessionTimeout = payload.InactiveSessionTimeout
- case "value.workspace_profile.announcement":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DASHBOARD_ANNOUNCEMENT); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- if payload.Announcement != nil {
- if err := validateAnnouncementTheme(payload.Announcement.Theme); err != nil {
- return nil, err
- }
- }
- oldSetting.Announcement = payload.Announcement
- case "value.workspace_profile.maximum_request_expiration":
- if payload.MaximumRequestExpiration != nil {
- // If the value is less than or equal to 0, we will remove the setting. AKA no limit.
- if payload.MaximumRequestExpiration.Seconds <= 0 {
- payload.MaximumRequestExpiration = nil
- }
- }
- oldSetting.MaximumRequestExpiration = payload.MaximumRequestExpiration
- case "value.workspace_profile.maximum_role_expiration":
- if payload.MaximumRoleExpiration != nil {
- // If the value is less than or equal to 0, we will remove the setting. AKA no limit.
- if payload.MaximumRoleExpiration.Seconds <= 0 {
- payload.MaximumRoleExpiration = nil
- }
- }
- oldSetting.MaximumRoleExpiration = payload.MaximumRoleExpiration
- case "value.workspace_profile.domains":
- if err := validateDomains(payload.Domains); err != nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid domains, error %v", err))
- }
- oldSetting.Domains = payload.Domains
- case "value.workspace_profile.enforce_identity_domain":
- if payload.EnforceIdentityDomain {
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_USER_EMAIL_DOMAIN_RESTRICTION); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- }
- oldSetting.EnforceIdentityDomain = payload.EnforceIdentityDomain
- case "value.workspace_profile.database_change_mode":
- oldSetting.DatabaseChangeMode = payload.DatabaseChangeMode
- case "value.workspace_profile.mcp_capability":
- if err := validateMCPCapability(payload.McpCapability); err != nil {
- return nil, err
- }
- oldSetting.McpCapability = payload.McpCapability
- case "value.workspace_profile.allow_email_code_signin":
- if s.profile.SaaS {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("allow_email_code_signin cannot be changed in SaaS mode"))
- }
- if payload.AllowEmailCodeSignin {
- emailSetting, err := s.store.GetSetting(ctx, workspaceID, storepb.SettingName_EMAIL)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to load email setting"))
- }
- if emailSetting == nil {
- return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("cannot enable email code signin without an EMAIL setting"))
- }
- }
- oldSetting.AllowEmailCodeSignin = payload.AllowEmailCodeSignin
- case "value.workspace_profile.disallow_password_signin":
- if s.profile.SaaS {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_password_signin cannot be changed in SaaS mode"))
- }
- if payload.DisallowPasswordSignin {
- // We should still allow users to turn it off.
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_PASSWORD_SIGNIN); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
-
- settingWsID := common.GetWorkspaceIDFromContext(ctx)
- identityProviders, err := s.store.ListIdentityProviders(ctx, &store.FindIdentityProviderMessage{Workspace: &settingWsID})
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to list identity providers: %v", err))
- }
- if len(identityProviders) == 0 {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot disallow password signin when no identity provider is set"))
- }
- }
- oldSetting.DisallowPasswordSignin = payload.DisallowPasswordSignin
- case "value.workspace_profile.enable_metric_collection":
- oldSetting.EnableMetricCollection = payload.EnableMetricCollection
- case "value.workspace_profile.enable_audit_log_stdout":
- if payload.EnableAuditLogStdout {
- // Require TEAM or ENTERPRISE license
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- }
- resetAuditLogStdout = true
- oldSetting.EnableAuditLogStdout = payload.EnableAuditLogStdout
- case "value.workspace_profile.watermark":
- if payload.Watermark {
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_WATERMARK); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- }
- oldSetting.Watermark = payload.Watermark
- case "value.workspace_profile.directory_sync_token":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DIRECTORY_SYNC); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- // Generate a new token if the payload is empty.
- // This handles both initial setup and token reset (when user explicitly sends empty string).
- if payload.DirectorySyncToken == "" {
- payload.DirectorySyncToken = uuid.New().String()
- }
- oldSetting.DirectorySyncToken = payload.DirectorySyncToken
- case "value.workspace_profile.password_restriction":
- if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_PASSWORD_RESTRICTIONS); err != nil {
- return nil, connect.NewError(connect.CodePermissionDenied, err)
- }
- if payload.PasswordRestriction != nil && payload.PasswordRestriction.MinLength < 8 {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid password minimum length, should no less than 8"))
- }
- oldSetting.PasswordRestriction = payload.PasswordRestriction
- case "value.workspace_profile.sql_result_size":
- oldSetting.SqlResultSize = payload.SqlResultSize
- case "value.workspace_profile.query_timeout":
- oldSetting.QueryTimeout = payload.QueryTimeout
- case "value.workspace_profile.sql_editor_theme_id":
- oldSetting.SqlEditorThemeId = payload.SqlEditorThemeId
- case "value.workspace_profile.sql_editor_custom_theme":
- if err := validateSQLEditorCustomTheme(payload.SqlEditorCustomTheme); err != nil {
- return nil, err
- }
- oldSetting.SqlEditorCustomTheme = payload.SqlEditorCustomTheme
- default:
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid update mask path %v", path))
- }
- }
-
- if len(oldSetting.Domains) == 0 && oldSetting.EnforceIdentityDomain {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("identity domain can be enforced only when workspace domains are set"))
- }
- storeSettingValue = oldSetting
+ return s.updateWorkspaceProfileSetting(ctx, request, workspaceID)
case storepb.SettingName_WORKSPACE_APPROVAL:
if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_APPROVAL_WORKFLOW); err != nil {
return nil, connect.NewError(connect.CodePermissionDenied, err)
@@ -736,15 +515,6 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Req
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to set setting: %v", err))
}
- // Dynamically update audit logger runtime flag if enable_audit_log_stdout was changed
- if resetAuditLogStdout {
- workspaceProfile, err := s.store.GetWorkspaceProfileSetting(ctx, workspaceID)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get workspace setting message: %v", err))
- }
- s.profile.RuntimeEnableAuditLogStdout.Store(workspaceProfile.EnableAuditLogStdout)
- }
-
// It's a temporary solution to map the classification to all projects before we support it in the UX.
if resetClassification {
classification, err := s.store.GetDataClassificationSetting(ctx, workspaceID)
@@ -784,6 +554,368 @@ func (s *SettingService) UpdateSetting(ctx context.Context, request *connect.Req
return connect.NewResponse(settingMessage), nil
}
+// updateWorkspaceProfileSetting handles the WORKSPACE_PROFILE branch of
+// UpdateSetting through the store's row-locking read-modify-write primitive:
+// the update-mask merge and its validations run inside the transaction against
+// the value of the locked row, so validation sees the true final state and a
+// concurrent write (another admin, emergency SQL) can never be silently
+// reverted by a merge based on a stale read. Validate-only requests run the
+// same merge and validations against an uncached snapshot and write nothing.
+func (s *SettingService) updateWorkspaceProfileSetting(ctx context.Context, request *connect.Request[v1pb.UpdateSettingRequest], workspaceID string) (*connect.Response[v1pb.Setting], error) {
+ if request.Msg.UpdateMask == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("update mask is required"))
+ }
+ payload := convertWorkspaceProfileSetting(request.Msg.Setting.Value.GetWorkspaceProfile())
+ // License, store, and profile checks run before the row-locking
+ // transaction so apply stays free of database reads.
+ if err := s.preflightWorkspaceProfilePaths(ctx, workspaceID, request, payload); err != nil {
+ return nil, err
+ }
+ var lockedBefore *storepb.WorkspaceProfileSetting
+ apply := func(current proto.Message) (proto.Message, error) {
+ oldSetting, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storepb.SettingName_WORKSPACE_PROFILE))
+ }
+ // The audit before-image is the row this merge actually ran against,
+ // not the possibly stale pre-lock snapshot captured by UpdateSetting.
+ lockedBefore = proto.CloneOf(oldSetting)
+ if err := mergeWorkspaceProfilePaths(request, payload, oldSetting); err != nil {
+ return nil, err
+ }
+ if len(oldSetting.Domains) == 0 && oldSetting.EnforceIdentityDomain {
+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("identity domain can be enforced only when workspace domains are set"))
+ }
+ return oldSetting, nil
+ }
+
+ if request.Msg.ValidateOnly {
+ freshSetting, err := s.store.GetSettingUncached(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find setting %s with error: %v", storepb.SettingName_WORKSPACE_PROFILE, err))
+ }
+ if freshSetting == nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("cannot find setting %v", storepb.SettingName_WORKSPACE_PROFILE))
+ }
+ profileValue, ok := freshSetting.Value.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("invalid setting value type for %s", storepb.SettingName_WORKSPACE_PROFILE))
+ }
+ // GetSettingUncached returns a fresh object today (uncached reads no
+ // longer populate the cache), but validate against a clone anyway so
+ // a validate-only request can never mutate shared state should the
+ // read path change.
+ if _, err := apply(proto.CloneOf(profileValue)); err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&v1pb.Setting{
+ Name: request.Msg.Setting.Name,
+ Value: request.Msg.Setting.Value,
+ }), nil
+ }
+
+ // Runtime derivatives (audit stdout flag, debug log level, pprof gate)
+ // reconcile from the freshly re-read committed state on EVERY successful
+ // profile publication — not only when this request touched those fields —
+ // so a publication skipped by an earlier failure is repaired by the next
+ // profile write, mirroring startup derivation (server.go). Skipped in
+ // SaaS, where workspace-scoped stored values must not drive process-global
+ // state on a shared replica (the corresponding mask paths are also
+ // rejected in preflight).
+ //
+ // The audit-log entitlement is computed here, outside the publish mutex:
+ // runtime audit stdout requires the stored flag AND a valid license, same
+ // as startup — and postCommit must not call IsFeatureEnabled itself,
+ // because it runs under settingPublishMu and a setting cache miss there
+ // would re-acquire that mutex.
+ auditLogFeatureEnabled := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG) == nil
+ postCommit := func(current *store.SettingMessage) {
+ if s.profile.SaaS {
+ return
+ }
+ profile, ok := current.Value.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return
+ }
+ s.profile.RuntimeEnableAuditLogStdout.Store(profile.EnableAuditLogStdout && auditLogFeatureEnabled)
+ s.profile.RuntimeDebug.Store(profile.EnableDebug)
+ level := slog.LevelInfo
+ if profile.EnableDebug {
+ level = slog.LevelDebug
+ }
+ log.LogLevel.Set(level)
+ }
+ setting, err := s.store.UpdateSettingAtomic(ctx, workspaceID, storepb.SettingName_WORKSPACE_PROFILE, apply, postCommit)
+ if err != nil {
+ var connectErr *connect.Error
+ if errors.As(err, &connectErr) {
+ return nil, err
+ }
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to set setting: %v", err))
+ }
+
+ // Re-capture the audit before-image from the locked row the merge ran
+ // against, overwriting UpdateSetting's earlier pre-lock snapshot.
+ if setServiceData, ok := common.GetSetServiceDataFromContext(ctx); ok && lockedBefore != nil {
+ v1pbSetting, err := convertToSettingMessage(&store.SettingMessage{
+ Name: storepb.SettingName_WORKSPACE_PROFILE,
+ Workspace: workspaceID,
+ Value: lockedBefore,
+ })
+ if err != nil {
+ slog.Warn("audit: failed to convert to v1.Setting", log.BBError(err))
+ }
+ p, err := anypb.New(v1pbSetting)
+ if err != nil {
+ slog.Warn("audit: failed to convert to anypb.Any", log.BBError(err))
+ }
+ setServiceData(p)
+ }
+
+ settingMessage, err := convertToSettingMessage(setting)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to convert setting message: %v", err))
+ }
+ return connect.NewResponse(settingMessage), nil
+}
+
+// preflightWorkspaceProfilePaths runs the update-mask validations that need
+// license, store, or profile lookups. It runs BEFORE the row-locking
+// transaction: apply must stay free of database reads, because it executes
+// while holding the transaction's pooled connection and the row lock, and a
+// nested read wanting a second connection is a bounded-pool starvation cycle.
+// None of these checks depend on the locked row's state, so hoisting them is
+// behavior-preserving. May normalize payload fields (e.g. external_url).
+func (s *SettingService) preflightWorkspaceProfilePaths(ctx context.Context, workspaceID string, request *connect.Request[v1pb.UpdateSettingRequest], payload *storepb.WorkspaceProfileSetting) error {
+ for _, path := range request.Msg.UpdateMask.Paths {
+ switch path {
+ case "value.workspace_profile.enable_debug":
+ // Debug flips the process-global log level and pprof exposure; on
+ // a shared SaaS replica that would affect other workspaces.
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the enable_debug cannot be changed in SaaS mode"))
+ }
+ case "value.workspace_profile.disallow_signup":
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_signup cannot be changed in SaaS mode"))
+ }
+ if payload.DisallowSignup {
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_SELF_SERVICE_SIGNUP); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ }
+ case "value.workspace_profile.external_url":
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the external_url cannot be changed in SaaS mode"))
+ }
+ // Prevent changing external URL via UI when it's set via command-line flag
+ if s.profile.ExternalURL != "" {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))
+ }
+ if payload.ExternalUrl != "" {
+ externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)
+ if err != nil {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))
+ }
+ payload.ExternalUrl = externalURL
+ }
+ case "value.workspace_profile.require_mfa":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TWO_FA); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ case "value.workspace_profile.access_token_duration":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ if payload.AccessTokenDuration != nil && payload.AccessTokenDuration.Seconds > 0 && payload.AccessTokenDuration.AsDuration() < time.Minute {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("access token duration should be at least one minute"))
+ }
+ case "value.workspace_profile.refresh_token_duration":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ if payload.RefreshTokenDuration != nil && payload.RefreshTokenDuration.Seconds > 0 && payload.RefreshTokenDuration.AsDuration() < time.Hour {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("refresh token duration should be at least one hour"))
+ }
+ case "value.workspace_profile.inactive_session_timeout":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_TOKEN_DURATION_CONTROL); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ case "value.workspace_profile.announcement":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DASHBOARD_ANNOUNCEMENT); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ if payload.Announcement != nil {
+ if err := validateAnnouncementTheme(payload.Announcement.Theme); err != nil {
+ return err
+ }
+ }
+ case "value.workspace_profile.enforce_identity_domain":
+ if payload.EnforceIdentityDomain {
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_USER_EMAIL_DOMAIN_RESTRICTION); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ }
+ case "value.workspace_profile.allow_email_code_signin":
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("allow_email_code_signin cannot be changed in SaaS mode"))
+ }
+ if payload.AllowEmailCodeSignin {
+ emailSetting, err := s.store.GetSetting(ctx, workspaceID, storepb.SettingName_EMAIL)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to load email setting"))
+ }
+ if emailSetting == nil {
+ return connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("cannot enable email code signin without an EMAIL setting"))
+ }
+ }
+ case "value.workspace_profile.disallow_password_signin":
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the disallow_password_signin cannot be changed in SaaS mode"))
+ }
+ if payload.DisallowPasswordSignin {
+ // We should still allow users to turn it off.
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DISALLOW_PASSWORD_SIGNIN); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+
+ settingWsID := common.GetWorkspaceIDFromContext(ctx)
+ identityProviders, err := s.store.ListIdentityProviders(ctx, &store.FindIdentityProviderMessage{Workspace: &settingWsID})
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, errors.Errorf("failed to list identity providers: %v", err))
+ }
+ if len(identityProviders) == 0 {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot disallow password signin when no identity provider is set"))
+ }
+ }
+ case "value.workspace_profile.enable_audit_log_stdout":
+ // Audit stdout output is process-global; on a shared SaaS replica
+ // it would affect other workspaces.
+ if s.profile.SaaS {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("the enable_audit_log_stdout cannot be changed in SaaS mode"))
+ }
+ if payload.EnableAuditLogStdout {
+ // Require TEAM or ENTERPRISE license
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_AUDIT_LOG); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ }
+ case "value.workspace_profile.watermark":
+ if payload.Watermark {
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_WATERMARK); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ }
+ case "value.workspace_profile.directory_sync_token":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_DIRECTORY_SYNC); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ case "value.workspace_profile.password_restriction":
+ if err := s.licenseService.IsFeatureEnabled(ctx, workspaceID, v1pb.PlanFeature_FEATURE_PASSWORD_RESTRICTIONS); err != nil {
+ return connect.NewError(connect.CodePermissionDenied, err)
+ }
+ if payload.PasswordRestriction != nil && payload.PasswordRestriction.MinLength < 8 {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid password minimum length, should no less than 8"))
+ }
+ default:
+ // Unknown paths are rejected by the merge pass.
+ }
+ }
+ return nil
+}
+
+// mergeWorkspaceProfilePaths applies the request's update-mask paths onto
+// oldSetting. It is pure apart from directory-sync token generation: it
+// performs no database or license reads — those ran in
+// preflightWorkspaceProfilePaths — because it executes inside the row-locking
+// transaction. Only validations that are payload-local or need the merged
+// state live here.
+func mergeWorkspaceProfilePaths(request *connect.Request[v1pb.UpdateSettingRequest], payload *storepb.WorkspaceProfileSetting, oldSetting *storepb.WorkspaceProfileSetting) error {
+ for _, path := range request.Msg.UpdateMask.Paths {
+ switch path {
+ case "value.workspace_profile.enable_debug":
+ oldSetting.EnableDebug = payload.EnableDebug
+ case "value.workspace_profile.disallow_signup":
+ oldSetting.DisallowSignup = payload.DisallowSignup
+ case "value.workspace_profile.external_url":
+ oldSetting.ExternalUrl = payload.ExternalUrl
+ case "value.workspace_profile.require_mfa":
+ oldSetting.Require_2Fa = payload.Require_2Fa
+ case "value.workspace_profile.access_token_duration":
+ oldSetting.AccessTokenDuration = payload.AccessTokenDuration
+ case "value.workspace_profile.refresh_token_duration":
+ oldSetting.RefreshTokenDuration = payload.RefreshTokenDuration
+ case "value.workspace_profile.inactive_session_timeout":
+ oldSetting.InactiveSessionTimeout = payload.InactiveSessionTimeout
+ case "value.workspace_profile.announcement":
+ oldSetting.Announcement = payload.Announcement
+ case "value.workspace_profile.maximum_request_expiration":
+ if payload.MaximumRequestExpiration != nil {
+ // If the value is less than or equal to 0, we will remove the setting. AKA no limit.
+ if payload.MaximumRequestExpiration.Seconds <= 0 {
+ payload.MaximumRequestExpiration = nil
+ }
+ }
+ oldSetting.MaximumRequestExpiration = payload.MaximumRequestExpiration
+ case "value.workspace_profile.maximum_role_expiration":
+ if payload.MaximumRoleExpiration != nil {
+ // If the value is less than or equal to 0, we will remove the setting. AKA no limit.
+ if payload.MaximumRoleExpiration.Seconds <= 0 {
+ payload.MaximumRoleExpiration = nil
+ }
+ }
+ oldSetting.MaximumRoleExpiration = payload.MaximumRoleExpiration
+ case "value.workspace_profile.domains":
+ if err := validateDomains(payload.Domains); err != nil {
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid domains, error %v", err))
+ }
+ oldSetting.Domains = payload.Domains
+ case "value.workspace_profile.enforce_identity_domain":
+ oldSetting.EnforceIdentityDomain = payload.EnforceIdentityDomain
+ case "value.workspace_profile.database_change_mode":
+ oldSetting.DatabaseChangeMode = payload.DatabaseChangeMode
+ case "value.workspace_profile.mcp_capability":
+ if err := validateMCPCapability(payload.McpCapability); err != nil {
+ return err
+ }
+ oldSetting.McpCapability = payload.McpCapability
+ case "value.workspace_profile.allow_email_code_signin":
+ oldSetting.AllowEmailCodeSignin = payload.AllowEmailCodeSignin
+ case "value.workspace_profile.disallow_password_signin":
+ oldSetting.DisallowPasswordSignin = payload.DisallowPasswordSignin
+ case "value.workspace_profile.enable_metric_collection":
+ oldSetting.EnableMetricCollection = payload.EnableMetricCollection
+ case "value.workspace_profile.enable_audit_log_stdout":
+ oldSetting.EnableAuditLogStdout = payload.EnableAuditLogStdout
+ case "value.workspace_profile.watermark":
+ oldSetting.Watermark = payload.Watermark
+ case "value.workspace_profile.directory_sync_token":
+ // Generate a new token if the payload is empty.
+ // This handles both initial setup and token reset (when user explicitly sends empty string).
+ if payload.DirectorySyncToken == "" {
+ payload.DirectorySyncToken = uuid.New().String()
+ }
+ oldSetting.DirectorySyncToken = payload.DirectorySyncToken
+ case "value.workspace_profile.password_restriction":
+ oldSetting.PasswordRestriction = payload.PasswordRestriction
+ case "value.workspace_profile.sql_result_size":
+ oldSetting.SqlResultSize = payload.SqlResultSize
+ case "value.workspace_profile.query_timeout":
+ oldSetting.QueryTimeout = payload.QueryTimeout
+ case "value.workspace_profile.sql_editor_theme_id":
+ oldSetting.SqlEditorThemeId = payload.SqlEditorThemeId
+ case "value.workspace_profile.sql_editor_custom_theme":
+ if err := validateSQLEditorCustomTheme(payload.SqlEditorCustomTheme); err != nil {
+ return err
+ }
+ oldSetting.SqlEditorCustomTheme = payload.SqlEditorCustomTheme
+ default:
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid update mask path %v", path))
+ }
+ }
+ return nil
+}
+
// TestEmailSetting sends a test email using the provided config.
func (s *SettingService) TestEmailSetting(ctx context.Context, req *connect.Request[v1pb.TestEmailSettingRequest]) (*connect.Response[v1pb.TestEmailSettingResponse], error) {
emailSetting := convertEmailSetting(req.Msg.EmailSetting)
diff --git a/backend/api/v1/setting_service_test.go b/backend/api/v1/setting_service_test.go
index 4269d1e06f..872a6c4489 100644
--- a/backend/api/v1/setting_service_test.go
+++ b/backend/api/v1/setting_service_test.go
@@ -4,10 +4,13 @@ import (
"context"
"testing"
+ "connectrpc.com/connect"
"github.com/stretchr/testify/require"
colorpb "google.golang.org/genproto/googleapis/type/color"
+ "google.golang.org/protobuf/types/known/fieldmaskpb"
"google.golang.org/protobuf/types/known/wrapperspb"
+ "github.com/bytebase/bytebase/backend/component/config"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
@@ -252,3 +255,30 @@ func TestValidateEnvironmentsColor(t *testing.T) {
})
}
}
+
+// TestPreflightWorkspaceProfileSaaSRestrictedPaths pins that the update-mask
+// paths controlling process-global runtime behavior (debug/pprof, audit-log
+// stdout) are rejected in SaaS mode: on a shared replica they would let one
+// workspace admin change process-wide behavior affecting other workspaces.
+func TestPreflightWorkspaceProfileSaaSRestrictedPaths(t *testing.T) {
+ newRequest := func(path string) *connect.Request[v1pb.UpdateSettingRequest] {
+ return connect.NewRequest(&v1pb.UpdateSettingRequest{
+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{path}},
+ })
+ }
+ saas := &SettingService{profile: &config.Profile{SaaS: true}}
+ selfHosted := &SettingService{profile: &config.Profile{}}
+
+ for _, path := range []string{
+ "value.workspace_profile.enable_debug",
+ "value.workspace_profile.enable_audit_log_stdout",
+ } {
+ err := saas.preflightWorkspaceProfilePaths(context.Background(), "ws", newRequest(path), &storepb.WorkspaceProfileSetting{})
+ require.Error(t, err, path)
+ require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), path)
+
+ // Self-hosted keeps accepting them (payload zero-values skip license checks).
+ err = selfHosted.preflightWorkspaceProfilePaths(context.Background(), "ws", newRequest(path), &storepb.WorkspaceProfileSetting{})
+ require.NoError(t, err, path)
+ }
+}
diff --git a/backend/store/export_test.go b/backend/store/export_test.go
new file mode 100644
index 0000000000..ccee7c0ea2
--- /dev/null
+++ b/backend/store/export_test.go
@@ -0,0 +1,8 @@
+package store
+
+// SetSettingPublishHookForTest installs a hook that runs between a setting
+// update's commit and its cache publish — test-only, to deterministically
+// expose the publish-ordering window.
+func (s *Store) SetSettingPublishHookForTest(hook func()) {
+ s.settingPublishHookForTest = hook
+}
diff --git a/backend/store/setting.go b/backend/store/setting.go
index 275bc58055..ae7fbe3cc1 100644
--- a/backend/store/setting.go
+++ b/backend/store/setting.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"math"
+ "time"
"github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson"
@@ -237,10 +238,27 @@ type FindSettingMessage struct {
// GetSetting returns the setting by name.
func (s *Store) GetSetting(ctx context.Context, workspace string, name storepb.SettingName) (*SettingMessage, error) {
- if v, ok := s.settingCache.Get(getSettingCacheKey(workspace, name)); ok && s.enableCache {
+ // With caching disabled (HA), every read goes straight to the database:
+ // there is no cache to fill, and taking the publish mutex here would
+ // serialize all setting reads in the process behind a single lock.
+ if !s.enableCache {
+ return s.GetSettingUncached(ctx, workspace, name)
+ }
+ if v, ok := s.settingCache.Get(getSettingCacheKey(workspace, name)); ok {
return v, nil
}
- return s.GetSettingUncached(ctx, workspace, name)
+ // Fill the cache under the publish mutex so the fill's content is current
+ // at publish time: an unordered fill could read a pre-commit snapshot and
+ // cache it after a concurrent writer already published a newer value. No
+ // row lock is held here, so the row-lock -> publish-lock order is kept.
+ s.settingPublishMu.Lock()
+ defer s.settingPublishMu.Unlock()
+ setting, err := s.GetSettingUncached(ctx, workspace, name)
+ if err != nil || setting == nil {
+ return setting, err
+ }
+ s.settingCache.Add(getSettingCacheKey(workspace, name), setting)
+ return setting, nil
}
// GetSettingUncached reads the setting directly from the database, bypassing
@@ -320,12 +338,129 @@ func (s *Store) ListSettings(ctx context.Context, find *FindSettingMessage) ([]*
return nil, err
}
- for _, setting := range settingMessages {
- s.settingCache.Add(getSettingCacheKey(setting.Workspace, setting.Name), setting)
- }
+ // Deliberately no cache publication: this path runs outside the publish
+ // mutex (GetSettingUncached callers hit it on every read), and an
+ // unordered fill could pin a pre-commit snapshot over a newer published
+ // value. Publication happens only in GetSetting's fill, UpsertSetting,
+ // and UpdateSettingAtomic — all ordered by settingPublishMu.
return settingMessages, nil
}
+// UpdateSettingAtomic performs a row-locking read-modify-write of a setting:
+// the current value is read under SELECT ... FOR UPDATE, apply transforms it,
+// and the result is written back in the same transaction — so a concurrent
+// write can never be silently reverted by a merge based on a stale read (the
+// lost-update hazard of read-then-UpsertSetting). apply receives the value
+// freshly unmarshaled from the locked row and may mutate it in place; an error
+// from apply aborts the transaction with no write and is returned unwrapped so
+// callers keep typed errors. After commit, the served state is refreshed via
+// publishSetting: the row is re-read under the publish mutex and the fresh
+// value goes to the cache and to postCommit (optional, for derived state) —
+// so publications always carry current truth regardless of the order in
+// which updaters reach the mutex. apply must stay free of database reads:
+// it runs while holding the transaction's pooled connection and the row
+// lock, and a nested read wanting a second connection is the bounded-pool
+// starvation cycle.
+func (s *Store) UpdateSettingAtomic(ctx context.Context, workspace string, name storepb.SettingName, apply func(current proto.Message) (proto.Message, error), postCommit func(current *SettingMessage)) (*SettingMessage, error) {
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to begin transaction")
+ }
+ defer tx.Rollback()
+
+ q := qb.Q().Space(`
+ SELECT value FROM setting WHERE workspace = ? AND name = ? FOR UPDATE
+ `, workspace, name.String())
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+ var valueString string
+ if err := tx.QueryRowContext(ctx, query, args...).Scan(&valueString); err != nil {
+ if err == sql.ErrNoRows {
+ return nil, errors.Errorf("setting %s not found", name)
+ }
+ return nil, errors.Wrapf(err, "failed to lock setting %s", name)
+ }
+ current, err := getSettingMessage(name)
+ if err != nil {
+ return nil, err
+ }
+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(valueString), current); err != nil {
+ return nil, errors.Wrapf(err, "failed to unmarshal setting %s", name)
+ }
+
+ next, err := apply(current)
+ if err != nil {
+ return nil, err
+ }
+
+ nextBytes, err := protojson.Marshal(next)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to marshal setting value")
+ }
+ q = qb.Q().Space(`
+ UPDATE setting SET value = ? WHERE workspace = ? AND name = ?
+ `, string(nextBytes), workspace, name.String())
+ query, args, err = q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {
+ return nil, errors.Wrapf(err, "failed to update setting %s", name)
+ }
+ // Commit BEFORE acquiring the publish mutex: commit returns the pooled
+ // connection and releases the row lock, so waiting for the mutex never
+ // holds pool resources (connection-holders never wait for the mutex, so
+ // the mutex/pool wait graph stays acyclic). Ordering is preserved because
+ // publishSetting re-reads the row under the mutex — whichever updater
+ // publishes last still publishes current truth.
+ if err := tx.Commit(); err != nil {
+ return nil, errors.Wrap(err, "failed to commit transaction")
+ }
+ s.publishSetting(ctx, workspace, name, postCommit)
+ return &SettingMessage{Name: name, Workspace: workspace, Value: next}, nil
+}
+
+// publishSetting refreshes the served state for a setting after a committed
+// write: under settingPublishMu it re-reads the row and publishes the fresh
+// value to the cache (when enabled) and to postCommit for derived state. It
+// must be called with no transaction, row lock, or pooled connection held —
+// the mutex-then-connection order here is what keeps the publish protocol
+// deadlock-free against the bounded metadata pool. If the re-read fails, the
+// cache entry is evicted rather than risk publishing stale state, and
+// postCommit is skipped (derived state converges on the next write).
+func (s *Store) publishSetting(ctx context.Context, workspace string, name storepb.SettingName, postCommit func(current *SettingMessage)) {
+ if !s.enableCache && postCommit == nil {
+ return
+ }
+ s.settingPublishMu.Lock()
+ defer s.settingPublishMu.Unlock()
+ if s.settingPublishHookForTest != nil {
+ s.settingPublishHookForTest()
+ }
+ // The write is already committed, so publication must not depend on the
+ // caller still listening: re-read on a bounded context detached from
+ // request cancellation. If the read still fails (database unreachable),
+ // evict and skip postCommit — the cache heals on the next fill, and
+ // derived runtime state heals on the next successful publication of this
+ // setting (postCommit callbacks must therefore reconcile from the fresh
+ // value unconditionally, not only for fields their request touched).
+ publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer cancel()
+ fresh, err := s.GetSettingUncached(publishCtx, workspace, name)
+ if err != nil || fresh == nil {
+ s.settingCache.Remove(getSettingCacheKey(workspace, name))
+ return
+ }
+ if s.enableCache {
+ s.settingCache.Add(getSettingCacheKey(workspace, name), fresh)
+ }
+ if postCommit != nil {
+ postCommit(fresh)
+ }
+}
+
// UpsertSetting upserts the setting by name.
func (s *Store) UpsertSetting(ctx context.Context, update *SettingMessage) (*SettingMessage, error) {
valueBytes, err := protojson.Marshal(update.Value)
@@ -373,7 +508,7 @@ func (s *Store) UpsertSetting(ctx context.Context, update *SettingMessage) (*Set
}
setting.Value = msg
- s.settingCache.Add(getSettingCacheKey(setting.Workspace, setting.Name), &setting)
+ s.publishSetting(ctx, setting.Workspace, setting.Name, nil)
return &setting, nil
}
@@ -389,6 +524,11 @@ func (s *Store) DeleteSetting(ctx context.Context, workspace string, name storep
return err
}
+ // Invalidate under the ordering mutex: a cache-miss fill reads and
+ // publishes atomically under the same mutex, so once serialized against
+ // it, a fill can no longer resurrect the deleted row.
+ s.settingPublishMu.Lock()
+ defer s.settingPublishMu.Unlock()
s.settingCache.Remove(getSettingCacheKey(workspace, name))
return nil
}
diff --git a/backend/store/setting_atomic_test.go b/backend/store/setting_atomic_test.go
new file mode 100644
index 0000000000..a3f27d6801
--- /dev/null
+++ b/backend/store/setting_atomic_test.go
@@ -0,0 +1,460 @@
+package store_test
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/migrator"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// newSettingAtomicFixture boots a real PostgreSQL, migrates the schema, seeds
+// the workspace and an empty WORKSPACE_PROFILE row, and returns a store with
+// the setting cache ENABLED — the production non-HA shape, so the tests also
+// pin that the cache reflects committed state.
+func newSettingAtomicFixture(t *testing.T) (context.Context, *sql.DB, *store.Store) {
+ t.Helper()
+ return newSettingAtomicFixtureWithCache(t, true)
+}
+
+func newSettingAtomicFixtureWithCache(t *testing.T, enableCache bool) (context.Context, *sql.DB, *store.Store) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ t.Cleanup(cancel)
+ container := testcontainer.GetTestPgContainer(ctx, t)
+ t.Cleanup(func() { container.Close(context.Background()) })
+ db := container.GetDB()
+ require.NoError(t, migrator.MigrateSchema(ctx, db))
+
+ _, err := db.ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('default');`)
+ require.NoError(t, err)
+
+ pgURL := fmt.Sprintf(
+ "host=%s port=%s user=postgres password=root-password database=postgres",
+ container.GetHost(), container.GetPort(),
+ )
+ s, err := store.New(ctx, pgURL, enableCache)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+ _, err = s.UpsertSetting(ctx, &store.SettingMessage{
+ Name: storepb.SettingName_WORKSPACE_PROFILE,
+ Workspace: "default",
+ Value: &storepb.WorkspaceProfileSetting{},
+ })
+ require.NoError(t, err)
+ return ctx, db, s
+}
+
+func mustProfile(t *testing.T, setting *store.SettingMessage) *storepb.WorkspaceProfileSetting {
+ t.Helper()
+ profile, ok := setting.Value.(*storepb.WorkspaceProfileSetting)
+ require.True(t, ok, "unexpected setting value type %T", setting.Value)
+ return profile
+}
+
+// TestUpdateSettingAtomicInterleaving pins the lost-update protection: while
+// one update holds the row lock inside its apply callback, a concurrent update
+// to a DIFFERENT field must wait behind it and then merge onto the committed
+// result — both mutations survive in the final row and in the setting cache.
+// Against a read-then-UpsertSetting implementation this fails: the second
+// update merges onto a stale read and silently reverts the first's field.
+func TestUpdateSettingAtomicInterleaving(t *testing.T) {
+ ctx, db, s := newSettingAtomicFixture(t)
+
+ entered := make(chan struct{})
+ release := make(chan struct{})
+ firstResult := make(chan error, 1)
+ go func() {
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.EnableMetricCollection = true
+ close(entered)
+ <-release
+ return profile, nil
+ }, nil)
+ firstResult <- err
+ }()
+ // The first update is now inside apply and holds the row lock.
+ select {
+ case <-entered:
+ case <-time.After(15 * time.Second):
+ t.Fatal("first update never reached its apply callback")
+ }
+
+ secondResult := make(chan error, 1)
+ go func() {
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.McpCapability = storepb.WorkspaceProfileSetting_DISABLED
+ return profile, nil
+ }, nil)
+ secondResult <- err
+ }()
+
+ // Deterministic barrier: the second update must be observed waiting behind
+ // the first's row lock before the first is released.
+ require.Eventually(t, func() bool {
+ var waiting bool
+ err := db.QueryRowContext(ctx, `
+ SELECT EXISTS (
+ SELECT 1
+ FROM pg_stat_activity
+ WHERE cardinality(pg_blocking_pids(pid)) > 0
+ AND query LIKE '%setting%FOR UPDATE%'
+ )
+ `).Scan(&waiting)
+ return err == nil && waiting
+ }, 10*time.Second, 10*time.Millisecond,
+ "the second update should wait behind the first's row lock")
+ select {
+ case err := <-secondResult:
+ t.Fatalf("second update completed while the first held the row lock: %v", err)
+ default:
+ }
+
+ close(release)
+ for _, result := range []chan error{firstResult, secondResult} {
+ select {
+ case err := <-result:
+ require.NoError(t, err)
+ case <-time.After(15 * time.Second):
+ t.Fatal("update did not complete; possible deadlock")
+ }
+ }
+
+ // Both mutations survive in the database ...
+ fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ freshProfile := mustProfile(t, fresh)
+ require.True(t, freshProfile.EnableMetricCollection, "first update's field must survive")
+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, freshProfile.McpCapability,
+ "second update's field must survive")
+ // ... and in the setting cache.
+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ cachedProfile := mustProfile(t, cached)
+ require.True(t, cachedProfile.EnableMetricCollection)
+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, cachedProfile.McpCapability)
+}
+
+// TestUpdateSettingAtomicApplyAbort pins that an error from apply aborts the
+// transaction with no write, leaves the cache untouched, and surfaces
+// unwrapped so callers keep typed errors.
+func TestUpdateSettingAtomicApplyAbort(t *testing.T) {
+ ctx, _, s := newSettingAtomicFixture(t)
+
+ sentinel := errors.New("validation failed")
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ // Mutate before failing: the mutation must not leak into the row
+ // or the served cache.
+ profile.EnableMetricCollection = true
+ return nil, sentinel
+ }, nil)
+ require.ErrorIs(t, err, sentinel)
+
+ fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ require.False(t, mustProfile(t, fresh).EnableMetricCollection)
+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ require.False(t, mustProfile(t, cached).EnableMetricCollection)
+}
+
+// TestUpdateSettingAtomicMissingRow pins that a missing setting row is an
+// error, not a silent create — the primitive updates existing state only.
+func TestUpdateSettingAtomicMissingRow(t *testing.T) {
+ ctx, _, s := newSettingAtomicFixture(t)
+
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_AI,
+ func(current proto.Message) (proto.Message, error) {
+ return current, nil
+ }, nil)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "not found")
+}
+
+// TestUpdateSettingAtomicPublishOrder pins that the setting cache is published
+// in commit order: while one update is paused between its commit and its cache
+// publish (test hook), a later update must not be able to commit and publish —
+// otherwise the paused update's older value would overwrite the newer one in
+// the cache, serving reverted state until the next refresh. Red against an
+// implementation whose commit releases the row lock before an unordered cache
+// write.
+func TestUpdateSettingAtomicPublishOrder(t *testing.T) {
+ ctx, _, s := newSettingAtomicFixture(t)
+
+ pauseFirst := make(chan struct{})
+ resumeFirst := make(chan struct{})
+ // Only the first update pauses; later publishes pass through WITHOUT
+ // blocking (sync.Once.Do would serialize concurrent callers and mask the
+ // very ordering bug this test pins).
+ var first atomic.Bool
+ s.SetSettingPublishHookForTest(func() {
+ if first.CompareAndSwap(false, true) {
+ close(pauseFirst)
+ <-resumeFirst
+ }
+ })
+
+ // postCommit callbacks run inside the ordered publish window and receive
+ // the freshly re-read value, so derived state converges on committed
+ // truth regardless of which updater reaches the mutex first.
+ var postCommitMu sync.Mutex
+ var postCommitSeen []*storepb.WorkspaceProfileSetting
+ recordPostCommit := func() func(current *store.SettingMessage) {
+ return func(current *store.SettingMessage) {
+ profile, ok := current.Value.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return
+ }
+ postCommitMu.Lock()
+ defer postCommitMu.Unlock()
+ postCommitSeen = append(postCommitSeen, profile)
+ }
+ }
+
+ firstResult := make(chan error, 1)
+ go func() {
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.EnableMetricCollection = true
+ return profile, nil
+ }, recordPostCommit())
+ firstResult <- err
+ }()
+ // The first update has committed and is paused before publishing.
+ select {
+ case <-pauseFirst:
+ case <-time.After(15 * time.Second):
+ t.Fatal("first update never reached the publish window")
+ }
+
+ secondResult := make(chan error, 1)
+ go func() {
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.McpCapability = storepb.WorkspaceProfileSetting_DISABLED
+ return profile, nil
+ }, recordPostCommit())
+ secondResult <- err
+ }()
+
+ // Deterministic barrier: wait until the second update's write is visible
+ // in the database. A writer must never retain its transaction (a
+ // bounded-pool connection and the row lock) while waiting for the publish
+ // mutex — that is the connection-pool deadlock cycle — so the commit must
+ // land while the first still holds the publish window.
+ require.Eventually(t, func() bool {
+ committed, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ if err != nil {
+ return false
+ }
+ profile, ok := committed.Value.(*storepb.WorkspaceProfileSetting)
+ return ok && profile.McpCapability == storepb.WorkspaceProfileSetting_DISABLED
+ }, 10*time.Second, 10*time.Millisecond,
+ "second update must commit while its publish waits on the mutex")
+
+ // Having committed, it must still not have completed: its publication is
+ // blocked behind the first's paused window — completing here would be
+ // exactly the out-of-order publish.
+ select {
+ case err := <-secondResult:
+ t.Fatalf("second update published while the first held the publish window: %v", err)
+ default:
+ }
+
+ close(resumeFirst)
+ for _, result := range []chan error{firstResult, secondResult} {
+ select {
+ case err := <-result:
+ require.NoError(t, err)
+ case <-time.After(15 * time.Second):
+ t.Fatal("update did not complete; possible deadlock")
+ }
+ }
+
+ // The served cache must hold the final committed state: the second update
+ // ran after the first's commit, so its merge carries BOTH fields.
+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ cachedProfile := mustProfile(t, cached)
+ require.True(t, cachedProfile.EnableMetricCollection, "cache must not serve pre-first-update state")
+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, cachedProfile.McpCapability,
+ "cache must not be overwritten by the earlier committer's stale publish")
+ postCommitMu.Lock()
+ defer postCommitMu.Unlock()
+ require.Len(t, postCommitSeen, 2, "both postCommit callbacks must run")
+ for _, seen := range postCommitSeen {
+ require.True(t, seen.EnableMetricCollection,
+ "postCommit must observe committed truth, not the caller's own write")
+ require.Equal(t, storepb.WorkspaceProfileSetting_DISABLED, seen.McpCapability,
+ "postCommit must observe committed truth, not the caller's own write")
+ }
+}
+
+// TestSettingCacheNoFillFromUncachedReads pins that uncached reads do not
+// populate the setting cache: GetSettingUncached (and ListSettings under it)
+// runs outside the publish-ordering mutex, so a fill from that path could
+// cache a pre-commit snapshot after a newer value was already published.
+// Uncached readers must leave publication to the ordered paths.
+func TestSettingCacheNoFillFromUncachedReads(t *testing.T) {
+ ctx, db, s := newSettingAtomicFixture(t)
+ // Drop the seed's cache entry so the next cached read must fill.
+ s.DeleteCache()
+
+ // An uncached read must not leave a cache entry behind ...
+ _, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+
+ // ... so after an out-of-band change, a cached read serves the new value.
+ _, err = db.ExecContext(ctx, `
+ UPDATE setting SET value = jsonb_set(value, '{enableMetricCollection}', 'true')
+ WHERE workspace = 'default' AND name = 'WORKSPACE_PROFILE';
+ `)
+ require.NoError(t, err)
+
+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ require.True(t, mustProfile(t, cached).EnableMetricCollection,
+ "GetSetting must not serve a snapshot cached by an uncached read")
+}
+
+// TestGetSettingCacheDisabledNotSerialized pins the HA read path: with the
+// cache disabled (store.New(..., !profile.HA)), GetSetting must go straight to
+// the database and never touch the publish-ordering mutex — otherwise every
+// setting read in an HA process serializes behind a single lock, including
+// while a writer sits in its commit-to-publish window.
+func TestGetSettingCacheDisabledNotSerialized(t *testing.T) {
+ ctx, _, s := newSettingAtomicFixtureWithCache(t, false)
+
+ pause := make(chan struct{})
+ resume := make(chan struct{})
+ var first atomic.Bool
+ s.SetSettingPublishHookForTest(func() {
+ if first.CompareAndSwap(false, true) {
+ close(pause)
+ <-resume
+ }
+ })
+
+ writerResult := make(chan error, 1)
+ go func() {
+ _, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.EnableMetricCollection = true
+ return profile, nil
+ }, func(*store.SettingMessage) {})
+ writerResult <- err
+ }()
+ select {
+ case <-pause:
+ case <-time.After(15 * time.Second):
+ t.Fatal("writer never reached the publish window")
+ }
+
+ // The writer holds the publish mutex; a cache-disabled read must still
+ // complete (it reads committed state and consults no cache).
+ readResult := make(chan error, 1)
+ go func() {
+ setting, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ if err == nil && !mustProfile(t, setting).EnableMetricCollection {
+ err = errors.New("read did not observe the committed value")
+ }
+ readResult <- err
+ }()
+ select {
+ case err := <-readResult:
+ require.NoError(t, err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("cache-disabled GetSetting blocked behind the publish mutex")
+ }
+
+ close(resume)
+ select {
+ case err := <-writerResult:
+ require.NoError(t, err)
+ case <-time.After(15 * time.Second):
+ t.Fatal("writer did not complete")
+ }
+}
+
+// TestUpdateSettingAtomicPublishSurvivesCancellation pins that publication
+// completes even when the request context dies right after commit: the write
+// is already durable, so the cache refresh and postCommit must not depend on
+// the caller still listening. Red against a publish path that re-reads on the
+// request context — the canceled read was swallowed and postCommit silently
+// skipped, leaving derived runtime state diverged from the database forever.
+func TestUpdateSettingAtomicPublishSurvivesCancellation(t *testing.T) {
+ ctx, _, s := newSettingAtomicFixture(t)
+
+ updateCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var once atomic.Bool
+ s.SetSettingPublishHookForTest(func() {
+ // Fires after commit, before the publish re-read.
+ if once.CompareAndSwap(false, true) {
+ cancel()
+ }
+ })
+
+ var observed *storepb.WorkspaceProfileSetting
+ _, err := s.UpdateSettingAtomic(updateCtx, "default", storepb.SettingName_WORKSPACE_PROFILE,
+ func(current proto.Message) (proto.Message, error) {
+ profile, ok := current.(*storepb.WorkspaceProfileSetting)
+ if !ok {
+ return nil, errors.Errorf("unexpected type %T", current)
+ }
+ profile.EnableMetricCollection = true
+ return profile, nil
+ }, func(current *store.SettingMessage) {
+ if profile, ok := current.Value.(*storepb.WorkspaceProfileSetting); ok {
+ observed = profile
+ }
+ })
+ require.NoError(t, err)
+
+ require.NotNil(t, observed, "postCommit must run despite request cancellation after commit")
+ require.True(t, observed.EnableMetricCollection, "postCommit must observe the committed state")
+
+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
+ require.NoError(t, err)
+ require.True(t, mustProfile(t, cached).EnableMetricCollection,
+ "the served cache must reflect the committed write despite request cancellation")
+}
diff --git a/backend/store/store.go b/backend/store/store.go
index d02eb6b14b..b6bc1e0267 100644
--- a/backend/store/store.go
+++ b/backend/store/store.go
@@ -2,6 +2,8 @@
package store
import (
+ "sync"
+
"context"
"database/sql"
"fmt"
@@ -20,19 +22,30 @@ type Store struct {
enableCache bool
// Cache.
- Secret string
- userEmailCache *lru.Cache[string, *UserMessage]
- instanceCache *lru.Cache[string, *InstanceMessage]
- databaseCache *lru.Cache[string, *DatabaseMessage]
- projectCache *lru.Cache[string, *ProjectMessage]
- policyCache *lru.Cache[string, *PolicyMessage]
- settingCache *lru.Cache[string, *SettingMessage]
- rolesCache *expirable.LRU[string, *RoleMessage]
- groupCache *expirable.LRU[string, *GroupMessage]
- groupMembersCache *expirable.LRU[string, map[string]bool]
- memberGroupsCache *expirable.LRU[string, []string]
- dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]
- iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]
+ Secret string
+ userEmailCache *lru.Cache[string, *UserMessage]
+ instanceCache *lru.Cache[string, *InstanceMessage]
+ databaseCache *lru.Cache[string, *DatabaseMessage]
+ projectCache *lru.Cache[string, *ProjectMessage]
+ policyCache *lru.Cache[string, *PolicyMessage]
+ settingCache *lru.Cache[string, *SettingMessage]
+
+ // settingPublishMu serializes all setting cache publications (writer
+ // republishes, cache-miss fills, invalidations). Publishers re-read the
+ // row under the mutex, so whichever publishes last publishes current
+ // truth. It must be acquired with no transaction, row lock, or pooled
+ // connection held — writers commit first, then publish — so the
+ // mutex/pool wait graph stays acyclic against the bounded metadata pool.
+ settingPublishMu sync.Mutex
+ // settingPublishHookForTest, when set, runs between commit and cache
+ // publish so tests can deterministically expose the ordering window.
+ settingPublishHookForTest func()
+ rolesCache *expirable.LRU[string, *RoleMessage]
+ groupCache *expirable.LRU[string, *GroupMessage]
+ groupMembersCache *expirable.LRU[string, map[string]bool]
+ memberGroupsCache *expirable.LRU[string, []string]
+ dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]
+ iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]
// Large objects.
sheetFullCache *lru.Cache[string, *SheetMessage]
@@ -116,7 +129,11 @@ func (s *Store) GetDB() *sql.DB {
// DeleteCache deletes the cache.
func (s *Store) DeleteCache() {
+ // The setting cache purge participates in the publish-ordering invariant
+ // so an in-flight fill cannot republish a just-purged snapshot.
+ s.settingPublishMu.Lock()
s.settingCache.Purge()
+ s.settingPublishMu.Unlock()
s.policyCache.Purge()
s.userEmailCache.Purge()
}