Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(recovery): non-interactive recovery beside a running Bytebase (#21602)
zchpeter6342304b464c
diff --git a/backend/api/v1/account_email_validation.go b/backend/api/v1/account_email_validation.goindex 8dae7d95b7..71ce99e56c 100644--- a/backend/api/v1/account_email_validation.go+++ b/backend/api/v1/account_email_validation.go@@ -1,19 +1,11 @@package v1import (- "strings"-"github.com/pkg/errors""github.com/bytebase/bytebase/backend/common")-// normalizeEmail canonicalizes a submitted email for lookups, lockout-// identities, and audit resources — these must all key the same string.-func normalizeEmail(email string) string {- return strings.ToLower(strings.TrimSpace(email))-}-func validateServiceAccountEmail(email string) error {if err := common.ValidateEmail(email); err != nil {return errdiff --git a/backend/api/v1/audit.go b/backend/api/v1/audit.goindex 35dbf0cbe9..612192617e 100644--- a/backend/api/v1/audit.go+++ b/backend/api/v1/audit.go@@ -460,17 +460,17 @@ func getRequestResource(request any, method string) string {case *v1pb.CreateUserRequest:return r.GetUser().GetName()case *v1pb.LoginRequest:- return normalizeEmail(r.GetEmail())+ return common.NormalizeEmail(r.GetEmail())case *v1pb.SignupRequest:return r.GetEmail()case *v1pb.ExchangeTokenRequest:return r.GetEmail()case *v1pb.RequestPasswordResetRequest:- return normalizeEmail(r.GetEmail())+ return common.NormalizeEmail(r.GetEmail())case *v1pb.ResetPasswordRequest:- return normalizeEmail(r.GetEmail())+ return common.NormalizeEmail(r.GetEmail())case *v1pb.SendEmailLoginCodeRequest:- return normalizeEmail(r.GetEmail())+ return common.NormalizeEmail(r.GetEmail())case *v1pb.CreateInstanceRequest:if r.GetParent() == "" {return common.FormatInstance(r.GetInstanceId())diff --git a/backend/api/v1/auth_service.go b/backend/api/v1/auth_service.goindex 69ef6f2416..d10d16b2f2 100644--- a/backend/api/v1/auth_service.go+++ b/backend/api/v1/auth_service.go@@ -359,7 +359,7 @@ func (s *AuthService) needResetPassword(ctx context.Context, user *store.UserMes// - Otherwise, creates a new workspace with the user as admin.func (s *AuthService) Signup(ctx context.Context, req *connect.Request[v1pb.SignupRequest]) (*connect.Response[v1pb.LoginResponse], error) {request := req.Msg- email := normalizeEmail(request.Email)+ email := common.NormalizeEmail(request.Email)if email == "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email must be set"))}@@ -641,7 +641,7 @@ func (s *AuthService) Refresh(ctx context.Context, req *connect.Request[v1pb.Ref}func (s *AuthService) getAndVerifyUser(ctx context.Context, request *v1pb.LoginRequest) (*store.UserMessage, error) {- email := normalizeEmail(request.Email)+ email := common.NormalizeEmail(request.Email)// An invalid-syntax email can never be an account; reject it before the// claim with the same error a wrong password gets.if !common.IsValidEmail(email) {diff --git a/backend/api/v1/auth_service_email_code.go b/backend/api/v1/auth_service_email_code.goindex 780d99db9f..3a4fff9636 100644--- a/backend/api/v1/auth_service_email_code.go+++ b/backend/api/v1/auth_service_email_code.go@@ -51,7 +51,7 @@ var errSendBudgetExhausted = errors.New("too many sign-in codes requested, pleas// RequestPasswordReset sends a password reset email. Always returns success to avoid leaking email existence.func (s *AuthService) RequestPasswordReset(ctx context.Context, req *connect.Request[v1pb.RequestPasswordResetRequest]) (*connect.Response[emptypb.Empty], error) {- email := normalizeEmail(req.Msg.Email)+ email := common.NormalizeEmail(req.Msg.Email)if email == "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email is required"))}@@ -82,7 +82,7 @@ func (s *AuthService) RequestPasswordReset(ctx context.Context, req *connect.Req// ResetPassword verifies the 6-digit code and updates the user's password.// Also revokes all refresh tokens to force re-login.func (s *AuthService) ResetPassword(ctx context.Context, req *connect.Request[v1pb.ResetPasswordRequest]) (*connect.Response[emptypb.Empty], error) {- email := normalizeEmail(req.Msg.Email)+ email := common.NormalizeEmail(req.Msg.Email)if email == "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email is required"))}@@ -148,7 +148,7 @@ func (s *AuthService) ResetPassword(ctx context.Context, req *connect.Request[v1// (no email enumeration). Rate limit: 60-sec resend cooldown enforced atomically// via the store — effective cap ≈ 60 sends/hour/email.func (s *AuthService) SendEmailLoginCode(ctx context.Context, req *connect.Request[v1pb.SendEmailLoginCodeRequest]) (*connect.Response[emptypb.Empty], error) {- email := normalizeEmail(req.Msg.Email)+ email := common.NormalizeEmail(req.Msg.Email)if email == "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email is required"))}@@ -215,7 +215,7 @@ func (s *AuthService) authenticateEmailCodeLogin(ctx context.Context, request *vif request.Password != "" || request.GetIdpName() != "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email_code is mutually exclusive with password and idp_name"))}- email := normalizeEmail(request.Email)+ email := common.NormalizeEmail(request.Email)if email == "" {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("email is required"))}diff --git a/backend/bin/server/cmd/recovery.go b/backend/bin/server/cmd/recovery.goindex 5af5ebcbbf..482e16d639 100644--- a/backend/bin/server/cmd/recovery.go+++ b/backend/bin/server/cmd/recovery.go@@ -9,9 +9,12 @@ import ("os""os/signal""path/filepath"+ "regexp""strconv""strings""syscall"+ "unicode"+ "unicode/utf8""github.com/pkg/errors""github.com/spf13/cobra"@@ -24,6 +27,7 @@ import ("github.com/bytebase/bytebase/backend/store")+// recoveryRunner is the recovery service as the command uses it.type recoveryRunner interface {GetWorkspaceID(context.Context) (string, error)EnablePasswordSignin(context.Context, string) (*recovery.EnablePasswordSigninResult, error)@@ -53,46 +57,188 @@ func init() {}))}+// oneActionCommands are the two forms that run one action without a terminal.+const oneActionCommands = `"bytebase recovery enable-password-signin" or "bytebase recovery reset-password --email <email> --password <password>"`+func newRecoveryCommand(dependencies recoveryCommandDependencies) *cobra.Command {command := &cobra.Command{- Use: "recovery",- Short: "Recover password sign-in for a self-hosted deployment",- Args: cobra.NoArgs,- SilenceUsage: true,- RunE: func(command *cobra.Command, _ []string) (retErr error) {+ Use: "recovery",+ Short: "Recover password sign-in for a self-hosted deployment",+ Args: noArgs,+ RunE: func(command *cobra.Command, _ []string) error {+ command.SilenceUsage = trueinput := command.InOrStdin()output := command.OutOrStdout()if !dependencies.isTerminal(input, output) {- return errors.New("recovery requires terminal input and output")- }- fmt.Fprintln(output, "Stop all Bytebase servers that use this metadata database before continuing.")- fmt.Fprintln(output, "Restart all Bytebase servers after recovery so they reload updated settings and IAM policies.")-- ctx, stop := signal.NotifyContext(command.Context(), os.Interrupt, syscall.SIGTERM)- defer stop()- backend, err := dependencies.open(ctx)- if err != nil {- return errors.Wrap(err, "failed to open recovery metadata store")+ return errors.New("recovery requires terminal input and output; without a terminal, run " + oneActionCommands)}- defer func() {- if err := backend.close(); err != nil && retErr == nil {- retErr = errors.Wrap(err, "failed to close recovery metadata store")+ return runRecoveryAction(command, dependencies, func(ctx context.Context, workspaceID string, runner recoveryRunner) error {+ return runRecoverySession(ctx, workspaceID, input, output, runner, dependencies.readPassword)+ })+ },+ }+ // The subcommands inherit this, so no flag error of theirs quotes the+ // command line either.+ command.SetFlagErrorFunc(redactFlagError)+ command.AddCommand(newEnablePasswordSigninCommand(dependencies), newResetPasswordCommand(dependencies))+ return command+}++func newEnablePasswordSigninCommand(dependencies recoveryCommandDependencies) *cobra.Command {+ return &cobra.Command{+ Use: "enable-password-signin",+ Short: "Enable password sign-in when a workspace administrator has a password",+ Args: noArgs,+ RunE: func(command *cobra.Command, _ []string) error {+ command.SilenceUsage = true+ return runRecoveryAction(command, dependencies, func(ctx context.Context, workspaceID string, runner recoveryRunner) error {+ result, err := runner.EnablePasswordSignin(ctx, workspaceID)+ if result != nil {+ printPasswordSigninEnabled(command.OutOrStdout())}- }()+ return err+ })+ },+ }+}- workspaceID, err := backend.runner.GetWorkspaceID(ctx)- if err != nil {- return errors.Wrap(err, "failed to find the recovery workspace")+func newResetPasswordCommand(dependencies recoveryCommandDependencies) *cobra.Command {+ var email, password string+ command := &cobra.Command{+ Use: "reset-password --email <email> --password <password>",+ Short: "Set a new password for an existing, active user",+ Args: noArgs,+ // A Job fills these flags from Secrets; the checks catch what that gets+ // wrong, before anything is read.+ PreRunE: func(*cobra.Command, []string) error {+ switch {+ case email == "" && password == "":+ return errors.New("--email and --password need a value")+ case email == "":+ return errors.New("--email needs a value")+ case password == "":+ return errors.New("--password needs a value")+ }+ if _, err := recovery.ParseEmail(email); err != nil {+ return errors.New("--email is not an email address")}- if workspaceID == "" {- return errors.New("no active workspace was found in the metadata database")+ if unexpandedReference.MatchString(password) {+ return errors.New("--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines")}- return runRecoverySession(ctx, workspaceID, input, output, backend.runner, dependencies.readPassword)+ if !utf8.ValidString(password) || strings.ContainsFunc(password, cannotBeTyped) {+ return errors.New("--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding")+ }+ return nil+ },+ RunE: func(command *cobra.Command, _ []string) error {+ command.SilenceUsage = true+ return runRecoveryAction(command, dependencies, func(ctx context.Context, workspaceID string, runner recoveryRunner) error {+ result, err := runner.ResetUserPassword(ctx, recovery.ResetUserPasswordRequest{+ WorkspaceID: workspaceID,+ Email: email,+ Password: []byte(password),+ })+ if result == nil {+ return err+ }+ output := command.OutOrStdout()+ printPasswordResetSuccess(output, result.Email)+ // The role lookup only informs the operator about the saved+ // password, so it runs even when a later step failed, and its own+ // failure is a warning.+ member, lookupErr := runner.IsUserInWorkspace(ctx, workspaceID, result.Email)+ if lookupErr != nil {+ fmt.Fprintf(command.ErrOrStderr(), "Warning: could not check whether user %s has a workspace role: %v\n", result.Email, lookupErr)+ } else if !member {+ fmt.Fprintf(output, "User %s has no workspace role. Before they try to sign in, run \"bytebase recovery\" to add one.\n", result.Email)+ }+ return err+ })},}+ command.Flags().StringVar(&email, "email", "", "email of an existing, active user")+ command.Flags().StringVar(&password, "password", "", "the new password; it shows in the process list and the shell history, so change it after signing in")return command}+// noArgs and redactFlagError replace cobra's errors, which quote what was+// typed: a password that lost or garbled its --password flag would be printed+// into the log. For the same reason, no other error here repeats a flag's value.+func noArgs(command *cobra.Command, args []string) error {+ if len(args) == 0 {+ return nil+ }+ if command.HasSubCommands() {+ return subcommandUsageError(command)+ }+ return errors.Errorf("%s takes no arguments", command.CommandPath())+}++func redactFlagError(command *cobra.Command, _ error) error {+ if command.HasSubCommands() {+ return subcommandUsageError(command)+ }+ return errors.Errorf("%s was given a flag it does not know or a flag without a valid value; the usage lists its flags", command.CommandPath())+}++// subcommandUsageError answers what `bytebase recovery` does not take, most+// often a misspelled subcommand, with the forms it does take.+func subcommandUsageError(command *cobra.Command) error {+ return errors.Errorf("%s was given an argument or a flag it does not take; to run one action, use %s", command.CommandPath(), oneActionCommands)+}++// unexpandedReference matches a whole value that is a variable reference left+// in place: $NAME and ${NAME} with a shell name, or $(NAME) with any name a+// Secret or ConfigMap key can hold (letters, digits, '-', '.' and '_'), which+// envFrom turns into variables. Kubernetes passes an undefined $(NAME) through+// unchanged. A value with other characters inside, such as $(Str0ng!), is more+// likely a password than a variable's name.+var unexpandedReference = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*|\{[A-Za-z_][A-Za-z0-9_]*\}|\([-._A-Za-z0-9]+\))$`)++// cannotBeTyped reports a character that a Secret file or a pasted document+// adds and a sign-in form cannot receive: control characters such as a+// trailing newline, line and paragraph separators, a byte-order mark, a+// zero-width space or word joiner, and the replacement character a decoder+// leaves for bytes in another encoding. Joiners that keyboards produce, as in+// Persian text or emoji, are allowed.+func cannotBeTyped(r rune) bool {+ switch r {+ case '\ufeff', '\u200b', '\u2060', utf8.RuneError:+ return true+ default:+ return unicode.IsControl(r) || unicode.In(r, unicode.Zl, unicode.Zp)+ }+}++// runRecoveryAction opens the metadata store, resolves its only workspace and+// runs action against it.+func runRecoveryAction(+ command *cobra.Command,+ dependencies recoveryCommandDependencies,+ action func(ctx context.Context, workspaceID string, runner recoveryRunner) error,+) (retErr error) {+ ctx, stop := signal.NotifyContext(command.Context(), os.Interrupt, syscall.SIGTERM)+ defer stop()+ backend, err := dependencies.open(ctx)+ if err != nil {+ return errors.Wrap(err, "failed to open recovery metadata store")+ }+ defer func() {+ if err := backend.close(); err != nil && retErr == nil {+ retErr = errors.Wrap(err, "failed to close recovery metadata store")+ }+ }()++ workspaceID, err := backend.runner.GetWorkspaceID(ctx)+ if err != nil {+ return errors.Wrap(err, "failed to find the recovery workspace")+ }+ if workspaceID == "" {+ return errors.New("no active workspace was found in the metadata database")+ }+ return action(ctx, workspaceID, backend.runner)+}+func openRecoveryBackend(ctx context.Context) (*recoveryBackend, error) {profile, err := resolveProfile()if err != nil {@@ -107,7 +253,8 @@ func openRecoveryBackend(ctx context.Context) (*recoveryBackend, error) {}pgURL := fmt.Sprintf("host=%s port=%d user=bb database=bb", postgres.GetPostgresSocketDir(), profile.DatastorePort)- if stores, err := store.New(ctx, pgURL, false); err == nil {+ stores, connectErr := store.New(ctx, pgURL, false)+ if connectErr == nil {return newRecoveryBackend(stores, nil), nil}@@ -118,11 +265,14 @@ func openRecoveryBackend(ctx context.Context) (*recoveryBackend, error) {}return nil, errors.Wrap(err, "failed to inspect embedded metadata database")}+ if err := refuseRunningEmbeddedPostgres(pgDataDir, connectErr); err != nil {+ return nil, err+ }stopEmbeddedPostgres, err := postgres.StartMetadataInstance(ctx, pgDataDir, profile.DatastorePort, profile.Mode)if err != nil {return nil, errors.Wrap(err, "failed to start embedded metadata database")}- stores, err := store.New(ctx, pgURL, false)+ stores, err = store.New(ctx, pgURL, false)if err != nil {stopEmbeddedPostgres()return nil, errors.Wrap(err, "failed to connect to embedded metadata database")@@ -130,6 +280,21 @@ func openRecoveryBackend(ctx context.Context) (*recoveryBackend, error) {return newRecoveryBackend(stores, stopEmbeddedPostgres), nil}+// refuseRunningEmbeddedPostgres stops recovery from starting PostgreSQL on a+// data directory that another postmaster may hold. PostgreSQL's own lock+// cannot see a postmaster in another container, so the two would write the+// same files.+func refuseRunningEmbeddedPostgres(pgDataDir string, connectErr error) error {+ _, err := os.Stat(filepath.Join(pgDataDir, "postmaster.pid"))+ if os.IsNotExist(err) {+ return nil+ }+ if err != nil {+ return errors.Wrap(err, "failed to inspect embedded metadata database")+ }+ return errors.Errorf("cannot connect to the embedded metadata database (%v), and its data directory %q may be in use: run recovery inside the Bytebase server's container, with the server's --data and --port; if no Bytebase server is running, start it once, then run recovery that way", connectErr, pgDataDir)+}+func newRecoveryBackend(stores *store.Store, stopEmbeddedPostgres func()) *recoveryBackend {return &recoveryBackend{runner: recovery.NewService(stores),@@ -216,14 +381,16 @@ func runEnablePasswordSignin(}fmt.Fprintln(output, "Enabling password sign-in...")- _, err = runner.EnablePasswordSignin(ctx, workspaceID)- if err != nil {+ result, err := runner.EnablePasswordSignin(ctx, workspaceID)+ if result == nil {fmt.Fprintf(output, "Failed to enable password sign-in: %v\n", err)return nil}- fmt.Fprintln(output, "Password sign-in is enabled.")- fmt.Fprintln(output, "Restart all Bytebase servers after exiting recovery.")- fmt.Fprintln(output, "After restarting, sign in at /auth with an existing user account.")+ printPasswordSigninEnabled(output)+ if err != nil {+ fmt.Fprintf(output, "Warning: %v\n", err)+ }+ fmt.Fprintln(output, "Sign in at /auth with an existing user account.")fmt.Fprintln(output, "After repairing SSO, re-enable Disallow password sign-in.")return nil}@@ -370,30 +537,18 @@ func resetUserPassword(Email: result.Email,Role: role.Name,})- if addErr != nil {- if addResult != nil && addResult.Changed {- fmt.Fprintf(- output,- "User %s was added as %s (%s), but recovery did not complete: %v\n",- result.Email,- role.Title,- role.Name,- addErr,- )- } else {- fmt.Fprintf(output, "Failed to add user to workspace: %v\n", addErr)- }+ if addResult == nil {+ fmt.Fprintf(output, "Failed to add user to workspace: %v\n", addErr)return nil}-- fmt.Fprintf(- output,- "User %s was added as %s (%s).\n",- result.Email,- role.Title,- role.Name,- )- fmt.Fprintln(output, "Restart all Bytebase servers after exiting recovery so they reload the updated IAM policy.")+ granted := "already has"+ if addResult.Changed {+ granted = "was added as"+ }+ fmt.Fprintf(output, "User %s %s %s (%s). Live within one minute.\n", result.Email, granted, role.Title, role.Name)+ if addErr != nil {+ fmt.Fprintf(output, "Warning: %v\n", addErr)+ }return nil}@@ -450,8 +605,12 @@ func selectRecoveryRole(ctx context.Context, reader *bufio.Reader, output io.Wri}}+func printPasswordSigninEnabled(output io.Writer) {+ fmt.Fprintln(output, "Password sign-in is enabled. Live within one minute.")+}+func printPasswordResetSuccess(output io.Writer, email string) {- fmt.Fprintf(output, "Password updated for user %s.\n", email)+ fmt.Fprintf(output, "Password updated for user %s. Live within one minute.\n", email)}func isRecoveryConfirmed(value string) bool {diff --git a/backend/bin/server/cmd/recovery_test.go b/backend/bin/server/cmd/recovery_test.goindex 9c84d0efcf..73acac10f4 100644--- a/backend/bin/server/cmd/recovery_test.go+++ b/backend/bin/server/cmd/recovery_test.go@@ -3,6 +3,7 @@ package cmdimport ("bytes""context"+ "fmt""io""os""path/filepath"@@ -14,8 +15,10 @@ import ("github.com/spf13/cobra""github.com/stretchr/testify/require"+ "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/component/recovery"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store")func TestResolveProfileUsesServerDataConfiguration(t *testing.T) {@@ -48,6 +51,24 @@ func TestResolveProfileUsesServerDataConfiguration(t *testing.T) {require.Equal(t, "postgres://bytebase.example/metadata", profile.PgURL)}+func TestRecoveryRefusesToStartPostgresOnADataDirectoryInUse(t *testing.T) {+ pgDataDir := t.TempDir()+ connectErr := errors.New("the database system is starting up")+ require.NoError(t, refuseRunningEmbeddedPostgres(pgDataDir, connectErr))++ require.NoError(t, os.WriteFile(filepath.Join(pgDataDir, "postmaster.pid"), []byte("12345\n"), 0o600))+ err := refuseRunningEmbeddedPostgres(pgDataDir, connectErr)++ require.ErrorContains(t, err, "the database system is starting up")+ require.ErrorContains(t, err, "may be in use")+ require.ErrorContains(t, err, "inside the Bytebase server's container")+}++// The result lines promise "Live within one minute."+func TestRecoveryPromisesTheStoreCacheLifetime(t *testing.T) {+ require.Equal(t, time.Minute, store.OutOfBandCacheLifetime)+}+func TestReadRecoveryPasswordInputInterrupt(t *testing.T) {terminal := struct {io.Reader@@ -83,11 +104,13 @@ type fakeRecoveryRunner struct {listWorkspaces []stringaddRequests []recovery.AddUserToWorkspaceRequestroles []*recovery.Role+ enableResult *recovery.EnablePasswordSigninResultenableErr errorresetResult *recovery.ResetUserPasswordResultresetErr errorresetErrs []errorlistErr error+ addResult *recovery.AddUserToWorkspaceResultaddErr errorpasswordRestriction *storepb.WorkspaceProfileSetting_PasswordRestrictionpasswordRestrictionErr error@@ -113,7 +136,7 @@ func (f *fakeRecoveryRunner) GetWorkspaceID(context.Context) (string, error) {func (f *fakeRecoveryRunner) EnablePasswordSignin(_ context.Context, workspaceID string) (*recovery.EnablePasswordSigninResult, error) {f.enableWorkspaces = append(f.enableWorkspaces, workspaceID)if f.enableErr != nil {- return nil, f.enableErr+ return f.enableResult, f.enableErr}return &recovery.EnablePasswordSigninResult{WorkspaceID: workspaceID}, nil}@@ -153,7 +176,7 @@ func (f *fakeRecoveryRunner) ResetUserPassword(_ context.Context, request recoveif f.resetResult != nil {return f.resetResult, nil}- return &recovery.ResetUserPasswordResult{WorkspaceID: request.WorkspaceID, Email: request.Email}, nil+ return &recovery.ResetUserPasswordResult{WorkspaceID: request.WorkspaceID, Email: common.NormalizeEmail(request.Email)}, nil}func (f *fakeRecoveryRunner) ListRoles(_ context.Context, workspaceID string) ([]*recovery.Role, error) {@@ -167,7 +190,10 @@ func (f *fakeRecoveryRunner) ListRoles(_ context.Context, workspaceID string) ([func (f *fakeRecoveryRunner) AddUserToWorkspace(_ context.Context, request recovery.AddUserToWorkspaceRequest) (*recovery.AddUserToWorkspaceResult, error) {f.addRequests = append(f.addRequests, request)if f.addErr != nil {- return nil, f.addErr+ return f.addResult, f.addErr+ }+ if f.addResult != nil {+ return f.addResult, nil}return &recovery.AddUserToWorkspaceResult{WorkspaceID: request.WorkspaceID,@@ -178,35 +204,31 @@ func (f *fakeRecoveryRunner) AddUserToWorkspace(_ context.Context, request recov}func TestRecoveryCommandContract(t *testing.T) {- t.Run("help exposes no workspace or action inputs", func(t *testing.T) {- command, output, _, _ := newTestRecoveryCommand(t, &fakeRecoveryRunner{}, "", nil)- command.SetArgs([]string{"--help"})-- require.NoError(t, command.Execute())- require.NotContains(t, output.String(), "--workspace")- require.NotContains(t, output.String(), "--email")- require.NotContains(t, output.String(), "--password")- require.NotContains(t, output.String(), "--yes")- require.Empty(t, command.Commands())+ t.Run("help lists the subcommands and no workspace or confirmation flag", func(t *testing.T) {+ help := recoveryHelp(t)++ require.Contains(t, help[""], "enable-password-signin")+ require.Contains(t, help[""], "reset-password")+ for _, text := range help {+ for _, rejected := range []string{"--workspace", "--yes", "--add-role", "--wait", "--password-file"} {+ require.NotContains(t, text, rejected)+ }+ }+ require.NotContains(t, help["enable-password-signin"], "--email")+ require.NotContains(t, help["enable-password-signin"], "--password")+ require.Contains(t, help["reset-password"], "--email string")+ require.Contains(t, help["reset-password"], "--password string")})- t.Run("rejects a non-terminal before opening metadata", func(t *testing.T) {- openCount := 0- command := newRecoveryCommand(recoveryCommandDependencies{- open: func(context.Context) (*recoveryBackend, error) {- openCount++- return nil, nil- },- isTerminal: func(io.Reader, io.Writer) bool { return false },- })- command.SetArgs(nil)- command.SilenceUsage = true- command.SilenceErrors = true+ t.Run("rejects a non-terminal before opening metadata and names the subcommands", func(t *testing.T) {+ command, _, openCount, _ := newTestRecoveryCommandWithTerminal(t, &fakeRecoveryRunner{}, false, "", nil)err := command.Execute()require.ErrorContains(t, err, "terminal input and output")- require.Zero(t, openCount)+ require.ErrorContains(t, err, `"bytebase recovery enable-password-signin"`)+ require.ErrorContains(t, err, `"bytebase recovery reset-password --email <email> --password <password>"`)+ require.Zero(t, *openCount)})t.Run("opens and closes metadata once for the session", func(t *testing.T) {@@ -215,8 +237,7 @@ func TestRecoveryCommandContract(t *testing.T) {require.NoError(t, command.Execute())require.Equal(t, 1, *openCount)require.Equal(t, 1, *closeCount)- require.Contains(t, output.String(), "Stop all Bytebase servers that use this metadata database before continuing.")- require.Contains(t, output.String(), "Restart all Bytebase servers after recovery")+ requireNoStopOrRestart(t, output.String())})t.Run("rejects metadata with multiple workspaces", func(t *testing.T) {@@ -264,12 +285,12 @@ func TestRecoveryCommandContract(t *testing.T) {require.NoError(t, command.Execute())require.Equal(t, []string{"resolved-workspace"}, runner.enableWorkspaces)require.Contains(t, output.String(), "Enabling password sign-in...")- require.Contains(t, output.String(), "Password sign-in is enabled.")+ require.Contains(t, output.String(), "Password sign-in is enabled. Live within one minute.")require.NotContains(t, output.String(), "resolved-workspace")- require.Contains(t, output.String(), "Restart all Bytebase servers after exiting recovery.")- require.Contains(t, output.String(), "After restarting, sign in at /auth with an existing user account.")+ require.Contains(t, output.String(), "Sign in at /auth with an existing user account.")require.NotContains(t, output.String(), "existing administrator account")- require.Less(t, strings.Index(output.String(), "Restart all Bytebase servers"), strings.Index(output.String(), "After restarting, sign in at /auth"))+ require.Less(t, strings.Index(output.String(), "Live within one minute."), strings.Index(output.String(), "Sign in at /auth"))+ requireNoStopOrRestart(t, output.String())require.Equal(t, 2, strings.Count(output.String(), "Select a recovery action:"))})@@ -282,6 +303,20 @@ func TestRecoveryCommandContract(t *testing.T) {require.Equal(t, 2, strings.Count(output.String(), "Select a recovery action:"))})+ t.Run("enable reports a saved setting with a warning when the audit record fails", func(t *testing.T) {+ runner := &fakeRecoveryRunner{+ enableResult: &recovery.EnablePasswordSigninResult{WorkspaceID: "acme", UsableAdminCount: 1},+ enableErr: errors.New("password sign-in was enabled, but failed to create the recovery audit log: audit write failed"),+ }+ command, output, _, _ := newTestRecoveryCommand(t, runner, "1\ny\nq\n", nil)++ require.NoError(t, command.Execute())+ require.Contains(t, output.String(), "Password sign-in is enabled. Live within one minute.\n"++ "Warning: password sign-in was enabled, but failed to create the recovery audit log: audit write failed\n")+ require.NotContains(t, output.String(), "Failed to enable password sign-in")+ require.Equal(t, 2, strings.Count(output.String(), "Select a recovery action:"))+ })+t.Run("blank reset email returns to the menu", func(t *testing.T) {runner := &fakeRecoveryRunner{}command, _, _, _ := newTestRecoveryCommand(t, runner, "2\n\nq\n", nil)@@ -414,8 +449,9 @@ func TestRecoveryCommandContract(t *testing.T) {require.Equal(t, make([]byte, len(firstPassword)), firstPassword)require.Equal(t, make([]byte, len(secondPassword)), secondPassword)require.Contains(t, output.String(), "Resetting user password...")- require.Contains(t, output.String(), "Password updated for user [email protected].")+ require.Contains(t, output.String(), "Password updated for user [email protected]. Live within one minute.")require.NotContains(t, output.String(), "NewPassword1!")+ requireNoStopOrRestart(t, output.String())})t.Run("missing member can decline the IAM grant", func(t *testing.T) {@@ -473,9 +509,10 @@ func TestRecoveryCommandContract(t *testing.T) {require.Contains(t, output.String(), "Invalid role selection")require.Contains(t, output.String(), "Add [email protected] as Workspace admin (roles/workspaceAdmin)? [y/N]")require.Contains(t, output.String(), "Adding user to workspace...")- require.Contains(t, output.String(), "Restart all Bytebase servers after exiting recovery so they reload the updated IAM policy.")+ require.Contains(t, output.String(), "User [email protected] was added as Workspace admin (roles/workspaceAdmin). Live within one minute.")require.Contains(t, output.String(), "Password updated for user [email protected].")require.NotContains(t, output.String(), "NewPassword1!")+ requireNoStopOrRestart(t, output.String())require.Equal(t, make([]byte, len(firstPassword)), firstPassword)require.Equal(t, make([]byte, len(secondPassword)), secondPassword)})@@ -500,6 +537,45 @@ func TestRecoveryCommandContract(t *testing.T) {require.Contains(t, output.String(), "Failed to add user to workspace: IAM write failed")})+ for _, tc := range []struct {+ name string+ runner *fakeRecoveryRunner+ wantLine string+ }{+ {+ name: "role grant reports a stored role with a warning when its audit record fails",+ runner: &fakeRecoveryRunner{+ addResult: &recovery.AddUserToWorkspaceResult{WorkspaceID: "acme", Email: "[email protected]", Role: "roles/workspaceAdmin", Changed: true},+ addErr: errors.New("user was added to the workspace, but failed to create the recovery audit log: audit write failed"),+ },+ wantLine: "User [email protected] was added as Workspace admin (roles/workspaceAdmin). Live within one minute.\n" ++ "Warning: user was added to the workspace, but failed to create the recovery audit log: audit write failed\n",+ },+ {+ name: "role grant says the user already has a role another server granted meanwhile",+ runner: &fakeRecoveryRunner{+ addResult: &recovery.AddUserToWorkspaceResult{WorkspaceID: "acme", Email: "[email protected]", Role: "roles/workspaceAdmin"},+ },+ wantLine: "User [email protected] already has Workspace admin (roles/workspaceAdmin). Live within one minute.\n",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ tc.runner.notInWorkspace = true+ tc.runner.roles = []*recovery.Role{{Name: "roles/workspaceAdmin", Title: "Workspace admin"}}+ passwords := [][]byte{[]byte("NewPassword1!"), []byte("NewPassword1!")}+ readCount := 0+ command, output, _, _ := newTestRecoveryCommand(t, tc.runner, "2\[email protected]\ny\n1\ny\nq\n", func(io.Reader) ([]byte, error) {+ password := passwords[readCount]+ readCount+++ return password, nil+ })++ require.NoError(t, command.Execute())+ require.Contains(t, output.String(), "Adding user to workspace...\n"+tc.wantLine)+ require.NotContains(t, output.String(), "Failed to add user to workspace")+ })+ }+t.Run("audit failure preserves the reset and continues membership recovery", func(t *testing.T) {runner := &fakeRecoveryRunner{resetResult: &recovery.ResetUserPasswordResult{@@ -596,11 +672,425 @@ func TestRecoveryCommandContract(t *testing.T) {})}+func TestRecoverySubcommands(t *testing.T) {+ resetArgs := []string{"reset-password", "--email", "[email protected]", "--password", "NewPassword1!"}+ const resetResult = "Password updated for user [email protected]. Live within one minute.\n"++ t.Run("enable-password-signin enables sign-in without a terminal", func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, output, openCount, closeCount := newTestRecoverySubcommand(t, runner, "enable-password-signin")++ require.NoError(t, command.Execute())+ require.Equal(t, []string{"acme"}, runner.enableWorkspaces)+ require.Equal(t, "Password sign-in is enabled. Live within one minute.\n", output.String())+ require.Equal(t, 1, *openCount)+ require.Equal(t, 1, *closeCount)+ })++ t.Run("reset-password sets the password without a terminal or a prompt", func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, output, openCount, closeCount := newTestRecoverySubcommand(t, runner, resetArgs...)++ require.NoError(t, command.Execute())+ require.Equal(t, []recovery.ResetUserPasswordRequest{{+ WorkspaceID: "acme",+ Email: "[email protected]",+ Password: []byte("NewPassword1!"),+ }}, runner.resetRequests)+ require.Equal(t, resetResult, output.String())+ require.Equal(t, 1, *openCount)+ require.Equal(t, 1, *closeCount)+ })++ for _, password := range []string{"می\u200cخواهمPw1!", "Dev👩\u200d💻Password1!"} {+ t.Run(fmt.Sprintf("reset-password accepts the typeable password %q", password), func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, _, _, _ := newTestRecoverySubcommand(t, runner,+ "reset-password", "--email", "[email protected]", "--password", password)++ require.NoError(t, command.Execute())+ require.Len(t, runner.resetRequests, 1)+ })+ }++ t.Run("reset-password accepts a password that only looks partly like a reference", func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, _, _, _ := newTestRecoverySubcommand(t, runner,+ "reset-password", "--email", "[email protected]", "--password", "$(Str0ng!)")++ require.NoError(t, command.Execute())+ require.Len(t, runner.resetRequests, 1)+ require.Equal(t, []byte("$(Str0ng!)"), runner.resetRequests[0].Password)+ })++ for _, email := range []string{"[email protected]", "[email protected]\n", " [email protected] "} {+ t.Run(fmt.Sprintf("reset-password reads the email %q as sign-in does", email), func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, output, _, _ := newTestRecoverySubcommand(t, runner,+ "reset-password", "--email", email, "--password", "NewPassword1!")++ require.NoError(t, command.Execute())+ require.Len(t, runner.resetRequests, 1)+ require.Equal(t, resetResult, output.String())+ })+ }++ t.Run("reset-password succeeds with a warning when the workspace role check fails", func(t *testing.T) {+ runner := &fakeRecoveryRunner{membershipErr: errors.New("membership read failed")}+ command, output, _, _ := newTestRecoverySubcommand(t, runner, resetArgs...)++ require.NoError(t, command.Execute())+ require.Equal(t, resetResult++ "Warning: could not check whether user [email protected] has a workspace role: membership read failed\n", output.String())+ })++ t.Run("reset-password points a user with no workspace role at the menu", func(t *testing.T) {+ runner := &fakeRecoveryRunner{notInWorkspace: true}+ command, output, _, _ := newTestRecoverySubcommand(t, runner, resetArgs...)++ require.NoError(t, command.Execute())+ require.Equal(t, resetResult++ "User [email protected] has no workspace role. Before they try to sign in, run \"bytebase recovery\" to add one.\n", output.String())+ require.Len(t, runner.resetRequests, 1)+ require.Empty(t, runner.listWorkspaces)+ require.Empty(t, runner.addRequests)+ })++ for _, tc := range []struct {+ name string+ args []string+ runner *fakeRecoveryRunner+ wantErr string+ wantResult string+ }{+ {+ name: "enable-password-signin when no administrator has a password",+ args: []string{"enable-password-signin"},+ runner: &fakeRecoveryRunner{enableErr: errors.New("no usable workspace administrator has an active end-user identity with a password credential")},+ wantErr: "no usable workspace administrator has an active end-user identity with a password credential",+ },+ {+ name: "enable-password-signin when the audit record fails after the save",+ args: []string{"enable-password-signin"},+ runner: &fakeRecoveryRunner{+ enableResult: &recovery.EnablePasswordSigninResult{WorkspaceID: "acme", UsableAdminCount: 1},+ enableErr: errors.New("password sign-in was enabled, but failed to create the recovery audit log: audit write failed"),+ },+ wantErr: "password sign-in was enabled, but failed to create the recovery audit log: audit write failed",+ wantResult: "Password sign-in is enabled. Live within one minute.\n",+ },+ {+ name: "reset-password for an unknown user",+ args: resetArgs,+ runner: &fakeRecoveryRunner{resetErr: errors.New("the user does not exist")},+ wantErr: "the user does not exist",+ },+ {+ name: "reset-password for a deactivated or non-user account",+ args: resetArgs,+ runner: &fakeRecoveryRunner{resetErr: errors.New(`user "[email protected]" is not an active end user`)},+ wantErr: `user "[email protected]" is not an active end user`,+ },+ {+ name: "reset-password with a password the workspace restriction refuses",+ args: resetArgs,+ runner: &fakeRecoveryRunner{resetErr: errors.New("password does not satisfy the workspace restriction: password length should no less than 16 characters")},+ wantErr: "password does not satisfy the workspace restriction: password length should no less than 16 characters",+ },+ {+ name: "reset-password for an email outside the allowed domains",+ args: resetArgs,+ runner: &fakeRecoveryRunner{resetErr: errors.New("the email does not belong to the workspace's allowed domains")},+ wantErr: "the email does not belong to the workspace's allowed domains",+ },+ {+ name: "reset-password when the audit record fails after the save",+ args: resetArgs,+ runner: &fakeRecoveryRunner{+ resetResult: &recovery.ResetUserPasswordResult{WorkspaceID: "acme", Email: "[email protected]", Changed: true},+ resetErr: errors.New("user password reset completed, but failed to create the recovery audit log: audit write failed"),+ },+ wantErr: "user password reset completed, but failed to create the recovery audit log: audit write failed",+ wantResult: resetResult,+ },+ {+ name: "reset-password names a missing workspace role before a later step's error",+ args: resetArgs,+ runner: &fakeRecoveryRunner{+ resetResult: &recovery.ResetUserPasswordResult{WorkspaceID: "acme", Email: "[email protected]", Changed: true},+ resetErr: errors.New("user password reset completed, but failed to create the recovery audit log: audit write failed"),+ notInWorkspace: true,+ },+ wantErr: "user password reset completed, but failed to create the recovery audit log: audit write failed",+ wantResult: resetResult + "User [email protected] has no workspace role. Before they try to sign in, run \"bytebase recovery\" to add one.\n",+ },+ {+ name: "reset-password with the password already set when clearing the lockout fails",+ args: resetArgs,+ runner: &fakeRecoveryRunner{+ resetResult: &recovery.ResetUserPasswordResult{WorkspaceID: "acme", Email: "[email protected]"},+ resetErr: errors.New("failed to clear the login attempt counter: database unavailable"),+ },+ wantErr: "failed to clear the login attempt counter: database unavailable",+ wantResult: resetResult,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ command, output, _, closeCount := newTestRecoverySubcommand(t, tc.runner, tc.args...)++ err := command.Execute()++ require.EqualError(t, err, tc.wantErr)+ require.Equal(t, tc.wantResult+"Error: "+tc.wantErr+"\n", output.String())+ require.Equal(t, 1, *closeCount)+ })+ }++ for _, tc := range []struct {+ name string+ args []string+ wantErr string+ }{+ {+ name: "reset-password without flags",+ args: []string{"reset-password"},+ wantErr: "--email and --password need a value",+ },+ {+ name: "reset-password without a password",+ args: []string{"reset-password", "--email", "[email protected]"},+ wantErr: "--password needs a value",+ },+ {+ name: "reset-password without an email",+ args: []string{"reset-password", "--password", "NewPassword1!"},+ wantErr: "--email needs a value",+ },+ {+ name: "reset-password with an empty password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", ""},+ wantErr: "--password needs a value",+ },+ {+ name: "reset-password with an empty email",+ args: []string{"reset-password", "--email", "", "--password", "NewPassword1!"},+ wantErr: "--email needs a value",+ },+ {+ name: "reset-password with a value that is not an email address",+ args: []string{"reset-password", "--email", "not-an-email", "--password", "NewPassword1!"},+ wantErr: "--email is not an email address",+ },+ {+ name: "reset-password with an unresolved Kubernetes reference as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "$(NEW_PASSWORD)"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with an unresolved Kubernetes reference to a dashed name as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "$(new-password)"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with an unresolved Kubernetes reference to a dotted name as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "$(db.password)"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with an unresolved Kubernetes reference to a name starting with a digit as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "$(1PASSWORD)"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with an unexpanded braced shell reference as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "${NEW_PASSWORD}"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with an unexpanded shell reference as the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "$NEW_PASSWORD"},+ wantErr: "--password looks like a variable reference that was not expanded ($NAME, ${NAME} or $(NAME)); in a Job's args, Kubernetes expands only $(NAME), and only for a variable the container defines",+ },+ {+ name: "reset-password with a trailing newline in the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "NewPassword1!\n"},+ wantErr: "--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding",+ },+ {+ name: "reset-password with a byte-order mark in the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "\ufeffNewPassword1!"},+ wantErr: "--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding",+ },+ {+ name: "reset-password with a zero-width space in the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "NewPassword1!\u200b"},+ wantErr: "--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding",+ },+ {+ name: "reset-password with a replacement character in the password",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "NewPassw\ufffdrd1!"},+ wantErr: "--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding",+ },+ {+ name: "reset-password with a password that is not valid UTF-8",+ args: []string{"reset-password", "--email", "[email protected]", "--password", "NewPassw\xf6rd1!"},+ wantErr: "--password contains a character that cannot be typed at sign-in, such as a trailing newline, a byte-order mark or bytes from a file in another encoding",+ },+ {+ name: "reset-password with positional arguments",+ args: []string{"reset-password", "[email protected]", "NewPassword1!"},+ wantErr: "bytebase recovery reset-password takes no arguments",+ },+ {+ name: "enable-password-signin with an argument",+ args: []string{"enable-password-signin", "now"},+ wantErr: "bytebase recovery enable-password-signin takes no arguments",+ },+ {+ name: "reset-password with an unknown flag",+ args: []string{"reset-password", "--emial", "[email protected]", "--password", "NewPassword1!"},+ wantErr: "bytebase recovery reset-password was given a flag it does not know or a flag without a valid value; the usage lists its flags",+ },+ {+ name: "reset-password with a flag that has no value",+ args: []string{"reset-password", "--email", "[email protected]", "--password"},+ wantErr: "bytebase recovery reset-password was given a flag it does not know or a flag without a valid value; the usage lists its flags",+ },+ {+ name: "a misspelled subcommand",+ args: []string{"enable-password-sign-in"},+ wantErr: `bytebase recovery was given an argument or a flag it does not take; to run one action, use "bytebase recovery enable-password-signin" or "bytebase recovery reset-password --email <email> --password <password>"`,+ },+ {+ name: "a misspelled subcommand with flags",+ args: []string{"reset-passwd", "--email", "[email protected]", "--password", "NewPassword1!"},+ wantErr: `bytebase recovery was given an argument or a flag it does not take; to run one action, use "bytebase recovery enable-password-signin" or "bytebase recovery reset-password --email <email> --password <password>"`,+ },+ } {+ t.Run(tc.name+" is a usage error that reads nothing", func(t *testing.T) {+ runner := &fakeRecoveryRunner{}+ command, output, openCount, _ := newTestRecoverySubcommand(t, runner, tc.args...)++ err := command.Execute()++ require.EqualError(t, err, tc.wantErr)+ require.Zero(t, *openCount)+ require.True(t, strings.HasPrefix(output.String(), "Error: "+tc.wantErr+"\nUsage:\n"), output.String())+ })+ }++ for _, tc := range []struct {+ name string+ args []string+ }{+ {name: "a password given without --password", args: []string{"reset-password", "--email", "[email protected]", "S3cretPassword1!"}},+ {name: "a password given as --email", args: []string{"reset-password", "--email", "S3cretPassword1!", "--password", "[email protected]"}},+ {name: "a password that starts with a dash given without --password", args: []string{"reset-password", "--email", "[email protected]", "-S3cretPassword1!"}},+ {name: "a password that starts with two dashes given without --password", args: []string{"reset-password", "--email", "[email protected]", "--S3cretPassword1!"}},+ {name: "--password and its value in one argument", args: []string{"reset-password", "--email", "[email protected]", "--password S3cretPassword1!"}},+ {name: "a single-dash --password", args: []string{"reset-password", "--email", "[email protected]", "-password=S3cretPassword1!"}},+ {name: "a whole command in one argument", args: []string{"reset-password --email [email protected] --password S3cretPassword1!"}},+ } {+ t.Run(tc.name+" is not repeated in the error", func(t *testing.T) {+ command, output, openCount, _ := newTestRecoverySubcommand(t, &fakeRecoveryRunner{}, tc.args...)++ err := command.Execute()++ require.Error(t, err)+ require.NotContains(t, err.Error(), "S3cretPassword1!")+ require.NotContains(t, output.String(), "S3cretPassword1!")+ require.Zero(t, *openCount)+ })+ }++ for _, args := range [][]string{{"enable-password-signin"}, resetArgs} {+ for _, tc := range []struct {+ name string+ runner *fakeRecoveryRunner+ wantErr string+ }{+ {+ name: "no workspace",+ runner: &fakeRecoveryRunner{noWorkspace: true},+ wantErr: "no active workspace was found in the metadata database",+ },+ {+ name: "more than one workspace",+ runner: &fakeRecoveryRunner{workspaceErr: errors.New("found 2 workspaces; recovery requires exactly one workspace")},+ wantErr: "failed to find the recovery workspace: found 2 workspaces; recovery requires exactly one workspace",+ },+ } {+ t.Run(args[0]+" refuses metadata with "+tc.name, func(t *testing.T) {+ command, output, _, closeCount := newTestRecoverySubcommand(t, tc.runner, args...)++ err := command.Execute()++ require.EqualError(t, err, tc.wantErr)+ require.Equal(t, "Error: "+tc.wantErr+"\n", output.String())+ require.Empty(t, tc.runner.enableWorkspaces)+ require.Empty(t, tc.runner.resetRequests)+ require.Equal(t, 1, *closeCount)+ })+ }+ }+}++func requireNoStopOrRestart(t *testing.T, output string) {+ t.Helper()+ require.NotContains(t, strings.ToLower(output), "stop all")+ require.NotContains(t, strings.ToLower(output), "restart")+}++// recoveryHelp returns the help of the recovery command, keyed "", and of each+// of its subcommands, keyed by name.+func recoveryHelp(t *testing.T) map[string]string {+ t.Helper()+ help := map[string]string{}+ for _, name := range []string{"", "enable-password-signin", "reset-password"} {+ args := []string{"--help"}+ if name != "" {+ args = []string{name, "--help"}+ }+ command, output, openCount, _ := newTestRecoverySubcommand(t, &fakeRecoveryRunner{}, args...)+ require.NoError(t, command.Execute())+ require.Zero(t, *openCount)+ help[name] = output.String()+ }+ return help+}++// newTestRecoverySubcommand returns a root command that runs+// `bytebase recovery <args>` the way a Kubernetes Job does: no terminal and+// nothing on standard input. The output is what the operator sees, including+// the errors and usage the root prints.+func newTestRecoverySubcommand(t *testing.T, runner recoveryRunner, args ...string) (*cobra.Command, *bytes.Buffer, *int, *int) {+ t.Helper()+ recoveryCommand, output, openCount, closeCount := newTestRecoveryCommandWithTerminal(t, runner, false, "", nil)+ recoveryCommand.SilenceErrors = false+ root := &cobra.Command{Use: "bytebase"}+ root.AddCommand(recoveryCommand)+ root.SetOut(output)+ root.SetErr(output)+ root.SetArgs(append([]string{"recovery"}, args...))+ return root, output, openCount, closeCount+}+func newTestRecoveryCommand(t *testing.T,runner recoveryRunner,input string,readPassword func(io.Reader) ([]byte, error),+) (*cobra.Command, *bytes.Buffer, *int, *int) {+ t.Helper()+ return newTestRecoveryCommandWithTerminal(t, runner, true, input, readPassword)+}++func newTestRecoveryCommandWithTerminal(+ t *testing.T,+ runner recoveryRunner,+ terminal bool,+ input string,+ readPassword func(io.Reader) ([]byte, error),) (*cobra.Command, *bytes.Buffer, *int, *int) {t.Helper()openCount := 0@@ -621,7 +1111,7 @@ func newTestRecoveryCommand(},}, nil},- isTerminal: func(io.Reader, io.Writer) bool { return true },+ isTerminal: func(io.Reader, io.Writer) bool { return terminal },readPassword: readPassword,})output := &bytes.Buffer{}@@ -629,7 +1119,6 @@ func newTestRecoveryCommand(command.SetOut(output)command.SetErr(output)command.SetArgs(nil)- command.SilenceUsage = truecommand.SilenceErrors = truereturn command, output, &openCount, &closeCount}diff --git a/backend/common/util.go b/backend/common/util.goindex 0d943a254e..7406780741 100644--- a/backend/common/util.go+++ b/backend/common/util.go@@ -104,6 +104,12 @@ func ValidatePhone(phone string) error {// Modified to only allow lowercase letters since Bytebase requires lowercase emails.var emailRegex = regexp.MustCompile(`^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$`)+// NormalizeEmail canonicalizes a submitted email for lookups, lockout+// identities, and audit resources — these must all key the same string.+func NormalizeEmail(email string) string {+ return strings.ToLower(strings.TrimSpace(email))+}+// ValidateEmail validates the email address using WHATWG HTML spec.// Only lowercase ASCII letters, digits, and standard email symbols are allowed.func ValidateEmail(email string) error {diff --git a/backend/component/recovery/service.go b/backend/component/recovery/service.goindex 164f6c0b30..eaa6764729 100644--- a/backend/component/recovery/service.go+++ b/backend/component/recovery/service.go@@ -25,6 +25,9 @@ const ()// Service performs self-hosted recovery operations against the metadata store.+// An action that stored what was asked for and then failed a later step, such+// as the audit record or clearing the lockout, returns its result together+// with the error.type Service struct {store *store.Store}@@ -175,9 +178,9 @@ func (s *Service) IsUserInWorkspace(ctx context.Context, workspaceID, email striif err := s.requireWorkspace(ctx, workspaceID); err != nil {return false, err}- email = strings.ToLower(email)- if err := common.ValidateEmail(email); err != nil {- return false, errors.Wrap(err, "invalid user email")+ email, err := ParseEmail(email)+ if err != nil {+ return false, err}workspace, err := s.store.FindWorkspace(ctx, &store.FindWorkspaceMessage{WorkspaceID: &workspaceID,@@ -237,16 +240,16 @@ func (s *Service) AddUserToWorkspace(ctx context.Context, request AddUserToWorksreturn nil, err}- email := strings.ToLower(request.Email)- if err := common.ValidateEmail(email); err != nil {- return nil, errors.Wrap(err, "invalid user email")+ email, err := ParseEmail(request.Email)+ if err != nil {+ return nil, err}profile, err := s.store.GetWorkspaceProfileSetting(ctx, request.WorkspaceID)if err != nil {return nil, errors.Wrap(err, "failed to get workspace profile setting")}if profile.EnforceIdentityDomain && !emailBelongsToDomains(email, profile.Domains) {- return nil, errors.Errorf("email %q does not belong to the workspace's allowed domains", email)+ return nil, errDomainNotAllowed}if _, err := s.getActiveEndUser(ctx, email); err != nil {return nil, err@@ -264,35 +267,25 @@ func (s *Service) AddUserToWorkspace(ctx context.Context, request AddUserToWorksreturn nil, errors.Errorf("role %q does not exist", request.Role)}- policy, err := s.store.GetWorkspaceIamPolicy(ctx, request.WorkspaceID)- if err != nil {- return nil, errors.Wrap(err, "failed to get workspace IAM policy")- }member := common.FormatUserEmail(email)- directRoles := make([]string, 0)- selected := false- for _, binding := range policy.Policy.Bindings {- if slices.Contains(binding.Members, member) {- directRoles = append(directRoles, binding.Role)- selected = selected || binding.Role == request.Role- }+ // Deciding on the locked policy keeps a role that a running server grants+ // this user while recovery runs.+ granted := false+ _, err = s.store.PatchIamPolicy(ctx, request.WorkspaceID, storepb.Policy_WORKSPACE, common.FormatWorkspace(request.WorkspaceID),+ func(policy *storepb.IamPolicy) error {+ if granted = addRoleMember(policy, request.Role, member); !granted {+ return errRoleAlreadyGranted+ }+ return nil+ })+ if err != nil && !errors.Is(err, errRoleAlreadyGranted) {+ return nil, errors.Wrap(err, "failed to add user to workspace IAM policy")}-result := &AddUserToWorkspaceResult{WorkspaceID: request.WorkspaceID,Email: email,Role: request.Role,- }- if !selected {- directRoles = append(directRoles, request.Role)- if _, err := s.store.PatchWorkspaceIamPolicy(ctx, &store.PatchIamPolicyMessage{- Workspace: request.WorkspaceID,- Member: member,- Roles: directRoles,- }); err != nil {- return nil, errors.Wrap(err, "failed to add user to workspace IAM policy")- }- result.Changed = true+ Changed: granted,}auditRequest, err := json.Marshal(struct {@@ -325,9 +318,9 @@ func (s *Service) ResetUserPassword(ctx context.Context, request ResetUserPassworeturn nil, err}- email := strings.ToLower(request.Email)- if err := common.ValidateEmail(email); err != nil {- return nil, errors.Wrap(err, "invalid user email")+ email, err := ParseEmail(request.Email)+ if err != nil {+ return nil, err}profile, err := s.store.GetWorkspaceProfileSetting(ctx, request.WorkspaceID)@@ -335,7 +328,7 @@ func (s *Service) ResetUserPassword(ctx context.Context, request ResetUserPassworeturn nil, errors.Wrap(err, "failed to get workspace profile setting")}if profile.EnforceIdentityDomain && !emailBelongsToDomains(email, profile.Domains) {- return nil, errors.Errorf("email %q does not belong to the workspace's allowed domains", email)+ return nil, errDomainNotAllowed}if err := common.ValidatePassword(string(request.Password), profile.PasswordRestriction); err != nil {return nil, errors.Wrap(err, "password does not satisfy the workspace restriction")@@ -372,15 +365,17 @@ func (s *Service) ResetUserPassword(ctx context.Context, request ResetUserPasswo// A lockout the user guessed themselves into must not outlive the reset:// without this, the operator hands over a password that Login keeps// refusing with ResourceExhausted until the window lapses.- if err := s.store.ClearLoginAttempt(ctx, email, storepb.LoginAttemptKind_PASSWORD); err != nil {+ clearErr := s.store.ClearLoginAttempt(ctx, email, storepb.LoginAttemptKind_PASSWORD)+ // The stored password is audited even when clearing the lockout failed.+ auditErr := s.createAuditLog(ctx, request.WorkspaceID, resetUserPasswordAuditMethod, string(auditRequest))+ if clearErr != nil {if result.Changed {- return result, errors.Wrap(err, "password was reset, but failed to clear the login attempt counter")+ return result, errors.Wrap(clearErr, "password was reset, but failed to clear the login attempt counter")}- return result, errors.Wrap(err, "failed to clear the login attempt counter")+ return result, errors.Wrap(clearErr, "failed to clear the login attempt counter")}-- if err := s.createAuditLog(ctx, request.WorkspaceID, resetUserPasswordAuditMethod, string(auditRequest)); err != nil {- return result, errors.Wrap(err, "user password reset completed, but failed to create the recovery audit log")+ if auditErr != nil {+ return result, errors.Wrap(auditErr, "user password reset completed, but failed to create the recovery audit log")}return result, nil}@@ -391,7 +386,7 @@ func (s *Service) getActiveEndUser(ctx context.Context, email string) (*store.Usreturn nil, errors.Wrap(err, "failed to find user identity")}if account == nil {- return nil, errors.Errorf("user %q does not exist", email)+ return nil, errors.New("the user does not exist")}if account.Type != storepb.PrincipalType_END_USER || account.MemberDeleted {return nil, errors.Errorf("user %q is not an active end user", email)@@ -417,6 +412,49 @@ func (s *Service) requireWorkspace(ctx context.Context, workspaceID string) erroreturn nil}+// ParseEmail returns email canonicalized as sign-in does, so that a reset+// clears the lockout sign-in counted, or an error when it is not an email+// address.+func ParseEmail(email string) (string, error) {+ email = common.NormalizeEmail(email)+ if err := common.ValidateEmail(email); err != nil {+ return "", errors.Wrap(err, "invalid user email")+ }+ return email, nil+}++// errRoleAlreadyGranted ends the IAM write without a change.+var errRoleAlreadyGranted = errors.New("the user already has the role")++// An error names the email only after it matched an account. Until then the+// value can be a password given as the email by mistake, and a password can+// look like an email address, so the checks before the match do not repeat it.+var errDomainNotAllowed = errors.New("the email does not belong to the workspace's allowed domains")++// addRoleMember adds member to the unconditional binding for role, creating+// it when there is none. It reports false when member is already there; a+// conditional binding does not count, because its condition can expire.+func addRoleMember(policy *storepb.IamPolicy, role, member string) bool {+ var target *storepb.Binding+ for _, binding := range policy.Bindings {+ if binding.Role != role || binding.GetCondition().GetExpression() != "" {+ continue+ }+ if slices.Contains(binding.Members, member) {+ return false+ }+ if target == nil {+ target = binding+ }+ }+ if target == nil {+ policy.Bindings = append(policy.Bindings, &storepb.Binding{Role: role, Members: []string{member}})+ return true+ }+ target.Members = append(target.Members, member)+ return true+}+func emailBelongsToDomains(email string, domains []string) bool {if len(domains) == 0 {return truediff --git a/backend/component/recovery/service_test.go b/backend/component/recovery/service_test.goindex 16aee72f34..e7510d8b84 100644--- a/backend/component/recovery/service_test.go+++ b/backend/component/recovery/service_test.go@@ -11,6 +11,7 @@ import ("github.com/stretchr/testify/require""golang.org/x/crypto/bcrypt"+ "google.golang.org/genproto/googleapis/type/expr""google.golang.org/protobuf/encoding/protojson""google.golang.org/protobuf/proto"@@ -160,6 +161,24 @@ func TestResetUserPassword(t *testing.T) {require.True(t, proto.Equal(profile.PasswordRestriction, restriction))})+ t.Run("reads the email as sign-in does", func(t *testing.T) {+ const (+ workspaceID = "recovery-reset-padded-email"+ email = "[email protected]"+ )+ createRecoveryWorkspace(ctx, t, stores, workspaceID, email, recoveryProfile())+ createRecoveryUser(ctx, t, stores, email, "OriginalPassword1!", false, nil, nil)++ result, err := service.ResetUserPassword(ctx, recovery.ResetUserPasswordRequest{+ WorkspaceID: workspaceID,+ Email: " [email protected]\n",+ Password: []byte("ReplacementPassword1!"),+ })++ require.NoError(t, err)+ require.Equal(t, email, result.Email)+ })+t.Run("resets an existing user password and only password metadata", func(t *testing.T) {const (workspaceID = "recovery-reset-success"@@ -382,7 +401,10 @@ func TestResetUserPassword(t *testing.T) {password stringcreateUser booldeleted bool- wantError string+ // matchedAccount is set when the error comes after the email matched+ // an account, the only errors that may repeat it.+ matchedAccount bool+ wantError string}{{name: "missing user",@@ -393,14 +415,23 @@ func TestResetUserPassword(t *testing.T) {wantError: "does not exist",},{- name: "deleted user",- workspaceID: "recovery-reset-deleted",- email: "[email protected]",+ name: "password given as the email",+ workspaceID: "recovery-reset-swapped",+ email: "Welcome@2024",profile: recoveryProfile(),- password: "ReplacementPassword1!",- createUser: true,- deleted: true,- wantError: "not an active end user",+ password: "[email protected]",+ wantError: "does not exist",+ },+ {+ name: "deleted user",+ workspaceID: "recovery-reset-deleted",+ email: "[email protected]",+ profile: recoveryProfile(),+ password: "ReplacementPassword1!",+ createUser: true,+ deleted: true,+ matchedAccount: true,+ wantError: "not an active end user",},{name: "outside enforced domain",@@ -437,6 +468,11 @@ func TestResetUserPassword(t *testing.T) {Password: []byte(tc.password),})require.ErrorContains(t, err, tc.wantError)+ if !tc.matchedAccount {+ // Until the email matches an account, it can be a password given+ // in the wrong flag, so the error must not repeat it.+ require.NotContains(t, strings.ToLower(err.Error()), strings.ToLower(tc.email))+ }require.Nil(t, result)user, getErr := stores.GetUserByEmail(ctx, tc.email)require.NoError(t, getErr)@@ -539,6 +575,40 @@ func TestAddUserToWorkspace(t *testing.T) {}})+ t.Run("restores a role whose only binding has expired", func(t *testing.T) {+ const (+ workspaceID = "recovery-add-expired-role"+ email = "[email protected]"+ )+ createRecoveryWorkspace(ctx, t, stores, workspaceID, "[email protected]", recoveryProfile())+ createRecoveryUser(ctx, t, stores, email, "OriginalPassword1!", false, nil, nil)+ _, err := stores.PatchIamPolicy(ctx, workspaceID, storepb.Policy_WORKSPACE, common.FormatWorkspace(workspaceID),+ func(policy *storepb.IamPolicy) error {+ policy.Bindings = append(policy.Bindings, &storepb.Binding{+ Role: "roles/workspaceAdmin",+ Members: []string{common.FormatUserEmail(email)},+ Condition: &expr.Expr{Expression: `request.time < timestamp("2020-01-01T00:00:00Z")`},+ })+ return nil+ })+ require.NoError(t, err)+ member, err := service.IsUserInWorkspace(ctx, workspaceID, email)+ require.NoError(t, err)+ require.False(t, member)++ result, err := service.AddUserToWorkspace(ctx, recovery.AddUserToWorkspaceRequest{+ WorkspaceID: workspaceID,+ Email: email,+ Role: "roles/workspaceAdmin",+ })++ require.NoError(t, err)+ require.True(t, result.Changed)+ member, err = service.IsUserInWorkspace(ctx, workspaceID, email)+ require.NoError(t, err)+ require.True(t, member)+ })+t.Run("rejects an unknown role", func(t *testing.T) {const (workspaceID = "recovery-add-unknown-role"@@ -573,6 +643,7 @@ func TestAddUserToWorkspace(t *testing.T) {Role: "roles/workspaceMember",})require.ErrorContains(t, err, "allowed domains")+ require.NotContains(t, err.Error(), email)require.Nil(t, result)})@@ -623,6 +694,149 @@ func TestAddUserToWorkspace(t *testing.T) {}}+// Each case renames a table in its own database to make the step after the+// save fail.+func TestRecoveryReturnsItsResultWhenAStepAfterTheSaveFails(t *testing.T) {+ ctx := context.Background()+ const (+ workspaceID = "recovery-partial"+ email = "[email protected]"+ password = "OriginalPassword1!"+ )+ setup := func(t *testing.T, brokenTable string) (*store.Store, *recovery.Service) {+ t.Helper()+ db, stores, _ := testcontainer.NewMetadataDB(t)+ createRecoveryWorkspace(ctx, t, stores, workspaceID, email, recoveryProfile())+ createRecoveryUser(ctx, t, stores, email, password, false, nil, nil)+ _, err := db.ExecContext(ctx, "ALTER TABLE "+brokenTable+" RENAME TO "+brokenTable+"_unavailable")+ require.NoError(t, err)+ return stores, recovery.NewService(stores)+ }++ t.Run("enable-password-signin when the audit record fails", func(t *testing.T) {+ stores, service := setup(t, "audit_log")++ result, err := service.EnablePasswordSignin(ctx, workspaceID)++ require.ErrorContains(t, err, "password sign-in was enabled, but failed to create the recovery audit log")+ require.NotNil(t, result)+ profile, err := stores.GetWorkspaceProfileSetting(ctx, workspaceID)+ require.NoError(t, err)+ require.False(t, profile.DisallowPasswordSignin)+ })++ t.Run("add-user-to-workspace when the audit record fails", func(t *testing.T) {+ stores, service := setup(t, "audit_log")++ result, err := service.AddUserToWorkspace(ctx, recovery.AddUserToWorkspaceRequest{+ WorkspaceID: workspaceID,+ Email: email,+ Role: "roles/workspaceDBA",+ })++ require.ErrorContains(t, err, "user was added to the workspace, but failed to create the recovery audit log")+ require.NotNil(t, result)+ require.True(t, result.Changed)+ policy, err := stores.GetWorkspaceIamPolicy(ctx, workspaceID)+ require.NoError(t, err)+ require.True(t, bindingContains(policy.Policy, "roles/workspaceDBA", common.FormatUserEmail(email)))+ })++ for _, tc := range []struct {+ name string+ brokenTable string+ newPassword string+ wantErr string+ wantChanged bool+ }{+ {+ name: "when the audit record fails",+ brokenTable: "audit_log",+ newPassword: "RecoveredPassword1!",+ wantErr: "user password reset completed, but failed to create the recovery audit log",+ wantChanged: true,+ },+ {+ name: "when clearing the lockout fails",+ brokenTable: "login_attempt",+ newPassword: "RecoveredPassword1!",+ wantErr: "password was reset, but failed to clear the login attempt counter",+ wantChanged: true,+ },+ {+ name: "with the same password when clearing the lockout fails",+ brokenTable: "login_attempt",+ newPassword: password,+ wantErr: "failed to clear the login attempt counter",+ },+ } {+ t.Run("reset-password "+tc.name, func(t *testing.T) {+ stores, service := setup(t, tc.brokenTable)++ result, err := service.ResetUserPassword(ctx, recovery.ResetUserPasswordRequest{+ WorkspaceID: workspaceID,+ Email: email,+ Password: []byte(tc.newPassword),+ })++ require.ErrorContains(t, err, tc.wantErr)+ require.NotNil(t, result)+ require.Equal(t, tc.wantChanged, result.Changed)+ user, err := stores.GetUserByEmail(ctx, email)+ require.NoError(t, err)+ require.NoError(t, bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(tc.newPassword)))+ if tc.brokenTable != "audit_log" {+ logs, err := stores.SearchAuditLogs(ctx, &store.AuditLogFind{Workspace: workspaceID})+ require.NoError(t, err)+ require.Len(t, logs, 1, "the stored reset must be audited even though a later step failed")+ require.Equal(t, "/bytebase.cli.Recovery/ResetUserPassword", logs[0].Payload.Method)+ }+ })+ }+}++// The service's store has its caches on and has already cached the IAM+// policy, so it misses a role that another server grants afterwards. Adding a+// role must keep that grant, which needs the policy read inside the write's+// lock rather than from the cache.+func TestAddUserToWorkspaceKeepsARoleAnotherServerGranted(t *testing.T) {+ ctx := context.Background()+ _, stores, dsn := testcontainer.NewMetadataDBWithCache(t, true)+ server, err := store.New(ctx, dsn, false)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, server.Close()) })+ service := recovery.NewService(stores)++ const (+ workspaceID = "recovery-add-user-concurrent"+ email = "[email protected]"+ )+ createRecoveryWorkspace(ctx, t, stores, workspaceID, "[email protected]", recoveryProfile())+ createRecoveryUser(ctx, t, stores, email, "OriginalPassword1!", false, nil, nil)+ _, err = stores.GetWorkspaceIamPolicy(ctx, workspaceID)+ require.NoError(t, err)++ _, err = server.PatchWorkspaceIamPolicy(ctx, &store.PatchIamPolicyMessage{+ Workspace: workspaceID,+ Member: common.FormatUserEmail(email),+ Roles: []string{"roles/workspaceDBA"},+ })+ require.NoError(t, err)++ result, err := service.AddUserToWorkspace(ctx, recovery.AddUserToWorkspaceRequest{+ WorkspaceID: workspaceID,+ Email: email,+ Role: "roles/workspaceMember",+ })+ require.NoError(t, err)+ require.True(t, result.Changed)++ policy, err := server.GetWorkspaceIamPolicy(ctx, workspaceID)+ require.NoError(t, err)+ require.True(t, bindingContains(policy.Policy, "roles/workspaceDBA", common.FormatUserEmail(email)))+ require.True(t, bindingContains(policy.Policy, "roles/workspaceMember", common.FormatUserEmail(email)))+}+func setupRecoveryStore(_ context.Context, t *testing.T) *store.Store {t.Helper()_, stores, _ := testcontainer.NewMetadataDB(t)diff --git a/backend/store/cache_lifetime_test.go b/backend/store/cache_lifetime_test.gonew file mode 100644index 0000000000..59e28bc415--- /dev/null+++ b/backend/store/cache_lifetime_test.go@@ -0,0 +1,166 @@+package store_test++import (+ "context"+ "slices"+ "testing"+ "time"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/proto"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++const (+ cacheLifetimeWorkspace = "cache-lifetime"+ cacheLifetimeEmail = "[email protected]"+)++var cacheLifetimeMember = common.FormatUserEmail(cacheLifetimeEmail)++// recoveryChanges write the three rows that `bytebase recovery` changes. seen+// reports whether a read through the given store returns the change; it+// returns an error rather than failing the test because require.Eventually+// calls it on another goroutine.+var recoveryChanges = []struct {+ name string+ apply func(ctx context.Context, t *testing.T, recovery *store.Store)+ seen func(ctx context.Context, server *store.Store) (bool, error)+}{+ {+ name: "user password",+ apply: func(ctx context.Context, t *testing.T, recovery *store.Store) {+ user, err := recovery.GetUserByEmail(ctx, cacheLifetimeEmail)+ require.NoError(t, err)+ _, err = recovery.UpdateUser(ctx, user, &store.UpdateUserMessage{PasswordHash: new("new-hash")})+ require.NoError(t, err)+ },+ seen: func(ctx context.Context, server *store.Store) (bool, error) {+ user, err := server.GetUserByEmail(ctx, cacheLifetimeEmail)+ if err != nil {+ return false, err+ }+ return user != nil && user.PasswordHash == "new-hash", nil+ },+ },+ {+ name: "password sign-in setting",+ apply: func(ctx context.Context, t *testing.T, recovery *store.Store) {+ setDisallowPasswordSignin(ctx, t, recovery, false)+ },+ seen: func(ctx context.Context, server *store.Store) (bool, error) {+ profile, err := server.GetWorkspaceProfileSetting(ctx, cacheLifetimeWorkspace)+ if err != nil {+ return false, err+ }+ return !profile.DisallowPasswordSignin, nil+ },+ },+ {+ name: "workspace role",+ apply: func(ctx context.Context, t *testing.T, recovery *store.Store) {+ _, err := recovery.PatchWorkspaceIamPolicy(ctx, &store.PatchIamPolicyMessage{+ Workspace: cacheLifetimeWorkspace,+ Member: cacheLifetimeMember,+ Roles: []string{"roles/workspaceMember"},+ })+ require.NoError(t, err)+ },+ seen: func(ctx context.Context, server *store.Store) (bool, error) {+ policy, err := server.GetWorkspaceIamPolicy(ctx, cacheLifetimeWorkspace)+ if err != nil {+ return false, err+ }+ return hasMember(policy.Policy, cacheLifetimeMember), nil+ },+ },+}++// newCacheLifetimeFixture returns a store shaped like a running server, with+// its caches on and expiring after lifetime, and the fixture's store, shaped+// like `bytebase recovery` (its own process, caches off), on the same+// database. The seeded workspace is locked out the way recovery finds it:+// password sign-in is disallowed and the one user has no workspace role.+func newCacheLifetimeFixture(t *testing.T, lifetime time.Duration) (context.Context, *store.Store, *store.Store) {+ t.Helper()+ fixture := newStorePostgresFixture(t, "")+ ctx, recovery := fixture.ctx, fixture.store+ server, err := store.NewWithOutOfBandCacheLifetimeForTest(ctx, fixture.pgURL, true, lifetime)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, server.Close()) })++ _, err = recovery.CreateWorkspace(ctx, &store.WorkspaceMessage{ResourceID: cacheLifetimeWorkspace}, "[email protected]")+ require.NoError(t, err)+ setDisallowPasswordSignin(ctx, t, recovery, true)+ _, err = recovery.CreateUser(ctx, &store.UserMessage{+ Email: cacheLifetimeEmail,+ Name: cacheLifetimeEmail,+ PasswordHash: "old-hash",+ })+ require.NoError(t, err)+ return ctx, server, recovery+}++func TestServerReadsRecoveryChangesAfterCacheLifetime(t *testing.T) {+ t.Parallel()+ ctx, server, recovery := newCacheLifetimeFixture(t, 500*time.Millisecond)+ warmThenApplyRecoveryChanges(ctx, t, server, recovery)++ for _, change := range recoveryChanges {+ require.Eventually(t, func() bool {+ seen, err := change.seen(ctx, server)+ return err == nil && seen+ }, 15*time.Second, 50*time.Millisecond, "%s: the server still serves the old value after the cache lifetime", change.name)+ }+}++func TestServerServesCachedValueWithinCacheLifetime(t *testing.T) {+ t.Parallel()+ ctx, server, recovery := newCacheLifetimeFixture(t, store.OutOfBandCacheLifetime)+ warmThenApplyRecoveryChanges(ctx, t, server, recovery)++ for _, change := range recoveryChanges {+ seen, err := change.seen(ctx, server)+ require.NoError(t, err)+ require.False(t, seen, "%s: the server read the change before the cache lifetime ended", change.name)+ }+}++func warmThenApplyRecoveryChanges(ctx context.Context, t *testing.T, server, recovery *store.Store) {+ t.Helper()+ for _, change := range recoveryChanges {+ seen, err := change.seen(ctx, server)+ require.NoError(t, err)+ require.False(t, seen, change.name)+ }+ for _, change := range recoveryChanges {+ change.apply(ctx, t, recovery)+ }+}++func hasMember(policy *storepb.IamPolicy, member string) bool {+ for _, binding := range policy.Bindings {+ if slices.Contains(binding.Members, member) {+ return true+ }+ }+ return false+}++func setDisallowPasswordSignin(ctx context.Context, t *testing.T, s *store.Store, disallow bool) {+ t.Helper()+ _, err := s.UpdateSettingAtomic(ctx, cacheLifetimeWorkspace, storepb.SettingName_WORKSPACE_PROFILE,+ func(current proto.Message) (proto.Message, error) {+ profile, ok := current.(*storepb.WorkspaceProfileSetting)+ if !ok {+ return nil, errors.Errorf("unexpected type %T", current)+ }+ profile.DisallowPasswordSignin = disallow+ return profile, nil+ }, nil)+ require.NoError(t, err)+}diff --git a/backend/store/export_test.go b/backend/store/export_test.goindex ccee7c0ea2..ae09e7f17b 100644--- a/backend/store/export_test.go+++ b/backend/store/export_test.go@@ -1,8 +1,38 @@package store+import (+ "context"+ "time"++ "github.com/pkg/errors"+)++// NewWithOutOfBandCacheLifetimeForTest is New with another lifetime for the+// caches that expire after OutOfBandCacheLifetime. expirable.LRU never expires+// a lifetime of zero and panics on one under 100ns, so it must be at least a+// millisecond.+func NewWithOutOfBandCacheLifetimeForTest(ctx context.Context, pgURL string, enableCache bool, lifetime time.Duration) (*Store, error) {+ if lifetime < time.Millisecond {+ return nil, errors.Errorf("lifetime %v is shorter than a millisecond", lifetime)+ }+ return newStore(ctx, pgURL, enableCache, lifetime)+}+// SetSettingPublishHookForTest installs a hook that runs between a setting// update's commit and its cache publish — test-only, to deterministically// expose the publish-ordering window.func (s *Store) SetSettingPublishHookForTest(hook func()) {s.settingPublishHookForTest = hook}++// SetSettingMissHookForTest installs a hook that runs after a setting read+// misses the cache and before it waits for the publish mutex.+func (s *Store) SetSettingMissHookForTest(hook func()) {+ s.settingMissHookForTest = hook+}++// SetSettingFillHookForTest installs a hook that runs right before a setting+// cache-miss fill reads the database.+func (s *Store) SetSettingFillHookForTest(hook func()) {+ s.settingFillHookForTest = hook+}diff --git a/backend/store/iam_policy_test.go b/backend/store/iam_policy_test.goindex c1af70d48e..f700903b0e 100644--- a/backend/store/iam_policy_test.go+++ b/backend/store/iam_policy_test.go@@ -390,10 +390,11 @@ func TestPatchIamPolicyRollsBackAFailedMutation(t *testing.T) {// A write must not publish what it just wrote into the cache. The advisory lock// ends with the commit, so two writers can commit in one order and reach the-// cache in the other, and the loser's snapshot would then sit in a cache with no-// expiry -- which is what permission checks read. Observed here without racing-// goroutines: after a write, the row is changed out of band, and a read that-// still saw the written value would prove the write had published it.+// cache in the other, and the loser's snapshot would then sit in the cache+// until the entry expires -- which is what permission checks read. Observed+// here without racing goroutines: after a write, the row is changed out of+// band, and a read that still saw the written value would prove the write had+// published it.func TestSetIamPolicyDoesNotPublishItsOwnWrite(t *testing.T) {t.Parallel()const seedSQL = `diff --git a/backend/store/policy.go b/backend/store/policy.goindex b9bd331f05..6862bb2dff 100644--- a/backend/store/policy.go+++ b/backend/store/policy.go@@ -191,9 +191,9 @@ func writeIamPolicyImpl(ctx context.Context, tx *sql.Tx, workspace string, resou// evictIamPolicyCache drops this process's cached copies of one IAM policy. A// write evicts rather than publishing what it just wrote: the advisory lock ends// with the commit, so two writers can commit in one order and reach the cache in-// the other, leaving the older policy cached with no expiry -- and permission-// checks read this cache. Eviction is safe under every interleaving, because it-// never installs a value; the next read repopulates it from the row.+// the other, leaving the older policy cached until the entry expires -- and+// permission checks read this cache. Eviction is safe under every interleaving,+// because it never installs a value; the next read repopulates it from the row.func (s *Store) evictIamPolicyCache(workspace string, resourceType storepb.Policy_Resource, resource string) {s.policyCache.Remove(getPolicyCacheKey(workspace, resourceType, resource, storepb.Policy_IAM))s.iamPolicyCache.Remove(getIamPolicyCacheKey(workspace, resourceType, resource))diff --git a/backend/store/setting.go b/backend/store/setting.goindex a423806dca..d4730934c8 100644--- a/backend/store/setting.go+++ b/backend/store/setting.go@@ -266,28 +266,39 @@ func (s *Store) GetSetting(ctx context.Context, workspace string, name storepb.Sif !s.enableCache {return s.GetSettingUncached(ctx, workspace, name)}- if v, ok := s.settingCache.Get(getSettingCacheKey(workspace, name)); ok {+ key := getSettingCacheKey(workspace, name)+ if v, ok := s.settingCache.Get(key); ok {return v, nil}+ if s.settingMissHookForTest != nil {+ s.settingMissHookForTest()+ }// Fill the cache under the publish mutex so the fill's content is current// at publish time: an unordered fill could read a pre-commit snapshot and// cache it after a concurrent writer already published a newer value. No// row lock is held here, so the row-lock -> publish-lock order is kept.s.settingPublishMu.Lock()defer s.settingPublishMu.Unlock()+ // A fill that waited for the lock can find what the fill before it stored.+ if v, ok := s.settingCache.Get(key); ok {+ return v, nil+ }+ if s.settingFillHookForTest != nil {+ s.settingFillHookForTest()+ }setting, err := s.GetSettingUncached(ctx, workspace, name)if err != nil || setting == nil {return setting, err}- s.settingCache.Add(getSettingCacheKey(workspace, name), setting)+ s.settingCache.Add(key, setting)return setting, nil}// GetSettingUncached reads the setting directly from the database, bypassing-// the setting cache. The cache has no TTL and only in-process writes refresh-// it, so callers that must observe out-of-band writes (enforcement gates) or-// that merge-and-write the value back (read-modify-write updates) use this to-// avoid acting on a stale cached copy.+// the setting cache. An out-of-band write reaches the cache only when the+// entry expires (OutOfBandCacheLifetime), so callers that must observe it at+// once (enforcement gates) or that merge-and-write the value back+// (read-modify-write updates) use this to avoid acting on a stale cached copy.func (s *Store) GetSettingUncached(ctx context.Context, workspace string, name storepb.SettingName) (*SettingMessage, error) {settings, err := s.ListSettings(ctx, &FindSettingMessage{Workspace: workspace, Name: &name})if err != nil {@@ -573,9 +584,9 @@ func (s *Store) DeleteSetting(ctx context.Context, workspace string, name storep}// GetMCPSettingsUncached reads the workspace's stored MCP setting straight from-// the database, bypassing the setting cache. The cache has no TTL and only-// in-process writes refresh it, so a setting flipped out of band — direct SQL,-// another replica — must still bite on the next request.+// the database, bypassing the setting cache, so a setting flipped out of band —+// direct SQL, another replica — bites on the next request, not when the cached+// entry expires.//// Returns nil with any error; no caller may act on partially resolved settings.//diff --git a/backend/store/setting_atomic_test.go b/backend/store/setting_atomic_test.goindex 4bb969c0c9..95dd0abcc9 100644--- a/backend/store/setting_atomic_test.go+++ b/backend/store/setting_atomic_test.go@@ -546,3 +546,91 @@ func TestUpdateSettingAtomicRefusesAnUnparsedRow(t *testing.T) {`SELECT value::text FROM setting WHERE name = 'WORKSPACE_PROFILE' AND workspace = 'default'`).Scan(&stored))require.Contains(t, stored, "externalUrl", "the refused write must leave the row as it found it")}++// The first fill is held inside the lock until every other read has missed+// the cache, so each of them waits for the lock and then looks again.+func TestSimultaneousSettingMissesReadTheDatabaseOnce(t *testing.T) {+ t.Parallel()+ ctx, _, s := newSettingAtomicFixture(t)+ s.DeleteCache()+ var misses, fills atomic.Int32+ firstFillStarted := make(chan struct{})+ releaseFirstFill := make(chan struct{})+ s.SetSettingMissHookForTest(func() { misses.Add(1) })+ s.SetSettingFillHookForTest(func() {+ if fills.Add(1) == 1 {+ close(firstFillStarted)+ <-releaseFirstFill+ }+ })+ results := make(chan error, 5)+ read := func() {+ _, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ results <- err+ }++ go read()+ <-firstFillStarted+ for range 4 {+ go read()+ }+ require.Eventually(t, func() bool { return misses.Load() == 5 }, 10*time.Second, time.Millisecond)+ close(releaseFirstFill)+ for range 5 {+ require.NoError(t, <-results)+ }++ require.Equal(t, int32(1), fills.Load())+}++// Colliding keys: WORKSPACE_PROFILE in two workspaces, with the second read+// waiting behind the first workspace's fill.+func TestSettingFillWaitingBehindAnotherWorkspaceReadsItsOwnRow(t *testing.T) {+ t.Parallel()+ ctx, db, s := newSettingAtomicFixture(t)+ _, err := db.ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('other');`)+ require.NoError(t, err)+ _, err = s.UpsertSetting(ctx, &store.SettingMessage{+ Name: storepb.SettingName_WORKSPACE_PROFILE,+ Workspace: "other",+ Value: &storepb.WorkspaceProfileSetting{DisallowPasswordSignin: true},+ })+ require.NoError(t, err)+ s.DeleteCache()+ var misses, fills atomic.Int32+ firstFillStarted := make(chan struct{})+ releaseFirstFill := make(chan struct{})+ s.SetSettingMissHookForTest(func() { misses.Add(1) })+ s.SetSettingFillHookForTest(func() {+ if fills.Add(1) == 1 {+ close(firstFillStarted)+ <-releaseFirstFill+ }+ })+ type readResult struct {+ setting *store.SettingMessage+ err error+ }+ read := func(workspace string, results chan<- readResult) {+ setting, err := s.GetSetting(ctx, workspace, storepb.SettingName_WORKSPACE_PROFILE)+ results <- readResult{setting: setting, err: err}+ }++ defaultResult, otherResult := make(chan readResult, 1), make(chan readResult, 1)+ go read("default", defaultResult)+ <-firstFillStarted+ go read("other", otherResult)+ require.Eventually(t, func() bool { return misses.Load() == 2 }, 10*time.Second, time.Millisecond)+ close(releaseFirstFill)++ other := <-otherResult+ require.NoError(t, other.err)+ require.True(t, mustProfile(t, other.setting).DisallowPasswordSignin)+ first := <-defaultResult+ require.NoError(t, first.err)+ require.False(t, mustProfile(t, first.setting).DisallowPasswordSignin)+ require.Equal(t, int32(2), fills.Load())+ cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)+ require.NoError(t, err)+ require.False(t, mustProfile(t, cached).DisallowPasswordSignin)+}diff --git a/backend/store/store.go b/backend/store/store.goindex 3965eae15b..50901cbb38 100644--- a/backend/store/store.go+++ b/backend/store/store.go@@ -24,12 +24,12 @@ type Store struct {// Cache.authSecret atomic.Pointer[string]- userEmailCache *lru.Cache[string, *UserMessage]+ userEmailCache *expirable.LRU[string, *UserMessage]instanceCache *lru.Cache[string, *InstanceMessage]databaseCache *lru.Cache[string, *DatabaseMessage]projectCache *lru.Cache[string, *ProjectMessage]- policyCache *lru.Cache[string, *PolicyMessage]- settingCache *lru.Cache[string, *SettingMessage]+ policyCache *expirable.LRU[string, *PolicyMessage]+ settingCache *expirable.LRU[string, *SettingMessage]// projectPublishMu prevents a cache fill that read an old database snapshot// from publishing after a concurrent project update invalidates the cache.@@ -44,24 +44,35 @@ type Store struct {// settingPublishHookForTest, when set, runs between commit and cache// publish so tests can deterministically expose the ordering window.settingPublishHookForTest func()- rolesCache *expirable.LRU[string, *RoleMessage]- groupCache *expirable.LRU[string, *GroupMessage]- groupMembersCache *expirable.LRU[string, map[string]bool]- memberGroupsCache *expirable.LRU[string, []string]- dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]- iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]+ // settingMissHookForTest and settingFillHookForTest, when set, run after a+ // setting read misses the cache and before it waits for the publish mutex,+ // and right before a cache-miss fill reads the database.+ settingMissHookForTest func()+ settingFillHookForTest func()+ rolesCache *expirable.LRU[string, *RoleMessage]+ groupCache *expirable.LRU[string, *GroupMessage]+ groupMembersCache *expirable.LRU[string, map[string]bool]+ memberGroupsCache *expirable.LRU[string, []string]+ dbSchemaCache *expirable.LRU[string, *model.DatabaseMetadata]+ iamPolicyCache *expirable.LRU[string, *IamPolicyMessage]// Large objects.sheetFullCache *lru.Cache[string, *SheetMessage]}+// OutOfBandCacheLifetime bounds how long a cached user, setting or policy+// misses a change another process made to its row, such as `bytebase recovery`+// running beside the server. An entry expires this long after it is stored; a+// read must not extend it.+const OutOfBandCacheLifetime = time.Minute+// New creates a new instance of Store.// pgURL can be either a direct PostgreSQL URL or a file path containing the URL.func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {- userEmailCache, err := lru.New[string, *UserMessage](32768)- if err != nil {- return nil, err- }+ return newStore(ctx, pgURL, enableCache, OutOfBandCacheLifetime)+}++func newStore(ctx context.Context, pgURL string, enableCache bool, outOfBandCacheLifetime time.Duration) (*Store, error) {instanceCache, err := lru.New[string, *InstanceMessage](32768)if err != nil {return nil, err@@ -74,14 +85,6 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {if err != nil {return nil, err}- policyCache, err := lru.New[string, *PolicyMessage](4096)- if err != nil {- return nil, err- }- settingCache, err := lru.New[string, *SettingMessage](1024)- if err != nil {- return nil, err- }rolesCache := expirable.NewLRU[string, *RoleMessage](128, nil, time.Minute)sheetFullCache, err := lru.New[string, *SheetMessage](10)if err != nil {@@ -104,12 +107,9 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {enableCache: enableCache,// Cache.- userEmailCache: userEmailCache,instanceCache: instanceCache,databaseCache: databaseCache,projectCache: projectCache,- policyCache: policyCache,- settingCache: settingCache,rolesCache: rolesCache,sheetFullCache: sheetFullCache,groupCache: groupCache,@@ -117,6 +117,9 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {memberGroupsCache: memberGroupsCache,dbSchemaCache: dbSchemaCache,iamPolicyCache: iamPolicyCache,+ userEmailCache: expirable.NewLRU[string, *UserMessage](32768, nil, outOfBandCacheLifetime),+ settingCache: expirable.NewLRU[string, *SettingMessage](1024, nil, outOfBandCacheLifetime),+ policyCache: expirable.NewLRU[string, *PolicyMessage](4096, nil, outOfBandCacheLifetime),}return s, nildiff --git a/docs/design/non-interactive-recovery.md b/docs/design/non-interactive-recovery.mdindex a570b5a418..db3edd7152 100644--- a/docs/design/non-interactive-recovery.md+++ b/docs/design/non-interactive-recovery.md@@ -1,6 +1,6 @@# Non-interactive recovery beside a running Bytebase-Status: proposal · 2026-10-13 · [BYT-10271](https://linear.app/bytebase/issue/BYT-10271)+Status: implemented in this PR · 2026-10-14 · [BYT-10271](https://linear.app/bytebase/issue/BYT-10271)`bytebase recovery` restores password sign-in for a self-hosted workspace nobody can sign in to.Today it runs only as an interactive menu, and its runbook stops every Bytebase server first and@@ -36,11 +36,16 @@ What recovery changes, and where a running server reads it:The lockout counter that a reset also clears is read from the database on every attempt.-Running beside a server is already safe for the data, under the two conditions in D4. Recovery-starts no server, runs no migration and writes no replica heartbeat. Its setting and IAM writes-lock the row and merge inside the transaction (`UpdateSettingAtomic`, `PatchIamPolicy`), as the-server's own writes to those rows do, and the password change updates one column. The stop-first-rule guarded an IAM lost-update race that #21285 has since fixed. What remains is the cache.+Running beside a server is safe for the data, under the two conditions in D4. Recovery starts no+server, runs no migration and writes no replica heartbeat. Each of its setting and IAM writes+locks the row and computes the change from the locked row (`UpdateSettingAtomic`,+`PatchIamPolicy`). The menu's role grant did not: it set the user's roles to a list read before+the lock, so a role another server granted in between was lost. It now adds the one role inside+the lock, and only an unconditional binding counts as already holding it, because a conditional+one can expire. The password change also rewrites the profile's last-change time from the row it+read just before, so a sign-in by the same user at that moment can lose its last-sign-in time;+that is accepted. The stop-first rule guarded an IAM lost-update race that #21285 has since fixed.+What remains is the cache.## Principle@@ -61,9 +66,10 @@ bytebase recovery reset-password --email <email> --password <password>A subcommand runs one action, prints the result and exits. It needs no terminal and asks for noconfirmation: naming the action is the confirmation. It keeps every check the menu runs (exactlyone workspace, a usable administrator for `enable-password-signin`, the domain and password-restrictions for `reset-password`) and writes the same audit record. The menu stays and still needs-a terminal. Rejected: one flat flag set on `bytebase recovery` with an `--action` flag, where which-flags are required would depend on another flag's value.+restrictions for `reset-password`) and writes the same audit record. It reads the email as sign-in+does, trimmed and lowercased, so a reset clears the lockout that sign-in counted. The menu stays and+still needs a terminal. Rejected: one flat flag set on `bytebase recovery` with an `--action` flag,+where which flags are required would depend on another flag's value.**D2 · The password is a flag.** `--password` takes the new password on the command line, as`redis-cli -a`, `docker login --password`, `grafana cli admin reset-admin-password` and@@ -73,12 +79,26 @@ value on the command line is visible in the process list and the shell history,tells the user to change the password after signing in. Rejected: a password file or standardinput only, as `psql` and `docker login --password-stdin` do. It is safer, but it adds a step toevery use, and it can be added beside the flag later. This replaces the original recovery design's-rule that the password never comes from an argument.+rule that the password never comes from an argument. No error repeats the password, so a+mistyped `bytebase recovery` command cannot print it into a Job's log:++- A usage error names at most the command and a flag, never a value or an argument from the+ command line. The flag parser's own errors quote the token, so recovery replaces them.+- An error names the email only after it matched an account. Until then the value in `--email`+ can be a password given in the wrong flag, and a password can look like an email address+ (`Welcome@2024`).++A whole command given as one argument fails in `bytebase` or in the container runtime before+recovery runs, and that error repeats it, so the runbook says to give each word as its own item.**D3 · The subcommands do not add a workspace role.** After a reset, the menu offers a role from anumbered list to a user who has none. A flag would need the role's ID (`roles/workspaceAdmin`, or acustom role's ID), which nobody has to hand during an outage. `reset-password` reports that the-user has no workspace role and points at the menu. Rejected: `--add-role <role>`.+user has no workspace role and points at the menu, to use before the user tries to sign in. On a+self-hosted deployment, a sign-in attempt with the right password adds a user with no role as a+workspace member before it checks whether the attempt is allowed, so even a refused attempt (while+password sign-in is disabled, say) does it, and the menu then offers no role. Rejected:+`--add-role <role>`.**D4 · Recovery runs beside the running server.** The runbook and the command stop telling theoperator to stop and restart Bytebase. Recovery reaches the metadata database exactly as the server@@ -96,8 +116,13 @@ one. Two conditions stay:tag from the manifest. A tag can move: the Helm chart defaults to `latest` and always pulls, soa Job started later can get a newer build than the server runs.- **With the embedded database, run the command inside the server's container.** A second- container cannot reach the embedded PostgreSQL. If it mounts the same data directory, it starts- its own PostgreSQL on it.+ container cannot reach the embedded PostgreSQL through the server's socket. If it mounts the+ same data directory, it would start its own PostgreSQL there, and PostgreSQL's lock cannot see a+ postmaster in another container, so two would write the same files. Recovery therefore refuses+ to start PostgreSQL on a data directory that has a `postmaster.pid`, with the connection error+ it got and a note that the directory may be in use (the file can be left over from a crash).+ When no server runs and recovery starts PostgreSQL itself, do not start Bytebase until recovery+ exits: a server in another container cannot see recovery's PostgreSQL either.**D5 · The user, setting and policy caches expire after one minute.** `userEmailCache`,`settingCache` and `policyCache` become `expirable.LRU` with a one-minute lifetime, the type and@@ -142,13 +167,30 @@ saved. Recovery cannot confirm that a server applied the change; the limit holdsRejected: a `--wait` flag that sleeps for a minute so that a finished Job means the change is live.The caller can wait.+The result line is how a failure says what was saved. It is printed whenever what was asked for is+stored, including when a second run finds it already stored, and before the error of any later+step: the audit record, or clearing the lockout. The service marks this by returning its result+together with the error, and the subcommands and the menu's three actions all read it that way. A+failed lockout clear does not skip the audit record.++Waiting matters for a reset password. A server that has not read it yet rejects the new password,+and each rejected sign-in counts toward the password lockout: five within 10 minutes lock the+user's password sign-in for 10 minutes. The runbook says to wait the minute before signing in.+Running the same `reset-password` again clears a lock and changes nothing else. Rejected: reading+the user's password hash from the database again when the cached one does not match, so that a new+password works at once. Every rejected sign-in, including each guess in a password-guessing attack,+would then cost a database read.+**D7 · Simultaneous misses each reload.** A `userEmailCache` miss reloads every user-(`listAndCacheAllUsers`). With expiry that reload runs about once a minute, and requests that miss-at the same moment each run their own. `settingCache` serializes its fills, so simultaneous misses-on one setting read one after another. That is accepted for now: a server with `--ha` already-reloads the user list on every request. Rejected for now: sharing one reload between simultaneous-misses, as `LicenseService` does with `singleflight`. It is the first thing to add if a large-workspace shows a spike.+(`listAndCacheAllUsers`). With expiry that reload runs about once a minute for the users that+exist, and requests that miss at the same moment each run their own. An email with no user is never+cached, so each sign-in with one reloads the list; that is so today, and this design does not+change it. `settingCache` serializes its fills, and a fill that waited for the lock looks at the+cache again, so simultaneous misses on one stored setting cost one read. A setting with no row is+not cached, and the fills of all settings wait on the same lock. The user-list reloads are+accepted for now: a server with `--ha` already reloads the user list on every request. Rejected+for now: sharing one reload between simultaneous misses, as `LicenseService` does with+`singleflight`. It is the first thing to add if a large workspace shows a spike.## States@@ -160,13 +202,17 @@ What each command prints and returns:| `enable-password-signin`, already enabled | The same | 0 || `enable-password-signin`, no administrator has a password | The existing error | 1 || `reset-password`, an active user with a workspace role | `Password updated for user <email>. Live within one minute.` | 0 |-| `reset-password`, an active user with no workspace role | The same, then `User <email> has no workspace role. Run "bytebase recovery" to add one.` | 0 |-| `reset-password`, an unknown, deactivated or non-user account | The existing error | 1 |-| `reset-password`, the password breaks the workspace restriction, or the email is outside the allowed domains | The existing error | 1 |-| Either, a required flag is missing | A usage error. Nothing is read or written. | 1 |-| Either, the change is saved but the audit record fails | The result line, then the error | 1 |+| `reset-password`, the password is already the one given | The same | 0 |+| `reset-password`, an active user with no workspace role | The same, then `User <email> has no workspace role. Before they try to sign in, run "bytebase recovery" to add one.` | 0 |+| `reset-password`, the password is saved but the lookup of the user's workspace role fails | The same, then `Warning: could not check whether user <email> has a workspace role: <reason>`. The lookup only informs the operator. | 0 |+| `reset-password`, a deactivated or non-user account | The existing error | 1 |+| `reset-password`, no account has the email, or the email is outside the allowed domains | `the user does not exist`, or `the email does not belong to the workspace's allowed domains`: the email has matched no account, so the error does not repeat it | 1 |+| `reset-password`, the password breaks the workspace restriction | The existing error | 1 |+| Either, a flag is unknown or has no valid value, a required flag is missing or empty, `--email` is not an email address, `--password` looks like a variable reference that a shell or Kubernetes did not expand (`$NAME` or `${NAME}` with a shell name, or `$(NAME)` with any name a Secret or ConfigMap key can hold, since `envFrom` turns keys into variables and Kubernetes leaves an undefined one in place; `$(Str0ng!)` is taken as a password) or holds a character that cannot be typed (a control character such as a trailing newline, a line or paragraph separator, a byte-order mark, a zero-width space or word joiner, the replacement character U+FFFD, or invalid UTF-8), or an argument is unexpected | A usage error that repeats no value or argument from the command line. Nothing is read or written. | 1 |+| Either, what was asked for is stored but a later step fails: the audit record, or clearing the lockout | The result line, then, for `reset-password`, the note about a missing workspace role, then the error | 1 || Either, the metadata database has no workspace, or more than one | The existing error | 1 || `bytebase recovery` without a terminal | The existing error, now naming the two subcommands | 1 |+| `bytebase recovery` with an argument or a flag it does not take, most often a misspelled subcommand | A usage error that names the two subcommands, then the usage. It repeats nothing from the command line. | 1 |When a recovery change is live on a running server:@@ -222,7 +268,8 @@ container every time it exits and repeat the action without end.## What changes outside recovery- **Every server with the caches on reads each cached entry again once a minute.** Settings and- policies are single-row reads. The user list is one query a minute (D7).+ policies are single-row reads. The user list is reloaded about once a minute, once by each+ request that misses while a reload runs (D7).- **What these three caches hold is live within a minute after a direct change to the metadatadatabase.** Today it waits for a restart. That bounds a stale entry in any of them, includingthe policies that permission checks read. It does not reach what the server copies out of the@@ -230,7 +277,9 @@ container every time it exits and repeat the action without end.the caches that still have no expiry (projects, instances, databases). Recovery changes none ofthose.- **The menu keeps its prompts and its checks.** It loses only the lines that say to stop and- restart Bytebase.+ restart Bytebase. Where it said to restart, its result lines end with `Live within one minute.`,+ as the subcommands' do. Its enable action reports a setting saved before a failed audit record as+ saved, with a warning, as its reset already did; it used to report it as a failure.## Scope@@ -238,7 +287,8 @@ container every time it exits and repeat the action without end.|---|---|| `backend/bin/server/cmd/recovery.go` | The two subcommands (D1 to D3), the result lines and exit status (D6), and no stop or restart lines (D4) || `backend/store/store.go` | `userEmailCache`, `settingCache` and `policyCache` as `expirable.LRU` with a one-minute lifetime that tests can shorten (D5) |-| `backend/store/setting.go` | The comments that say the setting cache has no TTL |+| `backend/store/setting.go`, `backend/store/policy.go` | The comments that say the setting cache has no TTL, or that a stale policy stays cached with no expiry; a setting fill that waited for the lock looks at the cache again (D7) |+| `backend/component/recovery/service.go` | The menu's role grant computed inside the IAM lock; one email normalization shared with the subcommands || `docs/operations/admin-recovery.md`, `docs/operations/high-availability.md` | The runbook without the stop and the restart, with the one-minute wait, the subcommands, and the Job for each way a deployment supplies `PG_URL` |Tests:@@ -246,6 +296,15 @@ Tests:- `TestRecoveryCommandContract` changes. The menu still refuses a non-terminal, and the help liststhe two subcommands and their flags. New cases cover every row of the first States table throughthe existing `fakeRecoveryRunner`.+- Contract tests on the real recovery service for what the fake models: a result returned with the+ error when the audit record or the lockout clear fails after the save, for each of the three+ actions; a role grant that keeps a role another writer granted in between; and a role grant that+ does not count an expired binding as the role.+- On the real recovery service, the errors that come before the email matched an account (no such+ user, outside the allowed domains) do not repeat it, including for a password given as the+ email.+- A collision test for the setting fill: the same setting name in two workspaces, one read waiting+ behind the other's fill.- A store test with the caches on and a short lifetime. It warms the three caches, changes the rowsthrough a second store as recovery does, waits past the lifetime, and reads the new values. Asecond case reads before the lifetime ends and gets the cached value.@@ -254,8 +313,11 @@ Tests:## Not in this PR-- The implementation.- The public recovery page (bytebase/bytebase.com, `docs/administration/admin-recovery.mdx`), whichneeds the same changes as the runbook.- A message or a counter that makes the change instant or confirmed (D5).- A role flag, a password file, a wait flag and a shared user reload (D2, D3, D6, D7).+- A menu role for a user who already holds one. Since recovery was added (#21182), the menu has+ offered a role only to a user with none. With no usable administrator left, recovery therefore+ cannot promote a user who holds another role, such as Member after directory sync deactivated+ the only administrator, or after the user tried to sign in (D3). The runbook states the limit.diff --git a/docs/operations/admin-recovery.md b/docs/operations/admin-recovery.mdindex 0a01e5f498..db057ec4b3 100644--- a/docs/operations/admin-recovery.md+++ b/docs/operations/admin-recovery.md@@ -5,6 +5,10 @@ deployment. Recovery changes metadata through the Bytebase store; it does notprovide an alternate authentication endpoint or bypass the workspace passwordpolicy.+Recovery runs beside the running Bytebase servers. Nothing is stopped or+restarted: every server reads the change from the metadata database within one+minute.+## Support boundaryUse this procedure only when all of the following are true:@@ -13,26 +17,35 @@ Use this procedure only when all of the following are true:database.- The metadata database contains exactly one workspace, and it is active.- At least one existing, active end-user identity remains.-- You can run the same Bytebase binary version against the same metadata- database from a terminal.+- You can run the same Bytebase build as the running servers, and it reaches+ the metadata database exactly as they do.The recovery workflow does not create or reactivate identities, create or editroles, clear MFA, revoke tokens, or repair an identity provider. When anexisting user is no longer a workspace member, the operator can explicitly add-one existing role. Direct metadata SQL changes are not a supported recovery-procedure.+one existing role from the interactive menu. The menu offers that role only to+a user with no workspace role at all. If no usable administrator remains and+every user who could become one already holds a workspace role, for example+Member after directory sync deactivated the only administrator, recovery+cannot restore an administrator. Direct metadata SQL changes are not a+supported recovery procedure.After recovery, sign in through the ordinary`/auth` page.## Choose the action-| Situation | Recovery action |-| --- | --- |-| An existing administrator password is known, but password sign-in is disabled | **Enable password sign-in** |-| An existing user's password is unknown | **Reset user password** |-| An existing user was removed from the workspace IAM policy | **Reset user password**, confirm adding the user, and select a role |-| An administrator password is unknown and password sign-in is disabled | **Reset user password**, then **Enable password sign-in** in the same session |+| Situation | Command | Menu action |+| --- | --- | --- |+| An existing administrator password is known, but password sign-in is disabled | `bytebase recovery enable-password-signin` | **Enable password sign-in** |+| An existing user's password is unknown | `bytebase recovery reset-password --email <email> --password <password>` | **Reset user password** |+| An existing user was removed from the workspace IAM policy | Menu only | **Reset user password**, confirm adding the user, and select a role |+| An administrator password is unknown and password sign-in is disabled | `reset-password`, then `enable-password-signin` | **Reset user password**, then **Enable password sign-in** in the same session |++Each command runs one action, prints the result and exits. It needs no+terminal and asks for no confirmation: naming the action is the confirmation.+`bytebase recovery` without a subcommand opens the interactive menu, which+needs a terminal.## Before recovery@@ -42,23 +55,227 @@ After recovery, sign in through the ordinary- For embedded PostgreSQL, do not copy a live `pgdata` directory as a filebackup. Use a consistent PostgreSQL backup, or stop Bytebase before takinga file-level snapshot.-2. Use the same Bytebase binary version as the deployment. The recovery command- neither initializes nor migrates the metadata schema.-3. Stop every Bytebase server instance or replica connected to the metadata- database. Keep them stopped until the recovery session exits so no process- uses stale cached settings or IAM policies during recovery.-4. Reuse the deployment's metadata connection configuration. Recovery does not- define a separate format. Follow the existing+2. Use the same Bytebase build as the running servers. The recovery command+ neither initializes nor migrates the metadata schema, and an older build+ drops the workspace setting fields it does not know when it saves the+ setting. On Kubernetes, take the image the running pod reports, as described+ in [Run recovery on Kubernetes](#run-recovery-on-kubernetes).+3. Leave Bytebase running. Recovery is safe beside running servers, and they+ need no restart afterwards: each server reads the recovered credential,+ workspace setting and IAM policy again within one minute. A server started+ with `--ha` reads them at once.+4. Reach the metadata database exactly as the server does. Recovery does not+ define a separate connection format. Follow the existing[Docker deployment guide](https://docs.bytebase.com/get-started/self-host/deploy-with-docker)for embedded data and port configuration, or[Configure External PostgreSQL](https://docs.bytebase.com/get-started/self-host/external-postgres)- for `PG_URL`.-5. Run the command from an interactive terminal. It rejects redirected or- otherwise non-terminal input and output.+ for `PG_URL`. The connection can also depend on a `PG_URL` that is a file+ path, on the identity the server runs as (AWS RDS and Cloud SQL IAM+ authentication), or on mounted TLS files; recovery needs the same.+5. With embedded PostgreSQL, run recovery inside the server's container, with+ the server's `--data` and `--port`. Do not run it in a second container that+ mounts the data directory: two PostgreSQL servers on one data directory+ corrupt it, so recovery refuses to start its own while a `postmaster.pid`+ shows the directory may be in use.++## Run one action++```bash+bytebase recovery enable-password-signin+```++```bash+bytebase recovery reset-password --email <email> --password <password>+```++For external PostgreSQL, make the deployment's `PG_URL` available to the+command. For embedded PostgreSQL, include the deployment's `--data` and any+custom `--port`.++| Situation | Output | Exit status |+| --- | --- | --- |+| `enable-password-signin` succeeds, or password sign-in was already enabled | `Password sign-in is enabled. Live within one minute.` | 0 |+| `reset-password` succeeds | `Password updated for user <email>. Live within one minute.` | 0 |+| `reset-password` succeeds for a user with no workspace role | The same, then `User <email> has no workspace role. Before they try to sign in, run "bytebase recovery" to add one.` | 0 |+| `reset-password` succeeds, but looking up the user's workspace role fails | The same, then `Warning: could not check whether user <email> has a workspace role: <reason>` | 0 |+| What you asked for is stored, but a later step fails: writing the audit record, or clearing the lockout | The result line, then `Error: <reason>` | 1 |+| The action is refused or fails, and nothing is saved | `Error: <reason>` | 1 |+| The subcommand is misspelled, a flag is unknown, missing or empty, `--email` is not an email address, `--password` looks like a variable reference that nothing expanded or contains a character that cannot be typed, or an argument is unexpected | `Error: <reason>` and the usage. Nothing is read or written, and the error repeats no value or argument from the command line. | 1 |++The result line appears only when what you asked for is stored, including on a+second run that finds it already stored, and it comes before the error when a+later step fails. Both commands are safe to repeat: a second run changes nothing+and writes one more audit record.++**Live within one minute.** Each server drops its cached copy of a user, a+setting or a policy one minute after it stored it, and then reads the metadata+database again. Recovery cannot confirm that a server has done so, and does not+wait. A server that was reading the old value at the moment recovery saved the+change is late by the length of that read.++**Wait the minute before signing in.** Until a server reads a reset password,+it rejects the new password, and every rejected sign-in counts toward the+lockout: five within 10 minutes lock the user's password sign-in for 10+minutes. If that happens, run the same `reset-password` command again. It+clears the lock and changes nothing else.++**The password is a flag.** `--password` puts the new password on the command+line, where it is visible in the process list and the shell history. Pass it+from a secret instead of writing it into a script or manifest, and change the+password after signing in. The command refuses a value that a secret commonly+gets wrong: an empty one, one that looks like a variable reference that nothing+expanded (`$NAME`, `${NAME}` or `$(NAME)`), and one with a character that cannot+be typed at sign-in, such as the trailing newline or byte-order mark of a+Secret created from a file.++**No role is added.** `reset-password` does not add a workspace role. When the+user has none, it says so; run `bytebase recovery` from a terminal to add one+with the menu before the user tries to sign in. A sign-in attempt with the+right password adds a user with no role as a workspace member, even an attempt+that is then refused, for example while password sign-in is disabled, and the+menu then offers no role.++## Run recovery on Kubernetes++Run the command as a Job beside the running server. The Job's pod must run the+same build as the server and reach the metadata database exactly as the+server's pod does.++1. Find the image the running pod reports, by digest:++ ```bash+ kubectl get pod bytebase-0 -n <namespace> -o jsonpath='{.status.containerStatuses[?(@.name=="bytebase")].imageID}'+ ```++ Use that reference, with its registry, repository and digest. Drop a+ `docker-pullable://` prefix if the container runtime reports one. Do not+ reuse the tag from the manifest: the chart defaults to `latest` and always+ pulls, so a Job started later can get a newer build than the server runs.+2. For `reset-password`, put the new password in a Secret, so that it is not+ written in the manifest:++ ```bash+ kubectl create secret generic bytebase-recovery -n <namespace> --from-literal=password='<new password>'+ ```++3. Create the Job. This one is for a server that reads `PG_URL` from a Secret+ (the chart's `bytebase.option.externalPg.existingPgURLSecret`):++ ```yaml+ apiVersion: batch/v1+ kind: Job+ metadata:+ name: bytebase-recovery+ namespace: <the server's namespace>+ spec:+ backoffLimit: 0+ template:+ spec:+ restartPolicy: Never+ serviceAccountName: <the server pod's service account>+ containers:+ - name: recovery+ image: <the image from step 1>+ args: ["recovery", "reset-password", "--email", "<email>", "--password", "$(NEW_PASSWORD)"]+ env:+ - name: PG_URL+ valueFrom:+ secretKeyRef: { name: <existingPgURLSecret>, key: <existingPgURLSecretKey> }+ - name: NEW_PASSWORD+ valueFrom:+ secretKeyRef: { name: bytebase-recovery, key: password }+ ```++ Give each word as its own item, as above. A command given as one item fails+ before recovery runs, and that error repeats the command, password included.+ For `enable-password-signin`, use `args: ["recovery", "enable-password-signin"]`+ and leave out `NEW_PASSWORD`. Kubernetes substitutes `$(NEW_PASSWORD)` in+ `args` itself, without a shell. If the container defines no variable of that+ name, Kubernetes passes `$(NEW_PASSWORD)` through unchanged, and the command+ refuses it.+4. Read the result. The Job succeeds only when the change is saved.++ ```bash+ kubectl logs job/bytebase-recovery -n <namespace>+ ```++5. Delete the Job and the Secret once sign-in works:++ ```bash+ kubectl delete job/bytebase-recovery secret/bytebase-recovery -n <namespace> --ignore-not-found+ ```++`backoffLimit: 0` stops Kubernetes from retrying a failed run. A pod deleted+before the Job records its result can still be replaced, so an action can run+twice, which is safe. Do not run recovery as a Deployment or StatefulSet: they+restart the container every time it exits and repeat the action without end.++The Job above covers a server that reads `PG_URL` from a Secret. For the other+ways the chart supplies the connection, change the Job as follows. In every+case, the server pod's spec (`kubectl get pod bytebase-0 -n <namespace> -o yaml`)+holds the values to copy.++| How the server connects (chart values under `bytebase.option.externalPg`) | What the Job needs |+| --- | --- |+| `existingPgURLSecret` | The Job above, as written |+| `url`, or `pgHost`, `pgUsername` and `pgPassword` | The server container's `PG_URL` entry, copied as it is |+| `existingPgPasswordSecret` | The server container's `PG_PASSWORD` and `PG_URL` entries, in that order: `PG_URL` refers to `$(PG_PASSWORD)` |+| `escapePassword` | The server pod's init container, its `bytebase-shared-volume`, the mount of that volume in the container, and the server container's start script, which builds `PG_URL`. See below. |+| `awsRdsIam` or `gcpCloudSqlIam` | The server container's `PG_URL` entry and the server pod's service account, which carries the cloud identity |+| A `PG_URL` that is a file path, such as a mounted Secret | The volume and the mount that hold the file, at the same path |+| Mounted TLS files or other files (`bytebase.extraSecretMounts`) | The same volumes and mounts |++With `escapePassword`, the init container writes the escaped password to the+shared volume, and the start script reads it from there. Keep the server+container's `command`, its mount of the shared volume, and the line of its start+script that exports `PG_URL`, and replace the lines that start the server. In+the recovery container:++```yaml+command: ["/bin/sh", "-c"]+args:+ - |+ export PG_URL=postgres://<username>:$(cat /mnt/bytebase-shared-volume/pg_escaped_password)@<host>:<port>/<database>+ exec bytebase recovery reset-password --email '<email>' --password "$NEW_PASSWORD"+volumeMounts:+ - name: bytebase-shared-volume+ mountPath: /mnt/bytebase-shared-volume+```++Here the shell reads `"$NEW_PASSWORD"`, without parentheses, so that no+character of the password is interpreted as part of the script. Use this form+only inside the script; in plain `args`, Kubernetes expands only+`$(NEW_PASSWORD)`.++With embedded PostgreSQL, there is no Job. Run the command in the server's+container:++```bash+kubectl exec bytebase-0 -n <namespace> -c bytebase -- bytebase recovery enable-password-signin --data <data> --port <port>+```++## Run recovery with Docker++With embedded PostgreSQL, run the command in the server's container. Without+`-it`, `docker exec` provides no terminal, which the commands do not need:++```bash+docker exec <server container> bytebase recovery reset-password --email <email> --password <password>+```++With external PostgreSQL, start a container from the server's own image, on the+server's network and with the server's `PG_URL`.+`docker inspect --format '{{.Image}}' <server container>` prints the image ID+to use:++```bash+docker run --rm --network <the server's network> --env-file <file holding PG_URL=...> <image ID> recovery enable-password-signin+```## Start the recovery session-Run recovery with the same metadata connection configuration as the Bytebase+The interactive menu covers every action, including adding a role. Run+recovery with the same metadata connection configuration as the Bytebaseserver:```bash@@ -67,15 +284,19 @@ bytebase recoveryFor embedded PostgreSQL, include the deployment's `--data` and any custom`--port`. For external PostgreSQL, make the deployment's `PG_URL` available to-the recovery process.+the recovery process. In a container, allocate an interactive terminal, such as+`kubectl exec -it` or `docker exec -it`. With the chart's `escapePassword`, a+process started by `kubectl exec` does not get the `PG_URL` that the server's+start script builds, so build it the same way:++```bash+kubectl exec -it bytebase-0 -n <namespace> -c bytebase -- /bin/sh -c 'export PG_URL=postgres://<username>:$(cat /mnt/bytebase-shared-volume/pg_escaped_password)@<host>:<port>/<database>; exec bytebase recovery'+```The command requires exactly one workspace in the metadata database, verifiesthat it is active, resolves it automatically, and displays:```text-Stop all Bytebase servers that use this metadata database before continuing.-Restart all Bytebase servers after recovery so they reload updated settings and IAM policies.-Select a recovery action:1. Enable password sign-in2. Reset user password@@ -84,8 +305,8 @@ q. ExitFor an embedded deployment, the command connects to the metadata database whenavailable. Otherwise it starts only an existing initialized `pgdata` and stops-only the PostgreSQL instance it started itself. All Bytebase server processes-must remain stopped throughout the recovery session.+only the PostgreSQL instance it started itself. While it runs that instance,+do not start Bytebase on the same data directory: select `q. Exit` first.## Enable password sign-in@@ -95,7 +316,7 @@ must remain stopped throughout the recovery session.an effective, active end-user administrator with a password credential, thensets only `DisallowPasswordSignin` to `false` and records a warning auditevent.-4. Restart all Bytebase servers so authentication reloads the updated workspace+4. Wait up to one minute while the running servers read the updated workspacesetting.5. Sign in at `/auth` using the known administrator email and password.6. Repair and verify the configured identity provider.@@ -111,9 +332,9 @@ change any user, MFA configuration, IAM policy, or identity-provider setting.1. Select `2. Reset user password`.2. Enter the existing user's email. A blank email cancels the action.3. Review the displayed workspace password requirements.-4. Enter the new password. Password input is hidden and is never accepted- through a flag or environment variable. A policy violation explains the- problem and prompts for the password again before confirmation.+4. Enter the new password. The menu hides password input. A policy violation+ explains the problem and prompts for the password again before+ confirmation.5. Confirm the valid new password. A mismatch prompts for both entries again.Press `Ctrl+C` at either password prompt to exit recovery immediately.6. Wait for the password completion message. The command validates the password@@ -125,8 +346,9 @@ change any user, MFA configuration, IAM policy, or identity-provider setting.entered manually. Declining leaves the completed password reset intact.8. If password sign-in is disabled and this user is an administrator needed forrecovery, select **Enable password sign-in** in the same session.-9. Restart all Bytebase servers so authentication reloads the updated- credential, workspace setting, and IAM policy.+9. Wait one minute before signing in, while the running servers read the+ updated credential, workspace setting, and IAM policy. A sign-in attempt+ made earlier can be rejected and counts toward the lockout.10. Sign in at `/auth` with the new password, repair and verify the identityprovider, then re-enable **Disallow password sign-in**.11. Select `q. Exit`.@@ -138,32 +360,61 @@ direct role while preserving unrelated IAM bindings. It does not create orreactivate the user, clear MFA, change profile data, revoke refresh tokens, ormodify workspace settings.-Keep every Bytebase server stopped until the recovery session exits. Restarted-servers read the recovered credential, workspace setting, and IAM policy from-the metadata database without retaining a stale process-local cache.-## Troubleshooting### Recovery requires terminal input and output-Run the command from a terminal. When running inside a container or remote-execution environment, allocate an interactive TTY. Do not pipe input into the-command because password entry must remain hidden.+The menu needs a terminal. Without one, run+`bytebase recovery enable-password-signin` or+`bytebase recovery reset-password --email <email> --password <password>`. To+use the menu inside a container or remote execution environment, allocate an+interactive TTY. Do not pipe input into the menu, because its password entry+must remain hidden.++### The embedded metadata database may be in use++Recovery could not connect to the embedded metadata database and found a+`postmaster.pid` in the data directory, so a PostgreSQL server may be running+on it, possibly in another container, or the file is left over from a crash.+Run recovery inside the Bytebase server's container, with the server's `--data`+and `--port`. If no Bytebase server is running, start it once, then run+recovery that way.++### The change is not live after one minute++Check that recovery reached the same metadata database as the servers: the same+`PG_URL`, including the database name. A change saved to another database has+no effect on the servers.++### Sign-in is locked after a password reset++Sign-in attempts made before the servers read the new password are rejected+and count toward the lockout. Run the same+`bytebase recovery reset-password --email <email> --password <password>`+command, or the menu's reset with the same password, again: it clears the lock+and changes nothing else. Then wait one minute and sign in.### No usable workspace administrator existsThe enable action requires an effective, active end-user administrator with apassword credential. If such an administrator identity still exists but itspassword is unknown, reset that identity's password and retry the enable action.-If an existing active user lost its administrator binding, reset the password,-confirm adding the user, and select **Workspace admin**. Recovery cannot create-or reactivate an identity.+If an existing active user has no workspace role left, reset the password in+the menu, confirm adding the user, and select **Workspace admin**, before the+user tries to sign in. The menu offers a role only to a user with no workspace+role at all, so recovery cannot restore the administrator role of a user who+still holds another workspace role, including the member role that a sign-in+attempt adds. If another active user has no workspace role, make that user the+administrator instead. Recovery cannot create or reactivate an identity.### The user does not belong to the workspace-Verify the email. For an existing active end user, the reset action offers to-add one selected role. Missing, deleted, service-account, and workload-identity-targets remain rejected.+Verify the email. For an existing active end user, the menu's reset action+offers to add one selected role, and `reset-password` reports that the user+has no workspace role. Add the role before the user tries to sign in: an+attempt with the right password makes them a workspace member, even when it is+refused. Missing, deleted, service-account, and+workload-identity targets remain rejected.### The password is rejected@@ -173,10 +424,12 @@ allowed domain.### The metadata database cannot be opened-Verify that the command uses the deployment's current Bytebase binary and the-same metadata connection configuration described in the deployment guides.-Recovery refuses an uninitialized embedded data directory and does not migrate-an old metadata schema.+Verify that the command uses the deployment's current Bytebase build and+reaches the metadata database as the server does, as described in the+deployment guides. On Kubernetes, compare the Job with the server pod: the+service account, the environment, the mounted files and, with+`escapePassword`, the start script. Recovery refuses an uninitialized embedded+data directory and does not migrate an old metadata schema.### Multiple workspaces are found@@ -188,14 +441,16 @@ connected to the intended self-hosted metadata database.## After recovery-1. Restart all Bytebase servers, then confirm the recovered administrator can- sign in through `/auth`.+1. Wait one minute after the command reports success, then confirm the+ recovered administrator can sign in through `/auth`. No restart is needed.2. Verify the identity-provider configuration and federated sign-in.3. Re-enable **Disallow password sign-in** if it was enabled before the outage.-4. Review the warning-level recovery audit events. They identify the operation+4. If the password was passed with `--password`, change it after signing in,+ and delete the recovery Job and its Secret.+5. Review the warning-level recovery audit events. They identify the operationand user email. IAM recovery events also identify the selected role. Theymust not contain a password, password hash, or metadata credential.-5. Confirm unrelated MFA configuration, IAM bindings, refresh tokens, profiles,+6. Confirm unrelated MFA configuration, IAM bindings, refresh tokens, profiles,and workspace settings remain unchanged. When membership recovery was used,confirm only the selected direct IAM binding was added.diff --git a/docs/operations/high-availability.md b/docs/operations/high-availability.mdindex f6ab625465..9665c9d13a 100644--- a/docs/operations/high-availability.md+++ b/docs/operations/high-availability.md@@ -82,16 +82,17 @@ Use this checklist when enabling or validating HA:## Administrator recovery-Stop every Bytebase replica before running the interactive administrator-recovery command. Run `bytebase recovery` with the exact same shared `PG_URL`-used by every replica. The command connects directly to the metadata database-and does not start an HTTP server, scheduler, or background runner. Restart all-replicas after recovery so they reload the updated settings and IAM policy.+Run `bytebase recovery` beside the running replicas, with the exact same shared+`PG_URL` used by every replica; do not stop them. The command connects directly+to the metadata database and does not start an HTTP server, scheduler, or+background runner. Replicas started with `--ha` read users, settings and+policies from the metadata database on every request, so they see a recovery+change at once and need no restart.Follow the [administrator recovery runbook](./admin-recovery.md) for backup,-terminal, action-selection, login, and cleanup requirements. Recovery is not an-alternative to HA licensing or replica coordination, and it does not repair the-deployment topology.+action-selection, Kubernetes Job, login, and cleanup requirements. Recovery is+not an alternative to HA licensing or replica coordination, and it does not+repair the deployment topology.## Troubleshooting