PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (logical): Mark backup Failed, not Canceled, when SaveFile fails
Rostislav Dugin7366c20d0577
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.goindex 5104f530..d2be4bf8 100644--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go@@ -40,6 +40,11 @@ const (exitCodeConnectionError = 2)+var (+ errBackupShutdown = errors.New("backup cancelled due to shutdown")+ errBackupTimeout = errors.New("backup cancelled due to timeout")+)+type CreateMariadbBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -170,7 +175,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(uc.logger.Info("Streaming MariaDB backup to storage", "mariadbBin", mariadbBin)ctx, cancel := uc.createBackupContext(parentCtx)- defer cancel()+ defer cancel(nil)myCnfFile, err := uc.createTempMyCnfFile(mdbConfig, password)if err != nil {@@ -235,7 +240,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage()if saveErr != nil {_ = storageReader.CloseWithError(saveErr)- cancel()+ cancel(saveErr)}saveErrCh <- saveErr}()@@ -275,7 +280,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)- return nil, uc.checkCancellationReason()+ return nil, uc.classifyCancellation(ctx)default:}@@ -426,10 +431,14 @@ func (uc *CreateMariadbBackupUsecase) copyWithShutdownCheck(func (uc *CreateMariadbBackupUsecase) createBackupContext(parentCtx context.Context,-) (context.Context, context.CancelFunc) {- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)+) (context.Context, context.CancelCauseFunc) {+ ctx, cancel := context.WithCancelCause(parentCtx)++ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })go func() {+ defer timeout.Stop()+ticker := time.NewTicker(shutdownCheckInterval)defer ticker.Stop()@@ -438,11 +447,11 @@ func (uc *CreateMariadbBackupUsecase) createBackupContext(case <-ctx.Done():returncase <-parentCtx.Done():- cancel()+ cancel(context.Cause(parentCtx))returncase <-ticker.C:if config.IsShouldShutdown() {- cancel()+ cancel(errBackupShutdown)return}}@@ -555,11 +564,19 @@ func (uc *CreateMariadbBackupUsecase) closeWriters(return nil}-func (uc *CreateMariadbBackupUsecase) checkCancellationReason() error {- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+func (uc *CreateMariadbBackupUsecase) classifyCancellation(ctx context.Context) error {+ cause := context.Cause(ctx)++ switch {+ case errors.Is(cause, errBackupShutdown):+ return errors.New("backup cancelled due to shutdown")+ case errors.Is(cause, errBackupTimeout):+ return errors.New("backup cancelled due to timeout")+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):+ return errors.New("backup cancelled")+ default:+ return fmt.Errorf("save to storage: %w", cause)}- return fmt.Errorf("backup cancelled")}func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage(diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.goindex 925df7f7..9beed0e9 100644--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go@@ -34,6 +34,11 @@ const (progressReportIntervalMB = 1.0)+var (+ errBackupShutdown = errors.New("backup cancelled due to shutdown")+ errBackupTimeout = errors.New("backup cancelled due to timeout")+)+type CreateMongodbBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -133,7 +138,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(uc.logger.Info("Streaming MongoDB backup to storage", "mongodumpBin", mongodumpBin)ctx, cancel := uc.createBackupContext(parentCtx)- defer cancel()+ defer cancel(nil)cmd := exec.CommandContext(ctx, mongodumpBin, args...)@@ -193,7 +198,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage()if saveErr != nil {_ = storageReader.CloseWithError(saveErr)- cancel()+ cancel(saveErr)}saveErrCh <- saveErr}()@@ -232,7 +237,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)- return nil, uc.checkCancellationReason()+ return nil, uc.classifyCancellation(ctx)default:}@@ -263,10 +268,14 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(func (uc *CreateMongodbBackupUsecase) createBackupContext(parentCtx context.Context,-) (context.Context, context.CancelFunc) {- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)+) (context.Context, context.CancelCauseFunc) {+ ctx, cancel := context.WithCancelCause(parentCtx)++ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })go func() {+ defer timeout.Stop()+ticker := time.NewTicker(shutdownCheckInterval)defer ticker.Stop()for {@@ -275,7 +284,7 @@ func (uc *CreateMongodbBackupUsecase) createBackupContext(returncase <-ticker.C:if config.IsShouldShutdown() {- cancel()+ cancel(errBackupShutdown)return}}@@ -417,11 +426,19 @@ func (uc *CreateMongodbBackupUsecase) closeWriters(return nil}-func (uc *CreateMongodbBackupUsecase) checkCancellationReason() error {- if config.IsShouldShutdown() {+func (uc *CreateMongodbBackupUsecase) classifyCancellation(ctx context.Context) error {+ cause := context.Cause(ctx)++ switch {+ case errors.Is(cause, errBackupShutdown):return errors.New("backup cancelled due to shutdown")+ case errors.Is(cause, errBackupTimeout):+ return errors.New("backup cancelled due to timeout")+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):+ return errors.New("backup cancelled")+ default:+ return fmt.Errorf("save to storage: %w", cause)}- return errors.New("backup cancelled due to timeout")}func (uc *CreateMongodbBackupUsecase) buildMongodumpErrorMessage(diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.goindex 4b151a37..abdae4eb 100644--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go@@ -40,6 +40,11 @@ const (exitCodeConnectionError = 2)+var (+ errBackupShutdown = errors.New("backup cancelled due to shutdown")+ errBackupTimeout = errors.New("backup cancelled due to timeout")+)+type CreateMysqlBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -189,7 +194,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(uc.logger.Info("Streaming MySQL backup to storage", "mysqlBin", mysqlBin)ctx, cancel := uc.createBackupContext(parentCtx)- defer cancel()+ defer cancel(nil)myCnfFile, err := uc.createTempMyCnfFile(myConfig, password)if err != nil {@@ -254,7 +259,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage()if saveErr != nil {_ = storageReader.CloseWithError(saveErr)- cancel()+ cancel(saveErr)}saveErrCh <- saveErr}()@@ -294,7 +299,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)- return nil, uc.checkCancellationReason()+ return nil, uc.classifyCancellation(ctx)default:}@@ -445,10 +450,14 @@ func (uc *CreateMysqlBackupUsecase) copyWithShutdownCheck(func (uc *CreateMysqlBackupUsecase) createBackupContext(parentCtx context.Context,-) (context.Context, context.CancelFunc) {- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)+) (context.Context, context.CancelCauseFunc) {+ ctx, cancel := context.WithCancelCause(parentCtx)++ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })go func() {+ defer timeout.Stop()+ticker := time.NewTicker(shutdownCheckInterval)defer ticker.Stop()@@ -457,11 +466,11 @@ func (uc *CreateMysqlBackupUsecase) createBackupContext(case <-ctx.Done():returncase <-parentCtx.Done():- cancel()+ cancel(context.Cause(parentCtx))returncase <-ticker.C:if config.IsShouldShutdown() {- cancel()+ cancel(errBackupShutdown)return}}@@ -574,11 +583,19 @@ func (uc *CreateMysqlBackupUsecase) closeWriters(return nil}-func (uc *CreateMysqlBackupUsecase) checkCancellationReason() error {- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+func (uc *CreateMysqlBackupUsecase) classifyCancellation(ctx context.Context) error {+ cause := context.Cause(ctx)++ switch {+ case errors.Is(cause, errBackupShutdown):+ return errors.New("backup cancelled due to shutdown")+ case errors.Is(cause, errBackupTimeout):+ return errors.New("backup cancelled due to timeout")+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):+ return errors.New("backup cancelled")+ default:+ return fmt.Errorf("save to storage: %w", cause)}- return fmt.Errorf("backup cancelled")}func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage(diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.goindex cfe4bd7e..dac8718b 100644--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go@@ -42,6 +42,11 @@ const (exitCodeConnectionError = 2)+var (+ errBackupShutdown = errors.New("backup cancelled due to shutdown")+ errBackupTimeout = errors.New("backup cancelled due to timeout")+)+type CreatePostgresqlBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -134,7 +139,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(uc.logger.Info("Streaming PostgreSQL backup to storage", "pgBin", pgBin, "args", args)ctx, cancel := uc.createBackupContext(parentCtx)- defer cancel()+ defer cancel(nil)credentials, err := postgresql_shared.WriteCredentialFilesToTempDir(db.PostgresqlLogical.CredentialSpec(), password, uc.fieldEncryptor)@@ -201,7 +206,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage()if saveErr != nil {_ = storageReader.CloseWithError(saveErr)- cancel()+ cancel(saveErr)}saveErrCh <- saveErr}()@@ -242,7 +247,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)- return nil, uc.checkCancellationReason()+ return nil, uc.classifyCancellation(ctx)default:}@@ -404,10 +409,14 @@ func (uc *CreatePostgresqlBackupUsecase) isOlderPostgresVersion(func (uc *CreatePostgresqlBackupUsecase) createBackupContext(parentCtx context.Context,-) (context.Context, context.CancelFunc) {- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)+) (context.Context, context.CancelCauseFunc) {+ ctx, cancel := context.WithCancelCause(parentCtx)++ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })go func() {+ defer timeout.Stop()+ticker := time.NewTicker(shutdownCheckInterval)defer ticker.Stop()@@ -416,11 +425,11 @@ func (uc *CreatePostgresqlBackupUsecase) createBackupContext(case <-ctx.Done():returncase <-parentCtx.Done():- cancel()+ cancel(context.Cause(parentCtx))returncase <-ticker.C:if config.IsShouldShutdown() {- cancel()+ cancel(errBackupShutdown)return}}@@ -567,22 +576,26 @@ func (uc *CreatePostgresqlBackupUsecase) closeWriters(}func (uc *CreatePostgresqlBackupUsecase) checkCancellation(ctx context.Context) error {- select {- case <-ctx.Done():- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")- }- return fmt.Errorf("backup cancelled")- default:+ if ctx.Err() == nil {return nil}++ return uc.classifyCancellation(ctx)}-func (uc *CreatePostgresqlBackupUsecase) checkCancellationReason() error {- if config.IsShouldShutdown() {- return fmt.Errorf("backup cancelled due to shutdown")+func (uc *CreatePostgresqlBackupUsecase) classifyCancellation(ctx context.Context) error {+ cause := context.Cause(ctx)++ switch {+ case errors.Is(cause, errBackupShutdown):+ return errors.New("backup cancelled due to shutdown")+ case errors.Is(cause, errBackupTimeout):+ return errors.New("backup cancelled due to timeout")+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):+ return errors.New("backup cancelled")+ default:+ return fmt.Errorf("save to storage: %w", cause)}- return fmt.Errorf("backup cancelled")}func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(