Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (logical): Mark backup Failed, not Canceled, when SaveFile fails

Rostislav Dugin7366c20d0577
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
index 5104f530..d2be4bf8 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
@@ -40,6 +40,11 @@ const (
exitCodeConnectionError = 2
)
+var (
+ errBackupShutdown = errors.New("backup cancelled due to shutdown")
+ errBackupTimeout = errors.New("backup cancelled due to timeout")
+)
+
type CreateMariadbBackupUsecase struct {
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
@@ -170,7 +175,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
uc.logger.Info("Streaming MariaDB backup to storage", "mariadbBin", mariadbBin)
ctx, cancel := uc.createBackupContext(parentCtx)
- defer cancel()
+ defer cancel(nil)
myCnfFile, err := uc.createTempMyCnfFile(mdbConfig, password)
if err != nil {
@@ -235,7 +240,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
)
if saveErr != nil {
_ = storageReader.CloseWithError(saveErr)
- cancel()
+ cancel(saveErr)
}
saveErrCh <- saveErr
}()
@@ -275,7 +280,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
select {
case <-ctx.Done():
uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
- return nil, uc.checkCancellationReason()
+ return nil, uc.classifyCancellation(ctx)
default:
}
@@ -426,10 +431,14 @@ func (uc *CreateMariadbBackupUsecase) copyWithShutdownCheck(
func (uc *CreateMariadbBackupUsecase) createBackupContext(
parentCtx context.Context,
-) (context.Context, context.CancelFunc) {
- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)
+) (context.Context, context.CancelCauseFunc) {
+ ctx, cancel := context.WithCancelCause(parentCtx)
+
+ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })
go func() {
+ defer timeout.Stop()
+
ticker := time.NewTicker(shutdownCheckInterval)
defer ticker.Stop()
@@ -438,11 +447,11 @@ func (uc *CreateMariadbBackupUsecase) createBackupContext(
case <-ctx.Done():
return
case <-parentCtx.Done():
- cancel()
+ cancel(context.Cause(parentCtx))
return
case <-ticker.C:
if config.IsShouldShutdown() {
- cancel()
+ cancel(errBackupShutdown)
return
}
}
@@ -555,11 +564,19 @@ func (uc *CreateMariadbBackupUsecase) closeWriters(
return nil
}
-func (uc *CreateMariadbBackupUsecase) checkCancellationReason() error {
- if config.IsShouldShutdown() {
- return fmt.Errorf("backup cancelled due to shutdown")
+func (uc *CreateMariadbBackupUsecase) classifyCancellation(ctx context.Context) error {
+ cause := context.Cause(ctx)
+
+ switch {
+ case errors.Is(cause, errBackupShutdown):
+ return errors.New("backup cancelled due to shutdown")
+ case errors.Is(cause, errBackupTimeout):
+ return errors.New("backup cancelled due to timeout")
+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):
+ return errors.New("backup cancelled")
+ default:
+ return fmt.Errorf("save to storage: %w", cause)
}
- return fmt.Errorf("backup cancelled")
}
func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage(
diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
index 925df7f7..9beed0e9 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
@@ -34,6 +34,11 @@ const (
progressReportIntervalMB = 1.0
)
+var (
+ errBackupShutdown = errors.New("backup cancelled due to shutdown")
+ errBackupTimeout = errors.New("backup cancelled due to timeout")
+)
+
type CreateMongodbBackupUsecase struct {
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
@@ -133,7 +138,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
uc.logger.Info("Streaming MongoDB backup to storage", "mongodumpBin", mongodumpBin)
ctx, cancel := uc.createBackupContext(parentCtx)
- defer cancel()
+ defer cancel(nil)
cmd := exec.CommandContext(ctx, mongodumpBin, args...)
@@ -193,7 +198,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
)
if saveErr != nil {
_ = storageReader.CloseWithError(saveErr)
- cancel()
+ cancel(saveErr)
}
saveErrCh <- saveErr
}()
@@ -232,7 +237,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
select {
case <-ctx.Done():
uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
- return nil, uc.checkCancellationReason()
+ return nil, uc.classifyCancellation(ctx)
default:
}
@@ -263,10 +268,14 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
func (uc *CreateMongodbBackupUsecase) createBackupContext(
parentCtx context.Context,
-) (context.Context, context.CancelFunc) {
- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)
+) (context.Context, context.CancelCauseFunc) {
+ ctx, cancel := context.WithCancelCause(parentCtx)
+
+ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })
go func() {
+ defer timeout.Stop()
+
ticker := time.NewTicker(shutdownCheckInterval)
defer ticker.Stop()
for {
@@ -275,7 +284,7 @@ func (uc *CreateMongodbBackupUsecase) createBackupContext(
return
case <-ticker.C:
if config.IsShouldShutdown() {
- cancel()
+ cancel(errBackupShutdown)
return
}
}
@@ -417,11 +426,19 @@ func (uc *CreateMongodbBackupUsecase) closeWriters(
return nil
}
-func (uc *CreateMongodbBackupUsecase) checkCancellationReason() error {
- if config.IsShouldShutdown() {
+func (uc *CreateMongodbBackupUsecase) classifyCancellation(ctx context.Context) error {
+ cause := context.Cause(ctx)
+
+ switch {
+ case errors.Is(cause, errBackupShutdown):
return errors.New("backup cancelled due to shutdown")
+ case errors.Is(cause, errBackupTimeout):
+ return errors.New("backup cancelled due to timeout")
+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):
+ return errors.New("backup cancelled")
+ default:
+ return fmt.Errorf("save to storage: %w", cause)
}
- return errors.New("backup cancelled due to timeout")
}
func (uc *CreateMongodbBackupUsecase) buildMongodumpErrorMessage(
diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
index 4b151a37..abdae4eb 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
@@ -40,6 +40,11 @@ const (
exitCodeConnectionError = 2
)
+var (
+ errBackupShutdown = errors.New("backup cancelled due to shutdown")
+ errBackupTimeout = errors.New("backup cancelled due to timeout")
+)
+
type CreateMysqlBackupUsecase struct {
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
@@ -189,7 +194,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
uc.logger.Info("Streaming MySQL backup to storage", "mysqlBin", mysqlBin)
ctx, cancel := uc.createBackupContext(parentCtx)
- defer cancel()
+ defer cancel(nil)
myCnfFile, err := uc.createTempMyCnfFile(myConfig, password)
if err != nil {
@@ -254,7 +259,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
)
if saveErr != nil {
_ = storageReader.CloseWithError(saveErr)
- cancel()
+ cancel(saveErr)
}
saveErrCh <- saveErr
}()
@@ -294,7 +299,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
select {
case <-ctx.Done():
uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
- return nil, uc.checkCancellationReason()
+ return nil, uc.classifyCancellation(ctx)
default:
}
@@ -445,10 +450,14 @@ func (uc *CreateMysqlBackupUsecase) copyWithShutdownCheck(
func (uc *CreateMysqlBackupUsecase) createBackupContext(
parentCtx context.Context,
-) (context.Context, context.CancelFunc) {
- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)
+) (context.Context, context.CancelCauseFunc) {
+ ctx, cancel := context.WithCancelCause(parentCtx)
+
+ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })
go func() {
+ defer timeout.Stop()
+
ticker := time.NewTicker(shutdownCheckInterval)
defer ticker.Stop()
@@ -457,11 +466,11 @@ func (uc *CreateMysqlBackupUsecase) createBackupContext(
case <-ctx.Done():
return
case <-parentCtx.Done():
- cancel()
+ cancel(context.Cause(parentCtx))
return
case <-ticker.C:
if config.IsShouldShutdown() {
- cancel()
+ cancel(errBackupShutdown)
return
}
}
@@ -574,11 +583,19 @@ func (uc *CreateMysqlBackupUsecase) closeWriters(
return nil
}
-func (uc *CreateMysqlBackupUsecase) checkCancellationReason() error {
- if config.IsShouldShutdown() {
- return fmt.Errorf("backup cancelled due to shutdown")
+func (uc *CreateMysqlBackupUsecase) classifyCancellation(ctx context.Context) error {
+ cause := context.Cause(ctx)
+
+ switch {
+ case errors.Is(cause, errBackupShutdown):
+ return errors.New("backup cancelled due to shutdown")
+ case errors.Is(cause, errBackupTimeout):
+ return errors.New("backup cancelled due to timeout")
+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):
+ return errors.New("backup cancelled")
+ default:
+ return fmt.Errorf("save to storage: %w", cause)
}
- return fmt.Errorf("backup cancelled")
}
func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage(
diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
index cfe4bd7e..dac8718b 100644
--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
@@ -42,6 +42,11 @@ const (
exitCodeConnectionError = 2
)
+var (
+ errBackupShutdown = errors.New("backup cancelled due to shutdown")
+ errBackupTimeout = errors.New("backup cancelled due to timeout")
+)
+
type CreatePostgresqlBackupUsecase struct {
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
@@ -134,7 +139,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
uc.logger.Info("Streaming PostgreSQL backup to storage", "pgBin", pgBin, "args", args)
ctx, cancel := uc.createBackupContext(parentCtx)
- defer cancel()
+ defer cancel(nil)
credentials, err := postgresql_shared.WriteCredentialFilesToTempDir(
db.PostgresqlLogical.CredentialSpec(), password, uc.fieldEncryptor)
@@ -201,7 +206,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
)
if saveErr != nil {
_ = storageReader.CloseWithError(saveErr)
- cancel()
+ cancel(saveErr)
}
saveErrCh <- saveErr
}()
@@ -242,7 +247,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
select {
case <-ctx.Done():
uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
- return nil, uc.checkCancellationReason()
+ return nil, uc.classifyCancellation(ctx)
default:
}
@@ -404,10 +409,14 @@ func (uc *CreatePostgresqlBackupUsecase) isOlderPostgresVersion(
func (uc *CreatePostgresqlBackupUsecase) createBackupContext(
parentCtx context.Context,
-) (context.Context, context.CancelFunc) {
- ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)
+) (context.Context, context.CancelCauseFunc) {
+ ctx, cancel := context.WithCancelCause(parentCtx)
+
+ timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })
go func() {
+ defer timeout.Stop()
+
ticker := time.NewTicker(shutdownCheckInterval)
defer ticker.Stop()
@@ -416,11 +425,11 @@ func (uc *CreatePostgresqlBackupUsecase) createBackupContext(
case <-ctx.Done():
return
case <-parentCtx.Done():
- cancel()
+ cancel(context.Cause(parentCtx))
return
case <-ticker.C:
if config.IsShouldShutdown() {
- cancel()
+ cancel(errBackupShutdown)
return
}
}
@@ -567,22 +576,26 @@ func (uc *CreatePostgresqlBackupUsecase) closeWriters(
}
func (uc *CreatePostgresqlBackupUsecase) checkCancellation(ctx context.Context) error {
- select {
- case <-ctx.Done():
- if config.IsShouldShutdown() {
- return fmt.Errorf("backup cancelled due to shutdown")
- }
- return fmt.Errorf("backup cancelled")
- default:
+ if ctx.Err() == nil {
return nil
}
+
+ return uc.classifyCancellation(ctx)
}
-func (uc *CreatePostgresqlBackupUsecase) checkCancellationReason() error {
- if config.IsShouldShutdown() {
- return fmt.Errorf("backup cancelled due to shutdown")
+func (uc *CreatePostgresqlBackupUsecase) classifyCancellation(ctx context.Context) error {
+ cause := context.Cause(ctx)
+
+ switch {
+ case errors.Is(cause, errBackupShutdown):
+ return errors.New("backup cancelled due to shutdown")
+ case errors.Is(cause, errBackupTimeout):
+ return errors.New("backup cancelled due to timeout")
+ case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):
+ return errors.New("backup cancelled")
+ default:
+ return fmt.Errorf("save to storage: %w", cause)
}
- return fmt.Errorf("backup cancelled")
}
func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(