PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (rclone): Add retry for the case of rclone failure over restore
Rostislav Dugin83c454336c53
diff --git a/backend/internal/features/restores/core/stream_failure.go b/backend/internal/features/restores/core/stream_failure.gonew file mode 100644index 00000000..9102f1c9--- /dev/null+++ b/backend/internal/features/restores/core/stream_failure.go@@ -0,0 +1,22 @@+package restores_core++import (+ "fmt"++ io_utils "databasus-backend/internal/util/io"+)++// GetBackupStreamFailure must be consulted before the restore client's own exit error:+// a truncated download makes the client fail on the half-written statement it was fed,+// and that downstream syntax error would otherwise be reported as the cause.+func GetBackupStreamFailure(backupStreamReader *io_utils.FailureTrackingReader) error {+ readError := backupStreamReader.GetFirstReadError()+ if readError == nil {+ return nil+ }++ return fmt.Errorf(+ "backup stream from storage failed mid-restore, target database is left partially restored: %w",+ readError,+ )+}diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.goindex fc720592..90033731 100644--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go@@ -28,6 +28,7 @@ import (restores_core "databasus-backend/internal/features/restores/core""databasus-backend/internal/features/storages"util_encryption "databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io""databasus-backend/internal/util/tools")@@ -202,7 +203,8 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(}defer zstdReader.Close()- cmd.Stdin = zstdReader+ backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)+ cmd.Stdin = backupStreamReadercmd.Env = os.Environ()cmd.Env = append(cmd.Env,@@ -242,6 +244,10 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(return fmt.Errorf("restore cancelled due to shutdown")}+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ return streamErr+ }+if waitErr != nil {return uc.handleMariadbRestoreError(database, waitErr, stderrOutput, mariadbBin)}diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.goindex b94a582c..db0db1a5 100644--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go@@ -24,6 +24,7 @@ import (restores_core "databasus-backend/internal/features/restores/core""databasus-backend/internal/features/storages"util_encryption "databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io""databasus-backend/internal/util/tools")@@ -199,7 +200,9 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(inputReader = decryptReader}- cmd.Stdin = inputReader+ backupStreamReader := io_utils.NewFailureTrackingReader(inputReader)+ cmd.Stdin = backupStreamReader+cmd.Env = os.Environ()cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8", "LANG=C.UTF-8")@@ -234,6 +237,10 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(return fmt.Errorf("restore cancelled due to shutdown")}+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ return streamErr+ }+if waitErr != nil {return uc.handleMongoRestoreError(waitErr, stderrOutput, mongorestoreBin)}diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.goindex 898b284b..ad16e0bb 100644--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go@@ -28,6 +28,7 @@ import (restores_core "databasus-backend/internal/features/restores/core""databasus-backend/internal/features/storages"util_encryption "databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io""databasus-backend/internal/util/tools")@@ -183,7 +184,8 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(}defer zstdReader.Close()- cmd.Stdin = zstdReader+ backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)+ cmd.Stdin = backupStreamReadercmd.Env = os.Environ()cmd.Env = append(cmd.Env,@@ -223,6 +225,10 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(return fmt.Errorf("restore cancelled due to shutdown")}+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ return streamErr+ }+if waitErr != nil {return uc.handleMysqlRestoreError(database, waitErr, stderrOutput, mysqlBin)}diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.gonew file mode 100644index 00000000..e93d7b49--- /dev/null+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go@@ -0,0 +1,104 @@+package usecases_mysql++import (+ "bytes"+ "io"+ "log/slog"+ "os"+ "path/filepath"+ "runtime"+ "testing"++ "github.com/klauspost/compress/zstd"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ "databasus-backend/internal/features/databases"+)++// The masking this test covers only happens when the client process is fed a real OS pipe+// by os/exec, so there is no HTTP-level way to reach it — hence a stub client binary and a+// deliberately truncated stream instead of a controller test.+func writeFailingMysqlClientStub(t *testing.T) string {+ t.Helper()++ if runtime.GOOS == "windows" {+ t.Skip("stub client relies on a POSIX shell")+ }++ stubPath := filepath.Join(t.TempDir(), "mysql-stub")+ stubScript := "#!/bin/sh\ncat > /dev/null\necho \"ERROR 1064 (42000) at line 53465: You have an error in your SQL syntax\" >&2\nexit 1\n"++ require.NoError(t, os.WriteFile(stubPath, []byte(stubScript), 0o700))++ return stubPath+}++func compressToZstd(t *testing.T, payload []byte) []byte {+ t.Helper()++ var compressed bytes.Buffer++ zstdWriter, err := zstd.NewWriter(&compressed)+ require.NoError(t, err)++ _, err = zstdWriter.Write(payload)+ require.NoError(t, err)+ require.NoError(t, zstdWriter.Close())++ return compressed.Bytes()+}++func Test_ExecuteMysqlRestore_WhenBackupStreamTruncates_ReportsStreamFailureNotSyntaxError(+ t *testing.T,+) {+ dumpStatements := bytes.Repeat(+ []byte("INSERT INTO orders VALUES ('2020-07-13 18:49:41');\n"),+ 200_000,+ )+ compressedDump := compressToZstd(t, dumpStatements)+ truncatedDump := compressedDump[:len(compressedDump)*3/4]++ restoreUsecase := &RestoreMysqlBackupUsecase{+ slog.New(slog.DiscardHandler),+ nil,+ }++ restoreErr := restoreUsecase.executeMysqlRestore(+ t.Context(),+ &databases.Database{},+ writeFailingMysqlClientStub(t),+ nil,+ filepath.Join(t.TempDir(), ".my.cnf"),+ io.NopCloser(bytes.NewReader(truncatedDump)),+ &backups_core_logical.LogicalBackup{},+ )++ require.Error(t, restoreErr)+ assert.Contains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")+ assert.NotContains(t, restoreErr.Error(), "SQL syntax")+}++func Test_ExecuteMysqlRestore_WhenBackupStreamIsComplete_ReportsClientExitError(t *testing.T) {+ compressedDump := compressToZstd(t, []byte("INSERT INTO orders VALUES (1);\n"))++ restoreUsecase := &RestoreMysqlBackupUsecase{+ slog.New(slog.DiscardHandler),+ nil,+ }++ restoreErr := restoreUsecase.executeMysqlRestore(+ t.Context(),+ &databases.Database{},+ writeFailingMysqlClientStub(t),+ nil,+ filepath.Join(t.TempDir(), ".my.cnf"),+ io.NopCloser(bytes.NewReader(compressedDump)),+ &backups_core_logical.LogicalBackup{},+ )++ require.Error(t, restoreErr)+ assert.NotContains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")+ assert.Contains(t, restoreErr.Error(), "SQL syntax")+}diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.goindex a85703a9..ad50e7ff 100644--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go@@ -28,6 +28,7 @@ import (restores_core "databasus-backend/internal/features/restores/core""databasus-backend/internal/features/storages"util_encryption "databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io""databasus-backend/internal/util/tools")@@ -311,10 +312,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(return fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)}- // Copy backup data to stdin in a separate goroutine with proper error handling+ backupStreamReader := io_utils.NewFailureTrackingReader(backupReader)+copyErrCh := make(chan error, 1)go func() {- _, copyErr := io.Copy(stdinPipe, backupReader)+ _, copyErr := io.Copy(stdinPipe, backupStreamReader)// Close stdin pipe to signal EOF to pg_restore - critical for proper terminationcloseErr := stdinPipe.Close()switch {@@ -346,6 +348,10 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(return fmt.Errorf("restore cancelled due to shutdown")}+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ return streamErr+ }+// Check for copy errors first - these indicate issues with decryption or data readingif copyErr != nil {return fmt.Errorf("failed to stream backup data to pg_restore: %w", copyErr)diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.goindex 5a86b4d2..ef1bd270 100644--- a/backend/internal/features/storages/models/rclone/model.go+++ b/backend/internal/features/storages/models/rclone/model.go@@ -107,7 +107,10 @@ func (r *RcloneStorage) GetFile(return nil, fmt.Errorf("failed to get object from rclone: %w", err)}- reader, err := obj.Open(ctx)+ // operations.Open resumes from the last byte read when the transfer drops, so a+ // multi-gigabyte restore survives transient remote failures instead of handing the+ // caller a truncated stream.+ reader, err := operations.Open(ctx, obj)if err != nil {return nil, fmt.Errorf("failed to open object from rclone: %w", err)}diff --git a/backend/internal/util/io/failure_tracking_reader.go b/backend/internal/util/io/failure_tracking_reader.gonew file mode 100644index 00000000..65434303--- /dev/null+++ b/backend/internal/util/io/failure_tracking_reader.go@@ -0,0 +1,42 @@+package io_utils++import (+ "errors"+ "io"+ "sync"+)++// FailureTrackingReader exists because os/exec discards the stdin copy error whenever the+// child process exits non-zero, so a download that dies mid-stream surfaces as the child's+// own downstream parse error instead of the transport failure that actually caused it.+// Tracking only the read side keeps the EPIPE from an early-dying child out of the result.+type FailureTrackingReader struct {+ reader io.Reader+ mu sync.Mutex+ firstReadError error+}++func NewFailureTrackingReader(reader io.Reader) *FailureTrackingReader {+ return &FailureTrackingReader{reader: reader}+}++func (ftr *FailureTrackingReader) Read(p []byte) (int, error) {+ n, err := ftr.reader.Read(p)++ if err != nil && !errors.Is(err, io.EOF) {+ ftr.mu.Lock()+ if ftr.firstReadError == nil {+ ftr.firstReadError = err+ }+ ftr.mu.Unlock()+ }++ return n, err+}++func (ftr *FailureTrackingReader) GetFirstReadError() error {+ ftr.mu.Lock()+ defer ftr.mu.Unlock()++ return ftr.firstReadError+}diff --git a/backend/internal/util/io/failure_tracking_reader_test.go b/backend/internal/util/io/failure_tracking_reader_test.gonew file mode 100644index 00000000..0fd2df7c--- /dev/null+++ b/backend/internal/util/io/failure_tracking_reader_test.go@@ -0,0 +1,72 @@+package io_utils++import (+ "errors"+ "io"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++type truncatingReader struct {+ payload string+ bytesServed int+ truncateAfter int+ failure error+}++func (tr *truncatingReader) Read(p []byte) (int, error) {+ if tr.bytesServed >= tr.truncateAfter {+ return 0, tr.failure+ }++ n := copy(p, tr.payload[tr.bytesServed:tr.truncateAfter])+ tr.bytesServed += n++ return n, nil+}++func Test_FailureTrackingReader_WhenSourceFailsMidStream_FirstReadErrorRecorded(t *testing.T) {+ networkFailure := errors.New("connection reset by peer")+ source := &truncatingReader{+ payload: "INSERT INTO orders VALUES (1);",+ truncateAfter: 10,+ failure: networkFailure,+ }++ trackingReader := NewFailureTrackingReader(source)+ copiedBytes, copyErr := io.Copy(io.Discard, trackingReader)++ require.ErrorIs(t, copyErr, networkFailure)+ assert.EqualValues(t, 10, copiedBytes)+ assert.ErrorIs(t, trackingReader.GetFirstReadError(), networkFailure)+}++func Test_FailureTrackingReader_WhenSourceEndsCleanly_NoReadErrorRecorded(t *testing.T) {+ payload := "INSERT INTO orders VALUES (1);"++ trackingReader := NewFailureTrackingReader(strings.NewReader(payload))+ copiedBytes, copyErr := io.Copy(io.Discard, trackingReader)++ require.NoError(t, copyErr)+ assert.EqualValues(t, len(payload), copiedBytes)+ assert.NoError(t, trackingReader.GetFirstReadError())+}++func Test_FailureTrackingReader_WhenSourceFailsRepeatedly_KeepsFirstReadError(t *testing.T) {+ originalFailure := errors.New("connection reset by peer")+ source := &truncatingReader{truncateAfter: 0, failure: originalFailure}++ trackingReader := NewFailureTrackingReader(source)++ _, firstErr := trackingReader.Read(make([]byte, 8))+ require.ErrorIs(t, firstErr, originalFailure)++ source.failure = errors.New("file already closed")+ _, secondErr := trackingReader.Read(make([]byte, 8))+ require.Error(t, secondErr)++ assert.ErrorIs(t, trackingReader.GetFirstReadError(), originalFailure)+}