Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (rclone): Add retry for the case of rclone failure over restore

Rostislav Dugin83c454336c53
diff --git a/backend/internal/features/restores/core/stream_failure.go b/backend/internal/features/restores/core/stream_failure.go
new file mode 100644
index 00000000..9102f1c9
--- /dev/null
+++ b/backend/internal/features/restores/core/stream_failure.go
@@ -0,0 +1,22 @@
+package restores_core
+
+import (
+ "fmt"
+
+ io_utils "databasus-backend/internal/util/io"
+)
+
+// GetBackupStreamFailure must be consulted before the restore client's own exit error:
+// a truncated download makes the client fail on the half-written statement it was fed,
+// and that downstream syntax error would otherwise be reported as the cause.
+func GetBackupStreamFailure(backupStreamReader *io_utils.FailureTrackingReader) error {
+ readError := backupStreamReader.GetFirstReadError()
+ if readError == nil {
+ return nil
+ }
+
+ return fmt.Errorf(
+ "backup stream from storage failed mid-restore, target database is left partially restored: %w",
+ readError,
+ )
+}
diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
index fc720592..90033731 100644
--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
@@ -28,6 +28,7 @@ import (
restores_core "databasus-backend/internal/features/restores/core"
"databasus-backend/internal/features/storages"
util_encryption "databasus-backend/internal/util/encryption"
+ io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/tools"
)
@@ -202,7 +203,8 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(
}
defer zstdReader.Close()
- cmd.Stdin = zstdReader
+ backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)
+ cmd.Stdin = backupStreamReader
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env,
@@ -242,6 +244,10 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(
return fmt.Errorf("restore cancelled due to shutdown")
}
+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {
+ return streamErr
+ }
+
if waitErr != nil {
return uc.handleMariadbRestoreError(database, waitErr, stderrOutput, mariadbBin)
}
diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
index b94a582c..db0db1a5 100644
--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
@@ -24,6 +24,7 @@ import (
restores_core "databasus-backend/internal/features/restores/core"
"databasus-backend/internal/features/storages"
util_encryption "databasus-backend/internal/util/encryption"
+ io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/tools"
)
@@ -199,7 +200,9 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(
inputReader = decryptReader
}
- cmd.Stdin = inputReader
+ backupStreamReader := io_utils.NewFailureTrackingReader(inputReader)
+ cmd.Stdin = backupStreamReader
+
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8", "LANG=C.UTF-8")
@@ -234,6 +237,10 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(
return fmt.Errorf("restore cancelled due to shutdown")
}
+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {
+ return streamErr
+ }
+
if waitErr != nil {
return uc.handleMongoRestoreError(waitErr, stderrOutput, mongorestoreBin)
}
diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
index 898b284b..ad16e0bb 100644
--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
@@ -28,6 +28,7 @@ import (
restores_core "databasus-backend/internal/features/restores/core"
"databasus-backend/internal/features/storages"
util_encryption "databasus-backend/internal/util/encryption"
+ io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/tools"
)
@@ -183,7 +184,8 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(
}
defer zstdReader.Close()
- cmd.Stdin = zstdReader
+ backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)
+ cmd.Stdin = backupStreamReader
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env,
@@ -223,6 +225,10 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(
return fmt.Errorf("restore cancelled due to shutdown")
}
+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {
+ return streamErr
+ }
+
if waitErr != nil {
return uc.handleMysqlRestoreError(database, waitErr, stderrOutput, mysqlBin)
}
diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go
new file mode 100644
index 00000000..e93d7b49
--- /dev/null
+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go
@@ -0,0 +1,104 @@
+package usecases_mysql
+
+import (
+ "bytes"
+ "io"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
+ "databasus-backend/internal/features/databases"
+)
+
+// The masking this test covers only happens when the client process is fed a real OS pipe
+// by os/exec, so there is no HTTP-level way to reach it — hence a stub client binary and a
+// deliberately truncated stream instead of a controller test.
+func writeFailingMysqlClientStub(t *testing.T) string {
+ t.Helper()
+
+ if runtime.GOOS == "windows" {
+ t.Skip("stub client relies on a POSIX shell")
+ }
+
+ stubPath := filepath.Join(t.TempDir(), "mysql-stub")
+ stubScript := "#!/bin/sh\ncat > /dev/null\necho \"ERROR 1064 (42000) at line 53465: You have an error in your SQL syntax\" >&2\nexit 1\n"
+
+ require.NoError(t, os.WriteFile(stubPath, []byte(stubScript), 0o700))
+
+ return stubPath
+}
+
+func compressToZstd(t *testing.T, payload []byte) []byte {
+ t.Helper()
+
+ var compressed bytes.Buffer
+
+ zstdWriter, err := zstd.NewWriter(&compressed)
+ require.NoError(t, err)
+
+ _, err = zstdWriter.Write(payload)
+ require.NoError(t, err)
+ require.NoError(t, zstdWriter.Close())
+
+ return compressed.Bytes()
+}
+
+func Test_ExecuteMysqlRestore_WhenBackupStreamTruncates_ReportsStreamFailureNotSyntaxError(
+ t *testing.T,
+) {
+ dumpStatements := bytes.Repeat(
+ []byte("INSERT INTO orders VALUES ('2020-07-13 18:49:41');\n"),
+ 200_000,
+ )
+ compressedDump := compressToZstd(t, dumpStatements)
+ truncatedDump := compressedDump[:len(compressedDump)*3/4]
+
+ restoreUsecase := &RestoreMysqlBackupUsecase{
+ slog.New(slog.DiscardHandler),
+ nil,
+ }
+
+ restoreErr := restoreUsecase.executeMysqlRestore(
+ t.Context(),
+ &databases.Database{},
+ writeFailingMysqlClientStub(t),
+ nil,
+ filepath.Join(t.TempDir(), ".my.cnf"),
+ io.NopCloser(bytes.NewReader(truncatedDump)),
+ &backups_core_logical.LogicalBackup{},
+ )
+
+ require.Error(t, restoreErr)
+ assert.Contains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")
+ assert.NotContains(t, restoreErr.Error(), "SQL syntax")
+}
+
+func Test_ExecuteMysqlRestore_WhenBackupStreamIsComplete_ReportsClientExitError(t *testing.T) {
+ compressedDump := compressToZstd(t, []byte("INSERT INTO orders VALUES (1);\n"))
+
+ restoreUsecase := &RestoreMysqlBackupUsecase{
+ slog.New(slog.DiscardHandler),
+ nil,
+ }
+
+ restoreErr := restoreUsecase.executeMysqlRestore(
+ t.Context(),
+ &databases.Database{},
+ writeFailingMysqlClientStub(t),
+ nil,
+ filepath.Join(t.TempDir(), ".my.cnf"),
+ io.NopCloser(bytes.NewReader(compressedDump)),
+ &backups_core_logical.LogicalBackup{},
+ )
+
+ require.Error(t, restoreErr)
+ assert.NotContains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")
+ assert.Contains(t, restoreErr.Error(), "SQL syntax")
+}
diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
index a85703a9..ad50e7ff 100644
--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
@@ -28,6 +28,7 @@ import (
restores_core "databasus-backend/internal/features/restores/core"
"databasus-backend/internal/features/storages"
util_encryption "databasus-backend/internal/util/encryption"
+ io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/tools"
)
@@ -311,10 +312,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
return fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
}
- // Copy backup data to stdin in a separate goroutine with proper error handling
+ backupStreamReader := io_utils.NewFailureTrackingReader(backupReader)
+
copyErrCh := make(chan error, 1)
go func() {
- _, copyErr := io.Copy(stdinPipe, backupReader)
+ _, copyErr := io.Copy(stdinPipe, backupStreamReader)
// Close stdin pipe to signal EOF to pg_restore - critical for proper termination
closeErr := stdinPipe.Close()
switch {
@@ -346,6 +348,10 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
return fmt.Errorf("restore cancelled due to shutdown")
}
+ if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {
+ return streamErr
+ }
+
// Check for copy errors first - these indicate issues with decryption or data reading
if copyErr != nil {
return fmt.Errorf("failed to stream backup data to pg_restore: %w", copyErr)
diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.go
index 5a86b4d2..ef1bd270 100644
--- a/backend/internal/features/storages/models/rclone/model.go
+++ b/backend/internal/features/storages/models/rclone/model.go
@@ -107,7 +107,10 @@ func (r *RcloneStorage) GetFile(
return nil, fmt.Errorf("failed to get object from rclone: %w", err)
}
- reader, err := obj.Open(ctx)
+ // operations.Open resumes from the last byte read when the transfer drops, so a
+ // multi-gigabyte restore survives transient remote failures instead of handing the
+ // caller a truncated stream.
+ reader, err := operations.Open(ctx, obj)
if err != nil {
return nil, fmt.Errorf("failed to open object from rclone: %w", err)
}
diff --git a/backend/internal/util/io/failure_tracking_reader.go b/backend/internal/util/io/failure_tracking_reader.go
new file mode 100644
index 00000000..65434303
--- /dev/null
+++ b/backend/internal/util/io/failure_tracking_reader.go
@@ -0,0 +1,42 @@
+package io_utils
+
+import (
+ "errors"
+ "io"
+ "sync"
+)
+
+// FailureTrackingReader exists because os/exec discards the stdin copy error whenever the
+// child process exits non-zero, so a download that dies mid-stream surfaces as the child's
+// own downstream parse error instead of the transport failure that actually caused it.
+// Tracking only the read side keeps the EPIPE from an early-dying child out of the result.
+type FailureTrackingReader struct {
+ reader io.Reader
+ mu sync.Mutex
+ firstReadError error
+}
+
+func NewFailureTrackingReader(reader io.Reader) *FailureTrackingReader {
+ return &FailureTrackingReader{reader: reader}
+}
+
+func (ftr *FailureTrackingReader) Read(p []byte) (int, error) {
+ n, err := ftr.reader.Read(p)
+
+ if err != nil && !errors.Is(err, io.EOF) {
+ ftr.mu.Lock()
+ if ftr.firstReadError == nil {
+ ftr.firstReadError = err
+ }
+ ftr.mu.Unlock()
+ }
+
+ return n, err
+}
+
+func (ftr *FailureTrackingReader) GetFirstReadError() error {
+ ftr.mu.Lock()
+ defer ftr.mu.Unlock()
+
+ return ftr.firstReadError
+}
diff --git a/backend/internal/util/io/failure_tracking_reader_test.go b/backend/internal/util/io/failure_tracking_reader_test.go
new file mode 100644
index 00000000..0fd2df7c
--- /dev/null
+++ b/backend/internal/util/io/failure_tracking_reader_test.go
@@ -0,0 +1,72 @@
+package io_utils
+
+import (
+ "errors"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type truncatingReader struct {
+ payload string
+ bytesServed int
+ truncateAfter int
+ failure error
+}
+
+func (tr *truncatingReader) Read(p []byte) (int, error) {
+ if tr.bytesServed >= tr.truncateAfter {
+ return 0, tr.failure
+ }
+
+ n := copy(p, tr.payload[tr.bytesServed:tr.truncateAfter])
+ tr.bytesServed += n
+
+ return n, nil
+}
+
+func Test_FailureTrackingReader_WhenSourceFailsMidStream_FirstReadErrorRecorded(t *testing.T) {
+ networkFailure := errors.New("connection reset by peer")
+ source := &truncatingReader{
+ payload: "INSERT INTO orders VALUES (1);",
+ truncateAfter: 10,
+ failure: networkFailure,
+ }
+
+ trackingReader := NewFailureTrackingReader(source)
+ copiedBytes, copyErr := io.Copy(io.Discard, trackingReader)
+
+ require.ErrorIs(t, copyErr, networkFailure)
+ assert.EqualValues(t, 10, copiedBytes)
+ assert.ErrorIs(t, trackingReader.GetFirstReadError(), networkFailure)
+}
+
+func Test_FailureTrackingReader_WhenSourceEndsCleanly_NoReadErrorRecorded(t *testing.T) {
+ payload := "INSERT INTO orders VALUES (1);"
+
+ trackingReader := NewFailureTrackingReader(strings.NewReader(payload))
+ copiedBytes, copyErr := io.Copy(io.Discard, trackingReader)
+
+ require.NoError(t, copyErr)
+ assert.EqualValues(t, len(payload), copiedBytes)
+ assert.NoError(t, trackingReader.GetFirstReadError())
+}
+
+func Test_FailureTrackingReader_WhenSourceFailsRepeatedly_KeepsFirstReadError(t *testing.T) {
+ originalFailure := errors.New("connection reset by peer")
+ source := &truncatingReader{truncateAfter: 0, failure: originalFailure}
+
+ trackingReader := NewFailureTrackingReader(source)
+
+ _, firstErr := trackingReader.Read(make([]byte, 8))
+ require.ErrorIs(t, firstErr, originalFailure)
+
+ source.failure = errors.New("file already closed")
+ _, secondErr := trackingReader.Read(make([]byte, 8))
+ require.Error(t, secondErr)
+
+ assert.ErrorIs(t, trackingReader.GetFirstReadError(), originalFailure)
+}