Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (tests): Keep database tool error output until the process wait ends, start the dump tool before the storage write and make test containers wait for their protocol greeting, so restores and backups report the tool's real error and CI stops failing on load-dependent races

Rostislav Dugin42972ae87a8e
diff --git a/agent/verification/internal/features/api/idle_timeout_reader_test.go b/agent/verification/internal/features/api/idle_timeout_reader_test.go
index 17d52ffc..2cf5f1a7 100644
--- a/agent/verification/internal/features/api/idle_timeout_reader_test.go
+++ b/agent/verification/internal/features/api/idle_timeout_reader_test.go
@@ -17,7 +17,9 @@ func Test_ReadThroughIdleTimeoutReader_WhenBytesFlowContinuously_DoesNotCancelCo
pr, pw := io.Pipe()
- idleReader := NewIdleTimeoutReader(pr, 200*time.Millisecond, cancel)
+ // The writer pauses 50 ms between writes, so a timeout of seconds leaves a margin no
+ // scheduler pause on a loaded runner reaches.
+ idleReader := NewIdleTimeoutReader(pr, 5*time.Second, cancel)
defer idleReader.Stop()
go func() {
@@ -45,9 +47,8 @@ func Test_ReadThroughIdleTimeoutReader_WhenNoBytesTransmitted_CancelsContext(t *
idleReader := NewIdleTimeoutReader(pr, 100*time.Millisecond, cancel)
defer idleReader.Stop()
- time.Sleep(200 * time.Millisecond)
-
- assert.Error(t, ctx.Err(), "context should be cancelled when no bytes are transmitted")
+ require.Eventually(t, func() bool { return ctx.Err() != nil }, 5*time.Second, 10*time.Millisecond,
+ "context should be cancelled when no bytes are transmitted")
assert.Contains(t, context.Cause(ctx).Error(), "upload idle timeout")
}
@@ -69,9 +70,8 @@ func Test_ReadThroughIdleTimeoutReader_WhenBytesStopMidStream_CancelsContext(t *
n, _ := idleReader.Read(buf)
assert.Equal(t, "initial", string(buf[:n]))
- time.Sleep(200 * time.Millisecond)
-
- assert.Error(t, ctx.Err(), "context should be cancelled when bytes stop mid-stream")
+ require.Eventually(t, func() bool { return ctx.Err() != nil }, 5*time.Second, 10*time.Millisecond,
+ "context should be cancelled when bytes stop mid-stream")
assert.Contains(t, context.Cause(ctx).Error(), "upload idle timeout")
}
diff --git a/backend/internal/features/backups/backups/core/logical/constants.go b/backend/internal/features/backups/backups/core/logical/constants.go
new file mode 100644
index 00000000..a7af4feb
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/logical/constants.go
@@ -0,0 +1,7 @@
+package backups_core_logical
+
+import "time"
+
+// DumpToolOutputWaitDelay bounds how long a backup waits for the dump tool's output to end after
+// the tool itself exited. A process the tool left behind can hold that output open.
+const DumpToolOutputWaitDelay = 10 * time.Second
diff --git a/backend/internal/features/backups/backups/core/logical/testing.go b/backend/internal/features/backups/backups/core/logical/testing.go
index ad614eb4..3cf46e7d 100644
--- a/backend/internal/features/backups/backups/core/logical/testing.go
+++ b/backend/internal/features/backups/backups/core/logical/testing.go
@@ -5,6 +5,9 @@ import (
"errors"
"fmt"
"io"
+ "os"
+ "path/filepath"
+ "runtime"
"sync/atomic"
"testing"
@@ -45,6 +48,20 @@ func (s *FailingBackupFileStore) GetReadBytes() int64 {
return s.readBytes.Load()
}
+// discardingBackupFileStore reads every write to the end and keeps nothing, for tests whose
+// backup fails for a reason other than the storage.
+type discardingBackupFileStore struct{}
+
+func (s *discardingBackupFileStore) WriteFile(
+ _ context.Context,
+ _ storage_files.StoredFileReference,
+ file io.Reader,
+) (storage_files.WriteReceipt, error) {
+ _, err := io.Copy(io.Discard, file)
+
+ return storage_files.WriteReceipt{}, err
+}
+
// ExecuteBackupInto runs one engine's real backup use case against a test database, writing into
// the given file store.
type ExecuteBackupInto func(
@@ -67,7 +84,7 @@ func AssertStorageTimeoutReportedAsStorageFailure(t *testing.T, executeBackupInt
backupErr := executeBackupInto(
t.Context(),
- newStorageFailureTestBackup(),
+ newTestBackup(),
newUnencryptedTestBackupConfig(),
fileStore,
)
@@ -81,7 +98,7 @@ func AssertStorageTimeoutReportedAsStorageFailure(t *testing.T, executeBackupInt
backupErr := executeBackupInto(
t.Context(),
- newStorageFailureTestBackup(),
+ newTestBackup(),
newUnencryptedTestBackupConfig(),
fileStore,
)
@@ -90,6 +107,47 @@ func AssertStorageTimeoutReportedAsStorageFailure(t *testing.T, executeBackupInt
})
}
+// ExecuteBackupWithDumpTool runs one engine's backup use case with the given binary in place of
+// its dump tool, writing into the given file store.
+type ExecuteBackupWithDumpTool func(
+ ctx context.Context,
+ backup *LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ fileStore BackupFileStore,
+ dumpToolBin string,
+) error
+
+// AssertDumpToolErrorReported runs an engine's backup with a dump tool that writes an error and
+// exits before producing any data, and checks the engine reports that error.
+func AssertDumpToolErrorReported(t *testing.T, executeBackupWithDumpTool ExecuteBackupWithDumpTool) {
+ t.Helper()
+
+ if runtime.GOOS == "windows" {
+ t.Skip("stub dump tool relies on a POSIX shell")
+ }
+
+ const dumpToolError = "error: query failed: ERROR: permission denied for sequence orders_id_seq"
+
+ // The error reaches the error output only after the dump tool has exited, which is how a
+ // fast-exiting tool looks when the reader of its output is scheduled late, as on a loaded CI
+ // runner. A capture that stops reading when the process exits loses it every time.
+ dumpToolBin := filepath.Join(t.TempDir(), "dump-tool")
+ dumpToolScript := "#!/bin/sh\n(sleep 0.2; echo '" + dumpToolError + "' >&2) >/dev/null &\nexit 1\n"
+ require.NoError(t, os.WriteFile(dumpToolBin, []byte(dumpToolScript), 0o700))
+
+ backupErr := executeBackupWithDumpTool(
+ t.Context(),
+ newTestBackup(),
+ newUnencryptedTestBackupConfig(),
+ &discardingBackupFileStore{},
+ dumpToolBin,
+ )
+
+ require.Error(t, backupErr)
+ assert.Contains(t, backupErr.Error(), dumpToolError)
+ assert.NotContains(t, backupErr.Error(), "save to storage")
+}
+
func assertStorageErrorReported(t *testing.T, backupErr, storageErr error) {
t.Helper()
@@ -99,14 +157,14 @@ func assertStorageErrorReported(t *testing.T, backupErr, storageErr error) {
assert.NotContains(t, backupErr.Error(), "backup cancelled")
}
-func newStorageFailureTestBackup() *LogicalBackup {
+func newTestBackup() *LogicalBackup {
backupID := uuid.New()
return &LogicalBackup{
ID: backupID,
DatabaseID: uuid.New(),
StorageID: uuid.New(),
- FileName: "storage-failure-" + backupID.String(),
+ FileName: "test-backup-" + backupID.String(),
Status: BackupStatusInProgress,
}
}
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
index ba4f420f..789c4800 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_logical_mariadb
import (
+ "bytes"
"context"
"errors"
"fmt"
@@ -215,21 +216,14 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
"LANG=C.UTF-8",
)
- pgStdout, err := cmd.StdoutPipe()
+ dumpStdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("stdout pipe: %w", err)
}
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return nil, fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = backups_core_logical.DumpToolOutputWaitDelay
storageReader, storageWriter := io.Pipe()
@@ -250,6 +244,12 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
+ // The dump tool starts before the storage write, so a storage that fails at once cannot
+ // cancel the context the tool is started with and hide its own error.
+ if err = cmd.Start(); err != nil {
+ return nil, fmt.Errorf("start %s: %w", filepath.Base(mariadbBin), err)
+ }
+
fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
@@ -258,17 +258,11 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
cancel,
)
- if err = cmd.Start(); err != nil {
- uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
-
- return nil, fmt.Errorf("start %s: %w", filepath.Base(mariadbBin), err)
- }
-
copyResultCh := make(chan error, 1)
go func() {
copyResultCh <- uc.copyWithShutdownCheck(ctx, compressedCopySpec{
Destination: zstdWriter,
- Source: pgStdout,
+ Source: dumpStdout,
CompressedBytesCounter: compressedBytesCounter,
ProgressListener: backupProgressListener,
})
@@ -304,7 +298,6 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
}
saveErr := <-fileWrite.Errors
- stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
compressedSizeMB := float64(compressedBytesCounter.GetBytesWritten()) / (1024 * 1024)
@@ -313,7 +306,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
switch {
case waitErr != nil:
- return nil, uc.buildMariadbDumpErrorMessage(waitErr, stderrOutput, mariadbBin)
+ return nil, uc.buildMariadbDumpErrorMessage(waitErr, stderrOutput.Bytes(), mariadbBin)
case copyErr != nil:
return nil, fmt.Errorf("copy to storage: %w", copyErr)
case saveErr != nil:
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go
index f886465f..e098c946 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go
@@ -173,3 +173,20 @@ func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue81
return err
})
}
+
+func Test_Execute_WhenMariadbDumpFailsBeforeProducingData_ReportsMariadbDumpError(t *testing.T) {
+ backups_core_logical.AssertDumpToolErrorReported(t, func(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ fileStore backups_core_logical.BackupFileStore,
+ dumpToolBin string,
+ ) error {
+ _, err := GetCreateMariadbBackupUsecase().streamToStorage(
+ ctx, backup, backupConfig, dumpToolBin, nil, "", fileStore, func(float64) {},
+ &mariadbtypes.MariadbDatabase{Version: tools.MariadbVersion1011, Database: new("orders_db")},
+ )
+
+ return err
+ })
+}
diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
index 7a15120f..bff98a70 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_logical_mongodb
import (
+ "bytes"
"context"
"errors"
"fmt"
@@ -157,21 +158,14 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
"LANG=C.UTF-8",
)
- pgStdout, err := cmd.StdoutPipe()
+ dumpStdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("stdout pipe: %w", err)
}
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return nil, fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = backups_core_logical.DumpToolOutputWaitDelay
storageReader, storageWriter := io.Pipe()
@@ -184,6 +178,12 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
return nil, err
}
+ // The dump tool starts before the storage write, so a storage that fails at once cannot
+ // cancel the context the tool is started with and hide its own error.
+ if err = cmd.Start(); err != nil {
+ return nil, fmt.Errorf("start %s: %w", filepath.Base(mongodumpBin), err)
+ }
+
fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
@@ -192,19 +192,13 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
cancel,
)
- if err = cmd.Start(); err != nil {
- uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
-
- return nil, fmt.Errorf("start %s: %w", filepath.Base(mongodumpBin), err)
- }
-
copyResultCh := make(chan error, 1)
bytesWrittenCh := make(chan int64, 1)
go func() {
bytesWritten, copyErr := uc.copyWithShutdownCheck(
ctx,
finalWriter,
- pgStdout,
+ dumpStdout,
backupProgressListener,
)
bytesWrittenCh <- bytesWritten
@@ -238,7 +232,6 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
}
saveErr := <-fileWrite.Errors
- stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
sizeMB := float64(bytesWritten) / (1024 * 1024)
@@ -247,7 +240,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
switch {
case waitErr != nil:
- return nil, uc.buildMongodumpErrorMessage(waitErr, stderrOutput, mongodumpBin)
+ return nil, uc.buildMongodumpErrorMessage(waitErr, stderrOutput.Bytes(), mongodumpBin)
case copyErr != nil:
return nil, fmt.Errorf("copy to storage: %w", copyErr)
case saveErr != nil:
diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go
index b1af3d41..715a8f71 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go
@@ -33,3 +33,19 @@ func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue81
return err
})
}
+
+func Test_Execute_WhenMongodumpFailsBeforeProducingData_ReportsMongodumpError(t *testing.T) {
+ backups_core_logical.AssertDumpToolErrorReported(t, func(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ fileStore backups_core_logical.BackupFileStore,
+ dumpToolBin string,
+ ) error {
+ _, err := GetCreateMongodbBackupUsecase().streamToStorage(
+ ctx, backup, backupConfig, dumpToolBin, nil, fileStore, func(float64) {},
+ )
+
+ return err
+ })
+}
diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
index bb332d7c..6d7b0471 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_logical_mysql
import (
+ "bytes"
"context"
"errors"
"fmt"
@@ -212,21 +213,14 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
"LANG=C.UTF-8",
)
- pgStdout, err := cmd.StdoutPipe()
+ dumpStdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("stdout pipe: %w", err)
}
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return nil, fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = backups_core_logical.DumpToolOutputWaitDelay
storageReader, storageWriter := io.Pipe()
@@ -247,6 +241,12 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
+ // The dump tool starts before the storage write, so a storage that fails at once cannot
+ // cancel the context the tool is started with and hide its own error.
+ if err = cmd.Start(); err != nil {
+ return nil, fmt.Errorf("start %s: %w", filepath.Base(mysqlBin), err)
+ }
+
fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
@@ -255,17 +255,11 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
cancel,
)
- if err = cmd.Start(); err != nil {
- uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
-
- return nil, fmt.Errorf("start %s: %w", filepath.Base(mysqlBin), err)
- }
-
copyResultCh := make(chan error, 1)
go func() {
copyResultCh <- uc.copyWithShutdownCheck(ctx, compressedCopySpec{
Destination: zstdWriter,
- Source: pgStdout,
+ Source: dumpStdout,
CompressedBytesCounter: compressedBytesCounter,
ProgressListener: backupProgressListener,
})
@@ -301,7 +295,6 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
}
saveErr := <-fileWrite.Errors
- stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
compressedSizeMB := float64(compressedBytesCounter.GetBytesWritten()) / (1024 * 1024)
@@ -310,7 +303,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
switch {
case waitErr != nil:
- return nil, uc.buildMysqldumpErrorMessage(waitErr, stderrOutput, mysqlBin)
+ return nil, uc.buildMysqldumpErrorMessage(waitErr, stderrOutput.Bytes(), mysqlBin)
case copyErr != nil:
return nil, fmt.Errorf("copy to storage: %w", copyErr)
case saveErr != nil:
diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go
index a65d7494..f9fbd0ef 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go
@@ -173,3 +173,20 @@ func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue81
return err
})
}
+
+func Test_Execute_WhenMysqldumpFailsBeforeProducingData_ReportsMysqldumpError(t *testing.T) {
+ backups_core_logical.AssertDumpToolErrorReported(t, func(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ fileStore backups_core_logical.BackupFileStore,
+ dumpToolBin string,
+ ) error {
+ _, err := GetCreateMysqlBackupUsecase().streamToStorage(
+ ctx, backup, backupConfig, dumpToolBin, nil, "", fileStore, func(float64) {},
+ &mysqltypes.MysqlDatabase{Version: tools.MysqlVersion80, Database: new("orders_db")},
+ )
+
+ return err
+ })
+}
diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
index b8c775af..2381601d 100644
--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_logical_postgresql
import (
+ "bytes"
"context"
"errors"
"fmt"
@@ -162,17 +163,9 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("stdout pipe: %w", err)
}
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return nil, fmt.Errorf("stderr pipe: %w", err)
- }
-
- // Capture stderr in a separate goroutine to ensure we don't miss any error output
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = backups_core_logical.DumpToolOutputWaitDelay
storageReader, storageWriter := io.Pipe()
@@ -185,6 +178,12 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
return nil, err
}
+ // The dump tool starts before the storage write, so a storage that fails at once cannot
+ // cancel the context the tool is started with and hide its own error.
+ if err = cmd.Start(); err != nil {
+ return nil, fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
+ }
+
fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
@@ -193,12 +192,6 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
cancel,
)
- if err = cmd.Start(); err != nil {
- uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
-
- return nil, fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
- }
-
copyResultCh := make(chan error, 1)
bytesWrittenCh := make(chan int64, 1)
go func() {
@@ -239,7 +232,6 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
}
saveErr := <-fileWrite.Errors
- stderrOutput := <-stderrCh
// Send final sizing after backup is completed
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
@@ -252,7 +244,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
if ctx.Err() != nil {
return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)
}
- return nil, uc.buildPgDumpErrorMessage(waitErr, stderrOutput, pgBin, args, password)
+ return nil, uc.buildPgDumpErrorMessage(waitErr, stderrOutput.Bytes(), pgBin, args, password)
case copyErr != nil:
if ctx.Err() != nil {
return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)
diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go
index e00a9b08..0bab56ab 100644
--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go
@@ -64,3 +64,21 @@ func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue81
return err
})
}
+
+func Test_Execute_WhenPgDumpFailsBeforeProducingData_ReportsPgDumpError(t *testing.T) {
+ backups_core_logical.AssertDumpToolErrorReported(t, func(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ fileStore backups_core_logical.BackupFileStore,
+ dumpToolBin string,
+ ) error {
+ _, err := GetCreatePostgresqlBackupUsecase().streamToStorage(
+ ctx, backup, backupConfig, dumpToolBin, nil, "", fileStore,
+ &databases.Database{PostgresqlLogical: &pgtypes.PostgresqlLogicalDatabase{}},
+ func(float64) {},
+ )
+
+ return err
+ })
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
index e3067f0d..77e92ab5 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
@@ -1,6 +1,7 @@
package usecases_physical_postgresql
import (
+ "bytes"
"context"
"fmt"
"log/slog"
@@ -80,12 +81,10 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
return receiverRunResult{Exit: receiverRetryable}
}
- stderrPipe, err := cmd.StderrPipe()
- if err != nil {
- logger.ErrorContext(ctx, "pg_receivewal stderr pipe", "error", err)
-
- return receiverRunResult{Exit: receiverRetryable}
- }
+ // Wait returns only after the whole error output is in the buffer, bounded by the command's
+ // WaitDelay, so reading it once exited delivers always sees what pg_receivewal wrote.
+ var stderr bytes.Buffer
+ cmd.Stderr = &stderr
if err := cmd.Start(); err != nil {
logger.ErrorContext(ctx, "start pg_receivewal", "error", err)
@@ -98,8 +97,6 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
s.receiverRunStartedAtUnixNano.Store(startedAt.UnixNano())
defer s.receiverRunStartedAtUnixNano.Store(0)
- stderr := newStderrCapture(stderrPipe)
-
logger.InfoContext(ctx, "pg_receivewal started", "watch_dir", s.watchDir)
exited := make(chan error, 1)
@@ -110,7 +107,6 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
case <-ctx.Done():
procCancel()
<-exited
- stderr.stop()
return receiverRunResult{Exit: receiverCtxCancelled}
@@ -118,12 +114,10 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
logger.InfoContext(ctx, "restarting pg_receivewal on internal signal (back pressure or slot stall)")
procCancel()
<-exited
- stderr.stop()
return receiverRunResult{Exit: receiverInternalRestart, RanFor: time.Since(startedAt)}
case waitErr := <-exited:
- stderr.stop()
ranFor := time.Since(startedAt)
if waitErr == nil || procCtx.Err() != nil {
@@ -132,7 +126,7 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
return receiverRunResult{Exit: receiverRetryable, RanFor: ranFor}
}
- stderrText := stderr.contents()
+ stderrText := stderr.Bytes()
if isResumeMismatchError(stderrText) {
logger.WarnContext(ctx, "pg_receivewal asked for WAL the server has recycled; realigning the resume point",
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
index 8fda6fd1..1525d922 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
@@ -1,6 +1,9 @@
package usecases_physical_postgresql
import (
+ "log/slog"
+ "os"
+ "path/filepath"
"strings"
"syscall"
"testing"
@@ -111,3 +114,35 @@ func Test_NewReceivewalCommand_WithoutATunnel_SendsNoPgHostAddr(t *testing.T) {
assertNoPgHostAddr(t, cmd.Env)
}
+
+// A rejected password exits pg_receivewal at once. The error reaches its error output only after
+// the process has exited, which is how that exit looks when the reader of the output is
+// scheduled late, as on a loaded CI runner. A capture that stops reading when the process exits
+// loses it and treats the exit as one worth retrying.
+func Test_SpawnAndSupervise_WhenReceiverExitsRightAfterAuthenticationFailure_EndsAsFatal(t *testing.T) {
+ receiverStub := filepath.Join(t.TempDir(), "pg_receivewal")
+ receiverScript := "#!/bin/sh\n" +
+ "(sleep 0.2; echo 'pg_receivewal: error: connection to server failed: " +
+ "FATAL: password authentication failed for user \"repl\"' >&2) >/dev/null &\n" +
+ "exit 1\n"
+ require.NoError(t, os.WriteFile(receiverStub, []byte(receiverScript), 0o700))
+
+ supervisor := &WalStreamSupervisor{
+ spec: WalStreamSpec{
+ SourceDB: &postgresql_physical.PostgresqlPhysicalDatabase{
+ ID: uuid.New(),
+ Host: "localhost",
+ Port: 5432,
+ Username: "repl",
+ },
+ },
+ watchDir: t.TempDir(),
+ slotName: "slot",
+ restartSignal: make(chan struct{}, 1),
+ }
+
+ runResult := supervisor.spawnAndSupervise(t.Context(), slog.New(slog.DiscardHandler), receiverStub)
+
+ require.Equal(t, receiverFatal, runResult.Exit)
+ require.ErrorContains(t, runResult.FatalErr, "FATAL: password authentication failed")
+}
diff --git a/backend/internal/features/restores/core/constants.go b/backend/internal/features/restores/core/constants.go
new file mode 100644
index 00000000..84d4e167
--- /dev/null
+++ b/backend/internal/features/restores/core/constants.go
@@ -0,0 +1,7 @@
+package restores_core
+
+import "time"
+
+// ToolOutputWaitDelay bounds how long a restore waits for the restore tool's output to end
+// after the tool itself exited. A process the tool left behind can hold that output open.
+const ToolOutputWaitDelay = 10 * time.Second
diff --git a/backend/internal/features/restores/core/testing.go b/backend/internal/features/restores/core/testing.go
new file mode 100644
index 00000000..497aeb5a
--- /dev/null
+++ b/backend/internal/features/restores/core/testing.go
@@ -0,0 +1,56 @@
+package restores_core
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/require"
+)
+
+// WriteToolStub writes a shell script that stands in for a restore tool. Error output is lost only
+// when the tool writes to a real OS pipe created by os/exec, so there is no HTTP-level way to reach
+// it, hence a stub binary instead of a controller test.
+func WriteToolStub(t *testing.T, toolName, stubScript string) string {
+ t.Helper()
+
+ if runtime.GOOS == "windows" {
+ t.Skip("stub restore tool relies on a POSIX shell")
+ }
+
+ stubPath := filepath.Join(t.TempDir(), toolName)
+ require.NoError(t, os.WriteFile(stubPath, []byte(stubScript), 0o700))
+
+ return stubPath
+}
+
+// WriteLateFailingToolStub writes a restore tool stub that reads the whole backup and exits with
+// status 1, while errorLine reaches its error output only after the stub has exited. That is how a
+// fast-exiting tool looks when the reader of its output is scheduled late, as on a loaded CI
+// runner, and a capture that stops reading when the process exits loses the line every time.
+// errorLine is quoted with single quotes in the script, so it must not contain one.
+func WriteLateFailingToolStub(t *testing.T, toolName, errorLine string) string {
+ t.Helper()
+
+ return WriteToolStub(t, toolName, "#!/bin/sh\ncat > /dev/null\n"+
+ "(sleep 0.2; echo '"+errorLine+"' >&2) &\n"+
+ "exit 1\n")
+}
+
+func CompressToZstd(t *testing.T, payload []byte) []byte {
+ t.Helper()
+
+ var compressed bytes.Buffer
+
+ zstdWriter, err := zstd.NewWriter(&compressed)
+ require.NoError(t, err)
+
+ _, err = zstdWriter.Write(payload)
+ require.NoError(t, err)
+ require.NoError(t, zstdWriter.Close())
+
+ return compressed.Bytes()
+}
diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
index 76567de7..f6c3c96c 100644
--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_mariadb
import (
+ "bytes"
"context"
"encoding/base64"
"errors"
@@ -221,23 +222,15 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(
"LANG=C.UTF-8",
)
- stderrPipe, err := cmd.StderrPipe()
- if err != nil {
- return fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- output, _ := io.ReadAll(stderrPipe)
- stderrCh <- output
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = restores_core.ToolOutputWaitDelay
if err = cmd.Start(); err != nil {
return fmt.Errorf("start mariadb: %w", err)
}
waitErr := cmd.Wait()
- stderrOutput := <-stderrCh
// Check for cancellation
select {
@@ -260,7 +253,7 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(
}
if waitErr != nil {
- return uc.handleMariadbRestoreError(database, waitErr, stderrOutput, mariadbBin)
+ return uc.handleMariadbRestoreError(database, waitErr, stderrOutput.Bytes(), mariadbBin)
}
return nil
diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc_test.go
new file mode 100644
index 00000000..18572ce4
--- /dev/null
+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc_test.go
@@ -0,0 +1,41 @@
+package usecases_mariadb
+
+import (
+ "bytes"
+ "io"
+ "log/slog"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
+ "databasus-backend/internal/features/databases"
+ restores_core "databasus-backend/internal/features/restores/core"
+)
+
+func Test_ExecuteMariadbRestore_WhenClientExitsRightAfterError_ReportsClientError(t *testing.T) {
+ compressedDump := restores_core.CompressToZstd(t, []byte("INSERT INTO orders VALUES (1);\n"))
+
+ restoreUsecase := &RestoreMariadbBackupUsecase{
+ slog.New(slog.DiscardHandler),
+ nil,
+ }
+
+ restoreErr := restoreUsecase.executeMariadbRestore(
+ t.Context(),
+ &databases.Database{},
+ restores_core.WriteLateFailingToolStub(
+ t, "mariadb-stub", "ERROR 1064 (42000) at line 1: You have an error in your SQL syntax",
+ ),
+ nil,
+ filepath.Join(t.TempDir(), ".my.cnf"),
+ io.NopCloser(bytes.NewReader(compressedDump)),
+ &backups_core_logical.LogicalBackup{},
+ )
+
+ require.Error(t, restoreErr)
+ assert.NotContains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")
+ assert.Contains(t, restoreErr.Error(), "SQL syntax")
+}
diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
index b90fd9d3..101a0bd2 100644
--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_mongodb
import (
+ "bytes"
"context"
"encoding/base64"
"errors"
@@ -210,23 +211,15 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8", "LANG=C.UTF-8")
- stderrPipe, err := cmd.StderrPipe()
- if err != nil {
- return fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- output, _ := io.ReadAll(stderrPipe)
- stderrCh <- output
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = restores_core.ToolOutputWaitDelay
- if err = cmd.Start(); err != nil {
+ if err := cmd.Start(); err != nil {
return fmt.Errorf("start mongorestore: %w", err)
}
waitErr := cmd.Wait()
- stderrOutput := <-stderrCh
// Check for cancellation
select {
@@ -249,7 +242,7 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(
}
if waitErr != nil {
- return uc.handleMongoRestoreError(waitErr, stderrOutput, mongorestoreBin)
+ return uc.handleMongoRestoreError(waitErr, stderrOutput.Bytes(), mongorestoreBin)
}
return nil
diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc_test.go
new file mode 100644
index 00000000..7ce54bd4
--- /dev/null
+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc_test.go
@@ -0,0 +1,35 @@
+package usecases_mongodb
+
+import (
+ "bytes"
+ "io"
+ "log/slog"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
+ restores_core "databasus-backend/internal/features/restores/core"
+)
+
+func Test_ExecuteMongoRestore_WhenToolExitsRightAfterError_ReportsToolError(t *testing.T) {
+ restoreUsecase := &RestoreMongodbBackupUsecase{
+ slog.New(slog.DiscardHandler),
+ nil,
+ }
+
+ restoreErr := restoreUsecase.executeMongoRestore(
+ t.Context(),
+ restores_core.WriteLateFailingToolStub(
+ t, "mongorestore-stub", "Failed: restore error: orders: error restoring from archive: unexpected EOF",
+ ),
+ nil,
+ io.NopCloser(bytes.NewReader([]byte("archive"))),
+ &backups_core_logical.LogicalBackup{},
+ )
+
+ require.Error(t, restoreErr)
+ assert.NotContains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")
+ assert.Contains(t, restoreErr.Error(), "error restoring from archive")
+}
diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
index 40a4cbbc..90e96298 100644
--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_mysql
import (
+ "bytes"
"context"
"encoding/base64"
"errors"
@@ -202,23 +203,15 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(
"LANG=C.UTF-8",
)
- stderrPipe, err := cmd.StderrPipe()
- if err != nil {
- return fmt.Errorf("stderr pipe: %w", err)
- }
-
- stderrCh := make(chan []byte, 1)
- go func() {
- output, _ := io.ReadAll(stderrPipe)
- stderrCh <- output
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = restores_core.ToolOutputWaitDelay
if err = cmd.Start(); err != nil {
return fmt.Errorf("start mysql: %w", err)
}
waitErr := cmd.Wait()
- stderrOutput := <-stderrCh
// Check for cancellation
select {
@@ -241,7 +234,7 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(
}
if waitErr != nil {
- return uc.handleMysqlRestoreError(database, waitErr, stderrOutput, mysqlBin)
+ return uc.handleMysqlRestoreError(database, waitErr, stderrOutput.Bytes(), mysqlBin)
}
return nil
diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go
index e93d7b49..26f99ec9 100644
--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go
+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc_test.go
@@ -4,52 +4,27 @@ import (
"bytes"
"io"
"log/slog"
- "os"
"path/filepath"
- "runtime"
"testing"
- "github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
"databasus-backend/internal/features/databases"
+ restores_core "databasus-backend/internal/features/restores/core"
)
-// The masking this test covers only happens when the client process is fed a real OS pipe
-// by os/exec, so there is no HTTP-level way to reach it — hence a stub client binary and a
-// deliberately truncated stream instead of a controller test.
func writeFailingMysqlClientStub(t *testing.T) string {
t.Helper()
- if runtime.GOOS == "windows" {
- t.Skip("stub client relies on a POSIX shell")
- }
-
- stubPath := filepath.Join(t.TempDir(), "mysql-stub")
- stubScript := "#!/bin/sh\ncat > /dev/null\necho \"ERROR 1064 (42000) at line 53465: You have an error in your SQL syntax\" >&2\nexit 1\n"
-
- require.NoError(t, os.WriteFile(stubPath, []byte(stubScript), 0o700))
-
- return stubPath
-}
-
-func compressToZstd(t *testing.T, payload []byte) []byte {
- t.Helper()
-
- var compressed bytes.Buffer
-
- zstdWriter, err := zstd.NewWriter(&compressed)
- require.NoError(t, err)
-
- _, err = zstdWriter.Write(payload)
- require.NoError(t, err)
- require.NoError(t, zstdWriter.Close())
-
- return compressed.Bytes()
+ return restores_core.WriteLateFailingToolStub(
+ t, "mysql-stub", "ERROR 1064 (42000) at line 53465: You have an error in your SQL syntax",
+ )
}
+// A truncated stream makes the client fail on a half-written statement, and that syntax error must
+// not mask the stream failure.
func Test_ExecuteMysqlRestore_WhenBackupStreamTruncates_ReportsStreamFailureNotSyntaxError(
t *testing.T,
) {
@@ -57,7 +32,7 @@ func Test_ExecuteMysqlRestore_WhenBackupStreamTruncates_ReportsStreamFailureNotS
[]byte("INSERT INTO orders VALUES ('2020-07-13 18:49:41');\n"),
200_000,
)
- compressedDump := compressToZstd(t, dumpStatements)
+ compressedDump := restores_core.CompressToZstd(t, dumpStatements)
truncatedDump := compressedDump[:len(compressedDump)*3/4]
restoreUsecase := &RestoreMysqlBackupUsecase{
@@ -81,7 +56,7 @@ func Test_ExecuteMysqlRestore_WhenBackupStreamTruncates_ReportsStreamFailureNotS
}
func Test_ExecuteMysqlRestore_WhenBackupStreamIsComplete_ReportsClientExitError(t *testing.T) {
- compressedDump := compressToZstd(t, []byte("INSERT INTO orders VALUES (1);\n"))
+ compressedDump := restores_core.CompressToZstd(t, []byte("INSERT INTO orders VALUES (1);\n"))
restoreUsecase := &RestoreMysqlBackupUsecase{
slog.New(slog.DiscardHandler),
diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
index f5da32b7..fc3cfd6d 100644
--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
@@ -1,6 +1,7 @@
package usecases_postgresql
import (
+ "bytes"
"context"
"encoding/base64"
"errors"
@@ -272,11 +273,30 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
logger.InfoContext(parentCtx, "using decryption for encrypted backup")
}
- cmd := exec.CommandContext(ctx, pgBin, args...)
- logger.InfoContext(parentCtx, "executing PostgreSQL restore command via stdin", "command", cmd.String())
+ return uc.executePgRestoreViaStdin(ctx, pgRestoreCommand{
+ database: originalDB,
+ pgBin: pgBin,
+ args: args,
+ credentials: credentials,
+ pgConfig: pg,
+ logger: logger,
+ }, backupReader, storageReadFailureTracker)
+}
+
+// executePgRestoreViaStdin pumps the decoded backup into pg_restore's standard input.
+func (uc *RestorePostgresqlBackupUsecase) executePgRestoreViaStdin(
+ ctx context.Context,
+ command pgRestoreCommand,
+ backupReader io.Reader,
+ storageReadFailureTracker *io_utils.FailureTrackingReader,
+) error {
+ pgBin := command.pgBin
+
+ cmd := exec.CommandContext(ctx, pgBin, command.args...)
+ command.logger.InfoContext(ctx, "executing PostgreSQL restore command via stdin", "command", cmd.String())
// Setup environment variables
- uc.setupPgRestoreEnvironment(cmd, credentials, pg)
+ uc.setupPgRestoreEnvironment(cmd, command.credentials, command.pgConfig)
// Verify executable exists and is accessible
if _, err := exec.LookPath(pgBin); err != nil {
@@ -293,18 +313,9 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
return fmt.Errorf("stdin pipe: %w", err)
}
- // Get stderr to capture any error output
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return fmt.Errorf("stderr pipe: %w", err)
- }
-
- // Capture stderr in a separate goroutine
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = restores_core.ToolOutputWaitDelay
// Start pg_restore
if err = cmd.Start(); err != nil {
@@ -330,7 +341,6 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
// Wait for the restore to finish
waitErr := cmd.Wait()
- stderrOutput := <-stderrCh
copyErr := <-copyErrCh
// Check for cancellation
@@ -354,11 +364,6 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
return streamErr
}
- // Check for copy errors first - these indicate issues with decryption or data reading
- if copyErr != nil {
- return fmt.Errorf("failed to stream backup data to pg_restore: %w", copyErr)
- }
-
if waitErr != nil {
// Check for cancellation again
select {
@@ -374,16 +379,22 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
}
return uc.resolvePgRestoreOutcome(pgRestoreFailure{
- database: originalDB,
+ database: command.database,
waitErr: waitErr,
- stderrOutput: stderrOutput,
- pgBin: pgBin,
- args: args,
- pgConfig: pg,
- logger: logger,
+ stderrOutput: stderrOutput.Bytes(),
+ pgBin: command.pgBin,
+ args: command.args,
+ pgConfig: command.pgConfig,
+ logger: command.logger,
})
}
+ // pg_restore exiting early breaks the pipe, so its own exit error is the cause and the copy
+ // error counts only when pg_restore succeeded.
+ if copyErr != nil {
+ return fmt.Errorf("failed to stream backup data to pg_restore: %w", copyErr)
+ }
+
return nil
}
@@ -689,27 +700,17 @@ func (uc *RestorePostgresqlBackupUsecase) executePgRestore(
)
}
- // Get stderr to capture any error output
- pgStderr, err := cmd.StderrPipe()
- if err != nil {
- return fmt.Errorf("stderr pipe: %w", err)
- }
-
- // Capture stderr in a separate goroutine
- stderrCh := make(chan []byte, 1)
- go func() {
- stderrOutput, _ := io.ReadAll(pgStderr)
- stderrCh <- stderrOutput
- }()
+ var stderrOutput bytes.Buffer
+ cmd.Stderr = &stderrOutput
+ cmd.WaitDelay = restores_core.ToolOutputWaitDelay
// Start pg_restore
- if err = cmd.Start(); err != nil {
+ if err := cmd.Start(); err != nil {
return fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
}
// Wait for the restore to finish
waitErr := cmd.Wait()
- stderrOutput := <-stderrCh
// Check for cancellation
select {
@@ -742,7 +743,7 @@ func (uc *RestorePostgresqlBackupUsecase) executePgRestore(
return uc.resolvePgRestoreOutcome(pgRestoreFailure{
database: command.database,
waitErr: waitErr,
- stderrOutput: stderrOutput,
+ stderrOutput: stderrOutput.Bytes(),
pgBin: command.pgBin,
args: command.args,
pgConfig: command.pgConfig,
diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc_test.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc_test.go
new file mode 100644
index 00000000..8a266844
--- /dev/null
+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc_test.go
@@ -0,0 +1,86 @@
+package usecases_postgresql
+
+import (
+ "bytes"
+ "log/slog"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/features/databases"
+ pgtypes "databasus-backend/internal/features/databases/databases/postgresql/logical"
+ postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
+ restores_core "databasus-backend/internal/features/restores/core"
+ io_utils "databasus-backend/internal/util/io"
+)
+
+const pgRestoreStubError = `pg_restore: error: could not execute query: ERROR: relation "orders" already exists`
+
+func newStubPgRestoreCommand(pgBin string) pgRestoreCommand {
+ return pgRestoreCommand{
+ database: &databases.Database{},
+ pgBin: pgBin,
+ credentials: &postgresql_shared.CredentialTempFiles{},
+ pgConfig: &pgtypes.PostgresqlLogicalDatabase{},
+ logger: slog.New(slog.DiscardHandler),
+ }
+}
+
+func newStubRestoreUsecase() *RestorePostgresqlBackupUsecase {
+ return &RestorePostgresqlBackupUsecase{
+ slog.New(slog.DiscardHandler),
+ nil,
+ }
+}
+
+func Test_ExecutePgRestore_WhenPgRestoreExitsRightAfterError_ReportsPgRestoreError(t *testing.T) {
+ restoreErr := newStubRestoreUsecase().executePgRestore(
+ t.Context(),
+ newStubPgRestoreCommand(restores_core.WriteLateFailingToolStub(t, "pg_restore", pgRestoreStubError)),
+ )
+
+ require.Error(t, restoreErr)
+ assert.Contains(t, restoreErr.Error(), `relation "orders" already exists`)
+}
+
+func Test_ExecutePgRestoreViaStdin_WhenPgRestoreExitsRightAfterError_ReportsPgRestoreError(
+ t *testing.T,
+) {
+ backupReader := bytes.NewReader([]byte("PGDMP"))
+
+ restoreErr := newStubRestoreUsecase().executePgRestoreViaStdin(
+ t.Context(),
+ newStubPgRestoreCommand(restores_core.WriteLateFailingToolStub(t, "pg_restore", pgRestoreStubError)),
+ backupReader,
+ io_utils.NewFailureTrackingReader(backupReader),
+ )
+
+ require.Error(t, restoreErr)
+ assert.NotContains(t, restoreErr.Error(), "backup stream from storage failed mid-restore")
+ assert.Contains(t, restoreErr.Error(), `relation "orders" already exists`)
+}
+
+func Test_ExecutePgRestoreViaStdin_WhenPgRestoreExitsBeforeReadingBackup_ReportsPgRestoreError(
+ t *testing.T,
+) {
+ // The backup is far larger than a pipe buffer, so pg_restore exits while most of it is
+ // still waiting to be written to its standard input.
+ backupReader := bytes.NewReader(bytes.Repeat([]byte("PGDMP"), 4*1024*1024))
+ pgRestoreStub := restores_core.WriteToolStub(
+ t,
+ "pg_restore",
+ "#!/bin/sh\necho '"+pgRestoreStubError+"' >&2\nexit 1\n",
+ )
+
+ restoreErr := newStubRestoreUsecase().executePgRestoreViaStdin(
+ t.Context(),
+ newStubPgRestoreCommand(pgRestoreStub),
+ backupReader,
+ io_utils.NewFailureTrackingReader(backupReader),
+ )
+
+ require.Error(t, restoreErr)
+ assert.NotContains(t, restoreErr.Error(), "failed to stream backup data to pg_restore")
+ assert.Contains(t, restoreErr.Error(), `relation "orders" already exists`)
+}
diff --git a/backend/internal/features/telemetry/background_service_test.go b/backend/internal/features/telemetry/background_service_test.go
index a4b28347..7f208b87 100644
--- a/backend/internal/features/telemetry/background_service_test.go
+++ b/backend/internal/features/telemetry/background_service_test.go
@@ -182,22 +182,13 @@ func Test_Run_WhenCalledTwice_Panics(t *testing.T) {
sender := &sequenceSender{}
bg := newBackgroundServiceUnderTest(t, sender)
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- finished := make(chan struct{})
- go func() {
- bg.Run(ctx)
- close(finished)
- }()
+ // A cancelled context makes the first Run return at once, after it has marked itself as run.
+ cancelledCtx, cancel := context.WithCancel(context.Background())
+ cancel()
- // Wait long enough to be sure the first Run has flipped hasRun to true.
- time.Sleep(10 * time.Millisecond)
+ bg.Run(cancelledCtx)
require.Panics(t, func() {
- bg.Run(ctx)
+ bg.Run(cancelledCtx)
})
-
- cancel()
- <-finished
}
diff --git a/backend/internal/util/testing/containers/ftp.go b/backend/internal/util/testing/containers/ftp.go
index 1472aa77..689e0648 100644
--- a/backend/internal/util/testing/containers/ftp.go
+++ b/backend/internal/util/testing/containers/ftp.go
@@ -53,7 +53,10 @@ func StartFtpContainer(t *testing.T) ContainerHandle {
HostConfigModifier: func(hc *container.HostConfig) {
hc.PortBindings = withPassivePortRange(hc.PortBindings)
},
- WaitingFor: wait.ForListeningPort(ftpControlPort).WithStartupTimeout(120 * time.Second),
+ WaitingFor: wait.ForAll(
+ wait.ForListeningPort(ftpControlPort).WithStartupTimeout(120*time.Second),
+ forGreeting(ftpControlPort, 120*time.Second, "220"),
+ ),
}
return startContainer(t, req, ftpControlPort)
diff --git a/backend/internal/util/testing/containers/readiness.go b/backend/internal/util/testing/containers/readiness.go
new file mode 100644
index 00000000..31bd892c
--- /dev/null
+++ b/backend/internal/util/testing/containers/readiness.go
@@ -0,0 +1,147 @@
+package containers
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "net"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/hirochachacha/go-smb2"
+ "github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+ readinessProbeInterval = 200 * time.Millisecond
+ readinessProbeTimeout = 5 * time.Second
+)
+
+// Docker's port proxy accepts on the mapped port before the service inside listens, so a port
+// check alone hands out a connection that is reset or closed during the handshake. A readiness
+// probe connects from the host through the mapped port, the way the tests do, and succeeds only
+// once the service answers its protocol.
+type readinessProbe func(ctx context.Context, address string) error
+
+type probeStrategy struct {
+ port string
+ startupTimeout time.Duration
+ probe readinessProbe
+}
+
+func forProbe(port string, startupTimeout time.Duration, probe readinessProbe) wait.Strategy {
+ return &probeStrategy{port: port, startupTimeout: startupTimeout, probe: probe}
+}
+
+// forGreeting waits until the service on port sends a first line that starts with greetingPrefix,
+// such as the SSH identification line or the FTP 220 reply.
+func forGreeting(port string, startupTimeout time.Duration, greetingPrefix string) wait.Strategy {
+ return forProbe(port, startupTimeout, func(ctx context.Context, address string) error {
+ return readGreeting(ctx, address, greetingPrefix)
+ })
+}
+
+// forSmbShare waits until an SMB session logged in with the given credentials mounts the share.
+func forSmbShare(port string, startupTimeout time.Duration, username, password, share string) wait.Strategy {
+ return forProbe(port, startupTimeout, func(ctx context.Context, address string) error {
+ return mountSmbShare(ctx, address, username, password, share)
+ })
+}
+
+func (s *probeStrategy) WaitUntilReady(ctx context.Context, target wait.StrategyTarget) error {
+ ctx, cancel := context.WithTimeout(ctx, s.startupTimeout)
+ defer cancel()
+
+ ticker := time.NewTicker(readinessProbeInterval)
+ defer ticker.Stop()
+
+ for {
+ lastErr := s.probeOnce(ctx, target)
+ if lastErr == nil {
+ return nil
+ }
+
+ select {
+ case <-ctx.Done():
+ return fmt.Errorf("%s did not become ready: %w", s.port, lastErr)
+ case <-ticker.C:
+ }
+ }
+}
+
+func (s *probeStrategy) probeOnce(ctx context.Context, target wait.StrategyTarget) error {
+ host, err := target.Host(ctx)
+ if err != nil {
+ return fmt.Errorf("get container host: %w", err)
+ }
+
+ mappedPort, err := target.MappedPort(ctx, s.port)
+ if err != nil {
+ return fmt.Errorf("get mapped port: %w", err)
+ }
+
+ probeCtx, cancel := context.WithTimeout(ctx, readinessProbeTimeout)
+ defer cancel()
+
+ return s.probe(probeCtx, net.JoinHostPort(host, strconv.Itoa(int(mappedPort.Num()))))
+}
+
+func readGreeting(ctx context.Context, address, greetingPrefix string) error {
+ conn, err := dialWithDeadline(ctx, address)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = conn.Close() }()
+
+ greeting, err := bufio.NewReader(conn).ReadString('\n')
+ if err != nil {
+ return fmt.Errorf("read greeting: %w", err)
+ }
+
+ if !strings.HasPrefix(greeting, greetingPrefix) {
+ return fmt.Errorf("greeting %q does not start with %q", strings.TrimSpace(greeting), greetingPrefix)
+ }
+
+ return nil
+}
+
+func mountSmbShare(ctx context.Context, address, username, password, share string) error {
+ conn, err := dialWithDeadline(ctx, address)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = conn.Close() }()
+
+ dialer := &smb2.Dialer{Initiator: &smb2.NTLMInitiator{User: username, Password: password}}
+
+ session, err := dialer.DialContext(ctx, conn)
+ if err != nil {
+ return fmt.Errorf("log in: %w", err)
+ }
+ defer func() { _ = session.Logoff() }()
+
+ mountedShare, err := session.Mount(share)
+ if err != nil {
+ return fmt.Errorf("mount share %s: %w", share, err)
+ }
+
+ return mountedShare.Umount()
+}
+
+func dialWithDeadline(ctx context.Context, address string) (net.Conn, error) {
+ conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", address)
+ if err != nil {
+ return nil, fmt.Errorf("dial %s: %w", address, err)
+ }
+
+ if deadline, ok := ctx.Deadline(); ok {
+ if err := conn.SetDeadline(deadline); err != nil {
+ _ = conn.Close()
+
+ return nil, fmt.Errorf("set deadline: %w", err)
+ }
+ }
+
+ return conn, nil
+}
diff --git a/backend/internal/util/testing/containers/samba.go b/backend/internal/util/testing/containers/samba.go
index bf064e56..037fef34 100644
--- a/backend/internal/util/testing/containers/samba.go
+++ b/backend/internal/util/testing/containers/samba.go
@@ -36,7 +36,10 @@ func StartSambaContainer(t *testing.T) ContainerHandle {
"-s", SambaShare + ";/shared;yes;no;no;" + SambaUsername,
"-p",
},
- WaitingFor: wait.ForListeningPort(sambaPort).WithStartupTimeout(120 * time.Second),
+ WaitingFor: wait.ForAll(
+ wait.ForListeningPort(sambaPort).WithStartupTimeout(120*time.Second),
+ forSmbShare(sambaPort, 120*time.Second, SambaUsername, SambaPassword, SambaShare),
+ ),
}
return startContainer(t, req, sambaPort)
diff --git a/backend/internal/util/testing/containers/ssh_bastion.go b/backend/internal/util/testing/containers/ssh_bastion.go
index 390492ca..0688c089 100644
--- a/backend/internal/util/testing/containers/ssh_bastion.go
+++ b/backend/internal/util/testing/containers/ssh_bastion.go
@@ -207,6 +207,9 @@ func sshBastionRequest(t *testing.T) testcontainers.ContainerRequest {
KeepImage: true,
},
ExposedPorts: []string{sshBastionPort},
- WaitingFor: wait.ForListeningPort(sshBastionPort).WithStartupTimeout(120 * time.Second),
+ WaitingFor: wait.ForAll(
+ wait.ForListeningPort(sshBastionPort).WithStartupTimeout(120*time.Second),
+ forGreeting(sshBastionPort, 120*time.Second, "SSH-2.0-"),
+ ),
}
}
diff --git a/docker/start.sh b/docker/start.sh
index 9a28472c..b2d65371 100755
--- a/docker/start.sh
+++ b/docker/start.sh
@@ -683,7 +683,7 @@ supervise_application() {
exit 1
fi
- echo "Databasus application exited with status ${exit_status}"
+ echo "Databasus application exited with status ${exit_status}" >&2
stop_postgresql_server
exit "${exit_status}"
}
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/.openspec.yaml b/openspec/changes/archive/2026-10-21-fix-flaky-tests/.openspec.yaml
new file mode 100644
index 00000000..dc36fb1b
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/.openspec.yaml
@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-10-21
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/design.md b/openspec/changes/archive/2026-10-21-fix-flaky-tests/design.md
new file mode 100644
index 00000000..2944fd8f
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/design.md
@@ -0,0 +1,97 @@
+## Context
+
+See proposal.md for why. The flaky failures come from five separate causes, so this design has one decision per cause. Each was found in CI logs from the last two months, and none reproduced in three local full runs on an idle machine.
+
+**Error output of child processes.** Ten places start a database tool, read its standard error through `cmd.StderrPipe()` in a goroutine, and call `cmd.Wait()` before that goroutine has finished reading:
+
+- Restores: `restores/usecases/mysql/restore_backup_uc.go:205-221`, `mariadb/restore_backup_uc.go:224-240`, `mongodb/restore_backup_uc.go:213-229`, and `postgresql/restore_backup_uc.go:297-333` and `:693-712`.
+- Logical backups: `backups/backups/usecases/logical/postgresql/create_backup_uc.go:165,217,242`, `mysql/create_backup_uc.go:220,275,304`, `mariadb/create_backup_uc.go:223,278,307` and `mongodb/create_backup_uc.go:165,216,241`. The stdout copy finishes before `Wait`, which narrows the window, but a tool that fails before writing any data still loses its message.
+- The WAL receiver: `backups/backups/usecases/physical/postgresql/receivewal_process.go:83-107` starts `cmd.Wait()` in a goroutine right after `cmd.Start()`. The fatal-error check at `:144` and the resume-mismatch check at `:131` then read whatever the capture got before `Wait` closed the pipe.
+
+The `os/exec` documentation forbids this order: `Wait` closes the read end of the pipe, and output not yet read is lost. The CI failure `"mysql-stub failed: exit status 1 – stderr: " does not contain "SQL syntax"` in `restores/usecases/mysql/restore_backup_uc_test.go:103` is this race. It failed four times. `pg_basebackup.go:229` and `stream.go:204-217` already drain the output correctly before `Wait` and are not touched.
+
+**Test container readiness.** `util/testing/containers/ssh_bastion.go:210`, `samba.go:39` and `ftp.go:56` wait with `wait.ForListeningPort` only. Docker's port proxy accepts on the mapped port before the service inside listens, so the first connection is reset during the handshake. SFTP had the same failure and now also waits for its server's log line (`sftp.go:23-31`). The FTP failure `failed to dial the container after adding egress delay: connection refused` (`netem.go:35`) dials the container's own address, which bypasses the proxy, and has the same cause: the server was not listening yet.
+
+**Storage failure before the dump starts.** Each logical backup starts the storage write, `storage_files.StartBackgroundWrite`, before it starts the dump tool, for example `logical/mysql/create_backup_uc.go:250` and `:258`. A storage that fails at once cancels the backup context, and `exec.CommandContext` then refuses to start the tool. The backup fails with `start mysqldump: context canceled` instead of the storage error. The shared assertion at `backups/backups/core/logical/testing.go:79-90` (`Test_Execute_WhenStorageFailsBeforeReading_ReturnsStorageError`) caught this once in CI.
+
+**Fixed waits in tests.**
+
+- `telemetry/background_service_test.go:180-202` starts `Run` in a goroutine, sleeps 10 ms and expects a second `Run` to panic. If the goroutine has not reached `hasRun.Swap(true)` (`background_service.go:55`) by then, the test's own call becomes the first one and blocks. The goroutine's call then panics outside `require.Panics`, and that kills the whole test binary of the package.
+- `agent/verification/internal/features/api/idle_timeout_reader_test.go:14-71` uses a 100–200 ms idle timeout against writes every 50 ms and sleeps of 200 ms. That leaves 100–150 ms of slack for a loaded runner.
+
+**Log order in the Docker filesystem suite.** `e2e/docker-storage/run.sh:1024-1040` compares line numbers in `docker logs 2>&1`. The supervisor writes `Databasus application exited with status <n>` to standard output (`docker/start.sh:686`), and PostgreSQL writes `received fast shutdown request` to standard error, its default log destination, because `start.sh:497-503` passes no log settings. Docker copies the two streams in separate goroutines and does not order them against each other.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- Every failure listed in the proposal becomes deterministic: it passes on a loaded runner because the code no longer depends on timing, not because a limit was raised.
+- Each bug fix starts with a test that fails on today's code for the reason described above.
+
+**Non-Goals:**
+
+- Bounding the size of captured error output. Today's capture is unbounded too.
+- Merging the four logical backup usecases or the restore usecases into shared code beyond what each decision below needs.
+- Changing `pg_basebackup.go` or `stream.go`, which already drain their output correctly.
+
+## Decisions
+
+### 1. Capture error output with `cmd.Stderr` set to a buffer
+
+Each of the ten places sets `cmd.Stderr` to a `bytes.Buffer` and drops the pipe and its goroutine. `os/exec` then copies the output itself, and `cmd.Wait()` returns only after the copy has finished. Reading the buffer after `Wait` therefore always sees all the output. The WAL receiver keeps its `cmd.Wait()` goroutine and its `select`, reads the buffer after `exited` delivers, and drops `newStderrCapture` from its file.
+
+When `Wait` waits for that copy, a grandchild process holding the inherited stderr open can keep `Wait` blocked after the tool itself exited. So every place that switches to the buffer also sets `cmd.WaitDelay` to 10 seconds, as `pg_basebackup.go:121` and `receivewal_process.go:248` already do with `killAfterCancel` (`pg_basebackup.go:27`). The restore and logical backup packages each get their own 10-second constant, because the physical backup package is not a dependency they should take. When the delay expires, `Wait` closes the pipes and returns `exec.ErrWaitDelay` even for a tool that exited with status 0. No special handling is added for that case, because none of these tools leaves a process behind after a successful exit. `pg_dump -j` and `pg_restore -j` wait for their workers before they exit, and `mysqldump`, `mysql`, the MariaDB tools, `mongodump`, `mongorestore` and `pg_receivewal` start no long-lived children.
+
+The PostgreSQL restore over standard input (`postgresql/restore_backup_uc.go:291-359`) needs one more change. It pumps the backup through `cmd.StdinPipe()`. When `pg_restore` exits before reading everything, that copy fails with a broken pipe, and `:357-359` returns `failed to stream backup data to pg_restore` before the code reaches `resolvePgRestoreOutcome` with the tool's output. After this change, a non-nil `waitErr` goes to `resolvePgRestoreOutcome` first, and the copy error is reported only when `pg_restore` itself succeeded. Read failures of the backup stream are not affected. They are caught earlier by `GetBackupStreamFailure` (`:351-355`), because the failure-tracking reader sits on the read side of that copy. The MySQL, MariaDB and MongoDB restores pass the stream through `cmd.Stdin`, where `os/exec` ignores a broken pipe and `Wait` reports the tool's exit status, so they need no such change.
+
+Alternatives rejected:
+
+- **Receive from the channel before calling `Wait`.** It is a one-line move in each place, but it keeps ten copies of the goroutine boilerplate. It also makes the caller wait for end of output with no bound, while `WaitDelay` gives a bound.
+- **Move `stderrCapture` from `pg_basebackup.go:196` to a shared package.** It exists for a caller that needs a drain with a timeout and a forced stop. None of the ten places reads output while the tool runs, so its extra surface buys them nothing.
+
+### 2. Readiness checks that speak each protocol
+
+Each of the three containers waits until a probe from the host gets a valid protocol answer through the mapped port, alongside the existing port wait:
+
+- **SSH bastion:** read the server's identification line and require the `SSH-2.0-` prefix.
+- **FTP:** read the greeting and require the `220` reply code.
+- **Samba:** an SMB session through the SMB client library the NAS storage already uses (`github.com/hirochachacha/go-smb2`, `storages/models/nas/model.go:247`), logged in as `SambaUsername` with `SambaPassword` from `samba.go`, which then mounts `SambaShare`. That library runs negotiation and login together and refuses anonymous sessions, and a mounted share is also what the NAS tests need.
+
+The probe is a `wait.Strategy` in `util/testing/containers`, written once for the line-greeting case and reused by SSH and FTP.
+
+Alternatives rejected:
+
+- **Wait for a log line, as `sftpReady` does.** The bastion is built from our own Dockerfile, and its sshd logging target would need checking. The Samba and FTP images are third-party, and their log lines are not a contract. A protocol answer is what the tests actually need.
+- **Retry the connection inside the tests or the storage code.** That spreads a test-fixture problem across production code and many call sites, and a retry in production code would hide real connection failures.
+
+### 3. Start the dump tool before the storage write
+
+Each logical backup calls `cmd.Start()` before `storage_files.StartBackgroundWrite`. A storage that fails at once, before reading any backup data, can then no longer stop the tool from starting. A storage failure after the start already reaches the `save to storage` path that the after-the-dump scenario tests (`create_backup_uc.go:219-226` in PostgreSQL and the same block in the other three).
+
+Alternative rejected: keep the order, and when `cmd.Start()` fails with a cancelled context, wait for the storage write's error and report it instead. That adds a branch to four files to explain a failure the reordering removes. The reordering also lets the start-failure path drop its storage cleanup.
+
+The reordering is safe because the tool's stdout is an OS pipe that nothing reads until the copy goroutine starts. The tool blocks on a full pipe until then, which is the same backpressure it has today.
+
+### 4. Tests wait for a condition or remove the race
+
+- **Telemetry:** the test calls `Run` synchronously first with an already cancelled context. `Run` sets `hasRun`, and `sleep` at `background_service.go:97` returns at once on a cancelled context, so the call returns. The second `Run` then must panic, and no goroutine or sleep is involved. A rejected alternative polled `hasRun` with `require.Eventually`. It still needs a goroutine, and it reads private state.
+- **Idle-timeout reader:** the checks that expect cancellation use `require.Eventually` with a bound of seconds. The check that expects no cancellation while bytes flow raises the idle timeout to seconds while keeping the 50 ms write interval. That keeps the test fast, because the writer still finishes in 250 ms, and leaves a margin no scheduler pause reaches. The check that `Stop` prevents cancellation keeps its fixed sleep. After `Stop` no cancellation can happen, so the sleep only gives a broken `Stop` the chance to show itself.
+
+### 5. The supervisor's exit line goes to standard error
+
+`docker/start.sh:686` writes its line with `>&2`. PostgreSQL inherits the same standard error, and the supervisor sends the shutdown signal only after the `echo` returned. The two lines therefore reach Docker through one pipe in order. The PostgreSQL-exit error at `:680` already goes to standard error.
+
+Alternative rejected: compare `docker logs --timestamps`. The daemon stamps each stream's lines as its copier reads them, so the timestamps can invert just as the line order does.
+
+## Risks / Trade-offs
+
+- [A process that writes huge error output fills memory] → The buffer is unbounded, as `io.ReadAll` is today. No behavior gets worse.
+- [A grandchild outlives the tool and holds its error output open] → After 10 seconds `Wait` closes the pipes and returns `exec.ErrWaitDelay`, so a tool that exited with status 0 would be reported as failed. Decision 1 explains why none of these tools leaves such a process behind. If one ever does, the failure names `exec.ErrWaitDelay` and points straight at the cause.
+- [A readiness probe is wrong for one image and the container never becomes ready] → The probe has the existing 120-second startup timeout and fails with the probe's last error, so a wrong probe shows up as a clear setup failure in the first local run of the touched package.
+- [The reordering in decision 3 changes behavior when the tool fails to start for its own reasons] → The storage write has not started yet, so there is nothing to clean up. The start error is returned as today.
+- [The tests for the races still pass on broken code if the race window does not open] → Repeating a fast-exiting stub does not open it reliably: even on one processor the output got lost about once in 1500 runs. So each error-output test uses a stub that writes its error from a background process shortly after the stub itself exited, which is how a fast exit looks when the reader is scheduled late. Today's code loses that output on every run, and each task verifies the failure before the fix.
+- [The storage-before-start race of decision 3 has no test that fails on today's code] → `exec.Cmd.Start` checks the context right after the storage write's goroutine is started, with no point in between where the scheduler can switch, so the store won the race in at most 6 of 3000 runs. A hook between the two calls would make it deterministic, but only for tests and in four files. The reordering removes the race by construction, and the existing `Test_Execute_WhenStorageFailsBeforeReading_ReturnsStorageError` keeps the scenario covered.
+
+## Migration Plan
+
+Nothing to migrate. The change ships with the next release, and rollback is a revert.
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/proposal.md b/openspec/changes/archive/2026-10-21-fix-flaky-tests/proposal.md
new file mode 100644
index 00000000..e8daaa50
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/proposal.md
@@ -0,0 +1,43 @@
+## Why
+
+The backend suite fails in CI often enough that a red run no longer means a regression: over the last two months most failed attempts passed on a rerun of the same commit. Three local full runs on an idle machine passed, so these failures depend on runner load rather than on the code under test. Two of the causes are also defects users see. A restore or backup whose database tool exits quickly can lose the tool's error output, and a backup whose storage fails at once reports `context canceled` instead of the storage error.
+
+## What Changes
+
+- Test containers for the SSH bastion, Samba and FTP count as ready only once the service inside answers its protocol, not as soon as Docker's proxy accepts on the mapped port. Today they wait for an open port only, which produced about a dozen `connection reset by peer` and `EOF` failures, plus the FTP `connection refused` after an egress delay is added.
+- Every place that runs a database client tool hands the tool's error output to a buffer that the process wait itself fills to the end, with a bounded wait for that output. Today restores for all four engines, logical backups for all four engines and the WAL receiver read the output through a pipe in a goroutine and wait for the process first, and waiting closes the pipe the output is still in. Restores and backups then report a failure without the tool's message, and the WAL receiver can treat a fatal error as one worth retrying.
+- A PostgreSQL restore whose tool exits before reading the whole backup reports the tool's error instead of the failure to pass the remaining data to it.
+- A logical backup whose storage fails at once, before reading any backup data, reports the storage error. Today the backup context is already cancelled when the tool starts, so the backup fails with `start <tool>: context canceled`.
+- Tests that wait a fixed time for another goroutine stop doing so: the telemetry test that calls `Run` twice, and the idle-timeout reader tests in the verification agent.
+- The container's supervisor writes `Databasus application exited with status <n>` to standard error, the same stream PostgreSQL logs to, so the Docker filesystem suite's check that this line precedes PostgreSQL's shutdown request stops depending on how Docker interleaves two streams.
+
+No caller, schema, configuration or API breaks.
+
+## Out of scope
+
+- The TimescaleDB restore failure `schema "public" already exists`. Its cause is unknown and needs its own investigation.
+- `Test_WalStream_WhenSourceIsIdle_KeepsReceiverAliveAndDoesNotRebuildSlot`, which failed once because the receiver dropped its replication connection on an idle source. Nothing points to a test-only cause, so it needs its own investigation before anything is changed.
+- Negative assertions that sleep before checking that nothing happened, in scheduler, notification and disk-watch tests. They pass reliably today and may miss bugs, but they are not flaky.
+- Fixed sleeps in test cleanup that wait for cascading deletes.
+- The 10-second budget for `docker stop` in the Docker filesystem suite, which has never failed in CI.
+- CI workflow changes, reruns and retry wrappers around flaky tests.
+
+## Capabilities
+
+### New Capabilities
+
+- `database-tool-failure-output`: what a restore and the WAL receiver do with the error output of the database tool they run when that tool fails.
+
+### Modified Capabilities
+
+- `logical-backup-outcome`: the fail message of a logical backup includes the dump tool's error output, and a storage that fails before reading any backup data is reported as the storage error.
+
+## Impact
+
+- Backend restore usecases for PostgreSQL, MySQL, MariaDB and MongoDB, logical backup usecases for the same engines, and the WAL receiver of physical backups.
+- The error-output pipe and its goroutine in each of those places, which give way to a buffer plus a bounded wait. The WAL receiver drops its use of the physical backup package's capture helper, which stays for `pg_basebackup`.
+- Backend test containers for the SSH bastion, Samba and FTP.
+- The telemetry background service test and the verification agent's idle-timeout reader tests.
+- `docker/start.sh` and the `clean-shutdown` case of the Docker filesystem suite.
+
+The change answers to the root `AGENTS.md`, `backend/AGENTS.md` and `agent/verification/AGENTS.md`.
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/database-tool-failure-output/spec.md b/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/database-tool-failure-output/spec.md
new file mode 100644
index 00000000..62dec4b4
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/database-tool-failure-output/spec.md
@@ -0,0 +1,43 @@
+## Purpose
+
+Defines what a restore and the WAL receiver of a physical backup do with the error output of the database tool they run, so a failure tells the user what the tool reported and the WAL receiver stops retrying a failure that a retry cannot fix.
+
+## ADDED Requirements
+
+### Requirement: A failed restore reports the restore tool's error output
+
+When the database tool that loads a backup into a PostgreSQL, MySQL, MariaDB or MongoDB database exits with an error, the restore SHALL be recorded as failed, and its failure message SHALL contain the error output the tool wrote before it exited. This SHALL hold however quickly the tool exits after writing that output, and whether or not the tool read the whole backup before it exited. One exception applies: a PostgreSQL restore whose only error is that the `public` schema already exists SHALL count as successful. When the backup stream from storage failed first, the failure message SHALL report that stream failure instead of the tool's output, because the tool's error is then only a consequence of the incomplete input.
+
+#### Scenario: Tool rejects the backup and exits immediately
+
+- **WHEN** a restore runs, the restore tool reads the whole backup, writes `ERROR 1064 (42000) at line 1: You have an error in your SQL syntax` to its error output and exits with a non-zero status at once
+- **THEN** the restore is recorded as failed, and its failure message contains `You have an error in your SQL syntax`
+
+#### Scenario: Tool exits before reading the whole backup
+
+- **WHEN** a restore runs, and the restore tool writes an error and exits with a non-zero status before it has read the whole backup
+- **THEN** the restore is recorded as failed, and its failure message contains the error the tool wrote, not only a failure to pass the backup data to the tool
+
+#### Scenario: Every engine keeps the tool's output
+
+- **WHEN** the restore tool of a PostgreSQL, a MySQL, a MariaDB or a MongoDB restore writes an error and exits at once
+- **THEN** each restore's failure message contains the error that tool wrote
+
+#### Scenario: Only the existing public schema fails
+
+- **WHEN** a PostgreSQL restore tool exits with an error whose only complaint is that the `public` schema already exists
+- **THEN** the restore is recorded as successful
+
+#### Scenario: Backup stream fails before the tool does
+
+- **WHEN** the backup stream from storage ends early, and the restore tool then fails on the incomplete input
+- **THEN** the failure message reports that the backup stream from storage failed mid-restore, and does not contain the tool's complaint about the incomplete input
+
+### Requirement: The WAL receiver recognizes a failure that a retry cannot fix
+
+When the process that receives WAL for a physical backup exits because the local disk is full or not writable, because the source rejected its credentials, or because another consumer holds its replication slot, and the source's network path is reachable, Databasus SHALL stop restarting it and SHALL mark the WAL stream as failed with an error that contains the receiver's error output. This SHALL hold however quickly the receiver exits after writing that output.
+
+#### Scenario: Credentials rejected
+
+- **WHEN** the source's network path is reachable, the WAL receiver connects, the source answers that password authentication failed, and the receiver exits at once
+- **THEN** the WAL stream is marked as failed with an error that contains `authentication failed`, and the receiver is not restarted on its retry schedule
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/logical-backup-outcome/spec.md b/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/logical-backup-outcome/spec.md
new file mode 100644
index 00000000..2c64ac44
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/specs/logical-backup-outcome/spec.md
@@ -0,0 +1,31 @@
+## ADDED Requirements
+
+### Requirement: The fail message includes the dump tool's error output
+
+When the dump tool of a logical PostgreSQL, MySQL, MariaDB or MongoDB backup exits with an error and the storage upload did not fail, the backup SHALL be recorded as failed, and its fail message SHALL contain the error output the tool wrote before it exited. This SHALL hold however quickly the tool exits after writing that output, including when it exits before it produced any backup data.
+
+#### Scenario: Dump tool fails before producing data
+
+- **WHEN** the dump tool writes an error such as `permission denied for sequence orders_id_seq` and exits with a non-zero status before producing any backup data
+- **THEN** the backup is recorded as failed, and its fail message contains `permission denied for sequence orders_id_seq`
+
+## MODIFIED Requirements
+
+### Requirement: The fail message names the storage error
+
+When a logical backup fails because its storage upload failed, the fail message SHALL contain the error the storage returned, so the user can see which storage operation failed and why. It SHALL NOT be reduced to a generic message such as "backup cancelled". This SHALL hold when the storage fails the upload at once, before it has read any backup data, and the fail message SHALL NOT then report that starting the dump tool was cancelled.
+
+#### Scenario: Fail message after a storage timeout
+
+- **WHEN** a logical backup fails because the upload to its S3 storage timed out while connecting to the endpoint
+- **THEN** the fail message shows that saving to storage failed and includes the storage's own description of the timeout, including the endpoint it tried to reach
+
+#### Scenario: Fail message is the same for every engine
+
+- **WHEN** the same storage timeout interrupts a PostgreSQL, a MySQL, a MariaDB and a MongoDB backup
+- **THEN** each backup is recorded as failed with a fail message that includes the storage error
+
+#### Scenario: Storage fails before reading any data
+
+- **WHEN** the storage fails the upload of a logical backup at once, before it has read any backup data
+- **THEN** the backup is recorded as failed, its fail message shows that saving to storage failed and includes the storage error, and it does not say that starting the dump tool was cancelled
diff --git a/openspec/changes/archive/2026-10-21-fix-flaky-tests/tasks.md b/openspec/changes/archive/2026-10-21-fix-flaky-tests/tasks.md
new file mode 100644
index 00000000..6f23f1fe
--- /dev/null
+++ b/openspec/changes/archive/2026-10-21-fix-flaky-tests/tasks.md
@@ -0,0 +1,40 @@
+## 1. Error output of restore tools
+
+- [x] 1.1 Make the MySQL restore regression test fail reliably on today's code: in `restores/usecases/mysql/restore_backup_uc_test.go`, let the client stub write its error from a background process shortly after the stub itself exited, which is how a fast exit looks when the output reader is scheduled late, and confirm that `just test-package ./internal/features/restores/usecases/mysql/...` in `backend/` fails with an empty `stderr:` before any fix.
+- [x] 1.2 Switch the MySQL restore to `cmd.Stderr` set to a `bytes.Buffer` with `cmd.WaitDelay`, as design decision 1 describes, and verify that `just test-package ./internal/features/restores/usecases/mysql/...` passes.
+- [x] 1.3 Add the same late-writing-stub regression test to the MariaDB, MongoDB and PostgreSQL restore packages, covering both PostgreSQL restore paths, and confirm each fails on today's code. Then apply design decision 1 in `mariadb/restore_backup_uc.go`, `mongodb/restore_backup_uc.go` and `postgresql/restore_backup_uc.go`, and verify that `just test-package ./internal/features/restores/...` passes.
+- [x] 1.4 Add a regression test for the PostgreSQL restore over standard input in which a `pg_restore` stub writes an error and exits before reading the whole backup, and confirm it fails on today's code with `failed to stream backup data to pg_restore`. Then send a non-nil `waitErr` to `resolvePgRestoreOutcome` before the copy error, as design decision 1 describes, and verify that `just test-package ./internal/features/restores/usecases/postgresql/...` passes, including the pre-existing `public` schema tests in that package.
+
+## 2. Error output of dump tools and the WAL receiver
+
+- [x] 2.1 Add a regression test, shared through `backups/backups/core/logical/testing.go` the way the storage-timeout assertion is, in which a dump tool stub writes an error and exits before producing data. Run it for all four engines, confirm it fails on today's code, apply design decision 1 to the four logical `create_backup_uc.go` files, and verify that `just test-package ./internal/features/backups/backups/usecases/logical/...` passes.
+- [x] 2.2 Add a regression test in `backups/backups/usecases/physical/postgresql/` in which a receiver stub writes `FATAL: password authentication failed` and exits at once, and assert the receiver run ends as fatal with that text in its error. Confirm it fails on today's code, apply design decision 1 to `receivewal_process.go`, remove `newStderrCapture` from that file, and verify that `just test-package ./internal/features/backups/backups/usecases/physical/postgresql/...` passes.
+
+## 3. Storage failure before the dump starts
+
+- [x] 3.1 Move `cmd.Start()` before `storage_files.StartBackgroundWrite` in the four logical `create_backup_uc.go` files, as design decision 3 describes, and verify that `just test-package ./internal/features/backups/backups/usecases/logical/...` passes.
+
+## 4. Test container readiness
+
+- [x] 4.1 Add a greeting-probe `wait.Strategy` to `util/testing/containers` and use it for the SSH bastion (`SSH-2.0-`) and FTP (`220`) next to their port waits. Verify that `just test-package ./internal/util/testing/containers/... ./internal/features/sshtunnel/... ./internal/features/storages/...` passes.
+- [x] 4.2 Add a Samba readiness probe that logs in with `SambaUsername` and `SambaPassword` through the NAS storage's SMB client library and mounts `SambaShare`, as design decision 2 describes, and verify that `just test-package ./internal/util/testing/containers/... ./internal/features/storages/...` passes.
+
+## 5. Fixed waits in tests
+
+- [x] 5.1 Rewrite `Test_Run_WhenCalledTwice_Panics` in `telemetry/background_service_test.go` to call `Run` first with a cancelled context and drop the goroutine and the sleep, as design decision 4 describes, and verify that `just test-package ./internal/features/telemetry/...` passes.
+- [x] 5.2 Rewrite the idle-timeout reader tests in `agent/verification/internal/features/api/idle_timeout_reader_test.go` as design decision 4 describes, and verify that `just --justfile agent/verification/Justfile test` passes.
+
+## 6. Log order in the Docker filesystem suite
+
+- [x] 6.1 Send the `Databasus application exited with status` line in `docker/start.sh` to standard error, as design decision 5 describes. Verify that `bash -n docker/start.sh` succeeds and that `git grep -n 'Databasus application exited with status' docker/start.sh` shows the line ending in `>&2`.
+
+## Change verification
+
+```sh
+just test-package ./internal/features/restores/...
+just test-package ./internal/features/backups/backups/usecases/logical/...
+just test-package ./internal/features/backups/backups/usecases/physical/postgresql/...
+just test-package ./internal/util/testing/containers/... ./internal/features/sshtunnel/... ./internal/features/storages/...
+just test-package ./internal/features/telemetry/...
+just --justfile agent/verification/Justfile test
+```
diff --git a/openspec/specs/database-tool-failure-output/spec.md b/openspec/specs/database-tool-failure-output/spec.md
new file mode 100644
index 00000000..74c5d0e8
--- /dev/null
+++ b/openspec/specs/database-tool-failure-output/spec.md
@@ -0,0 +1,45 @@
+# Database Tool Failure Output Specification
+
+## Purpose
+
+Defines what a restore and the WAL receiver of a physical backup do with the error output of the database tool they run, so a failure tells the user what the tool reported and the WAL receiver stops retrying a failure that a retry cannot fix.
+
+## Requirements
+
+### Requirement: A failed restore reports the restore tool's error output
+
+When the database tool that loads a backup into a PostgreSQL, MySQL, MariaDB or MongoDB database exits with an error, the restore SHALL be recorded as failed, and its failure message SHALL contain the error output the tool wrote before it exited. This SHALL hold however quickly the tool exits after writing that output, and whether or not the tool read the whole backup before it exited. One exception applies: a PostgreSQL restore whose only error is that the `public` schema already exists SHALL count as successful. When the backup stream from storage failed first, the failure message SHALL report that stream failure instead of the tool's output, because the tool's error is then only a consequence of the incomplete input.
+
+#### Scenario: Tool rejects the backup and exits immediately
+
+- **WHEN** a restore runs, the restore tool reads the whole backup, writes `ERROR 1064 (42000) at line 1: You have an error in your SQL syntax` to its error output and exits with a non-zero status at once
+- **THEN** the restore is recorded as failed, and its failure message contains `You have an error in your SQL syntax`
+
+#### Scenario: Tool exits before reading the whole backup
+
+- **WHEN** a restore runs, and the restore tool writes an error and exits with a non-zero status before it has read the whole backup
+- **THEN** the restore is recorded as failed, and its failure message contains the error the tool wrote, not only a failure to pass the backup data to the tool
+
+#### Scenario: Every engine keeps the tool's output
+
+- **WHEN** the restore tool of a PostgreSQL, a MySQL, a MariaDB or a MongoDB restore writes an error and exits at once
+- **THEN** each restore's failure message contains the error that tool wrote
+
+#### Scenario: Only the existing public schema fails
+
+- **WHEN** a PostgreSQL restore tool exits with an error whose only complaint is that the `public` schema already exists
+- **THEN** the restore is recorded as successful
+
+#### Scenario: Backup stream fails before the tool does
+
+- **WHEN** the backup stream from storage ends early, and the restore tool then fails on the incomplete input
+- **THEN** the failure message reports that the backup stream from storage failed mid-restore, and does not contain the tool's complaint about the incomplete input
+
+### Requirement: The WAL receiver recognizes a failure that a retry cannot fix
+
+When the process that receives WAL for a physical backup exits because the local disk is full or not writable, because the source rejected its credentials, or because another consumer holds its replication slot, and the source's network path is reachable, Databasus SHALL stop restarting it and SHALL mark the WAL stream as failed with an error that contains the receiver's error output. This SHALL hold however quickly the receiver exits after writing that output.
+
+#### Scenario: Credentials rejected
+
+- **WHEN** the source's network path is reachable, the WAL receiver connects, the source answers that password authentication failed, and the receiver exits at once
+- **THEN** the WAL stream is marked as failed with an error that contains `authentication failed`, and the receiver is not restarted on its retry schedule
diff --git a/openspec/specs/logical-backup-outcome/spec.md b/openspec/specs/logical-backup-outcome/spec.md
index 748e687a..d5dd511e 100644
--- a/openspec/specs/logical-backup-outcome/spec.md
+++ b/openspec/specs/logical-backup-outcome/spec.md
@@ -54,7 +54,7 @@ A cancelled backup SHALL NOT send a backup failure notification and SHALL NOT ma
### Requirement: The fail message names the storage error
-When a logical backup fails because its storage upload failed, the fail message SHALL contain the error the storage returned, so the user can see which storage operation failed and why. It SHALL NOT be reduced to a generic message such as "backup cancelled".
+When a logical backup fails because its storage upload failed, the fail message SHALL contain the error the storage returned, so the user can see which storage operation failed and why. It SHALL NOT be reduced to a generic message such as "backup cancelled". This SHALL hold when the storage fails the upload at once, before it has read any backup data, and the fail message SHALL NOT then report that starting the dump tool was cancelled.
#### Scenario: Fail message after a storage timeout
@@ -65,3 +65,17 @@ When a logical backup fails because its storage upload failed, the fail message
- **WHEN** the same storage timeout interrupts a PostgreSQL, a MySQL, a MariaDB and a MongoDB backup
- **THEN** each backup is recorded as failed with a fail message that includes the storage error
+
+#### Scenario: Storage fails before reading any data
+
+- **WHEN** the storage fails the upload of a logical backup at once, before it has read any backup data
+- **THEN** the backup is recorded as failed, its fail message shows that saving to storage failed and includes the storage error, and it does not say that starting the dump tool was cancelled
+
+### Requirement: The fail message includes the dump tool's error output
+
+When the dump tool of a logical PostgreSQL, MySQL, MariaDB or MongoDB backup exits with an error and the storage upload did not fail, the backup SHALL be recorded as failed, and its fail message SHALL contain the error output the tool wrote before it exited. This SHALL hold however quickly the tool exits after writing that output, including when it exits before it produced any backup data.
+
+#### Scenario: Dump tool fails before producing data
+
+- **WHEN** the dump tool writes an error such as `permission denied for sequence orders_id_seq` and exits with a non-zero status before producing any backup data
+- **THEN** the backup is recorded as failed, and its fail message contains `permission denied for sequence orders_id_seq`