PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Send S3, Azure and Google Drive traffic through the environment proxy and report storage timeouts as failed backups instead of cancelled ones
Rostislav Dugin4c1e2a50278b
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper.go b/backend/internal/features/backups/backups/backuping/logical/backuper.goindex 93b81167..6a7454c1 100644--- a/backend/internal/features/backups/backups/backuping/logical/backuper.go+++ b/backend/internal/features/backups/backups/backuping/logical/backuper.go@@ -4,11 +4,9 @@ import ("bytes""context""encoding/json"- "errors""fmt""log/slog""slices"- "strings""time""github.com/google/uuid"@@ -130,18 +128,11 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot"error", err,)- // Check if backup was cancelled (not due to shutdown)- isCancelled := strings.Contains(errMsg, "backup cancelled") ||- strings.Contains(errMsg, "context canceled") ||- errors.Is(err, context.Canceled)- isShutdown := strings.Contains(errMsg, "shutdown")-- if isCancelled && !isShutdown {- logger.WarnContext(ctx, "backup was cancelled by user or system",- "backup_id", backup.ID,- "is_cancelled", isCancelled,- "is_shutdown", isShutdown,- )+ // Only a cancel request cancels executionCtx. Shutdown, the time limit and a failed upload+ // end the use case's own context instead, and their errors can still read as a+ // cancellation, so the error decides nothing here.+ if executionCtx.Err() != nil {+ logger.WarnContext(ctx, "backup was cancelled on request", "backup_id", backup.ID)backup.Status = backups_core_logical.BackupStatusCanceledbackup.BackupDurationMs = time.Since(start).Milliseconds()diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go b/backend/internal/features/backups/backups/backuping/logical/backuper_test.goindex 81012eda..504feccc 100644--- a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go@@ -29,6 +29,7 @@ import ("databasus-backend/internal/util/cache""databasus-backend/internal/util/encryption"util_logger "databasus-backend/internal/util/logger"+ test_utils "databasus-backend/internal/util/testing")const partialLogicalArtifact = "partial logical backup"@@ -282,9 +283,15 @@ func (h *requestIDCapturingHandler) GetRequestIDForMessagePrefix(prefix string)// Cancellation and metadata failure are backuper concerns, not engine concerns:// every engine reaches storage through the same file store, so the discard logic// these cover is the same code for MySQL, MariaDB, MongoDB and PostgreSQL.-type createStoredArtifactThenCancelBackupUsecase struct{}+// createStoredArtifactThenBlockBackupUsecase stores an artifact, reports that it started, and holds+// the backup open until its context ends, the way a real engine sits in a running dump when the+// user cancels. It then returns errAfterCancel, or the cancellation an engine reports by default.+type createStoredArtifactThenBlockBackupUsecase struct {+ startedBackup chan struct{}+ errAfterCancel error+}-func (*createStoredArtifactThenCancelBackupUsecase) Execute(+func (uc *createStoredArtifactThenBlockBackupUsecase) Execute(ctx context.Context,backup *backups_core_logical.LogicalBackup,_ *backups_config_logical.LogicalBackupConfig,@@ -300,9 +307,31 @@ func (*createStoredArtifactThenCancelBackupUsecase) Execute(return nil, err}+ close(uc.startedBackup)+ <-ctx.Done()++ if uc.errAfterCancel != nil {+ return nil, uc.errAfterCancel+ }+return nil, fmt.Errorf("backup cancelled: %w", context.Canceled)}+type returnErrorBackupUsecase struct {+ backupErr error+}++func (uc *returnErrorBackupUsecase) Execute(+ _ context.Context,+ _ *backups_core_logical.LogicalBackup,+ _ *backups_config_logical.LogicalBackupConfig,+ _ *databases.Database,+ _ backups_core_logical.BackupFileStore,+ _ func(completedMBs float64),+) (*backups_core_logical.BackupArtifacts, error) {+ return nil, uc.backupErr+}+type createUnusableMetadataBackupUsecase struct{}func (*createUnusableMetadataBackupUsecase) Execute(@@ -332,19 +361,151 @@ func (*createUnusableMetadataBackupUsecase) Execute(func Test_MakeBackup_WhenCancelledAfterStoringArtifact_ArtifactRemoved(t *testing.T) {fixture := CreateBackupTestFixture(t, "Cancelled Backup Artifact Workspace")backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)+ notificationSender := createRecordingNotificationSender()- notificationSender := &MockNotificationSender{}- notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()+ backuper := CreateTestBackuperWithUseCase(&createStoredArtifactThenBlockBackupUsecase{+ startedBackup: make(chan struct{}),+ })+ backuper.notificationSender = notificationSender++ makeBackupCancellingThroughRegistry(t, backuper, backup.ID)++ assertBackupCancelledSilently(t, fixture, backup.ID, notificationSender)+ assertBackupFilesRemoved(t, fixture, backup.FileName)+}++func Test_MakeBackup_WhenCancelledAndStorageThenFails_BackupCancelledWithoutFailureNotification(t *testing.T) {+ fixture := CreateBackupTestFixture(t, "Cancelled Then Storage Failed Workspace")+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)+ notificationSender := createRecordingNotificationSender()++ backuper := CreateTestBackuperWithUseCase(&createStoredArtifactThenBlockBackupUsecase{+ startedBackup: make(chan struct{}),+ errAfterCancel: fmt.Errorf("save to storage: %w", test_utils.GetNetworkTimeoutError(t)),+ })+ backuper.notificationSender = notificationSender++ makeBackupCancellingThroughRegistry(t, backuper, backup.ID)++ assertBackupCancelledSilently(t, fixture, backup.ID, notificationSender)+}++// Test_MakeBackup_WhenStorageTimesOut_BackupFailedAndReported_RegressionForIssue816 pins issue+// #816: a storage upload that timed out was recorded as cancelled, so no failure notification+// went out.+func Test_MakeBackup_WhenStorageTimesOut_BackupFailedAndReported_RegressionForIssue816(t *testing.T) {+ storageErr := fmt.Errorf("save to storage: failed to upload file to S3: %w", test_utils.GetNetworkTimeoutError(t))++ assertBackupFailsAndIsReported(t, storageErr, storageErr.Error())+}++func Test_MakeBackup_WhenUsecaseErrorWrapsCanceledWithoutCancelRequest_BackupFailedAndReported(t *testing.T) {+ storageErr := fmt.Errorf("save to storage: upload aborted: %w", context.Canceled)++ assertBackupFailsAndIsReported(t, storageErr, storageErr.Error())+}++func Test_MakeBackup_WhenBackupExceedsTimeLimit_BackupFailedAndReported(t *testing.T) {+ assertBackupFailsAndIsReported(t,+ backups_core_logical.ErrBackupTimeout,+ "backup exceeded the time limit of 23 hours",+ )+}- backuper := CreateTestBackuperWithUseCase(&createStoredArtifactThenCancelBackupUsecase{})+func Test_MakeBackup_WhenApplicationShutsDown_BackupFailedNotCancelled(t *testing.T) {+ assertBackupFailsAndIsReported(t,+ backups_core_logical.ErrBackupShutdown,+ "backup interrupted by application shutdown",+ )+}++func assertBackupFailsAndIsReported(t *testing.T, backupErr error, expectedFailMessage string) {+ t.Helper()++ fixture := CreateBackupTestFixture(t, "Failed Backup Outcome Workspace")+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)++ notificationSender := &MockNotificationSender{}+ notificationSender.On("SendNotification",+ mock.Anything,+ mock.MatchedBy(func(notification notifier_models.Notification) bool {+ return notification.Type == notifier_models.NotificationTypeBackupFailed &&+ strings.Contains(notification.Message, expectedFailMessage)+ }),+ ).Once()++ backuper := CreateTestBackuperWithUseCase(&returnErrorBackupUsecase{backupErr})backuper.notificationSender = notificationSenderbackuper.MakeBackup(t.Context(), backup.ID, true)persistedBackup, err := backupRepository.FindByID(backup.ID)require.NoError(t, err)+ assert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status)+ require.NotNil(t, persistedBackup.FailMessage)+ assert.Equal(t, expectedFailMessage, *persistedBackup.FailMessage)++ persistedDatabase, err := databases.GetDatabaseService().GetDatabaseByID(fixture.Database.ID)+ require.NoError(t, err)+ require.NotNil(t, persistedDatabase.LastBackupErrorMessage, "a failed backup must mark its database as failing")+ assert.Equal(t, expectedFailMessage, *persistedDatabase.LastBackupErrorMessage)++ notificationSender.AssertExpectations(t)+}++func createRecordingNotificationSender() *MockNotificationSender {+ notificationSender := &MockNotificationSender{}+ notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()++ return notificationSender+}++// makeBackupCancellingThroughRegistry runs MakeBackup and, once the use case has started, cancels+// the backup the way the cancel endpoint does: through the task cancellation registry.+func makeBackupCancellingThroughRegistry(t *testing.T, backuper *Backuper, backupID uuid.UUID) {+ t.Helper()++ blockingUsecase, isBlockingUsecase := backuper.createBackupUseCase.(*createStoredArtifactThenBlockBackupUsecase)+ require.True(t, isBlockingUsecase, "cancelling through the registry needs a use case that waits for it")++ finishedBackup := make(chan struct{})++ go func() {+ defer close(finishedBackup)+ backuper.MakeBackup(t.Context(), backupID, true)+ }()++ select {+ case <-blockingUsecase.startedBackup:+ case <-time.After(30 * time.Second):+ t.Fatal("the backup use case never started")+ }++ taskCancellationRegistry.CancelRegisteredTask(backupID)++ select {+ case <-finishedBackup:+ case <-time.After(30 * time.Second):+ t.Fatal("the backup did not finish after it was cancelled")+ }+}++func assertBackupCancelledSilently(+ t *testing.T,+ fixture *BackupTestFixture,+ backupID uuid.UUID,+ notificationSender *MockNotificationSender,+) {+ t.Helper()++ persistedBackup, err := backupRepository.FindByID(backupID)+ require.NoError(t, err)assert.Equal(t, backups_core_logical.BackupStatusCanceled, persistedBackup.Status)- assertBackupFilesRemoved(t, fixture, backup.FileName)+ notificationSender.AssertNotCalled(t, "SendNotification", mock.Anything, mock.Anything)++ persistedDatabase, err := databases.GetDatabaseService().GetDatabaseByID(fixture.Database.ID)+ require.NoError(t, err)+ assert.Nil(t, persistedDatabase.LastBackupErrorMessage, "a cancelled backup must not mark its database as failing")}func Test_MakeBackup_WhenMetadataIsUnusable_BackupFailsAndArtifactRemoved(t *testing.T) {diff --git a/backend/internal/features/backups/backups/core/logical/backup_interruption.go b/backend/internal/features/backups/backups/core/logical/backup_interruption.gonew file mode 100644index 00000000..47e3a191--- /dev/null+++ b/backend/internal/features/backups/backups/core/logical/backup_interruption.go@@ -0,0 +1,64 @@+package backups_core_logical++import (+ "context"+ "errors"+ "fmt"+ "time"++ "databasus-backend/internal/config"+)++const (+ backupTimeLimit = 23 * time.Hour+ shutdownCheckInterval = 1 * time.Second+)++// NewBackupContext derives the context an engine runs one logical backup under. Besides ending+// with callerCtx, it ends with ErrBackupShutdown when the application starts shutting down and+// with ErrBackupTimeout once the backup time limit passes.+func NewBackupContext(callerCtx context.Context) (context.Context, context.CancelCauseFunc) {+ backupCtx, cancel := context.WithCancelCause(callerCtx)++ timeLimit := time.AfterFunc(backupTimeLimit, func() { cancel(ErrBackupTimeout) })++ go func() {+ defer timeLimit.Stop()++ ticker := time.NewTicker(shutdownCheckInterval)+ defer ticker.Stop()++ for {+ select {+ case <-backupCtx.Done():+ return+ case <-ticker.C:+ if config.IsShouldShutdown() {+ cancel(ErrBackupShutdown)+ return+ }+ }+ }+ }()++ return backupCtx, cancel+}++// DescribeBackupInterruption explains why backupCtx ended. A cancellation is reported only when+// callerCtx ended, because only the caller cancels on request. Any other cause comes from the+// storage write, which cancels backupCtx with its own error; that error can still match+// context.DeadlineExceeded or context.Canceled, so the shape of the cause decides nothing.+func DescribeBackupInterruption(callerCtx, backupCtx context.Context) error {+ cause := context.Cause(backupCtx)++ switch {+ case errors.Is(cause, ErrBackupShutdown):+ return ErrBackupShutdown+ case errors.Is(cause, ErrBackupTimeout):+ return ErrBackupTimeout+ case callerCtx.Err() != nil:+ return errors.New("backup cancelled")+ default:+ return fmt.Errorf("save to storage: %w", cause)+ }+}diff --git a/backend/internal/features/backups/backups/core/logical/backup_interruption_test.go b/backend/internal/features/backups/backups/core/logical/backup_interruption_test.gonew file mode 100644index 00000000..77b74182--- /dev/null+++ b/backend/internal/features/backups/backups/core/logical/backup_interruption_test.go@@ -0,0 +1,100 @@+package backups_core_logical++import (+ "context"+ "fmt"+ "testing"++ "github.com/stretchr/testify/assert"++ test_utils "databasus-backend/internal/util/testing"+)++func Test_DescribeBackupInterruption_ReportsWhoInterruptedTheBackup(t *testing.T) {+ storageTimeoutErr := fmt.Errorf("failed to upload file to S3: %w", test_utils.GetNetworkTimeoutError(t))+ storageCancelledErr := fmt.Errorf("failed to upload file to S3: %w", context.Canceled)++ testCases := []struct {+ name string+ backupCancelCause error+ isCallerCancelled bool+ expectedErr error+ expectedMessage string+ expectedWrappedCause error+ isReportedAsCancelled bool+ }{+ {+ name: "Test_DescribeBackupInterruption_WhenShuttingDown_ReturnsShutdownError",+ backupCancelCause: ErrBackupShutdown,+ expectedErr: ErrBackupShutdown,+ expectedMessage: "backup interrupted by application shutdown",+ },+ {+ name: "Test_DescribeBackupInterruption_WhenTimeLimitPassed_ReturnsTimeLimitError",+ backupCancelCause: ErrBackupTimeout,+ expectedErr: ErrBackupTimeout,+ expectedMessage: "backup exceeded the time limit of 23 hours",+ },+ {+ name: "Test_DescribeBackupInterruption_WhenCallerCancelled_ReturnsBackupCancelled",+ isCallerCancelled: true,+ expectedMessage: "backup cancelled",+ isReportedAsCancelled: true,+ },+ {+ name: "Test_DescribeBackupInterruption_WhenStorageTimedOut_ReturnsStorageError",+ backupCancelCause: storageTimeoutErr,+ expectedMessage: "save to storage: " + storageTimeoutErr.Error(),+ expectedWrappedCause: storageTimeoutErr,+ },+ {+ name: "Test_DescribeBackupInterruption_WhenStorageErrorWrapsCanceled_ReturnsStorageError",+ backupCancelCause: storageCancelledErr,+ expectedMessage: "save to storage: " + storageCancelledErr.Error(),+ expectedWrappedCause: storageCancelledErr,+ },+ {+ name: "Test_DescribeBackupInterruption_WhenCallerCancelledAfterStorageFailed_ReturnsBackupCancelled",+ backupCancelCause: storageTimeoutErr,+ isCallerCancelled: true,+ expectedMessage: "backup cancelled",+ isReportedAsCancelled: true,+ },+ }++ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ callerCtx, cancelCaller := context.WithCancel(t.Context())+ defer cancelCaller()++ backupCtx, cancelBackup := NewBackupContext(callerCtx)+ defer cancelBackup(nil)++ if testCase.backupCancelCause != nil {+ cancelBackup(testCase.backupCancelCause)+ }++ if testCase.isCallerCancelled {+ cancelCaller()+ }++ <-backupCtx.Done()++ interruptionErr := DescribeBackupInterruption(callerCtx, backupCtx)++ assert.EqualError(t, interruptionErr, testCase.expectedMessage)++ if testCase.expectedErr != nil {+ assert.ErrorIs(t, interruptionErr, testCase.expectedErr)+ }++ if testCase.expectedWrappedCause != nil {+ assert.ErrorIs(t, interruptionErr, testCase.expectedWrappedCause)+ }++ if !testCase.isReportedAsCancelled {+ assert.NotEqual(t, "backup cancelled", interruptionErr.Error())+ }+ })+ }+}diff --git a/backend/internal/features/backups/backups/core/logical/errors.go b/backend/internal/features/backups/backups/core/logical/errors.gonew file mode 100644index 00000000..b129fc93--- /dev/null+++ b/backend/internal/features/backups/backups/core/logical/errors.go@@ -0,0 +1,11 @@+package backups_core_logical++import (+ "errors"+ "fmt"+)++var (+ ErrBackupShutdown = errors.New("backup interrupted by application shutdown")+ ErrBackupTimeout = fmt.Errorf("backup exceeded the time limit of %d hours", int(backupTimeLimit.Hours()))+)diff --git a/backend/internal/features/backups/backups/core/logical/testing.go b/backend/internal/features/backups/backups/core/logical/testing.gonew file mode 100644index 00000000..ad614eb4--- /dev/null+++ b/backend/internal/features/backups/backups/core/logical/testing.go@@ -0,0 +1,116 @@+package backups_core_logical++import (+ "context"+ "errors"+ "fmt"+ "io"+ "sync/atomic"+ "testing"++ "github.com/google/uuid"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"+ backups_config_logical "databasus-backend/internal/features/backups/config/logical"+ storage_files "databasus-backend/internal/features/storages/files"+ test_utils "databasus-backend/internal/util/testing"+)++// FailingBackupFileStore fails every write with FailureErr. It either drains the whole stream+// first, so the dump runs to completion before the storage fails, or fails before reading a+// byte, so the storage fails while the dump is still producing data.+type FailingBackupFileStore struct {+ FailureErr error+ IsFailingBeforeReading bool++ readBytes atomic.Int64+}++func (s *FailingBackupFileStore) WriteFile(+ _ context.Context,+ _ storage_files.StoredFileReference,+ file io.Reader,+) (storage_files.WriteReceipt, error) {+ if !s.IsFailingBeforeReading {+ readBytes, _ := io.Copy(io.Discard, file)+ s.readBytes.Store(readBytes)+ }++ return storage_files.WriteReceipt{}, s.FailureErr+}++func (s *FailingBackupFileStore) GetReadBytes() int64 {+ return s.readBytes.Load()+}++// ExecuteBackupInto runs one engine's real backup use case against a test database, writing into+// the given file store.+type ExecuteBackupInto func(+ ctx context.Context,+ backup *LogicalBackup,+ backupConfig *backups_config_logical.LogicalBackupConfig,+ fileStore BackupFileStore,+) error++// AssertStorageTimeoutReportedAsStorageFailure runs an engine's real dump into a storage that+// times out, once after the dump finished and once while it is still running, and checks the+// engine reports the storage error rather than a cancellation.+func AssertStorageTimeoutReportedAsStorageFailure(t *testing.T, executeBackupInto ExecuteBackupInto) {+ t.Helper()++ storageErr := fmt.Errorf("failed to upload file to S3: %w", test_utils.GetNetworkTimeoutError(t))++ t.Run("Test_Execute_WhenStorageTimesOutAfterDumpFinished_ReturnsStorageError", func(t *testing.T) {+ fileStore := &FailingBackupFileStore{FailureErr: storageErr}++ backupErr := executeBackupInto(+ t.Context(),+ newStorageFailureTestBackup(),+ newUnencryptedTestBackupConfig(),+ fileStore,+ )++ require.Positive(t, fileStore.GetReadBytes(), "the dump never reached the storage")+ assertStorageErrorReported(t, backupErr, storageErr)+ })++ t.Run("Test_Execute_WhenStorageFailsBeforeReading_ReturnsStorageError", func(t *testing.T) {+ fileStore := &FailingBackupFileStore{FailureErr: storageErr, IsFailingBeforeReading: true}++ backupErr := executeBackupInto(+ t.Context(),+ newStorageFailureTestBackup(),+ newUnencryptedTestBackupConfig(),+ fileStore,+ )++ assertStorageErrorReported(t, backupErr, storageErr)+ })+}++func assertStorageErrorReported(t *testing.T, backupErr, storageErr error) {+ t.Helper()++ require.Error(t, backupErr)+ assert.True(t, errors.Is(backupErr, storageErr), "the backup error must wrap the storage error, got: %v", backupErr)+ assert.Contains(t, backupErr.Error(), "save to storage")+ assert.NotContains(t, backupErr.Error(), "backup cancelled")+}++func newStorageFailureTestBackup() *LogicalBackup {+ backupID := uuid.New()++ return &LogicalBackup{+ ID: backupID,+ DatabaseID: uuid.New(),+ StorageID: uuid.New(),+ FileName: "storage-failure-" + backupID.String(),+ Status: BackupStatusInProgress,+ }+}++func newUnencryptedTestBackupConfig() *backups_config_logical.LogicalBackupConfig {+ return &backups_config_logical.LogicalBackupConfig{Encryption: backups_core_enums.BackupEncryptionNone}+}diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.goindex 7923dcdd..ba4f420f 100644--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go@@ -11,7 +11,6 @@ import ("path/filepath""strconv""strings"- "time""github.com/google/uuid""github.com/klauspost/compress/zstd"@@ -32,8 +31,6 @@ import ()const (- backupTimeout = 23 * time.Hour- shutdownCheckInterval = 1 * time.SecondcopyBufferSize = 8 * 1024 * 1024progressReportIntervalMB = 1.0zstdStorageCompressionLevel = 5@@ -41,11 +38,6 @@ const (exitCodeConnectionError = 2)-var (- errBackupShutdown = errors.New("backup cancelled due to shutdown")- errBackupTimeout = errors.New("backup cancelled due to timeout")-)-type CreateMariadbBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -193,7 +185,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage() (*backups_core_logical.BackupArtifacts, error) {uc.logger.InfoContext(parentCtx, "streaming MariaDB backup to storage", "mariadb_bin", mariadbBin)- ctx, cancel := uc.createBackupContext(parentCtx)+ ctx, cancel := backups_core_logical.NewBackupContext(parentCtx)defer cancel(nil)myCnfFile, err := uc.createTempMyCnfFile(mdbConfig, password)@@ -299,7 +291,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)- return nil, uc.classifyCancellation(ctx)+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)default:}@@ -448,38 +440,6 @@ func (uc *CreateMariadbBackupUsecase) copyWithShutdownCheck(return nil}-func (uc *CreateMariadbBackupUsecase) createBackupContext(- parentCtx context.Context,-) (context.Context, context.CancelCauseFunc) {- ctx, cancel := context.WithCancelCause(parentCtx)-- timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })-- go func() {- defer timeout.Stop()-- ticker := time.NewTicker(shutdownCheckInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case <-parentCtx.Done():- cancel(context.Cause(parentCtx))- return- case <-ticker.C:- if config.IsShouldShutdown() {- cancel(errBackupShutdown)- return- }- }- }- }()-- return ctx, cancel-}-func (uc *CreateMariadbBackupUsecase) setupBackupEncryption(backupID uuid.UUID,backupConfig *backups_config_logical.LogicalBackupConfig,@@ -583,21 +543,6 @@ func (uc *CreateMariadbBackupUsecase) closeWriters(return nil}-func (uc *CreateMariadbBackupUsecase) classifyCancellation(ctx context.Context) error {- cause := context.Cause(ctx)-- switch {- case errors.Is(cause, errBackupShutdown):- return errors.New("backup cancelled due to shutdown")- case errors.Is(cause, errBackupTimeout):- return errors.New("backup cancelled due to timeout")- case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):- return errors.New("backup cancelled")- default:- return fmt.Errorf("save to storage: %w", cause)- }-}-func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage(waitErr error,stderrOutput []byte,diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.goindex 147e9b3a..f886465f 100644--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go@@ -1,11 +1,18 @@package usecases_logical_mariadbimport (+ "context""slices""strings""testing"+ "github.com/google/uuid"++ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ backups_config_logical "databasus-backend/internal/features/backups/config/logical"+ "databasus-backend/internal/features/databases"mariadbtypes "databasus-backend/internal/features/databases/databases/mariadb"+ "databasus-backend/internal/util/testing/containers""databasus-backend/internal/util/tools")@@ -144,3 +151,25 @@ func Test_BuildMariadbDumpArgs_WithoutExcludedTables_OmitsIgnoreTableArgs(t *tes}}}++// Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816 pins issue #816 for+// MariaDB: a storage timeout that ended the backup context was reported as "backup cancelled".+func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816(t *testing.T) {+ endpoint := containers.StartMariadb(t, "mariadb:10.11")+ database := &databases.Database{+ ID: uuid.New(),+ Type: databases.DatabaseTypeMariadb,+ Mariadb: databases.GetTestMariadbConfig(endpoint.Host, endpoint.Port),+ }++ backups_core_logical.AssertStorageTimeoutReportedAsStorageFailure(t, func(+ ctx context.Context,+ backup *backups_core_logical.LogicalBackup,+ backupConfig *backups_config_logical.LogicalBackupConfig,+ fileStore backups_core_logical.BackupFileStore,+ ) error {+ _, err := GetCreateMariadbBackupUsecase().Execute(ctx, backup, backupConfig, database, fileStore, func(float64) {})++ return err+ })+}diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.goindex da2f49d7..7a15120f 100644--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go@@ -9,7 +9,6 @@ import ("os""os/exec""path/filepath"- "time""github.com/google/uuid"@@ -28,17 +27,10 @@ import ()const (- backupTimeout = 23 * time.Hour- shutdownCheckInterval = 1 * time.SecondcopyBufferSize = 8 * 1024 * 1024progressReportIntervalMB = 1.0)-var (- errBackupShutdown = errors.New("backup cancelled due to shutdown")- errBackupTimeout = errors.New("backup cancelled due to timeout")-)-type CreateMongodbBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -144,7 +136,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage() (*backups_core_logical.BackupArtifacts, error) {uc.logger.InfoContext(parentCtx, "streaming MongoDB backup to storage", "mongodump_bin", mongodumpBin)- ctx, cancel := uc.createBackupContext(parentCtx)+ ctx, cancel := backups_core_logical.NewBackupContext(parentCtx)defer cancel(nil)cmd := exec.CommandContext(ctx, mongodumpBin, args...)@@ -236,7 +228,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)- return nil, uc.classifyCancellation(ctx)+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)default:}@@ -268,34 +260,6 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(}, nil}-func (uc *CreateMongodbBackupUsecase) createBackupContext(- parentCtx context.Context,-) (context.Context, context.CancelCauseFunc) {- ctx, cancel := context.WithCancelCause(parentCtx)-- timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })-- go func() {- defer timeout.Stop()-- ticker := time.NewTicker(shutdownCheckInterval)- defer ticker.Stop()- for {- select {- case <-ctx.Done():- return- case <-ticker.C:- if config.IsShouldShutdown() {- cancel(errBackupShutdown)- return- }- }- }- }()-- return ctx, cancel-}-func (uc *CreateMongodbBackupUsecase) setupBackupEncryption(backupID uuid.UUID,backupConfig *backups_config_logical.LogicalBackupConfig,@@ -428,21 +392,6 @@ func (uc *CreateMongodbBackupUsecase) closeWriters(return nil}-func (uc *CreateMongodbBackupUsecase) classifyCancellation(ctx context.Context) error {- cause := context.Cause(ctx)-- switch {- case errors.Is(cause, errBackupShutdown):- return errors.New("backup cancelled due to shutdown")- case errors.Is(cause, errBackupTimeout):- return errors.New("backup cancelled due to timeout")- case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):- return errors.New("backup cancelled")- default:- return fmt.Errorf("save to storage: %w", cause)- }-}-func (uc *CreateMongodbBackupUsecase) buildMongodumpErrorMessage(waitErr error,stderrOutput []byte,diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.gonew file mode 100644index 00000000..b1af3d41--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc_test.go@@ -0,0 +1,35 @@+package usecases_logical_mongodb++import (+ "context"+ "testing"++ "github.com/google/uuid"++ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ backups_config_logical "databasus-backend/internal/features/backups/config/logical"+ "databasus-backend/internal/features/databases"+ "databasus-backend/internal/util/testing/containers"+)++// Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816 pins issue #816 for+// MongoDB: a storage timeout that ended the backup context was reported as "backup cancelled".+func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816(t *testing.T) {+ endpoint := containers.StartMongodb(t, "mongo:7.0")+ database := &databases.Database{+ ID: uuid.New(),+ Type: databases.DatabaseTypeMongodb,+ Mongodb: databases.GetTestMongodbConfig(endpoint.Host, endpoint.Port),+ }++ backups_core_logical.AssertStorageTimeoutReportedAsStorageFailure(t, func(+ ctx context.Context,+ backup *backups_core_logical.LogicalBackup,+ backupConfig *backups_config_logical.LogicalBackupConfig,+ fileStore backups_core_logical.BackupFileStore,+ ) error {+ _, err := GetCreateMongodbBackupUsecase().Execute(ctx, backup, backupConfig, database, fileStore, func(float64) {})++ return err+ })+}diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.goindex 22d2d596..bb332d7c 100644--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go@@ -11,7 +11,6 @@ import ("path/filepath""strconv""strings"- "time""github.com/google/uuid""github.com/klauspost/compress/zstd"@@ -32,8 +31,6 @@ import ()const (- backupTimeout = 23 * time.Hour- shutdownCheckInterval = 1 * time.SecondcopyBufferSize = 8 * 1024 * 1024progressReportIntervalMB = 1.0zstdStorageCompressionLevel = 5@@ -41,11 +38,6 @@ const (exitCodeConnectionError = 2)-var (- errBackupShutdown = errors.New("backup cancelled due to shutdown")- errBackupTimeout = errors.New("backup cancelled due to timeout")-)-type CreateMysqlBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -191,7 +183,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage() (*backups_core_logical.BackupArtifacts, error) {uc.logger.InfoContext(parentCtx, "streaming MySQL backup to storage", "mysql_bin", mysqlBin)- ctx, cancel := uc.createBackupContext(parentCtx)+ ctx, cancel := backups_core_logical.NewBackupContext(parentCtx)defer cancel(nil)myCnfFile, err := uc.createTempMyCnfFile(myConfig, password)@@ -296,7 +288,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)- return nil, uc.classifyCancellation(ctx)+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)default:}@@ -445,38 +437,6 @@ func (uc *CreateMysqlBackupUsecase) copyWithShutdownCheck(return nil}-func (uc *CreateMysqlBackupUsecase) createBackupContext(- parentCtx context.Context,-) (context.Context, context.CancelCauseFunc) {- ctx, cancel := context.WithCancelCause(parentCtx)-- timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })-- go func() {- defer timeout.Stop()-- ticker := time.NewTicker(shutdownCheckInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case <-parentCtx.Done():- cancel(context.Cause(parentCtx))- return- case <-ticker.C:- if config.IsShouldShutdown() {- cancel(errBackupShutdown)- return- }- }- }- }()-- return ctx, cancel-}-func (uc *CreateMysqlBackupUsecase) setupBackupEncryption(backupID uuid.UUID,backupConfig *backups_config_logical.LogicalBackupConfig,@@ -580,21 +540,6 @@ func (uc *CreateMysqlBackupUsecase) closeWriters(return nil}-func (uc *CreateMysqlBackupUsecase) classifyCancellation(ctx context.Context) error {- cause := context.Cause(ctx)-- switch {- case errors.Is(cause, errBackupShutdown):- return errors.New("backup cancelled due to shutdown")- case errors.Is(cause, errBackupTimeout):- return errors.New("backup cancelled due to timeout")- case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):- return errors.New("backup cancelled")- default:- return fmt.Errorf("save to storage: %w", cause)- }-}-func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage(waitErr error,stderrOutput []byte,diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.goindex db1817ee..a65d7494 100644--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go@@ -1,11 +1,18 @@package usecases_logical_mysqlimport (+ "context""slices""strings""testing"+ "github.com/google/uuid"++ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ backups_config_logical "databasus-backend/internal/features/backups/config/logical"+ "databasus-backend/internal/features/databases"mysqltypes "databasus-backend/internal/features/databases/databases/mysql"+ "databasus-backend/internal/util/testing/containers""databasus-backend/internal/util/tools")@@ -144,3 +151,25 @@ func Test_BuildMysqldumpArgs_WithoutExcludedTables_OmitsIgnoreTableArgs(t *testi}}}++// Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816 pins issue #816 for+// MySQL: a storage timeout that ended the backup context was reported as "backup cancelled".+func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816(t *testing.T) {+ endpoint := containers.StartMysql(t, "mysql:8.0")+ database := &databases.Database{+ ID: uuid.New(),+ Type: databases.DatabaseTypeMysql,+ Mysql: databases.GetTestMysqlConfig(endpoint.Host, endpoint.Port),+ }++ backups_core_logical.AssertStorageTimeoutReportedAsStorageFailure(t, func(+ ctx context.Context,+ backup *backups_core_logical.LogicalBackup,+ backupConfig *backups_config_logical.LogicalBackupConfig,+ fileStore backups_core_logical.BackupFileStore,+ ) error {+ _, err := GetCreateMysqlBackupUsecase().Execute(ctx, backup, backupConfig, database, fileStore, func(float64) {})++ return err+ })+}diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.goindex 7dd4ca60..b8c775af 100644--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go@@ -11,7 +11,6 @@ import ("path/filepath""strconv""strings"- "time""github.com/google/uuid"@@ -31,8 +30,6 @@ import ()const (- backupTimeout = 23 * time.Hour- shutdownCheckInterval = 1 * time.SecondcopyBufferSize = 8 * 1024 * 1024progressReportIntervalMB = 1.0pgConnectTimeout = 30@@ -42,11 +39,6 @@ const (exitCodeConnectionError = 2)-var (- errBackupShutdown = errors.New("backup cancelled due to shutdown")- errBackupTimeout = errors.New("backup cancelled due to timeout")-)-type CreatePostgresqlBackupUsecase struct {logger *slog.LoggersecretKeyService *encryption_secrets.SecretKeyService@@ -141,7 +133,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage() (*backups_core_logical.BackupArtifacts, error) {uc.logger.InfoContext(parentCtx, "streaming PostgreSQL backup to storage", "pg_bin", pgBin, "args", args)- ctx, cancel := uc.createBackupContext(parentCtx)+ ctx, cancel := backups_core_logical.NewBackupContext(parentCtx)defer cancel(nil)credentials, err := postgresql_shared.WriteCredentialFilesToTempDir(@@ -237,7 +229,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)- return nil, uc.classifyCancellation(ctx)+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)default:}@@ -257,18 +249,18 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(switch {case waitErr != nil:- if err := uc.checkCancellation(ctx); err != nil {- return nil, err+ if ctx.Err() != nil {+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)}return nil, uc.buildPgDumpErrorMessage(waitErr, stderrOutput, pgBin, args, password)case copyErr != nil:- if err := uc.checkCancellation(ctx); err != nil {- return nil, err+ if ctx.Err() != nil {+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)}return nil, fmt.Errorf("copy to storage: %w", copyErr)case saveErr != nil:- if err := uc.checkCancellation(ctx); err != nil {- return nil, err+ if ctx.Err() != nil {+ return nil, backups_core_logical.DescribeBackupInterruption(parentCtx, ctx)}return nil, fmt.Errorf("save to storage: %w", saveErr)}@@ -400,38 +392,6 @@ func (uc *CreatePostgresqlBackupUsecase) isOlderPostgresVersion(version == tools.PostgresqlVersion15}-func (uc *CreatePostgresqlBackupUsecase) createBackupContext(- parentCtx context.Context,-) (context.Context, context.CancelCauseFunc) {- ctx, cancel := context.WithCancelCause(parentCtx)-- timeout := time.AfterFunc(backupTimeout, func() { cancel(errBackupTimeout) })-- go func() {- defer timeout.Stop()-- ticker := time.NewTicker(shutdownCheckInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case <-parentCtx.Done():- cancel(context.Cause(parentCtx))- return- case <-ticker.C:- if config.IsShouldShutdown() {- cancel(errBackupShutdown)- return- }- }- }- }()-- return ctx, cancel-}-func (uc *CreatePostgresqlBackupUsecase) setupPgEnvironment(cmd *exec.Cmd,credentials *postgresql_shared.CredentialTempFiles,@@ -567,29 +527,6 @@ func (uc *CreatePostgresqlBackupUsecase) closeWriters(return nil}-func (uc *CreatePostgresqlBackupUsecase) checkCancellation(ctx context.Context) error {- if ctx.Err() == nil {- return nil- }-- return uc.classifyCancellation(ctx)-}--func (uc *CreatePostgresqlBackupUsecase) classifyCancellation(ctx context.Context) error {- cause := context.Cause(ctx)-- switch {- case errors.Is(cause, errBackupShutdown):- return errors.New("backup cancelled due to shutdown")- case errors.Is(cause, errBackupTimeout):- return errors.New("backup cancelled due to timeout")- case cause == nil, errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):- return errors.New("backup cancelled")- default:- return fmt.Errorf("save to storage: %w", cause)- }-}-func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(waitErr error,stderrOutput []byte,diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.goindex 9baf80b4..e00a9b08 100644--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc_test.go@@ -1,12 +1,17 @@package usecases_logical_postgresqlimport (+ "context""io""log/slog""testing"+ "github.com/google/uuid""github.com/stretchr/testify/require"+ backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"+ backups_config_logical "databasus-backend/internal/features/backups/config/logical"+ "databasus-backend/internal/features/databases"pgtypes "databasus-backend/internal/features/databases/databases/postgresql/logical""databasus-backend/internal/util/tools")@@ -38,3 +43,24 @@ func Test_BuildPgDumpArgs_TreatsDuplicateParameterAsLiteralDatabaseName(t *testirequire.GreaterOrEqual(t, databaseArgumentIndex, 0)require.Equal(t, "dbname='postgres dbname=databasus'", args[databaseArgumentIndex])}++// Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816 pins issue #816:+// a storage timeout that ended the backup context was reported as "backup cancelled".+func Test_Execute_WhenStorageTimesOut_ReportsStorageFailure_RegressionForIssue816(t *testing.T) {+ database := &databases.Database{+ ID: uuid.New(),+ Type: databases.DatabaseTypePostgresLogical,+ PostgresqlLogical: databases.GetTestPostgresConfig(),+ }++ backups_core_logical.AssertStorageTimeoutReportedAsStorageFailure(t, func(+ ctx context.Context,+ backup *backups_core_logical.LogicalBackup,+ backupConfig *backups_config_logical.LogicalBackupConfig,+ fileStore backups_core_logical.BackupFileStore,+ ) error {+ _, err := GetCreatePostgresqlBackupUsecase().Execute(ctx, backup, backupConfig, database, fileStore, func(float64) {})++ return err+ })+}diff --git a/backend/internal/features/notifiers/models/mattermost/model.go b/backend/internal/features/notifiers/models/mattermost/model.goindex 823c2567..a6e30b28 100644--- a/backend/internal/features/notifiers/models/mattermost/model.go+++ b/backend/internal/features/notifiers/models/mattermost/model.go@@ -297,6 +297,7 @@ func (m *MattermostNotifier) newHTTPClient() *http.Client {if m.IsInsecureSkipVerify {client.Transport = &http.Transport{+ Proxy: http.ProxyFromEnvironment,TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12,InsecureSkipVerify: true,diff --git a/backend/internal/features/notifiers/models/mattermost/model_test.go b/backend/internal/features/notifiers/models/mattermost/model_test.goindex 28a7d3e7..df32c260 100644--- a/backend/internal/features/notifiers/models/mattermost/model_test.go+++ b/backend/internal/features/notifiers/models/mattermost/model_test.go@@ -1,10 +1,12 @@package mattermost_notifierimport (+ "crypto/tls""encoding/json""io""log/slog""net/http"+ "reflect""strings""testing"@@ -12,6 +14,7 @@ import ("github.com/stretchr/testify/require"notifier_models "databasus-backend/internal/features/notifiers/models"+ test_utils "databasus-backend/internal/util/testing")func send(t *testing.T, notifier *MattermostNotifier, notification notifier_models.Notification) error {@@ -362,3 +365,41 @@ func Test_Update_WithBlankCredentials_KeepsStoredOnes(t *testing.T) {assert.Equal(t, strings.Repeat("d", channelIDLength), storedNotifier.TargetChannelID)assert.Equal(t, "Databasus", storedNotifier.OverrideUsername)}++func Test_NewHTTPClient_WhenTLSVerificationSkipped_ResolvesProxyFromEnvironmentAndKeepsTLSSettings(t *testing.T) {+ transport := getInsecureMattermostTransport(t)++ require.NotNil(t, transport.Proxy)+ assert.Equal(t,+ reflect.ValueOf(http.ProxyFromEnvironment).Pointer(),+ reflect.ValueOf(transport.Proxy).Pointer(),+ )+ assert.True(t, transport.TLSClientConfig.InsecureSkipVerify)+ assert.Equal(t, uint16(tls.VersionTLS12), transport.TLSClientConfig.MinVersion)+}++func Test_NewHTTPClient_WhenTLSVerificationOn_UsesDefaultTransportWithEnvironmentProxy(t *testing.T) {+ client := (&MattermostNotifier{IsInsecureSkipVerify: false}).newHTTPClient()++ assert.Nil(t, client.Transport, "a nil transport means http.DefaultTransport, which reads the proxy environment")+}++func Test_NewHTTPClient_WhenTLSVerificationSkipped_CredentialsReachOnlyEndpointThroughProxy(t *testing.T) {+ exchange := test_utils.SendThroughTunnelProxy(t, getInsecureMattermostTransport(t), "Bearer bot-token")++ require.NoError(t, exchange.Err)+ test_utils.AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ require.Len(t, exchange.EndpointRequests, 1)+ assert.Equal(t, "Bearer bot-token", exchange.EndpointRequests[0].Header.Get("Authorization"))+}++func getInsecureMattermostTransport(t *testing.T) *http.Transport {+ t.Helper()++ client := (&MattermostNotifier{IsInsecureSkipVerify: true}).newHTTPClient()++ transport, isTransport := client.Transport.(*http.Transport)+ require.True(t, isTransport, "Mattermost with TLS verification skipped must use its own *http.Transport")++ return transport+}diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.goindex 58e573ff..c6e7ef6e 100644--- a/backend/internal/features/storages/models/azure_blob/model.go+++ b/backend/internal/features/storages/models/azure_blob/model.go@@ -502,6 +502,7 @@ func (s *AzureBlobStorage) getClient(encryptor encryption.FieldEncryptor) (*azblfunc (s *AzureBlobStorage) buildClientOptions() *azblob.ClientOptions {transport := &http.Transport{+ Proxy: http.ProxyFromEnvironment,DialContext: (&net.Dialer{Timeout: azureConnectTimeout,}).DialContext,diff --git a/backend/internal/features/storages/models/azure_blob/model_test.go b/backend/internal/features/storages/models/azure_blob/model_test.goindex 0651fb15..237029e7 100644--- a/backend/internal/features/storages/models/azure_blob/model_test.go+++ b/backend/internal/features/storages/models/azure_blob/model_test.go@@ -4,6 +4,8 @@ import ("bytes""encoding/base64""fmt"+ "net/http"+ "reflect""testing""github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"@@ -13,6 +15,7 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ test_utils "databasus-backend/internal/util/testing""databasus-backend/internal/util/testing/containers")@@ -123,3 +126,34 @@ func Test_DiscardUncommittedBlocks_WhenNothingExists_Succeeds(t *testing.T) {assert.NoError(t, storage.discardUncommittedBlocks(t.Context(), client, "absent-"+uuid.New().String()))}++func Test_BuildClientOptions_Transport_ResolvesProxyFromEnvironment(t *testing.T) {+ transport := getAzureTransport(t)++ require.NotNil(t, transport.Proxy)+ assert.Equal(t,+ reflect.ValueOf(http.ProxyFromEnvironment).Pointer(),+ reflect.ValueOf(transport.Proxy).Pointer(),+ )+}++func Test_BuildClientOptions_TransportThroughProxy_TunnelsAndRejectsUntrustedCertificate(t *testing.T) {+ exchange := test_utils.SendThroughTunnelProxy(t, getAzureTransport(t), "SharedKey account:signature")++ test_utils.AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ test_utils.AssertCertificateRejected(t, exchange)+}++func getAzureTransport(t *testing.T) *http.Transport {+ t.Helper()++ options := (&AzureBlobStorage{}).buildClientOptions()++ client, isClient := options.Transport.(*http.Client)+ require.True(t, isClient, "Azure client options must carry an *http.Client")++ transport, isTransport := client.Transport.(*http.Transport)+ require.True(t, isTransport, "Azure HTTP client must use an *http.Transport")++ return transport+}diff --git a/backend/internal/features/storages/models/google_drive/model.go b/backend/internal/features/storages/models/google_drive/model.goindex a8ebd512..afab4b64 100644--- a/backend/internal/features/storages/models/google_drive/model.go+++ b/backend/internal/features/storages/models/google_drive/model.go@@ -616,6 +616,7 @@ func (s *GoogleDriveStorage) getDriveService(func (s *GoogleDriveStorage) buildHTTPClient(tokenSource oauth2.TokenSource) *http.Client {transport := &http.Transport{+ Proxy: http.ProxyFromEnvironment,DialContext: (&net.Dialer{Timeout: gdConnectTimeout,}).DialContext,diff --git a/backend/internal/features/storages/models/google_drive/model_test.go b/backend/internal/features/storages/models/google_drive/model_test.gonew file mode 100644index 00000000..35f889eb--- /dev/null+++ b/backend/internal/features/storages/models/google_drive/model_test.go@@ -0,0 +1,46 @@+package google_drive_storage++import (+ "net/http"+ "reflect"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "golang.org/x/oauth2"++ test_utils "databasus-backend/internal/util/testing"+)++func Test_BuildHTTPClient_Transport_ResolvesProxyFromEnvironment(t *testing.T) {+ transport := getGoogleDriveBaseTransport(t)++ require.NotNil(t, transport.Proxy)+ assert.Equal(t,+ reflect.ValueOf(http.ProxyFromEnvironment).Pointer(),+ reflect.ValueOf(transport.Proxy).Pointer(),+ )+}++func Test_BuildHTTPClient_TransportThroughProxy_TunnelsAndRejectsUntrustedCertificate(t *testing.T) {+ exchange := test_utils.SendThroughTunnelProxy(t, getGoogleDriveBaseTransport(t), "Bearer access-token")++ test_utils.AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ test_utils.AssertCertificateRejected(t, exchange)+}++func getGoogleDriveBaseTransport(t *testing.T) *http.Transport {+ t.Helper()++ client := (&GoogleDriveStorage{}).buildHTTPClient(+ oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "access-token"}),+ )++ oauthTransport, isOAuthTransport := client.Transport.(*oauth2.Transport)+ require.True(t, isOAuthTransport, "Google Drive client must authenticate through oauth2.Transport")++ baseTransport, isTransport := oauthTransport.Base.(*http.Transport)+ require.True(t, isTransport, "Google Drive OAuth transport must sit on an *http.Transport")++ return baseTransport+}diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.goindex 7ceb7f69..8c0a81ef 100644--- a/backend/internal/features/storages/models/s3/model.go+++ b/backend/internal/features/storages/models/s3/model.go@@ -853,6 +853,7 @@ func (s *S3Storage) getClientParams(}transport = &http.Transport{+ Proxy: http.ProxyFromEnvironment,DialContext: (&net.Dialer{Timeout: s3ConnectTimeout,}).DialContext,diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.goindex 816d1849..8415d0e5 100644--- a/backend/internal/features/storages/models/s3/model_test.go+++ b/backend/internal/features/storages/models/s3/model_test.go@@ -8,6 +8,7 @@ import ("log/slog""net/http""net/http/httptest"+ "reflect""strconv""testing""time"@@ -20,6 +21,7 @@ import ("databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger"+ test_utils "databasus-backend/internal/util/testing""databasus-backend/internal/util/testing/containers")@@ -558,3 +560,63 @@ func (r *cancelAfterReader) Read(p []byte) (int, error) {return n, nil}++func Test_GetClientParams_Transport_ResolvesProxyFromEnvironment(t *testing.T) {+ transport := buildTestS3Transport(t, false)++ require.NotNil(t, transport.Proxy)+ assert.Equal(t,+ reflect.ValueOf(http.ProxyFromEnvironment).Pointer(),+ reflect.ValueOf(transport.Proxy).Pointer(),+ )+}++func Test_GetClientParams_Transport_FollowsSkipTLSVerify(t *testing.T) {+ assert.False(t, buildTestS3Transport(t, false).TLSClientConfig.InsecureSkipVerify)+ assert.True(t, buildTestS3Transport(t, true).TLSClientConfig.InsecureSkipVerify)+}++func Test_GetClientParams_TransportThroughProxy_WhenVerificationOn_TunnelsAndRejectsUntrustedCertificate(+ t *testing.T,+) {+ exchange := test_utils.SendThroughTunnelProxy(t, buildTestS3Transport(t, false), "AWS4-HMAC-SHA256 signature")++ test_utils.AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ test_utils.AssertCertificateRejected(t, exchange)+}++func Test_GetClientParams_TransportThroughProxy_WhenVerificationSkipped_CredentialsReachOnlyEndpoint(+ t *testing.T,+) {+ exchange := test_utils.SendThroughTunnelProxy(t, buildTestS3Transport(t, true), "AWS4-HMAC-SHA256 signature")++ require.NoError(t, exchange.Err)+ test_utils.AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ require.Len(t, exchange.EndpointRequests, 1)+ assert.Equal(t, "AWS4-HMAC-SHA256 signature", exchange.EndpointRequests[0].Header.Get("Authorization"))+}++func buildTestS3Transport(t *testing.T, isSkipTLSVerify bool) *http.Transport {+ t.Helper()++ encryptor := encryption.GetFieldEncryptor()++ accessKey, err := encryptor.Encrypt("access-key")+ require.NoError(t, err)+ secretKey, err := encryptor.Encrypt("secret-key")+ require.NoError(t, err)++ storage := &S3Storage{+ S3Bucket: "test-bucket",+ S3Region: "us-east-1",+ S3AccessKey: accessKey,+ S3SecretKey: secretKey,+ S3Endpoint: "https://s3.example.com",+ SkipTLSVerify: isSkipTLSVerify,+ }++ _, _, _, _, _, transport, err := storage.getClientParams(encryptor)+ require.NoError(t, err)++ return transport+}diff --git a/backend/internal/util/testing/network_timeout.go b/backend/internal/util/testing/network_timeout.gonew file mode 100644index 00000000..d3d00e84--- /dev/null+++ b/backend/internal/util/testing/network_timeout.go@@ -0,0 +1,36 @@+package testing++import (+ "net/http"+ "net/http/httptest"+ "testing"+ "time"+)++// GetNetworkTimeoutError returns the error a real HTTP client reports when an endpoint accepts the+// connection and never answers. Storage providers surface this same error type, and it matches+// context.DeadlineExceeded, without waiting out a production timeout.+func GetNetworkTimeoutError(t *testing.T) error {+ t.Helper()++ server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {+ <-r.Context().Done()+ }))+ defer server.Close()++ client := &http.Client{Transport: &http.Transport{ResponseHeaderTimeout: 50 * time.Millisecond}}+ defer client.CloseIdleConnections()++ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)+ if err != nil {+ t.Fatalf("build timeout request: %v", err)+ }++ response, err := client.Do(request)+ if err == nil {+ _ = response.Body.Close()+ t.Fatal("expected the silent endpoint to time out")+ }++ return err+}diff --git a/backend/internal/util/testing/tunnel_proxy.go b/backend/internal/util/testing/tunnel_proxy.gonew file mode 100644index 00000000..cefe8136--- /dev/null+++ b/backend/internal/util/testing/tunnel_proxy.go@@ -0,0 +1,232 @@+package testing++import (+ "crypto/tls"+ "errors"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "net/url"+ "sync"+ "testing"+ "time"+)++type RecordedRequest struct {+ Method string+ Host string+ Header http.Header+}++// TunnelProxy is an HTTP proxy that only tunnels: it answers CONNECT and refuses every other+// method, so a test can see exactly what a client hands to a proxy on the way to an https endpoint.+type TunnelProxy struct {+ URL *url.URL++ mutex sync.Mutex+ receivedRequests []RecordedRequest+}++func StartTunnelProxy(t *testing.T) *TunnelProxy {+ t.Helper()++ proxy := &TunnelProxy{}++ server := httptest.NewServer(http.HandlerFunc(proxy.serve))+ t.Cleanup(server.Close)++ proxyURL, err := url.Parse(server.URL)+ if err != nil {+ t.Fatalf("parse tunnel proxy URL: %v", err)+ }++ proxy.URL = proxyURL++ return proxy+}++func (p *TunnelProxy) GetReceivedRequests() []RecordedRequest {+ p.mutex.Lock()+ defer p.mutex.Unlock()++ return append([]RecordedRequest(nil), p.receivedRequests...)+}++func (p *TunnelProxy) serve(w http.ResponseWriter, r *http.Request) {+ p.record(r)++ if r.Method != http.MethodConnect {+ w.WriteHeader(http.StatusMethodNotAllowed)+ return+ }++ targetConnection, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(r.Context(), "tcp", r.Host)+ if err != nil {+ w.WriteHeader(http.StatusBadGateway)+ return+ }++ hijacker, isHijacker := w.(http.Hijacker)+ if !isHijacker {+ _ = targetConnection.Close()+ w.WriteHeader(http.StatusInternalServerError)++ return+ }++ clientConnection, _, err := hijacker.Hijack()+ if err != nil {+ _ = targetConnection.Close()+ return+ }++ if _, err := clientConnection.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {+ _ = clientConnection.Close()+ _ = targetConnection.Close()++ return+ }++ go pipeAndClose(targetConnection, clientConnection)+ go pipeAndClose(clientConnection, targetConnection)+}++func (p *TunnelProxy) record(r *http.Request) {+ p.mutex.Lock()+ defer p.mutex.Unlock()++ p.receivedRequests = append(p.receivedRequests, RecordedRequest{+ Method: r.Method,+ Host: r.Host,+ Header: r.Header.Clone(),+ })+}++func pipeAndClose(destination, source net.Conn) {+ _, _ = io.Copy(destination, source)+ _ = destination.Close()+ _ = source.Close()+}++// RecordingTLSServer is an https endpoint with a certificate no client trusts by default, which+// records the requests that reach it and answers each with 200 OK.+type RecordingTLSServer struct {+ URL string++ mutex sync.Mutex+ receivedRequests []RecordedRequest+}++func StartRecordingTLSServer(t *testing.T) *RecordingTLSServer {+ t.Helper()++ recordingServer := &RecordingTLSServer{}++ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ recordingServer.mutex.Lock()+ recordingServer.receivedRequests = append(recordingServer.receivedRequests, RecordedRequest{+ Method: r.Method,+ Host: r.Host,+ Header: r.Header.Clone(),+ })+ recordingServer.mutex.Unlock()++ w.WriteHeader(http.StatusOK)+ }))+ t.Cleanup(server.Close)++ recordingServer.URL = server.URL++ return recordingServer+}++func (s *RecordingTLSServer) GetReceivedRequests() []RecordedRequest {+ s.mutex.Lock()+ defer s.mutex.Unlock()++ return append([]RecordedRequest(nil), s.receivedRequests...)+}++func (s *RecordingTLSServer) GetHostPort() string {+ serverURL, err := url.Parse(s.URL)+ if err != nil {+ panic(err)+ }++ return serverURL.Host+}++type ProxiedExchange struct {+ ProxyRequests []RecordedRequest+ EndpointRequests []RecordedRequest+ EndpointHostPort string+ Err error+}++// SendThroughTunnelProxy replaces only the transport's Proxy with a fresh local tunnel proxy and+// sends one request carrying the given Authorization header to a fresh untrusted https endpoint.+// The transport keeps every other setting the code under test gave it, TLS included.+func SendThroughTunnelProxy(t *testing.T, transport *http.Transport, authorization string) ProxiedExchange {+ t.Helper()++ proxy := StartTunnelProxy(t)+ endpoint := StartRecordingTLSServer(t)++ transport.Proxy = http.ProxyURL(proxy.URL)+ defer transport.CloseIdleConnections()++ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, endpoint.URL+"/object", nil)+ if err != nil {+ t.Fatalf("build proxied request: %v", err)+ }++ request.Header.Set("Authorization", authorization)++ response, requestErr := (&http.Client{Transport: transport}).Do(request)+ if requestErr == nil {+ _ = response.Body.Close()+ }++ return ProxiedExchange{+ ProxyRequests: proxy.GetReceivedRequests(),+ EndpointRequests: endpoint.GetReceivedRequests(),+ EndpointHostPort: endpoint.GetHostPort(),+ Err: requestErr,+ }+}++func AssertProxyReceivedOnlyTunnelRequest(t *testing.T, exchange ProxiedExchange) {+ t.Helper()++ if len(exchange.ProxyRequests) != 1 {+ t.Fatalf("expected exactly one request at the proxy, got %d", len(exchange.ProxyRequests))+ }++ proxyRequest := exchange.ProxyRequests[0]++ if proxyRequest.Method != http.MethodConnect {+ t.Errorf("expected the proxy to receive CONNECT, got %s", proxyRequest.Method)+ }++ if proxyRequest.Host != exchange.EndpointHostPort {+ t.Errorf("expected CONNECT to %s, got %s", exchange.EndpointHostPort, proxyRequest.Host)+ }++ if proxyRequest.Header.Get("Authorization") != "" {+ t.Error("the proxy received the endpoint's Authorization header")+ }+}++func AssertCertificateRejected(t *testing.T, exchange ProxiedExchange) {+ t.Helper()++ var verificationErr *tls.CertificateVerificationError+ if !errors.As(exchange.Err, &verificationErr) {+ t.Fatalf("expected a certificate verification error, got %v", exchange.Err)+ }++ if len(exchange.EndpointRequests) != 0 {+ t.Error("a request reached the endpoint despite its untrusted certificate")+ }+}diff --git a/backend/internal/util/testing/tunnel_proxy_test.go b/backend/internal/util/testing/tunnel_proxy_test.gonew file mode 100644index 00000000..618d43db--- /dev/null+++ b/backend/internal/util/testing/tunnel_proxy_test.go@@ -0,0 +1,36 @@+package testing++import (+ "context"+ "crypto/tls"+ "errors"+ "net/http"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func Test_SendThroughTunnelProxy_WhenVerificationSkipped_ReachesEndpointThroughOneConnect(t *testing.T) {+ exchange := SendThroughTunnelProxy(t, &http.Transport{+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},+ }, "secret-signature")++ require.NoError(t, exchange.Err)+ AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ require.Len(t, exchange.EndpointRequests, 1)+ assert.Equal(t, "secret-signature", exchange.EndpointRequests[0].Header.Get("Authorization"))+}++func Test_SendThroughTunnelProxy_WhenVerificationOn_RejectsEndpointCertificate(t *testing.T) {+ exchange := SendThroughTunnelProxy(t, &http.Transport{}, "secret-signature")++ AssertProxyReceivedOnlyTunnelRequest(t, exchange)+ AssertCertificateRejected(t, exchange)+}++func Test_GetNetworkTimeoutError_ReturnsErrorMatchingDeadlineExceeded(t *testing.T) {+ timeoutErr := GetNetworkTimeoutError(t)++ assert.True(t, errors.Is(timeoutErr, context.DeadlineExceeded), "error: %v", timeoutErr)+}diff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/.openspec.yaml b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/.openspec.yamlnew file mode 100644index 00000000..53f62d70--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/.openspec.yaml@@ -0,0 +1,2 @@+schema: spec-driven+created: 2026-10-10diff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/design.md b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/design.mdnew file mode 100644index 00000000..0cb4fb14--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/design.md@@ -0,0 +1,125 @@+## Context++See proposal.md for why this change exists. This section covers only the current code the decisions rest on.++**Proxy.** Go's `http.DefaultTransport` reads the proxy variables through `http.ProxyFromEnvironment`. A transport built as `&http.Transport{...}` has no `Proxy` field and always connects directly. Four places build their own transport this way:++- S3: `backend/internal/features/storages/models/s3/model.go:855`, in `getClientParams`, used by every S3 operation and the connection test.+- Azure Blob: `backend/internal/features/storages/models/azure_blob/model.go:504`, in `buildClientOptions`.+- Google Drive: `backend/internal/features/storages/models/google_drive/model.go:618`, in `buildHTTPClient`, as the base under the OAuth transport.+- Mattermost: `backend/internal/features/notifiers/models/mattermost/model.go:299`, only when TLS verification is off.++Telegram clones `http.DefaultTransport` and replaces its proxy with the one from the notifier settings (`backend/internal/features/notifiers/models/telegram/proxies.go:43`). The other HTTP notifiers and telemetry use the default transport and already honour the variables.++**How a storage failure becomes "cancelled".** A logical backup streams the dump through a pipe into `storage_files.StartBackgroundWrite` (`backend/internal/features/storages/files/background_write.go:32`). When the upload fails, that goroutine cancels the backup's context with the storage error as its cause (line 36), so the dump process is killed. The engine use case then explains why the context ended. In PostgreSQL that is `classifyCancellation` (`backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go:578`), with identical copies in MySQL (`mysql/create_backup_uc.go:583`), MariaDB (`mariadb/create_backup_uc.go:586`) and MongoDB (`mongodb/create_backup_uc.go:431`). It returns the bare `"backup cancelled"` for any cause that matches `context.Canceled` or `context.DeadlineExceeded`, and `save to storage: <cause>` for anything else.++A network timeout matches `context.DeadlineExceeded`. Both a dial timeout (`dial tcp ...: i/o timeout`) and a response header timeout (`net/http: timeout awaiting response headers`) satisfy `errors.Is(err, context.DeadlineExceeded)` on Go 1.26.3, checked while preparing this change. So a storage timeout comes back as `"backup cancelled"`.++The use case reaches that classification on two paths:++- The dump finished before the upload failed. S3 buffers a 16 MiB part before sending anything (`defaultInnerPartSize`, `s3/model.go:45`, ADR-0004), so a small database dumps completely while S3 is still connecting. The use case then waits for the upload result, and its `saveErr` branch calls `checkCancellation` first (`postgresql/create_backup_uc.go:269`). The context has been cancelled by the upload failure, so the storage error is classified as a cancellation. This is the path in issue #816.+- Only PostgreSQL re-checks the context in that final branch. MySQL, MariaDB and MongoDB return `save to storage: ...` there, so this path misreports only PostgreSQL backups. Their regression tests therefore pass against the old classification too; they pin the outcome for every engine rather than reproduce a failure.+- The upload failed while the dump was still running. The early `earlySaveErr` check (`postgresql/create_backup_uc.go:226`) catches it only if the error is already in the channel; otherwise the `ctx.Done()` branch (line 236) classifies it the same wrong way.++The backuper then decides the status from the error text (`backend/internal/features/backups/backups/backuping/logical/backuper.go:133`). It treats any message containing `"backup cancelled"` or `"context canceled"`, or any error matching `context.Canceled`, as a cancellation unless the message also contains `"shutdown"`. A cancelled backup skips `SetBackupError` and the failure notification (lines 139-153). The time-limit message `"backup cancelled due to timeout"` contains `"backup cancelled"`, so a backup that hits the 23-hour limit is silently recorded as cancelled today.++The only thing that cancels a logical backup on purpose is the cancellation registry. `MakeBackup` creates `executionCtx` from `context.Background()` and registers its cancel function under the backup ID (`backuper.go:71-74`). The registry fires it for the user's cancel endpoint (`backend/internal/features/backups/backups/services/logical_service.go:265`) and for backups the scheduler finds left in progress after a restart (`backuping/logical/scheduler.go:316`). Shutdown and the time limit do not cancel `executionCtx`. The use cases detect them on their own context, through `createBackupContext` (`postgresql/create_backup_uc.go:403`), which is identical in PostgreSQL, MySQL and MariaDB. The MongoDB copy (`mongodb/create_backup_uc.go:271`) lacks the branch that copies the caller's cause, which `context.WithCancelCause` already propagates, so it behaves the same.++Physical backups (`backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go`) already check their own sentinel causes and then `saveErr` before anything is treated as a cancellation. Restores never cancel their context on a storage error; only `executionCtx` in `backend/internal/features/restores/restoring/restorer.go:97` does.++## Goals / Non-Goals++**Goals:**++- One rule decides "cancelled": a cancellation was requested for this backup. No error text takes part in that decision.+- The storage error that interrupted a backup reaches the fail message unchanged.+- The four engines cannot drift apart again on how they report an interruption.+- Tests reproduce the reported path with a real dump, not only with mocks.++**Non-Goals:**++- Retrying a failed upload, or changing any S3, Azure or Google Drive timeout.+- A per-storage proxy setting.+- Reworking the physical backup or restore outcome logic.++## Decisions++### 1. Set `Proxy: http.ProxyFromEnvironment` on the four transports++Each of the four transports gets the `Proxy` field and keeps its own timeouts and TLS settings.++Alternatives rejected:++- **Clone `http.DefaultTransport` and override the timeouts, as Telegram does.** The clone also brings `ForceAttemptHTTP2`, connection pool limits and `ExpectContinueTimeout`. HTTP/2 would then turn on for S3, Azure and Google Drive. That is a behaviour change on S3-compatible providers that this fix does not need and that nothing here tests.+- **A shared constructor for storage transports.** The four places use different timeouts, and Google Drive wraps the transport in an OAuth layer, so the constructor would take most of its fields as parameters to save one line per place. The tests in decision 6 guard against a fifth transport forgetting the field more cheaply.++Proxied `https` requests use the standard library's `CONNECT` tunnel. TLS is negotiated with the endpoint through the tunnel, and each transport's own `TLSClientConfig` keeps applying to the endpoint certificate. The end-to-end TLS requirement in `specs/outbound-http-proxy/spec.md` needs no code of ours beyond leaving those TLS settings in place, and decision 6 adds a test that proves it for each of the four transports. The standard library never proxies localhost or loopback addresses, which the spec states as well.++### 2. The backuper decides "cancelled" from `executionCtx`, not from the error++`MakeBackup` records the backup as cancelled exactly when `executionCtx.Err() != nil` after the use case returns. Every other error is a failure. The `strings.Contains` checks and the `errors.Is(err, context.Canceled)` check in `backuper.go:133-137` are deleted.++This makes three cases correct together. A storage error that wraps a cancellation becomes a failure. A backup stopped by the time limit becomes a failure with a notification. A user cancel on MongoDB that surfaces as a `mongodump` exit error becomes a cancellation; the MongoDB final `switch` (`mongodb/create_backup_uc.go:256`) never re-checks the context, so today that case depends on the dump's error text. Shutdown keeps its current outcome, a failure, because shutdown does not cancel `executionCtx`. If a user requests cancellation and the use case then returns a storage error, the backup is cancelled, which matches what the user asked for.++In practice only the user's cancel request reaches a running backup's `executionCtx`. The scheduler's `failBackupsInProgress` (`backuping/logical/scheduler.go:303`) also requests cancellation, but it runs at startup for backups the previous process left behind, and it records them as failed with `Backup failed due to application restart` itself. This change leaves that path alone; the spec pins its status and message so the new rule is not read as covering it.++Alternatives rejected:++- **Keep the text matching and only fix the use-case messages.** The status would still depend on wording. Any storage provider that wraps `context.Canceled`, or any message that happens to contain "backup cancelled", would hide a failure again.+- **A typed `ErrBackupCancelled` returned by the use cases.** It is better than text, but it still asks four use cases to classify correctly. The backuper already holds the one fact that defines a cancellation, so it needs no help from them.++### 3. One shared implementation of the backup context and its classification++The backup context setup and the classification move out of the four engines into `backups_core_logical`, which every engine already imports. The sentinels go to a new `errors.go` there: `ErrBackupShutdown` with the message `backup interrupted by application shutdown`, and `ErrBackupTimeout` with `backup exceeded the time limit of 23 hours`, the hours taken from the same constant as the limit. Today's texts, `backup cancelled due to shutdown` and `backup cancelled due to timeout`, both say "cancelled" for backups that are recorded as failed. The second also uses the word "timeout", which a storage network timeout produces too, so a user reading the fail message could not tell the two apart. The two functions go to a new file, with working names:++- `NewBackupContext(callerCtx)` returns the backup context and its cancel function. It is the current `createBackupContext`, with the 23-hour limit and the one-second shutdown check moved along with it.+- `DescribeBackupInterruption(callerCtx, backupCtx)` returns the error the use case reports once `backupCtx` has ended. It returns `ErrBackupShutdown` or `ErrBackupTimeout` when the cause matches them, `errors.New("backup cancelled")` when `callerCtx` itself was cancelled, and `fmt.Errorf("save to storage: %w", cause)` otherwise.++The `otherwise` branch is the storage failure, because `StartBackgroundWrite` is the only other code that cancels the backup context with a cause. The check on `callerCtx` replaces today's guess from the shape of the cause. A user cancel ends `callerCtx`; a storage timeout does not. Each engine's `checkCancellation` and `classifyCancellation` become calls to this function with the caller's context, and the four private copies of the sentinels, constants and both functions are deleted.++The message `"backup cancelled"` is kept for a real cancellation. It is no longer what the backuper reads (decision 2), but it still explains the use case's error in logs.++Alternatives rejected:++- **Fix the four copies in place.** The bug lives in four identical copies, and nothing keeps them identical. Fixing it four times keeps the duplication that let this rule go wrong in all of them at once.+- **Mark the cause in `StartBackgroundWrite`** with a sentinel such as `ErrStorageWrite`, and test for it before `context.DeadlineExceeded`. It works, but it changes a helper the physical stream shares, it depends on the order of the `switch` cases, and it still classifies by the shape of an error rather than by who cancelled.+- **Put the shared code in a new package under `usecases/logical`.** The dispatcher in `usecases/logical/create_backup_uc.go` imports the engine packages, so a sibling package would be needed to avoid an import cycle. `backups_core_logical` already holds the contracts between the backuper and the engines, such as `BackupFileStore` and `BackupArtifacts`.++### 4. No change to `StartBackgroundWrite` or the use cases' early-failure check++With decision 3, a storage failure produces the right error on every path the use case can take: the early `earlySaveErr` check, the `ctx.Done()` branch and the final `saveErr` branch. The helper keeps cancelling the backup context with the storage error as the cause, which is still what kills the dump quickly.++### 5. Regression tests per engine run a real dump into a failing file store++Each engine gets a test that runs its real use case against a real database and passes a `BackupFileStore` test double. Two cases cover the two paths in the context section:++- The store reads the whole stream, so the dump completes, and then returns a timeout error.+- The store fails before reading anything, while the dump is still producing data.++Both cases assert that the returned error wraps the storage error and is not a cancellation. The timeout error is a real one: an `http.Client` with a short `ResponseHeaderTimeout` against an `httptest` server that never answers, which produces the same error type the user saw without a 30-second wait. A helper in `internal/util/testing` builds it, so the engine tests and the classification test share one source. The tests live in each engine's `create_backup_uc_test.go`, with MongoDB getting the file. PostgreSQL uses the always-on logical fixture (`databases.GetTestPostgresConfig`); MySQL, MariaDB and MongoDB start one container each through `containers.StartMysql`, `StartMariadb` and `StartMongodb`, per ADR-0013.++Alternatives rejected:++- **An end-to-end test through the API with a real S3 storage that times out.** The S3 timeouts are 30 seconds and minio retries timed-out requests, so one test would take minutes per engine. The API path also cannot swap in a file store.+- **Mock-only tests.** The defect sits in how the real use case waits on its pipes and channels, which a mock usecase skips entirely.++### 6. Unit tests for the rest++- **Proxy wiring.** For each of the four builders, a test asserts that the resulting transport's `Proxy` is `http.ProxyFromEnvironment`, compared through `reflect.ValueOf(...).Pointer()`. Go reads the proxy variables once per process and caches them, and it never proxies loopback addresses. A test that sets `HTTPS_PROXY` and sends a request through a local proxy would therefore depend on test order and on which test in the binary first touched the proxy settings. The existing `Test_BuildHTTPClient_WhenProxyEnabled_UsesConfiguredProxy` in `notifiers/models/telegram/proxies_test.go` already pins the other side: a notifier proxy wins over the environment.+- **TLS through a proxy.** A test helper in `internal/util/testing` starts a local `CONNECT` proxy that records every request it receives and an `httptest` TLS server that records the requests reaching it. For each of the four builders, a test takes the transport the builder produces and replaces only its `Proxy` with `http.ProxyURL` pointing at that local proxy, which avoids the cached environment entirely. It then asserts three things. The proxy receives exactly one `CONNECT` naming the server's host and port, and no request line, headers or credentials. A request with TLS verification on fails with a certificate error, because the test server's certificate is not trusted. For S3 and Mattermost, the same request with TLS verification turned off reaches the server with its `Authorization` header intact.+- **Classification.** A table test of `DescribeBackupInterruption` covers shutdown, time limit, caller cancelled, storage timeout, storage error wrapping `context.Canceled`, and caller cancelled while the cause is a storage error.+- **Backuper.** Tests through `MakeBackup` with use case doubles. A storage timeout error ends as failed, with the storage error in the fail message, the database marked as failing and the failure notification sent. An error that wraps `context.Canceled` without a cancel request ends as failed. `ErrBackupTimeout` ends as failed with the fail message `backup exceeded the time limit of 23 hours`, the database marked as failing and a notification. `ErrBackupShutdown` ends as failed, not cancelled, with the fail message `backup interrupted by application shutdown`. Two tests cancel through the registry: in one the use case returns once its context ends, in the other it returns a storage error after the cancel; both end as cancelled, with no failure notification and the database not marked as failing. The existing `Test_MakeBackup_WhenCancelledAfterStoringArtifact_ArtifactRemoved` fakes a cancellation by returning an error text (`backuper_test.go:284-303`). It becomes the first of these two registry tests, since that text no longer means anything to the backuper.+- **Restart.** The existing `Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStatus` in `backuping/logical/scheduler_test.go` asserts the failed status and the restart fail message, which is all the restart scenario in the spec states. The notification the scheduler sends on that path is unchanged by this change and stays out of the spec.++## Risks / Trade-offs++- [An operator sets `HTTPS_PROXY` for some other tool in the container and does not list the storage host in `NO_PROXY`] → Storage traffic starts going through that proxy after the upgrade. This is the documented meaning of the variable and what the rest of Databasus already does. The pull request description states it so it reaches the release notes.+- [A proxy that only allows some hosts makes storage requests fail with a proxy error instead of a timeout] → The failure is now reported as failed with a notification, and the proxy's error is in the fail message, so the operator sees it at once.+- [Backups that hit the 23-hour limit start sending failure notifications] → Intended, and listed in proposal.md. Such a backup did not finish, and the user should know.+- [A use case path returns an unrelated error after a storage failure, for example `start pg_dump: context canceled` when the upload fails before the dump starts] → The status is still failed, because no cancellation was requested, but the message is less precise. S3 buffers a whole part before it connects, so the upload cannot fail before the dump starts producing data. Physical backups handle this case explicitly; logical backups do not need to yet.+- [The reflection-based proxy test checks a function identity rather than reading the environment] → It is the only order-independent check of the environment wiring in a shared test binary. The tunnel test covers the behaviour of each transport once a proxy is chosen.+- [Fail messages for shutdown and the time limit change wording] → Nothing parses them after decision 2. Old backups keep their stored messages.++## Migration Plan++No data migration. Backups recorded as cancelled before the upgrade keep that status. Rolling back restores the old proxy and status behaviour; nothing persisted depends on the new code.diff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/proposal.md b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/proposal.mdnew file mode 100644index 00000000..9ebd79e0--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/proposal.md@@ -0,0 +1,49 @@+## Why++GitHub issue #816 reports two faults that together left a user's backups broken for hours without anyone being told.++The first fault is about outbound proxies. The user runs Databasus in a network where outbound traffic may only leave through a proxy, configured with the standard `HTTPS_PROXY` variable. Most of Databasus honours that variable, but S3 uploads ignore it and connect directly. The firewall drops those connections, and every backup fails with a timeout. Azure Blob and Google Drive storages build their HTTP connections the same way and fail the same way. So do Mattermost notifications when the notifier skips TLS verification.++The second fault hides the first. When a storage upload fails with a network timeout, the logical backup is recorded as cancelled instead of failed. A cancelled backup sends no failure notification and does not mark the database as failing, because cancellation is normally something the user did on purpose. This happens with any storage timeout: an unreachable S3 endpoint or a dropped network connection is enough, no proxy required. A storage error that is not a timeout, such as a refused access to the bucket, is already reported as a failure.++## What Changes++- S3, Azure Blob and Google Drive storages send their requests through the proxy named by `HTTP_PROXY` / `HTTPS_PROXY` and skip it for hosts listed in `NO_PROXY`, the same way the rest of Databasus already does. When none of these variables is set, connections stay direct, as today.+- A Mattermost notifier with TLS verification turned off honours the same variables. With verification on, it already does.+- A logical backup is recorded as cancelled only when a user asked to cancel that backup. Every other interruption is a failure: the backup gets the failed status and a fail message, the database is marked as failing, and the failure notification goes out. This covers storage timeouts, storage errors that wrap a cancellation, and the backup time limit.+- The fail message of a backup interrupted by a storage error carries the storage error itself instead of the bare text "backup cancelled".+- The four logical engines (PostgreSQL, MySQL, MariaDB, MongoDB) share one implementation of the backup's time limit and shutdown watch, and one function that decides how an interrupted backup is reported, instead of four identical copies of each.+- A backup that runs into the backup time limit is now reported as a failed backup with a failure notification. Today it is silently recorded as cancelled. This changes observable behavior; it is not an API, schema or config break.+- The fail messages for the time limit and for shutdown stop saying "cancelled", since both backups are recorded as failed. They become `backup exceeded the time limit of 23 hours` and `backup interrupted by application shutdown`. Backups already stored keep their old messages.++Out of scope:++- Physical PostgreSQL backups and WAL archiving. Their stream already reports a storage failure as a failure before considering cancellation.+- Restores. A restore does not cancel its own context when a storage read fails; it tracks storage read failures separately, so the reported fault does not reach it. Its error-text matching stays as it is.+- Rclone storages. Rclone's own HTTP layer already reads the proxy variables.+- SFTP, FTP and NAS storages. They do not speak HTTP, and HTTP proxy variables do not apply to them.+- A proxy setting in the storage form. Operators configure the proxy for the whole container through the environment, as the issue does.+- Notifications for backups the user cancelled. They stay silent.+- Backups the previous run left in progress. The scheduler keeps recording them as failed with a restart message and a notification at startup.+- Changing the backup time limit, the S3 timeouts or the shutdown handling, which keeps reporting a backup interrupted by shutdown as failed.+- Other code the four engines duplicate, such as the copy loop and the writer cleanup. This change merges only the part that decides how an interruption is reported.++## Capabilities++### New Capabilities++- `outbound-http-proxy`: storages and notifiers that talk HTTP reach their endpoints through the proxy configured in the process environment, and respect its exclusion list.+- `logical-backup-outcome`: how a logical backup that did not complete is reported, as failed or cancelled, and what each outcome triggers.++### Modified Capabilities++None.++## Impact++- Backend storages: `backend/internal/features/storages/models/s3`, `azure_blob` and `google_drive`.+- Backend notifiers: `backend/internal/features/notifiers/models/mattermost`.+- Logical backups: the four engine use cases under `backend/internal/features/backups/backups/usecases/logical`, the shared logical backup core in `backend/internal/features/backups/backups/core/logical`, and the backuper in `backend/internal/features/backups/backups/backuping/logical`.+- Tests: unit tests next to each changed HTTP client, a shared test helper with a local tunnelling proxy in `backend/internal/util/testing`, a classification test in the logical backup core, backuper tests, and one regression test per logical engine that runs a real dump against the test fixtures. The existing backuper test that simulates a cancellation by returning an error text is rewritten to cancel the backup for real.+- No API, schema, migration, config or frontend change.+- Rules this change answers to: `AGENTS.md`, `backend/AGENTS.md`.diff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/logical-backup-outcome/spec.md b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/logical-backup-outcome/spec.mdnew file mode 100644index 00000000..928113d4--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/logical-backup-outcome/spec.md@@ -0,0 +1,65 @@+## Purpose++Decides how a logical backup that did not complete is reported, so that a backup the user stopped on purpose stays quiet while every other unfinished backup is reported as a failure that reaches the user through notifications.++## ADDED Requirements++### Requirement: Only a cancellation request makes a backup cancelled++A logical backup of a PostgreSQL, MySQL, MariaDB or MongoDB database SHALL be recorded as cancelled only when a user requested cancellation of that backup before it ended. Every other backup that does not complete SHALL be recorded as failed. This SHALL hold whatever the underlying error says, including an error that describes a timeout or a cancellation inside a storage provider.++A cancelled backup SHALL NOT send a backup failure notification and SHALL NOT mark its database as failing. A failed backup SHALL get a fail message, SHALL mark its database as failing, and SHALL send the backup failure notification to the notifiers configured for it.++#### Scenario: Storage times out after the dump has finished++- **WHEN** the database dump finishes, and the upload of the backup to its storage then fails with a network timeout+- **THEN** the backup is recorded as failed, its database is marked as failing, and the backup failure notification is sent++#### Scenario: Storage times out while the dump is still running++- **WHEN** the upload of the backup fails with a network timeout while the database dump is still producing data+- **THEN** the backup is recorded as failed and the backup failure notification is sent++#### Scenario: Storage error that wraps a cancellation++- **WHEN** the storage provider fails the upload with an error that reports its own request as cancelled, and nobody requested cancellation of the backup+- **THEN** the backup is recorded as failed and the backup failure notification is sent++#### Scenario: User cancels a running backup++- **WHEN** a user cancels a logical backup while it is running+- **THEN** the backup is recorded as cancelled, no backup failure notification is sent, and the database is not marked as failing++#### Scenario: User cancels while the storage is failing++- **WHEN** a user requests cancellation of a running logical backup, and the backup then ends with a storage error instead of a cancellation error+- **THEN** the backup is recorded as cancelled, no backup failure notification is sent, and the database is not marked as failing++#### Scenario: Backup exceeds the time limit++- **WHEN** a logical backup runs longer than the backup time limit of 23 hours and is stopped for that reason+- **THEN** the backup is recorded as failed with the fail message `backup exceeded the time limit of 23 hours`, its database is marked as failing, and the backup failure notification is sent++#### Scenario: Application shuts down during a backup++- **WHEN** the application shuts down while a logical backup is running+- **THEN** the backup is recorded as failed with the fail message `backup interrupted by application shutdown`, not as cancelled++#### Scenario: Backup left in progress by a previous run++- **WHEN** the application starts and finds a logical backup still marked as in progress from its previous run+- **THEN** the backup is recorded as failed with the fail message `Backup failed due to application restart`, not as cancelled, even though cancellation is requested for it++### Requirement: The fail message names the storage error++When a logical backup fails because its storage upload failed, the fail message SHALL contain the error the storage returned, so the user can see which storage operation failed and why. It SHALL NOT be reduced to a generic message such as "backup cancelled".++#### Scenario: Fail message after a storage timeout++- **WHEN** a logical backup fails because the upload to its S3 storage timed out while connecting to the endpoint+- **THEN** the fail message shows that saving to storage failed and includes the storage's own description of the timeout, including the endpoint it tried to reach++#### Scenario: Fail message is the same for every engine++- **WHEN** the same storage timeout interrupts a PostgreSQL, a MySQL, a MariaDB and a MongoDB backup+- **THEN** each backup is recorded as failed with a fail message that includes the storage errordiff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/outbound-http-proxy/spec.md b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/outbound-http-proxy/spec.mdnew file mode 100644index 00000000..30aa6b2e--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/specs/outbound-http-proxy/spec.md@@ -0,0 +1,63 @@+## Purpose++Lets Databasus run in networks where outbound traffic must leave through a proxy. Every storage and notifier that talks HTTP reaches its endpoint through the proxy the operator configured in the process environment, and goes direct for the hosts the operator excluded.++## ADDED Requirements++### Requirement: HTTP storages and notifiers use the proxy from the environment++S3, Azure Blob and Google Drive storages, and every notifier that talks HTTP, SHALL send their requests through the proxy named by the standard proxy variables of the process environment: `HTTPS_PROXY` for `https` endpoints and `HTTP_PROXY` for `http` endpoints, in upper or lower case. They SHALL connect directly to a host that matches `NO_PROXY`, and to localhost and loopback addresses. When no proxy variable is set, they SHALL connect directly. This SHALL hold for every request these storages make: uploads, downloads, existence checks, deletions and the connection test. It SHALL hold for a Mattermost notifier whether TLS verification is on or off. A notifier configured with its own proxy, such as a Telegram notifier with a proxy enabled, SHALL use that proxy instead of the environment's.++#### Scenario: S3 upload behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy, an S3 storage points at an `https` endpoint outside `NO_PROXY`, and a backup is uploaded to it+- **THEN** the upload goes through the proxy, and no direct connection to the endpoint is attempted++#### Scenario: Storage endpoint excluded from the proxy++- **WHEN** `HTTPS_PROXY` names a proxy and `NO_PROXY` lists the host of an S3 storage endpoint+- **THEN** requests to that storage connect to the endpoint directly++#### Scenario: No proxy configured++- **WHEN** no proxy variable is set in the process environment+- **THEN** S3, Azure Blob and Google Drive storages and all HTTP notifiers connect to their endpoints directly, as before this change++#### Scenario: Azure Blob and Google Drive behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy and a backup is uploaded to an Azure Blob or Google Drive storage+- **THEN** the upload goes through the proxy++#### Scenario: Mattermost notifier with TLS verification turned off++- **WHEN** `HTTPS_PROXY` names a proxy and a Mattermost notifier with TLS verification turned off sends a notification+- **THEN** the notification goes through the proxy++#### Scenario: Notifier with its own proxy++- **WHEN** `HTTPS_PROXY` names one proxy and a Telegram notifier has a different proxy enabled in its settings+- **THEN** the notification goes through the proxy from the notifier's settings++#### Scenario: Storage connection test behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy and the user tests the connection of an S3 storage+- **THEN** the test request goes through the proxy, so the result matches what scheduled backups will experience++### Requirement: A proxied HTTPS connection keeps TLS end to end++When an S3, Azure Blob or Google Drive storage or a Mattermost notifier sends a request to an `https` endpoint through a proxy, it SHALL ask the proxy only for a tunnel to the endpoint's host and port, and SHALL establish the TLS session with the endpoint inside that tunnel. The proxy SHALL NOT receive the request line, the headers, the body or the credentials sent to the endpoint. The endpoint's certificate SHALL be verified exactly as it is without a proxy: with TLS verification on, a certificate that is not trusted for the endpoint fails the request; with TLS verification turned off in the storage or notifier settings, the request succeeds as it does without a proxy.++#### Scenario: Credentials stay inside the tunnel++- **WHEN** a storage or notifier with an `https` endpoint sends a request with credentials through a proxy+- **THEN** the proxy receives a single tunnel request naming the endpoint's host and port, and the request with its credentials reaches only the endpoint++#### Scenario: Untrusted endpoint certificate through a proxy++- **WHEN** a storage or notifier with TLS verification on connects through a proxy to an endpoint whose certificate is not trusted+- **THEN** the request fails with a certificate error, as it would without a proxy++#### Scenario: TLS verification turned off through a proxy++- **WHEN** an S3 storage or a Mattermost notifier with TLS verification turned off connects through a proxy to an endpoint whose certificate is not trusted+- **THEN** the request reaches the endpoint, as it would without a proxydiff --git a/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/tasks.md b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/tasks.mdnew file mode 100644index 00000000..d36cb02f--- /dev/null+++ b/openspec/changes/archive/2026-10-10-fix-storage-proxy-and-backup-failure-status/tasks.md@@ -0,0 +1,47 @@+## 1. Proxy from the environment++- [x] 1.1 Add `Proxy: http.ProxyFromEnvironment` to the S3 transport in `getClientParams` and add a test in `storages/models/s3/model_test.go` asserting that the returned transport's `Proxy` is `http.ProxyFromEnvironment` (compared through `reflect.ValueOf(...).Pointer()`) and that `TLSClientConfig.InsecureSkipVerify` still follows `SkipTLSVerify`; verify the test fails before the change and passes after it with `just test-package ./internal/features/storages/models/s3/...` in `backend/`+- [x] 1.2 Do the same for Azure Blob in `buildClientOptions`, reaching the transport through the `*http.Client` in the returned options, and verify with `just test-package ./internal/features/storages/models/azure_blob/...`+- [x] 1.3 Do the same for Google Drive in `buildHTTPClient`, reaching the transport through the `oauth2.Transport` base, and verify with `just test-package ./internal/features/storages/models/google_drive/...`+- [x] 1.4 Do the same for the Mattermost transport built when TLS verification is off in `newHTTPClient`, assert that `InsecureSkipVerify` and `MinVersion` are unchanged, and verify with `just test-package ./internal/features/notifiers/models/mattermost/...`+- [x] 1.5 Add a test helper to `backend/internal/util/testing` that starts a local `CONNECT` proxy recording every request it receives and an `httptest` TLS server recording the requests that reach it, and verify it with its own test that a plain `http.Transport{Proxy: http.ProxyURL(...)}` with `InsecureSkipVerify` reaches the server through one `CONNECT`, using `just test-package ./internal/util/testing/...`+- [x] 1.6 For each of the four builders from tasks 1.1-1.4, add a test that replaces only the built transport's `Proxy` with `http.ProxyURL` to the helper's proxy and asserts: the proxy receives exactly one `CONNECT` naming the server's host and port and no request line, headers or credentials; with TLS verification on the request fails with a certificate error; for S3 and Mattermost with TLS verification off the request reaches the server with its `Authorization` header; verify with the package commands from tasks 1.1-1.4+- [x] 1.7 Verify with `git grep -n "&http.Transport{" -- backend/internal` that every remaining transport literal either sets `Proxy` or is outside the scope listed in proposal.md, and that the existing Telegram test `Test_BuildHTTPClient_WhenProxyEnabled_UsesConfiguredProxy` still passes with `just test-package ./internal/features/notifiers/models/telegram/...`++## 2. Shared backup context and interruption classification++- [x] 2.1 Add `errors.go` to `backups_core_logical` with `ErrBackupShutdown` (`backup interrupted by application shutdown`) and `ErrBackupTimeout` (`backup exceeded the time limit of 23 hours`, the hours taken from the time limit constant), and verify the package builds with `go build ./internal/features/backups/backups/core/logical/...` in `backend/`+- [x] 2.2 Move the backup time limit, the shutdown check interval and the context setup from the engines into `backups_core_logical` as `NewBackupContext(callerCtx)`, matching today's PostgreSQL `createBackupContext`, and verify with `go build ./internal/features/backups/backups/core/logical/...`+- [x] 2.3 Add `DescribeBackupInterruption(callerCtx, backupCtx)` with the order from design.md decision 3: shutdown cause, time limit cause, caller cancelled, otherwise `save to storage: %w` with the cause; verify with `go build ./internal/features/backups/backups/core/logical/...` and with the table test in task 2.5+- [x] 2.4 Add a helper to `backend/internal/util/testing` that returns a real network timeout error, produced by an `http.Client` with a short `ResponseHeaderTimeout` against an `httptest` server that never answers, and verify with its own test that the error matches `context.DeadlineExceeded`, using `just test-package ./internal/util/testing/...`+- [x] 2.5 Add a table test for `DescribeBackupInterruption` covering shutdown, time limit, caller cancelled, a storage timeout from the helper in task 2.4, a storage error wrapping `context.Canceled`, and a caller cancelled while the cause is a storage error; assert the storage cases wrap the storage error and are not `backup cancelled`, and verify with `just test-package ./internal/features/backups/backups/core/logical/...`++## 3. Engines use the shared code++- [x] 3.1 In the PostgreSQL use case, replace `createBackupContext`, `checkCancellation`, `classifyCancellation`, the two sentinels and the moved constants with the shared code, passing the caller's context to `DescribeBackupInterruption`; delete the private copies and verify with `go build ./...` in `backend/`+- [x] 3.2 Do the same in the MySQL use case and verify with `go build ./...`+- [x] 3.3 Do the same in the MariaDB use case and verify with `go build ./...`+- [x] 3.4 Do the same in the MongoDB use case and verify with `go build ./...`+- [x] 3.5 Verify with `git grep -n "classifyCancellation\|createBackupContext\|errBackupShutdown\|errBackupTimeout" -- backend/internal/features/backups/backups/usecases/logical` that no private copy is left++## 4. Backuper decides cancellation from its own context++- [x] 4.1 Write backuper tests through `MakeBackup` with use case doubles in `backuping/logical/backuper_test.go`, none of which cancels through the registry: a storage timeout error ends as failed, with the storage error in the fail message, the database marked as failing and the failure notification sent; an error wrapping `context.Canceled` ends as failed with a failure notification; `ErrBackupTimeout` ends as failed with the fail message `backup exceeded the time limit of 23 hours`, the database marked as failing and a failure notification; `ErrBackupShutdown` ends as failed, not cancelled, with the fail message `backup interrupted by application shutdown`; name the storage-timeout test after issue #816; verify the storage-timeout, `context.Canceled` and time-limit tests fail against the current backuper+- [x] 4.2 Rewrite `Test_MakeBackup_WhenCancelledAfterStoringArtifact_ArtifactRemoved` so its use case stores the artifact, signals the test, and blocks until its context ends, while the test cancels the backup through the cancellation registry; assert the backup is cancelled, no failure notification is sent, the database is not marked as failing and the artifact is removed+- [x] 4.3 Add a registry-cancel variant whose use case, after its context ends, returns a storage timeout error instead of a cancellation error; assert the backup is cancelled, no failure notification is sent and the database is not marked as failing+- [x] 4.4 In `backuper.go`, record the backup as cancelled exactly when `executionCtx.Err() != nil` after the use case returns, delete the `strings.Contains` and `errors.Is(err, context.Canceled)` checks with the `isShutdown` flag, and verify all backuper and scheduler tests pass with `just test-package ./internal/features/backups/backups/backuping/logical/...`, including `Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStatus`, which covers the restart scenario's status and fail message++## 5. Engine regression tests with a real dump++- [x] 5.1 Add a `BackupFileStore` test double to the logical backup test helpers with two modes: read the whole stream and then fail with a real timeout error, or fail before reading anything; take the timeout error from the helper in task 2.4+- [x] 5.2 Add a PostgreSQL test to `usecases/logical/postgresql/create_backup_uc_test.go` that runs the real use case against the always-on logical fixture (`databases.GetTestPostgresConfig`) with both modes of the double, and asserts each error wraps the storage error and is not `backup cancelled`; verify the "after the dump finished" case fails when task 3.1 is reverted locally, then passes with `just test-package ./internal/features/backups/backups/usecases/logical/postgresql/...`+- [x] 5.3 Add the same test for MySQL in `usecases/logical/mysql/create_backup_uc_test.go`, starting one container with `containers.StartMysql`, and verify with `just test-package ./internal/features/backups/backups/usecases/logical/mysql/...`+- [x] 5.4 Add the same test for MariaDB with `containers.StartMariadb` and verify with `just test-package ./internal/features/backups/backups/usecases/logical/mariadb/...`+- [x] 5.5 Add the same test for MongoDB in a new `usecases/logical/mongodb/create_backup_uc_test.go` with `containers.StartMongodb` and verify with `just test-package ./internal/features/backups/backups/usecases/logical/mongodb/...`++## 6. Verification++- [x] 6.1 Run the logical backup packages and the end-to-end hang regressions with `just test-package ./internal/features/backups/... ./internal/features/tests/logical/...` in `backend/`, and confirm the issue #582 hang tests still see backups to a missing bucket end as failed+- [x] 6.2 Run the storage and notifier packages with `just test-package ./internal/features/storages/... ./internal/features/notifiers/...` in `backend/`+- [x] 6.3 Run `just lint` and `just format` in `backend/`+- [x] 6.4 Run `openspec validate fix-storage-proxy-and-backup-failure-status --strict`diff --git a/openspec/specs/logical-backup-outcome/spec.md b/openspec/specs/logical-backup-outcome/spec.mdnew file mode 100644index 00000000..748e687a--- /dev/null+++ b/openspec/specs/logical-backup-outcome/spec.md@@ -0,0 +1,67 @@+# Logical Backup Outcome Specification++## Purpose++Decides how a logical backup that did not complete is reported, so that a backup the user stopped on purpose stays quiet while every other unfinished backup is reported as a failure that reaches the user through notifications.++## Requirements++### Requirement: Only a cancellation request makes a backup cancelled++A logical backup of a PostgreSQL, MySQL, MariaDB or MongoDB database SHALL be recorded as cancelled only when a user requested cancellation of that backup before it ended. Every other backup that does not complete SHALL be recorded as failed. This SHALL hold whatever the underlying error says, including an error that describes a timeout or a cancellation inside a storage provider.++A cancelled backup SHALL NOT send a backup failure notification and SHALL NOT mark its database as failing. A failed backup SHALL get a fail message, SHALL mark its database as failing, and SHALL send the backup failure notification to the notifiers configured for it.++#### Scenario: Storage times out after the dump has finished++- **WHEN** the database dump finishes, and the upload of the backup to its storage then fails with a network timeout+- **THEN** the backup is recorded as failed, its database is marked as failing, and the backup failure notification is sent++#### Scenario: Storage times out while the dump is still running++- **WHEN** the upload of the backup fails with a network timeout while the database dump is still producing data+- **THEN** the backup is recorded as failed and the backup failure notification is sent++#### Scenario: Storage error that wraps a cancellation++- **WHEN** the storage provider fails the upload with an error that reports its own request as cancelled, and nobody requested cancellation of the backup+- **THEN** the backup is recorded as failed and the backup failure notification is sent++#### Scenario: User cancels a running backup++- **WHEN** a user cancels a logical backup while it is running+- **THEN** the backup is recorded as cancelled, no backup failure notification is sent, and the database is not marked as failing++#### Scenario: User cancels while the storage is failing++- **WHEN** a user requests cancellation of a running logical backup, and the backup then ends with a storage error instead of a cancellation error+- **THEN** the backup is recorded as cancelled, no backup failure notification is sent, and the database is not marked as failing++#### Scenario: Backup exceeds the time limit++- **WHEN** a logical backup runs longer than the backup time limit of 23 hours and is stopped for that reason+- **THEN** the backup is recorded as failed with the fail message `backup exceeded the time limit of 23 hours`, its database is marked as failing, and the backup failure notification is sent++#### Scenario: Application shuts down during a backup++- **WHEN** the application shuts down while a logical backup is running+- **THEN** the backup is recorded as failed with the fail message `backup interrupted by application shutdown`, not as cancelled++#### Scenario: Backup left in progress by a previous run++- **WHEN** the application starts and finds a logical backup still marked as in progress from its previous run+- **THEN** the backup is recorded as failed with the fail message `Backup failed due to application restart`, not as cancelled, even though cancellation is requested for it++### Requirement: The fail message names the storage error++When a logical backup fails because its storage upload failed, the fail message SHALL contain the error the storage returned, so the user can see which storage operation failed and why. It SHALL NOT be reduced to a generic message such as "backup cancelled".++#### Scenario: Fail message after a storage timeout++- **WHEN** a logical backup fails because the upload to its S3 storage timed out while connecting to the endpoint+- **THEN** the fail message shows that saving to storage failed and includes the storage's own description of the timeout, including the endpoint it tried to reach++#### Scenario: Fail message is the same for every engine++- **WHEN** the same storage timeout interrupts a PostgreSQL, a MySQL, a MariaDB and a MongoDB backup+- **THEN** each backup is recorded as failed with a fail message that includes the storage errordiff --git a/openspec/specs/outbound-http-proxy/spec.md b/openspec/specs/outbound-http-proxy/spec.mdnew file mode 100644index 00000000..7531d8da--- /dev/null+++ b/openspec/specs/outbound-http-proxy/spec.md@@ -0,0 +1,65 @@+# Outbound HTTP Proxy Specification++## Purpose++Lets Databasus run in networks where outbound traffic must leave through a proxy. Every storage and notifier that talks HTTP reaches its endpoint through the proxy the operator configured in the process environment, and goes direct for the hosts the operator excluded.++## Requirements++### Requirement: HTTP storages and notifiers use the proxy from the environment++S3, Azure Blob and Google Drive storages, and every notifier that talks HTTP, SHALL send their requests through the proxy named by the standard proxy variables of the process environment: `HTTPS_PROXY` for `https` endpoints and `HTTP_PROXY` for `http` endpoints, in upper or lower case. They SHALL connect directly to a host that matches `NO_PROXY`, and to localhost and loopback addresses. When no proxy variable is set, they SHALL connect directly. This SHALL hold for every request these storages make: uploads, downloads, existence checks, deletions and the connection test. It SHALL hold for a Mattermost notifier whether TLS verification is on or off. A notifier configured with its own proxy, such as a Telegram notifier with a proxy enabled, SHALL use that proxy instead of the environment's.++#### Scenario: S3 upload behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy, an S3 storage points at an `https` endpoint outside `NO_PROXY`, and a backup is uploaded to it+- **THEN** the upload goes through the proxy, and no direct connection to the endpoint is attempted++#### Scenario: Storage endpoint excluded from the proxy++- **WHEN** `HTTPS_PROXY` names a proxy and `NO_PROXY` lists the host of an S3 storage endpoint+- **THEN** requests to that storage connect to the endpoint directly++#### Scenario: No proxy configured++- **WHEN** no proxy variable is set in the process environment+- **THEN** S3, Azure Blob and Google Drive storages and all HTTP notifiers connect to their endpoints directly, as before this change++#### Scenario: Azure Blob and Google Drive behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy and a backup is uploaded to an Azure Blob or Google Drive storage+- **THEN** the upload goes through the proxy++#### Scenario: Mattermost notifier with TLS verification turned off++- **WHEN** `HTTPS_PROXY` names a proxy and a Mattermost notifier with TLS verification turned off sends a notification+- **THEN** the notification goes through the proxy++#### Scenario: Notifier with its own proxy++- **WHEN** `HTTPS_PROXY` names one proxy and a Telegram notifier has a different proxy enabled in its settings+- **THEN** the notification goes through the proxy from the notifier's settings++#### Scenario: Storage connection test behind a proxy++- **WHEN** `HTTPS_PROXY` names a proxy and the user tests the connection of an S3 storage+- **THEN** the test request goes through the proxy, so the result matches what scheduled backups will experience++### Requirement: A proxied HTTPS connection keeps TLS end to end++When an S3, Azure Blob or Google Drive storage or a Mattermost notifier sends a request to an `https` endpoint through a proxy, it SHALL ask the proxy only for a tunnel to the endpoint's host and port, and SHALL establish the TLS session with the endpoint inside that tunnel. The proxy SHALL NOT receive the request line, the headers, the body or the credentials sent to the endpoint. The endpoint's certificate SHALL be verified exactly as it is without a proxy: with TLS verification on, a certificate that is not trusted for the endpoint fails the request; with TLS verification turned off in the storage or notifier settings, the request succeeds as it does without a proxy.++#### Scenario: Credentials stay inside the tunnel++- **WHEN** a storage or notifier with an `https` endpoint sends a request with credentials through a proxy+- **THEN** the proxy receives a single tunnel request naming the endpoint's host and port, and the request with its credentials reaches only the endpoint++#### Scenario: Untrusted endpoint certificate through a proxy++- **WHEN** a storage or notifier with TLS verification on connects through a proxy to an endpoint whose certificate is not trusted+- **THEN** the request fails with a certificate error, as it would without a proxy++#### Scenario: TLS verification turned off through a proxy++- **WHEN** an S3 storage or a Mattermost notifier with TLS verification turned off connects through a proxy to an endpoint whose certificate is not trusted+- **THEN** the request reaches the endpoint, as it would without a proxy