PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (storages): Resume interrupted restore reads on every network backend (issue #714)
Rostislav Duginb63ab2950b51
diff --git a/CLAUDE.md b/CLAUDE.mdindex fd16d11d..44dac6d1 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -130,6 +130,12 @@ After each change run linting and formatting depending on folder you are workingThe default is **no comment**. Before writing one, reach for a clearer name or a smaller function first — self-explanatory code beats commented code every time. A comment is justified only when it carries something the code cannot: a *why* (business rule, hidden cross-system constraint, non-obvious algorithm or optimisation, ADR reference). Never write a comment that restates *what* the code does or narrates the obvious — if a `// Foo does X` comment sits above the code, that's a naming bug: rename until the comment is redundant, then delete it. This applies everywhere, tests included — a well-named test needs no header comment explaining what it checks.+### No issue numbers in production code++Fixing an issue does not put its number in the code. Production files, comments and error messages state the constraint itself — an issue number is a pointer to a discussion the reader has to leave the code to follow, and it rots the moment the tracker moves.++The one exception is a test written specifically to pin that issue's behaviour: there the number belongs next to the test, because it says *why this case exists* and where the reproduction came from.+### No "how it was" comments, no unrequested backward compatibilityDon't write comments that explain previous behavior ("used to be X", "was renamed from Y", "kept for legacy callers"). Code shows the current state; history lives in git.diff --git a/backend/internal/features/restores/core/stream_failure.go b/backend/internal/features/restores/core/stream_failure.goindex 9102f1c9..feba2fdf 100644--- a/backend/internal/features/restores/core/stream_failure.go+++ b/backend/internal/features/restores/core/stream_failure.go@@ -6,11 +6,23 @@ import (io_utils "databasus-backend/internal/util/io")+type BackupStreamTrackers struct {+ StorageRead *io_utils.FailureTrackingReader+ DecodedStream *io_utils.FailureTrackingReader+}+// GetBackupStreamFailure must be consulted before the restore client's own exit error:// a truncated download makes the client fail on the half-written statement it was fed,// and that downstream syntax error would otherwise be reported as the cause.-func GetBackupStreamFailure(backupStreamReader *io_utils.FailureTrackingReader) error {- readError := backupStreamReader.GetFirstReadError()+//+// The transport failure wins over the decode failure for the same reason: a severed download+// surfaces one layer up as a decompression error, and reporting that would hide the real cause.+func GetBackupStreamFailure(trackers BackupStreamTrackers) error {+ readError := trackers.StorageRead.GetFirstReadError()+ if readError == nil {+ readError = trackers.DecodedStream.GetFirstReadError()+ }+if readError == nil {return nil}diff --git a/backend/internal/features/restores/core/stream_failure_test.go b/backend/internal/features/restores/core/stream_failure_test.gonew file mode 100644index 00000000..a6bba100--- /dev/null+++ b/backend/internal/features/restores/core/stream_failure_test.go@@ -0,0 +1,113 @@+package restores_core++import (+ "bytes"+ "errors"+ "io"+ "testing"++ "github.com/klauspost/compress/zstd"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ io_utils "databasus-backend/internal/util/io"+)++func Test_GetBackupStreamFailure_StorageDropsMidStream_ReportsTheTransportErrorNotTheDecodeError(t *testing.T) {+ errConnectionReset := errors.New("connection reset by peer")+ compressedBackup := compressBackup(t, bytes.Repeat([]byte("INSERT INTO users VALUES (1);\n"), 1000))++ storageReadFailureTracker := io_utils.NewFailureTrackingReader(&severingReader{+ content: compressedBackup,+ failAfter: len(compressedBackup) / 2,+ failWith: errConnectionReset,+ readAtOnce: 64,+ })++ zstdReader, err := zstd.NewReader(storageReadFailureTracker)+ require.NoError(t, err)+ defer zstdReader.Close()++ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(zstdReader)++ _, copyErr := io.Copy(io.Discard, decodedStreamFailureTracker)+ require.Error(t, copyErr, "the truncated download must break decompression")++ streamFailure := GetBackupStreamFailure(BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ })++ require.ErrorIs(t, streamFailure, errConnectionReset)+}++func Test_GetBackupStreamFailure_StorageIntactButDecodeFails_ReportsTheDecodeError(t *testing.T) {+ errCorruptArchive := errors.New("corrupt zstd frame")++ storageReadFailureTracker := io_utils.NewFailureTrackingReader(bytes.NewReader([]byte("payload")))+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(&severingReader{+ content: []byte("payload"),+ failAfter: 2,+ failWith: errCorruptArchive,+ readAtOnce: 1,+ })++ _, _ = io.Copy(io.Discard, storageReadFailureTracker)+ _, _ = io.Copy(io.Discard, decodedStreamFailureTracker)++ streamFailure := GetBackupStreamFailure(BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ })++ require.ErrorIs(t, streamFailure, errCorruptArchive)+}++func Test_GetBackupStreamFailure_BothStreamsCompleted_ReturnsNil(t *testing.T) {+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(bytes.NewReader([]byte("payload")))+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(bytes.NewReader([]byte("payload")))++ _, _ = io.Copy(io.Discard, storageReadFailureTracker)+ _, _ = io.Copy(io.Discard, decodedStreamFailureTracker)++ assert.NoError(t, GetBackupStreamFailure(BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ }))+}++func compressBackup(t *testing.T, backupContent []byte) []byte {+ t.Helper()++ var compressed bytes.Buffer++ zstdWriter, err := zstd.NewWriter(&compressed)+ require.NoError(t, err)++ _, err = zstdWriter.Write(backupContent)+ require.NoError(t, err)+ require.NoError(t, zstdWriter.Close())++ return compressed.Bytes()+}++type severingReader struct {+ content []byte+ failAfter int+ failWith error+ readAtOnce int+ deliveredBytes int+}++func (r *severingReader) Read(p []byte) (int, error) {+ if r.deliveredBytes >= r.failAfter {+ return 0, r.failWith+ }++ readable := min(len(p), r.readAtOnce, r.failAfter-r.deliveredBytes)++ copy(p, r.content[r.deliveredBytes:r.deliveredBytes+readable])+ r.deliveredBytes += readable++ return readable, nil+}diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.goindex 8ba552ca..b1482bb2 100644--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go@@ -188,10 +188,12 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(cmd := exec.CommandContext(ctx, mariadbBin, fullArgs...)uc.logger.Info("Executing MariaDB restore command", "command", cmd.String())- var inputReader io.Reader = backupReader+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(backupReader)++ var inputReader io.Reader = storageReadFailureTrackerif backup.Encryption == backups_core_enums.BackupEncryptionEncrypted {- decryptReader, err := uc.setupDecryption(backupReader, backup)+ decryptReader, err := uc.setupDecryption(storageReadFailureTracker, backup)if err != nil {return fmt.Errorf("failed to setup decryption: %w", err)}@@ -204,8 +206,8 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(}defer zstdReader.Close()- backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)- cmd.Stdin = backupStreamReader+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(zstdReader)+ cmd.Stdin = decodedStreamFailureTrackercmd.Env = os.Environ()cmd.Env = append(cmd.Env,@@ -245,7 +247,10 @@ func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore(return fmt.Errorf("restore cancelled due to shutdown")}- if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ if streamErr := restores_core.GetBackupStreamFailure(restores_core.BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ }); streamErr != nil {return streamErr}diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.goindex 44fb8beb..8b4cba54 100644--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go@@ -191,18 +191,20 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(safeArgs,)- var inputReader io.Reader = backupReader+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(backupReader)++ var inputReader io.Reader = storageReadFailureTrackerif backup.Encryption == backups_core_enums.BackupEncryptionEncrypted {- decryptReader, err := uc.setupDecryption(backupReader, backup)+ decryptReader, err := uc.setupDecryption(storageReadFailureTracker, backup)if err != nil {return fmt.Errorf("failed to setup decryption: %w", err)}inputReader = decryptReader}- backupStreamReader := io_utils.NewFailureTrackingReader(inputReader)- cmd.Stdin = backupStreamReader+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(inputReader)+ cmd.Stdin = decodedStreamFailureTrackercmd.Env = os.Environ()cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8", "LANG=C.UTF-8")@@ -238,7 +240,10 @@ func (uc *RestoreMongodbBackupUsecase) executeMongoRestore(return fmt.Errorf("restore cancelled due to shutdown")}- if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ if streamErr := restores_core.GetBackupStreamFailure(restores_core.BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ }); streamErr != nil {return streamErr}diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.goindex f66e44e2..d4acd8d1 100644--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go@@ -169,10 +169,12 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(cmd := exec.CommandContext(ctx, mysqlBin, fullArgs...)uc.logger.Info("Executing MySQL restore command", "command", cmd.String())- var inputReader io.Reader = backupReader+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(backupReader)++ var inputReader io.Reader = storageReadFailureTrackerif backup.Encryption == backups_core_enums.BackupEncryptionEncrypted {- decryptReader, err := uc.setupDecryption(backupReader, backup)+ decryptReader, err := uc.setupDecryption(storageReadFailureTracker, backup)if err != nil {return fmt.Errorf("failed to setup decryption: %w", err)}@@ -185,8 +187,8 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(}defer zstdReader.Close()- backupStreamReader := io_utils.NewFailureTrackingReader(zstdReader)- cmd.Stdin = backupStreamReader+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(zstdReader)+ cmd.Stdin = decodedStreamFailureTrackercmd.Env = os.Environ()cmd.Env = append(cmd.Env,@@ -226,7 +228,10 @@ func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore(return fmt.Errorf("restore cancelled due to shutdown")}- if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ if streamErr := restores_core.GetBackupStreamFailure(restores_core.BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ }); streamErr != nil {return streamErr}diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.goindex e833514f..b985e3ed 100644--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go@@ -233,20 +233,19 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(}}()- var backupReader io.Reader = rawReader+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(rawReader)++ var backupReader io.Reader = storageReadFailureTrackerif backup.Encryption == backups_core_enums.BackupEncryptionEncrypted {- // Validate encryption metadataif backup.EncryptionSalt == nil || backup.EncryptionIV == nil {return fmt.Errorf("backup is encrypted but missing encryption metadata")}- // Get master keymasterKey, err := uc.secretKeyService.GetSecretKey()if err != nil {return fmt.Errorf("failed to get master key for decryption: %w", err)}- // Decode salt and IV from base64salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)if err != nil {return fmt.Errorf("failed to decode encryption salt: %w", err)@@ -257,9 +256,8 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(return fmt.Errorf("failed to decode encryption IV: %w", err)}- // Create decryption readerdecryptReader, err := encryption.NewDecryptionReader(- rawReader,+ storageReadFailureTracker,masterKey,backup.ID,salt,@@ -312,11 +310,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(return fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)}- backupStreamReader := io_utils.NewFailureTrackingReader(backupReader)+ decodedStreamFailureTracker := io_utils.NewFailureTrackingReader(backupReader)copyErrCh := make(chan error, 1)go func() {- _, copyErr := io.Copy(stdinPipe, backupStreamReader)+ _, copyErr := io.Copy(stdinPipe, decodedStreamFailureTracker)// Close stdin pipe to signal EOF to pg_restore - critical for proper terminationcloseErr := stdinPipe.Close()switch {@@ -348,7 +346,10 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(return fmt.Errorf("restore cancelled due to shutdown")}- if streamErr := restores_core.GetBackupStreamFailure(backupStreamReader); streamErr != nil {+ if streamErr := restores_core.GetBackupStreamFailure(restores_core.BackupStreamTrackers{+ StorageRead: storageReadFailureTracker,+ DecodedStream: decodedStreamFailureTracker,+ }); streamErr != nil {return streamErr}@@ -572,23 +573,21 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(}}()- // Create a reader that handles decryption if needed- var backupReader io.Reader = rawReader+ storageReadFailureTracker := io_utils.NewFailureTrackingReader(rawReader)++ var backupReader io.Reader = storageReadFailureTrackerif backup.Encryption == backups_core_enums.BackupEncryptionEncrypted {- // Validate encryption metadataif backup.EncryptionSalt == nil || backup.EncryptionIV == nil {cleanupFunc()return "", nil, fmt.Errorf("backup is encrypted but missing encryption metadata")}- // Get master keymasterKey, err := uc.secretKeyService.GetSecretKey()if err != nil {cleanupFunc()return "", nil, fmt.Errorf("failed to get master key for decryption: %w", err)}- // Decode salt and IV from base64salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)if err != nil {cleanupFunc()@@ -601,9 +600,8 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(return "", nil, fmt.Errorf("failed to decode encryption IV: %w", err)}- // Create decryption readerdecryptReader, err := encryption.NewDecryptionReader(- rawReader,+ storageReadFailureTracker,masterKey,backup.ID,salt,diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.goindex 6427d17a..ea322a10 100644--- a/backend/internal/features/storages/model_test.go+++ b/backend/internal/features/storages/model_test.go@@ -208,6 +208,40 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoiassert.Equal(t, fileData, content, "File content should match the original")})+ // Reads run through a ResumingReader that stops at the size the backend declared, so a+ // backend reporting the wrong length truncates the download. One inline byte of test+ // data is too small to catch that.+ t.Run("Test_SaveAndGetLargeFile_ReturnsCompleteContent", func(t *testing.T) {+ largeFileData := generateFileContent(256 * 1024)+ fileID := uuid.New()++ err := tc.storage.SaveFile(+ t.Context(),+ encryptor,+ logger.GetLogger(),+ fileID.String(),+ bytes.NewReader(largeFileData),+ )+ require.NoError(t, err, "SaveFile should succeed")++ t.Cleanup(func() {+ _ = tc.storage.DeleteFile(encryptor, fileID.String())+ })++ file, err := tc.storage.GetFile(+ t.Context(),+ encryptor,+ logger.GetLogger(),+ fileID.String(),+ )+ require.NoError(t, err, "GetFile should succeed")+ defer file.Close()++ content, err := io.ReadAll(file)+ require.NoError(t, err, "Should be able to read file")+ assert.Equal(t, largeFileData, content, "File content should match the original")+ })+t.Run("Test_TestDeleteFile_RemovesFileFromDisk", func(t *testing.T) {fileData, err := os.ReadFile(testFilePath)require.NoError(t, err, "Should be able to read test file")@@ -245,6 +279,15 @@ func addressOf(endpoint containers.Endpoint) string {return fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port)}+func generateFileContent(size int) []byte {+ generated := make([]byte, size)+ for i := range generated {+ generated[i] = byte(i % 251)+ }++ return generated+}+func setupTestFile() (string, error) {tempDir := os.TempDir()testFilePath := filepath.Join(tempDir, "test_file.txt")diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.goindex c1b890dc..1bc63047 100644--- a/backend/internal/features/storages/models/azure_blob/model.go+++ b/backend/internal/features/storages/models/azure_blob/model.go@@ -16,10 +16,12 @@ import ("github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/azcore/policy""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob""github.com/google/uuid""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -159,7 +161,7 @@ func (s *AzureBlobStorage) SaveFile(func (s *AzureBlobStorage) GetFile(ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {client, err := s.getClient(encryptor)@@ -169,17 +171,43 @@ func (s *AzureBlobStorage) GetFile(blobName := s.buildBlobName(fileName)- response, err := client.DownloadStream(- ctx,- s.ContainerName,- blobName,- nil,- )+ properties, err := client.ServiceClient().+ NewContainerClient(s.ContainerName).+ NewBlobClient(blobName).+ GetProperties(ctx, nil)if err != nil {- return nil, fmt.Errorf("failed to download blob from Azure: %w", err)+ return nil, fmt.Errorf("failed to stat blob in Azure: %w", err)}- return response.Body, nil+ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: getBlobSizeOrUnknown(properties.ContentLength),+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ return s.downloadBlobFromOffset(attemptCtx, client, blobName, offsetBytes)+ },+ IsRetryableError: isRetryableAzureError,+ }), nil+}++func isRetryableAzureError(err error) bool {+ var responseError *azcore.ResponseError+ if errors.As(err, &responseError) {+ return responseError.StatusCode == http.StatusRequestTimeout ||+ responseError.StatusCode == http.StatusTooManyRequests ||+ responseError.StatusCode >= http.StatusInternalServerError+ }++ return true+}++func getBlobSizeOrUnknown(contentLength *int64) int64 {+ if contentLength == nil {+ return io_utils.UnknownTotalBytes+ }++ return *contentLength}func (s *AzureBlobStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {@@ -419,3 +447,35 @@ func (s *AzureBlobStorage) buildAccountURL() string {return fmt.Sprintf("https://%s.blob.core.windows.net/", s.AccountName)}++func (s *AzureBlobStorage) downloadBlobFromOffset(+ attemptCtx context.Context,+ client *azblob.Client,+ blobName string,+ offsetBytes int64,+) (io.ReadCloser, error) {+ response, err := client.DownloadStream(+ attemptCtx,+ s.ContainerName,+ blobName,+ &azblob.DownloadStreamOptions{Range: blob.HTTPRange{Offset: offsetBytes}},+ )+ if err != nil {+ return nil, fmt.Errorf("failed to download blob from Azure: %w", err)+ }++ if offsetBytes > 0 {+ contentRange := ""+ if response.ContentRange != nil {+ contentRange = *response.ContentRange+ }++ if err := io_utils.VerifyContentRangeStart(contentRange, offsetBytes); err != nil {+ _ = response.Body.Close()++ return nil, err+ }+ }++ return response.Body, nil+}diff --git a/backend/internal/features/storages/models/ftp/model.go b/backend/internal/features/storages/models/ftp/model.goindex 51d9dc36..77d0bacf 100644--- a/backend/internal/features/storages/models/ftp/model.go+++ b/backend/internal/features/storages/models/ftp/model.go@@ -7,6 +7,7 @@ import ("fmt""io""log/slog"+ "net/textproto""strings""time"@@ -14,6 +15,7 @@ import ("github.com/jlaffaye/ftp""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -113,9 +115,9 @@ func (f *FTPStorage) SaveFile(}func (f *FTPStorage) GetFile(- _ context.Context,+ ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {conn, err := f.connect(encryptor, ftpConnectTimeout)@@ -125,16 +127,35 @@ func (f *FTPStorage) GetFile(filePath := f.getFilePath(fileName)- resp, err := conn.Retr(filePath)+ totalBytes, err := conn.FileSize(filePath)if err != nil {- _ = conn.Quit()- return nil, fmt.Errorf("failed to retrieve file from FTP: %w", err)+ // A server that does not implement SIZE still serves the file, so an unsupported command+ // costs the completeness check rather than the whole restore. Any other reply, 550 above+ // all, means the file itself is unreachable.+ if !isUnsupportedFtpCommand(err) {+ _ = conn.Quit()++ return nil, fmt.Errorf("failed to stat file on FTP: %w", err)+ }++ logger.Warn("storage does not support the FTP SIZE command", "file_name", fileName, "error", err)++ totalBytes = io_utils.UnknownTotalBytes}- return &ftpFileReader{- response: resp,- conn: conn,- }, nil+ if err := conn.Quit(); err != nil {+ logger.Warn("failed to close the FTP connection used for stat", "error", err)+ }++ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: totalBytes,+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ return f.retrieveFileFromOffset(attemptCtx, encryptor, filePath, offsetBytes)+ },+ }), nil}func (f *FTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {@@ -330,8 +351,9 @@ func (f *FTPStorage) getFilePath(filename string) string {}type ftpFileReader struct {- response *ftp.Response- conn *ftp.ServerConn+ response *ftp.Response+ conn *ftp.ServerConn+ stopDeadlineOnCancel func() bool}func (r *ftpFileReader) Read(p []byte) (n int, err error) {@@ -341,6 +363,10 @@ func (r *ftpFileReader) Read(p []byte) (n int, err error) {func (r *ftpFileReader) Close() error {var errs []error+ if r.stopDeadlineOnCancel != nil {+ r.stopDeadlineOnCancel()+ }+if r.response != nil {if err := r.response.Close(); err != nil {errs = append(errs, fmt.Errorf("failed to close response: %w", err))@@ -373,3 +399,47 @@ func (r *contextReader) Read(p []byte) (n int, err error) {return r.reader.Read(p)}}++// Each attempt dials a fresh control connection: an FTP data transfer dies together with the+// control channel that opened it, so a resumed REST cannot reuse the broken one.+func (f *FTPStorage) retrieveFileFromOffset(+ attemptCtx context.Context,+ encryptor encryption.FieldEncryptor,+ filePath string,+ offsetBytes int64,+) (io.ReadCloser, error) {+ conn, err := f.connect(encryptor, ftpConnectTimeout)+ if err != nil {+ return nil, fmt.Errorf("failed to connect to FTP: %w", err)+ }++ response, err := conn.RetrFrom(filePath, uint64(offsetBytes))+ if err != nil {+ _ = conn.Quit()++ return nil, fmt.Errorf("failed to retrieve file from FTP: %w", err)+ }++ // The FTP client takes no context, so a read blocked on a silent server would ignore the+ // caller's cancellation; an expired deadline is what unblocks it.+ stopDeadlineOnCancel := context.AfterFunc(attemptCtx, func() {+ _ = response.SetDeadline(time.Now())+ })++ return &ftpFileReader{+ response: response,+ conn: conn,+ stopDeadlineOnCancel: stopDeadlineOnCancel,+ }, nil+}++func isUnsupportedFtpCommand(err error) bool {+ var protocolError *textproto.Error+ if !errors.As(err, &protocolError) {+ return false+ }++ return protocolError.Code == ftp.StatusNotImplemented ||+ protocolError.Code == ftp.StatusBadCommand ||+ protocolError.Code == ftp.StatusCommandNotImplemented+}diff --git a/backend/internal/features/storages/models/google_drive/model.go b/backend/internal/features/storages/models/google_drive/model.goindex 3ef42cc8..6e5bc719 100644--- a/backend/internal/features/storages/models/google_drive/model.go+++ b/backend/internal/features/storages/models/google_drive/model.go@@ -20,6 +20,7 @@ import ("google.golang.org/api/option""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -151,10 +152,12 @@ func (r *backpressureReader) Read(p []byte) (n int, err error) {func (s *GoogleDriveStorage) GetFile(ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- var result io.ReadCloser+ var driveFileID string+ var totalBytes int64+err := s.withRetryOnAuth(ctx,encryptor,@@ -164,22 +167,52 @@ func (s *GoogleDriveStorage) GetFile(return fmt.Errorf("failed to find backups folder: %w", err)}- fileIDGoogle, err := s.lookupFileID(driveService, fileName, folderID)+ driveFileID, err = s.lookupFileID(driveService, fileName, folderID)if err != nil {return err}- resp, err := driveService.Files.Get(fileIDGoogle).Context(ctx).Download() //nolint:bodyclose+ metadata, err := driveService.Files.Get(driveFileID).Fields("size").Context(ctx).Do()if err != nil {- return fmt.Errorf("failed to download file from Google Drive: %w", err)+ return fmt.Errorf("failed to stat file in Google Drive: %w", err)}- result = resp.Body+ totalBytes = metadata.Size+return nil},)+ if err != nil {+ return nil, err+ }++ // Drive omits the size for files it did not store verbatim, and a zero would end the stream+ // before its first byte.+ if totalBytes <= 0 {+ totalBytes = io_utils.UnknownTotalBytes+ }- return result, err+ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: totalBytes,+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ return s.downloadFileFromOffset(attemptCtx, encryptor, driveFileID, offsetBytes)+ },+ IsRetryableError: isRetryableGoogleDriveError,+ }), nil+}++func isRetryableGoogleDriveError(err error) bool {+ var apiError *googleapi.Error+ if errors.As(err, &apiError) {+ return apiError.Code == http.StatusRequestTimeout ||+ apiError.Code == http.StatusTooManyRequests ||+ apiError.Code >= http.StatusInternalServerError+ }++ return true}func (s *GoogleDriveStorage) DeleteFile(@@ -689,3 +722,48 @@ func (s *GoogleDriveStorage) findBackupsFolder(driveService *drive.Service) (strreturn results.Files[0].Id, nil}++func (s *GoogleDriveStorage) downloadFileFromOffset(+ attemptCtx context.Context,+ encryptor encryption.FieldEncryptor,+ driveFileID string,+ offsetBytes int64,+) (io.ReadCloser, error) {+ var body io.ReadCloser++ err := s.withRetryOnAuth(+ attemptCtx,+ encryptor,+ func(driveService *drive.Service) error {+ downloadCall := driveService.Files.Get(driveFileID).Context(attemptCtx)+ if offsetBytes > 0 {+ downloadCall.Header().Set("Range", fmt.Sprintf("bytes=%d-", offsetBytes))+ }++ response, err := downloadCall.Download() //nolint:bodyclose+ if err != nil {+ return fmt.Errorf("failed to download file from Google Drive: %w", err)+ }++ if offsetBytes > 0 {+ if err := io_utils.VerifyContentRangeStart(+ response.Header.Get("Content-Range"),+ offsetBytes,+ ); err != nil {+ _ = response.Body.Close()++ return err+ }+ }++ body = response.Body++ return nil+ },+ )+ if err != nil {+ return nil, err+ }++ return body, nil+}diff --git a/backend/internal/features/storages/models/nas/model.go b/backend/internal/features/storages/models/nas/model.goindex 00a3da66..5a069381 100644--- a/backend/internal/features/storages/models/nas/model.go+++ b/backend/internal/features/storages/models/nas/model.go@@ -16,6 +16,7 @@ import ("github.com/hirochachacha/go-smb2""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -156,9 +157,9 @@ func (n *NASStorage) SaveFile(}func (n *NASStorage) GetFile(- _ context.Context,+ ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {session, err := n.createSession(encryptor)@@ -166,35 +167,37 @@ func (n *NASStorage) GetFile(return nil, fmt.Errorf("failed to create NAS session: %w", err)}- fs, err := session.Mount(n.Share)+ share, err := session.Mount(n.Share)if err != nil {_ = session.Logoff()+return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)}filePath := n.getFilePath(fileName)- // Check if file exists- _, err = fs.Stat(filePath)+ fileInfo, err := share.Stat(filePath)if err != nil {- _ = fs.Umount()+ _ = share.Umount()_ = session.Logoff()+return nil, fmt.Errorf("file not found: %s", fileName)}- nasFile, err := fs.Open(filePath)- if err != nil {- _ = fs.Umount()- _ = session.Logoff()- return nil, fmt.Errorf("failed to open file from NAS: %w", err)- }+ totalBytes := fileInfo.Size()- // Return a wrapped reader that cleans up resources when closed- return &nasFileReader{- file: nasFile,- fs: fs,- session: session,- }, nil+ _ = share.Umount()+ _ = session.Logoff()++ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: totalBytes,+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ return n.openFileAtOffset(attemptCtx, encryptor, filePath, offsetBytes)+ },+ }), nil}func (n *NASStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {@@ -424,11 +427,11 @@ func (n *NASStorage) getFilePath(filename string) string {return cleanPath + "/" + filename}-// nasFileReader wraps the NAS file and handles cleanup of resourcestype nasFileReader struct {- file *smb2.File- fs *smb2.Share- session *smb2.Session+ file *smb2.File+ share *smb2.Share+ session *smb2.Session+ stopCloseOnCancel func() bool}func (r *nasFileReader) Read(p []byte) (n int, err error) {@@ -436,30 +439,32 @@ func (r *nasFileReader) Read(p []byte) (n int, err error) {}func (r *nasFileReader) Close() error {- // Close resources in reverse order- var errors []error+ var closeErrors []error++ if r.stopCloseOnCancel != nil {+ r.stopCloseOnCancel()+ }if r.file != nil {if err := r.file.Close(); err != nil {- errors = append(errors, fmt.Errorf("failed to close file: %w", err))+ closeErrors = append(closeErrors, fmt.Errorf("failed to close file: %w", err))}}- if r.fs != nil {- if err := r.fs.Umount(); err != nil {- errors = append(errors, fmt.Errorf("failed to unmount share: %w", err))+ if r.share != nil {+ if err := r.share.Umount(); err != nil {+ closeErrors = append(closeErrors, fmt.Errorf("failed to unmount share: %w", err))}}if r.session != nil {if err := r.session.Logoff(); err != nil {- errors = append(errors, fmt.Errorf("failed to logoff session: %w", err))+ closeErrors = append(closeErrors, fmt.Errorf("failed to logoff session: %w", err))}}- if len(errors) > 0 {- // Return the first error, but log others if needed- return errors[0]+ if len(closeErrors) > 0 {+ return closeErrors[0]}return nil@@ -532,3 +537,55 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64,return written, nil}++// Each attempt logs in again: a resumed read follows a dropped one, and the SMB session that+// carried it is no longer usable.+func (n *NASStorage) openFileAtOffset(+ attemptCtx context.Context,+ encryptor encryption.FieldEncryptor,+ filePath string,+ offsetBytes int64,+) (io.ReadCloser, error) {+ session, err := n.createSessionWithContext(attemptCtx, encryptor)+ if err != nil {+ return nil, fmt.Errorf("failed to create NAS session: %w", err)+ }++ share, err := session.Mount(n.Share)+ if err != nil {+ _ = session.Logoff()++ return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)+ }++ nasFile, err := share.Open(filePath)+ if err != nil {+ _ = share.Umount()+ _ = session.Logoff()++ return nil, fmt.Errorf("failed to open file from NAS: %w", err)+ }++ if offsetBytes > 0 {+ if _, err := nasFile.Seek(offsetBytes, io.SeekStart); err != nil {+ _ = nasFile.Close()+ _ = share.Umount()+ _ = session.Logoff()++ return nil, fmt.Errorf("failed to resume NAS read at offset %d: %w", offsetBytes, err)+ }+ }++ // The SMB file takes no context, so a read blocked on a silent server would ignore the+ // caller's cancellation; closing the file is what unblocks it.+ stopCloseOnCancel := context.AfterFunc(attemptCtx, func() {+ _ = nasFile.Close()+ })++ return &nasFileReader{+ file: nasFile,+ share: share,+ session: session,+ stopCloseOnCancel: stopCloseOnCancel,+ }, nil+}diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.goindex fbfd79e4..2efa1e58 100644--- a/backend/internal/features/storages/models/rclone/model.go+++ b/backend/internal/features/storages/models/rclone/model.go@@ -18,6 +18,7 @@ import ("github.com/rclone/rclone/fs/operations""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -92,7 +93,7 @@ func (r *RcloneStorage) SaveFile(func (r *RcloneStorage) GetFile(ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {remoteFs, err := r.getFs(ctx, encryptor)@@ -107,15 +108,27 @@ func (r *RcloneStorage) GetFile(return nil, fmt.Errorf("failed to get object from rclone: %w", err)}- // operations.Open resumes from the last byte read when the transfer drops, so a- // multi-gigabyte restore survives transient remote failures instead of handing the- // caller a truncated stream.- reader, err := operations.Open(ctx, obj)- if err != nil {- return nil, fmt.Errorf("failed to open object from rclone: %w", err)- }+ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: obj.Size(),+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ if offsetBytes == 0 {+ return obj.Open(attemptCtx)+ }++ return obj.Open(attemptCtx, &fs.RangeOption{Start: offsetBytes, End: -1})+ },+ IsRetryableError: isRetryableRcloneError,+ }), nil+}- return reader, nil+func isRetryableRcloneError(err error) bool {+ return !errors.Is(err, fs.ErrorObjectNotFound) &&+ !errors.Is(err, fs.ErrorDirNotFound) &&+ !errors.Is(err, fs.ErrorNotAFile) &&+ !errors.Is(err, fs.ErrorPermissionDenied)}func (r *RcloneStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {diff --git a/backend/internal/features/storages/models/s3/errors.go b/backend/internal/features/storages/models/s3/errors.goindex 65a260b8..9a499131 100644--- a/backend/internal/features/storages/models/s3/errors.go+++ b/backend/internal/features/storages/models/s3/errors.go@@ -7,13 +7,4 @@ import "errors"var (ErrChunkSizeMismatch = errors.New("s3 chunk size mismatch")ErrChunkChecksumMismatch = errors.New("s3 chunk checksum mismatch")-- // ErrRangeNotHonoured means a resumed GET was answered with a different byte span than the- // Range header asked for, which some S3-compatible backends do by replying 200 with the whole- // object. Resuming from byte 0 mid-stream is what this guards against.- ErrRangeNotHonoured = errors.New("s3 range request not honoured")-- ErrReadRetryBudgetExhausted = errors.New("s3 chunk read retries exhausted")-- ErrReaderClosed = errors.New("s3 chunk reader is closed"))diff --git a/backend/internal/features/storages/models/s3/reassembling_reader.go b/backend/internal/features/storages/models/s3/reassembling_reader.goindex 93bbc9e1..88429b0b 100644--- a/backend/internal/features/storages/models/s3/reassembling_reader.go+++ b/backend/internal/features/storages/models/s3/reassembling_reader.go@@ -10,23 +10,11 @@ import ("io""log/slog""net/http"- "strconv"- "strings""time""github.com/minio/minio-go/v7"-)--const (- // Once headers have arrived the transport's ResponseHeaderTimeout no longer applies, so a backend- // that accepts the GET and then goes silent would block until the OS TCP timeout. Armed around- // each blocking body read only: the gap between a consumer's reads is its own business, and a- // restore target that pauses on a slow statement must not lose its connection over it.- s3ReadStallTimeout = 60 * time.Second- s3MaxReadAttemptsWithoutProgress = 10- s3ReadRetryBaseDelay = 1 * time.Second- s3ReadRetryMaxDelay = 30 * time.Second+ io_utils "databasus-backend/internal/util/io")type reassemblingReaderSpec struct {@@ -59,33 +47,23 @@ type reassemblingReader struct {bucket stringparts []objectSpan- hasManifestChecksum bool- stallTimeout time.Duration- maxAttemptsWithoutProgress int- retryBaseDelay time.Duration-- isClosed bool- partIndex int- partBody io.ReadCloser- partHasher hash.Hash- attemptCancel context.CancelFunc- stallTimer *time.Timer- partReadBytes int64- attemptReadBytes int64- attemptsSinceProgress int+ hasManifestChecksum bool+ stallTimeout time.Duration+ retryPolicy *io_utils.ReadRetryPolicy++ isClosed bool+ partIndex int+ partBody io.ReadCloser+ partHasher hash.Hash+ attemptCancel context.CancelFunc+ stallTimer *time.Timer+ partReadBytes int64+ attemptReadBytes int64}func newReassemblingReader(spec reassemblingReaderSpec) *reassemblingReader {if spec.stallTimeout == 0 {- spec.stallTimeout = s3ReadStallTimeout- }-- if spec.maxAttemptsWithoutProgress == 0 {- spec.maxAttemptsWithoutProgress = s3MaxReadAttemptsWithoutProgress- }-- if spec.retryBaseDelay == 0 {- spec.retryBaseDelay = s3ReadRetryBaseDelay+ spec.stallTimeout = io_utils.DefaultReadStallTimeout}// The retry path is the only thing that logs, so a nil logger would panic exactly during an@@ -95,23 +73,25 @@ func newReassemblingReader(spec reassemblingReaderSpec) *reassemblingReader {}return &reassemblingReader{- streamCtx: spec.streamCtx,- logger: spec.logger,- coreClient: spec.coreClient,- bucket: spec.bucket,- parts: spec.parts,- hasManifestChecksum: spec.hasManifestChecksum,- stallTimeout: spec.stallTimeout,- maxAttemptsWithoutProgress: spec.maxAttemptsWithoutProgress,- retryBaseDelay: spec.retryBaseDelay,- partHasher: sha256.New(),+ streamCtx: spec.streamCtx,+ logger: spec.logger,+ coreClient: spec.coreClient,+ bucket: spec.bucket,+ parts: spec.parts,+ hasManifestChecksum: spec.hasManifestChecksum,+ stallTimeout: spec.stallTimeout,+ retryPolicy: io_utils.NewReadRetryPolicy(io_utils.ReadRetryPolicySpec{+ MaxAttemptsWithoutProgress: spec.maxAttemptsWithoutProgress,+ BaseDelay: spec.retryBaseDelay,+ }),+ partHasher: sha256.New(),}}func (r *reassemblingReader) Read(p []byte) (int, error) {for {if r.isClosed {- return 0, ErrReaderClosed+ return 0, io_utils.ErrReaderClosed}if err := r.streamCtx.Err(); err != nil {@@ -205,7 +185,13 @@ func (r *reassemblingReader) openCurrentPart() error {if err := options.SetRange(r.partReadBytes, part.Size-1); err != nil {cancelAttempt()- return fmt.Errorf("%w: chunk %s at offset %d: %w", ErrRangeNotHonoured, part.Key, r.partReadBytes, err)+ return fmt.Errorf(+ "%w: chunk %s at offset %d: %w",+ io_utils.ErrRangeNotHonoured,+ part.Key,+ r.partReadBytes,+ err,+ )}}@@ -243,11 +229,11 @@ func (r *reassemblingReader) verifyResponseSpan(expectedBytes := part.Size - r.partReadBytesif r.partReadBytes > 0 {- if rangeStart, isParsed := rangeStartOfContentRange(responseHeader.Get("Content-Range")); isParsed {+ if rangeStart, isParsed := io_utils.GetRangeStartOfContentRange(responseHeader.Get("Content-Range")); isParsed {if rangeStart != r.partReadBytes {return fmt.Errorf("%w: chunk %s resumed at offset %d but the response starts at %d",- ErrRangeNotHonoured, part.Key, r.partReadBytes, rangeStart,+ io_utils.ErrRangeNotHonoured, part.Key, r.partReadBytes, rangeStart,)}@@ -257,7 +243,7 @@ func (r *reassemblingReader) verifyResponseSpan(if objectInfo.Size != expectedBytes {return fmt.Errorf("%w: chunk %s resumed at offset %d answered %d bytes, expected %d",- ErrRangeNotHonoured, part.Key, r.partReadBytes, objectInfo.Size, expectedBytes,+ io_utils.ErrRangeNotHonoured, part.Key, r.partReadBytes, objectInfo.Size, expectedBytes,)}@@ -277,25 +263,6 @@ func (r *reassemblingReader) verifyResponseSpan(return nil}-func rangeStartOfContentRange(contentRange string) (int64, bool) {- span, isBytesUnit := strings.CutPrefix(contentRange, "bytes ")- if !isBytesUnit {- return 0, false- }-- start, _, isSplit := strings.Cut(span, "-")- if !isSplit {- return 0, false- }-- parsed, err := strconv.ParseInt(strings.TrimSpace(start), 10, 64)- if err != nil {- return 0, false- }-- return parsed, true-}-func (r *reassemblingReader) finishCurrentPart() error {part := r.parts[r.partIndex]@@ -316,14 +283,12 @@ func (r *reassemblingReader) finishCurrentPart() error {r.partIndex++r.partReadBytes = 0- r.attemptsSinceProgress = 0+ r.retryPolicy.ResetAttemptsAfterCompletedRead()r.partHasher = sha256.New()return nil}-// The budget counts attempts since the last byte delivered, not attempts per part: a multi-hour-// restore with sparse drops must not exhaust it.func (r *reassemblingReader) resumeAfterFailure(cause error) error {hasDeliveredBytes := r.attemptReadBytes > 0partKey := r.parts[r.partIndex].Key@@ -338,23 +303,21 @@ func (r *reassemblingReader) resumeAfterFailure(cause error) error {return cause}- if hasDeliveredBytes {- r.attemptsSinceProgress = 1- } else {- r.attemptsSinceProgress++- }+ r.retryPolicy.RegisterFailedAttempt(hasDeliveredBytes)- if r.attemptsSinceProgress > r.maxAttemptsWithoutProgress {+ if r.retryPolicy.IsExhausted() {return fmt.Errorf("%w: chunk %s stuck at offset %d: %w",- ErrReadRetryBudgetExhausted, partKey, r.partReadBytes, cause,+ io_utils.ErrReadRetryBudgetExhausted, partKey, r.partReadBytes, cause,)}r.logger.Warn(fmt.Sprintf("s3 chunk read failed, resuming from offset %d (attempt %d of %d)",- r.partReadBytes, r.attemptsSinceProgress, r.maxAttemptsWithoutProgress,+ r.partReadBytes,+ r.retryPolicy.GetAttemptsSinceProgress(),+ r.retryPolicy.GetMaxAttemptsWithoutProgress(),),"part_key", partKey,"error", cause,@@ -364,24 +327,7 @@ func (r *reassemblingReader) resumeAfterFailure(cause error) error {return nil}- return r.waitBeforeRetry()-}--func (r *reassemblingReader) waitBeforeRetry() error {- delay := s3ReadRetryMaxDelay- if shift := r.attemptsSinceProgress - 1; shift < 32 {- delay = min(r.retryBaseDelay<<shift, s3ReadRetryMaxDelay)- }-- retryTimer := time.NewTimer(delay)- defer retryTimer.Stop()-- select {- case <-r.streamCtx.Done():- return r.streamCtx.Err()- case <-retryTimer.C:- return nil- }+ return r.retryPolicy.WaitBeforeRetry(r.streamCtx)}func (r *reassemblingReader) abandonAttempt() {@@ -404,7 +350,7 @@ func (r *reassemblingReader) abandonAttempt() {}func isRetryableReadError(err error) bool {- if errors.Is(err, ErrRangeNotHonoured) ||+ if errors.Is(err, io_utils.ErrRangeNotHonoured) ||errors.Is(err, ErrChunkSizeMismatch) ||errors.Is(err, ErrChunkChecksumMismatch) {return falsediff --git a/backend/internal/features/storages/models/s3/reassembling_reader_test.go b/backend/internal/features/storages/models/s3/reassembling_reader_test.goindex 58523b51..268d73d9 100644--- a/backend/internal/features/storages/models/s3/reassembling_reader_test.go+++ b/backend/internal/features/storages/models/s3/reassembling_reader_test.go@@ -16,6 +16,8 @@ import ("github.com/minio/minio-go/v7/pkg/credentials""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"++ io_utils "databasus-backend/internal/util/io")func Test_ReadChunkedStream_ConnectionDropsMidPart_ResumesFromOffsetAndReturnsFullStream(t *testing.T) {@@ -60,7 +62,7 @@ func Test_ReadChunkedStream_ServerIgnoresRange_ReturnsErrorWithoutCorruptingStredeliveredBytes, err := io.ReadAll(reader)- require.ErrorIs(t, err, ErrRangeNotHonoured)+ require.ErrorIs(t, err, io_utils.ErrRangeNotHonoured)assert.Equal(t,partBytes[:3000],@@ -124,7 +126,7 @@ func Test_ReadChunkedStream_ResumeAnsweredFromTheStartOfTheObject_ReturnsError(tdeliveredBytes, err := io.ReadAll(reader)- require.ErrorIs(t, err, ErrRangeNotHonoured)+ require.ErrorIs(t, err, io_utils.ErrRangeNotHonoured)assert.Equal(t, partBytes[:3000], deliveredBytes)}@@ -283,7 +285,7 @@ func Test_ReadChunkedStream_RetryBudgetExhausted_ReturnsError(t *testing.T) {_, err := io.ReadAll(reader)- require.ErrorIs(t, err, ErrReadRetryBudgetExhausted)+ require.ErrorIs(t, err, io_utils.ErrReadRetryBudgetExhausted)assert.Equal(t, 3, transport.attempts, "the first attempt plus the two retries the budget allows")}@@ -320,7 +322,7 @@ func Test_ReadChunkedStream_ReopenKeepsFailingAfterAProductiveAttempt_ExhaustsBurequire.ErrorIs(t,err,- ErrReadRetryBudgetExhausted,+ io_utils.ErrReadRetryBudgetExhausted,"bytes delivered by an earlier attempt must not keep crediting later re-open failures with progress",)}@@ -392,7 +394,7 @@ func Test_ReadChunkedStream_ReadAfterClose_ReturnsClosedError(t *testing.T) {require.NoError(t, reader.Close())_, err = reader.Read(buffer)- assert.ErrorIs(t, err, ErrReaderClosed, "a closed reader must not silently re-open the chunk")+ assert.ErrorIs(t, err, io_utils.ErrReaderClosed, "a closed reader must not silently re-open the chunk")}func Test_ReadChunkedStream_SpecWithoutALogger_RetriesWithoutPanicking(t *testing.T) {diff --git a/backend/internal/features/storages/models/sftp/model.go b/backend/internal/features/storages/models/sftp/model.goindex 845a611c..bb351334 100644--- a/backend/internal/features/storages/models/sftp/model.go+++ b/backend/internal/features/storages/models/sftp/model.go@@ -15,6 +15,7 @@ import ("golang.org/x/crypto/ssh""databasus-backend/internal/util/encryption"+ io_utils "databasus-backend/internal/util/io")const (@@ -131,9 +132,9 @@ func (s *SFTPStorage) SaveFile(}func (s *SFTPStorage) GetFile(- _ context.Context,+ ctx context.Context,encryptor encryption.FieldEncryptor,- _ *slog.Logger,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)@@ -143,18 +144,28 @@ func (s *SFTPStorage) GetFile(filePath := s.getFilePath(fileName)- remoteFile, err := client.Open(filePath)+ fileInfo, err := client.Stat(filePath)if err != nil {_ = client.Close()_ = sshConn.Close()- return nil, fmt.Errorf("failed to open file from SFTP: %w", err)++ return nil, fmt.Errorf("failed to stat file on SFTP: %w", err)}- return &sftpFileReader{- file: remoteFile,- client: client,- sshConn: sshConn,- }, nil+ totalBytes := fileInfo.Size()++ _ = client.Close()+ _ = sshConn.Close()++ return io_utils.NewResumingReader(io_utils.ResumingReaderSpec{+ StreamCtx: ctx,+ Logger: logger,+ FileName: fileName,+ TotalBytes: totalBytes,+ OpenAtOffset: func(attemptCtx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ return s.openFileAtOffset(attemptCtx, encryptor, filePath, offsetBytes)+ },+ }), nil}func (s *SFTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {@@ -380,9 +391,10 @@ func (s *SFTPStorage) getFilePath(filename string) string {}type sftpFileReader struct {- file *sftp.File- client *sftp.Client- sshConn *ssh.Client+ file *sftp.File+ client *sftp.Client+ sshConn *ssh.Client+ stopCloseOnCancel func() bool}func (r *sftpFileReader) Read(p []byte) (n int, err error) {@@ -392,6 +404,10 @@ func (r *sftpFileReader) Read(p []byte) (n int, err error) {func (r *sftpFileReader) Close() error {var errs []error+ if r.stopCloseOnCancel != nil {+ r.stopCloseOnCancel()+ }+if r.file != nil {if err := r.file.Close(); err != nil {errs = append(errs, fmt.Errorf("failed to close file: %w", err))@@ -430,3 +446,48 @@ func (r *contextReader) Read(p []byte) (n int, err error) {return r.reader.Read(p)}}++// Each attempt dials a fresh SSH connection: a resumed read follows a dropped one, and the client+// that carried it is no longer usable.+func (s *SFTPStorage) openFileAtOffset(+ attemptCtx context.Context,+ encryptor encryption.FieldEncryptor,+ filePath string,+ offsetBytes int64,+) (io.ReadCloser, error) {+ client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)+ if err != nil {+ return nil, fmt.Errorf("failed to connect to SFTP: %w", err)+ }++ remoteFile, err := client.Open(filePath)+ if err != nil {+ _ = client.Close()+ _ = sshConn.Close()++ return nil, fmt.Errorf("failed to open file from SFTP: %w", err)+ }++ if offsetBytes > 0 {+ if _, err := remoteFile.Seek(offsetBytes, io.SeekStart); err != nil {+ _ = remoteFile.Close()+ _ = client.Close()+ _ = sshConn.Close()++ return nil, fmt.Errorf("failed to resume SFTP read at offset %d: %w", offsetBytes, err)+ }+ }++ // The SFTP client takes no context, so a read blocked on a silent server would ignore the+ // caller's cancellation; closing the connection is what unblocks it.+ stopCloseOnCancel := context.AfterFunc(attemptCtx, func() {+ _ = sshConn.Close()+ })++ return &sftpFileReader{+ file: remoteFile,+ client: client,+ sshConn: sshConn,+ stopCloseOnCancel: stopCloseOnCancel,+ }, nil+}diff --git a/backend/internal/util/io/content_range.go b/backend/internal/util/io/content_range.gonew file mode 100644index 00000000..4fd35f20--- /dev/null+++ b/backend/internal/util/io/content_range.go@@ -0,0 +1,48 @@+package io_utils++import (+ "fmt"+ "strconv"+ "strings"+)++func GetRangeStartOfContentRange(contentRange string) (int64, bool) {+ span, isBytesUnit := strings.CutPrefix(contentRange, "bytes ")+ if !isBytesUnit {+ return 0, false+ }++ start, _, isSplit := strings.Cut(span, "-")+ if !isSplit {+ return 0, false+ }++ parsed, err := strconv.ParseInt(strings.TrimSpace(start), 10, 64)+ if err != nil {+ return 0, false+ }++ return parsed, true+}++// A resumed read must not begin before the offset already handed out: replaying bytes into a+// stateful decryptor corrupts the stream. Content-Range is the positive proof the backend applied+// the range, so a reply that cannot be checked is refused rather than trusted.+func VerifyContentRangeStart(contentRange string, expectedOffsetBytes int64) error {+ rangeStart, isParsed := GetRangeStartOfContentRange(contentRange)+ if !isParsed {+ return fmt.Errorf(+ "%w: resumed at offset %d but the response carries no usable Content-Range",+ ErrRangeNotHonoured, expectedOffsetBytes,+ )+ }++ if rangeStart != expectedOffsetBytes {+ return fmt.Errorf(+ "%w: resumed at offset %d but the response starts at %d",+ ErrRangeNotHonoured, expectedOffsetBytes, rangeStart,+ )+ }++ return nil+}diff --git a/backend/internal/util/io/errors.go b/backend/internal/util/io/errors.gonew file mode 100644index 00000000..9f027fa1--- /dev/null+++ b/backend/internal/util/io/errors.go@@ -0,0 +1,18 @@+package io_utils++import "errors"++var (+ ErrReadRetryBudgetExhausted = errors.New("read retries exhausted")++ ErrReaderClosed = errors.New("reader is closed")++ // ErrRangeNotHonoured means a resumed read was answered with a different byte span than it+ // asked for, which some backends do by replying with the whole object. Replaying bytes that+ // were already handed out corrupts the stream for a stateful decryptor, so it is terminal.+ ErrRangeNotHonoured = errors.New("range request not honoured")++ // ErrStreamTruncated means the source ended before the size it declared. Retrying is still+ // worthwhile: the usual cause is a severed connection, not a shorter file.+ ErrStreamTruncated = errors.New("stream ended before the declared size")+)diff --git a/backend/internal/util/io/read_retry_policy.go b/backend/internal/util/io/read_retry_policy.gonew file mode 100644index 00000000..00ecb57d--- /dev/null+++ b/backend/internal/util/io/read_retry_policy.go@@ -0,0 +1,84 @@+package io_utils++import (+ "context"+ "time"+)++const (+ defaultMaxReadAttemptsWithoutProgress = 10+ defaultReadRetryBaseDelay = 1 * time.Second+ readRetryMaxDelay = 30 * time.Second+)++type ReadRetryPolicySpec struct {+ MaxAttemptsWithoutProgress int+ BaseDelay time.Duration+}++type ReadRetryPolicy struct {+ maxAttemptsWithoutProgress int+ baseDelay time.Duration+ attemptsSinceProgress int+}++func NewReadRetryPolicy(spec ReadRetryPolicySpec) *ReadRetryPolicy {+ if spec.MaxAttemptsWithoutProgress == 0 {+ spec.MaxAttemptsWithoutProgress = defaultMaxReadAttemptsWithoutProgress+ }++ if spec.BaseDelay == 0 {+ spec.BaseDelay = defaultReadRetryBaseDelay+ }++ return &ReadRetryPolicy{+ maxAttemptsWithoutProgress: spec.MaxAttemptsWithoutProgress,+ baseDelay: spec.BaseDelay,+ }+}++// The budget counts attempts since the last byte delivered, not attempts per transfer: a restore+// runs at the target's ingestion rate, so a multi-hour stream collects far more sparse drops than+// any per-transfer budget can cover.+func (p *ReadRetryPolicy) RegisterFailedAttempt(hasDeliveredBytes bool) {+ if hasDeliveredBytes {+ p.attemptsSinceProgress = 1++ return+ }++ p.attemptsSinceProgress+++}++func (p *ReadRetryPolicy) IsExhausted() bool {+ return p.attemptsSinceProgress > p.maxAttemptsWithoutProgress+}++func (p *ReadRetryPolicy) GetAttemptsSinceProgress() int {+ return p.attemptsSinceProgress+}++func (p *ReadRetryPolicy) GetMaxAttemptsWithoutProgress() int {+ return p.maxAttemptsWithoutProgress+}++func (p *ReadRetryPolicy) ResetAttemptsAfterCompletedRead() {+ p.attemptsSinceProgress = 0+}++func (p *ReadRetryPolicy) WaitBeforeRetry(ctx context.Context) error {+ delay := readRetryMaxDelay+ if shift := p.attemptsSinceProgress - 1; shift < 32 {+ delay = min(p.baseDelay<<shift, readRetryMaxDelay)+ }++ retryTimer := time.NewTimer(delay)+ defer retryTimer.Stop()++ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-retryTimer.C:+ return nil+ }+}diff --git a/backend/internal/util/io/read_retry_policy_test.go b/backend/internal/util/io/read_retry_policy_test.gonew file mode 100644index 00000000..7f5b3f8a--- /dev/null+++ b/backend/internal/util/io/read_retry_policy_test.go@@ -0,0 +1,83 @@+package io_utils++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func Test_ReadRetryPolicy_EveryFailedAttemptDeliveredBytes_NeverExhausts(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 3})++ for range 100 {+ policy.RegisterFailedAttempt(true)+ require.False(t, policy.IsExhausted())+ }++ assert.Equal(t, 1, policy.GetAttemptsSinceProgress())+}++func Test_ReadRetryPolicy_FailedAttemptsWithoutProgress_ExhaustsAfterTheBudget(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 3})++ for range 3 {+ policy.RegisterFailedAttempt(false)+ require.False(t, policy.IsExhausted())+ }++ policy.RegisterFailedAttempt(false)++ assert.True(t, policy.IsExhausted())+}++func Test_ReadRetryPolicy_ProgressAfterFailures_ResetsTheSpentBudget(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 3})++ policy.RegisterFailedAttempt(false)+ policy.RegisterFailedAttempt(false)+ policy.RegisterFailedAttempt(true)++ assert.Equal(t, 1, policy.GetAttemptsSinceProgress())+}++func Test_ReadRetryPolicy_CompletedRead_ResetsAttempts(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 3})++ policy.RegisterFailedAttempt(false)+ policy.ResetAttemptsAfterCompletedRead()++ assert.Equal(t, 0, policy.GetAttemptsSinceProgress())+}++func Test_ReadRetryPolicy_WaitBeforeRetryWithCancelledContext_ReturnsWithoutSleeping(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{BaseDelay: time.Hour})+ policy.RegisterFailedAttempt(false)++ cancelledCtx, cancel := context.WithCancel(t.Context())+ cancel()++ waitStart := time.Now()+ err := policy.WaitBeforeRetry(cancelledCtx)++ require.ErrorIs(t, err, context.Canceled)+ assert.Less(t, time.Since(waitStart), time.Second)+}++func Test_ReadRetryPolicy_ConsecutiveFailures_BackoffGrowsExponentially(t *testing.T) {+ policy := NewReadRetryPolicy(ReadRetryPolicySpec{+ MaxAttemptsWithoutProgress: 10,+ BaseDelay: 20 * time.Millisecond,+ })++ for range 3 {+ policy.RegisterFailedAttempt(false)+ }++ waitStart := time.Now()+ require.NoError(t, policy.WaitBeforeRetry(t.Context()))++ assert.GreaterOrEqual(t, time.Since(waitStart), 80*time.Millisecond)+}diff --git a/backend/internal/util/io/resuming_reader.go b/backend/internal/util/io/resuming_reader.gonew file mode 100644index 00000000..f2e4f778--- /dev/null+++ b/backend/internal/util/io/resuming_reader.go@@ -0,0 +1,263 @@+package io_utils++import (+ "context"+ "errors"+ "fmt"+ "io"+ "log/slog"+ "time"+)++// Once headers have arrived the transport's response timeout no longer applies, so a backend that+// accepts the read and then goes silent would block until the OS TCP timeout. It is armed around+// each blocking read only: the gap between a consumer's reads is its own business, and a restore+// target that pauses on a slow statement must not lose its connection over it.+const DefaultReadStallTimeout = 60 * time.Second++// UnknownTotalBytes skips the completeness check rather than guessing a size: a wrong expected+// size would reject an intact file.+const UnknownTotalBytes = -1++// The offset is where the stream must continue from; a backend that cannot honour it must fail+// rather than answer from the start, because replaying bytes corrupts a stateful decryptor.+type OpenFileAtOffsetFunc func(ctx context.Context, offsetBytes int64) (io.ReadCloser, error)++type ResumingReaderSpec struct {+ StreamCtx context.Context+ Logger *slog.Logger+ FileName string+ OpenAtOffset OpenFileAtOffsetFunc++ TotalBytes int64++ StallTimeout time.Duration+ RetryPolicySpec ReadRetryPolicySpec++ // Left nil, every failure is retried. Backends that can tell a missing file from a severed+ // connection should say so here, otherwise a permanent failure costs the whole budget.+ IsRetryableError func(error) bool+}++// A dropped or stalled connection is retried in place, resuming from the last byte handed out.+// It has to recover here rather than downstream: backups are an encrypted, compressed stream with+// no end-of-stream marker, so a caller that sees a short read cannot resynchronise and would feed+// the restore target a silently truncated dump.+type ResumingReader struct {+ streamCtx context.Context+ logger *slog.Logger+ fileName string+ openAtOffset OpenFileAtOffsetFunc+ totalBytes int64+ stallTimeout time.Duration+ retryPolicy *ReadRetryPolicy+ isRetryableError func(error) bool++ isClosed bool+ isFinished bool+ body io.ReadCloser+ attemptCancel context.CancelFunc+ stallTimer *time.Timer+ deliveredBytes int64+ attemptReadBytes int64+}++func NewResumingReader(spec ResumingReaderSpec) *ResumingReader {+ if spec.StallTimeout == 0 {+ spec.StallTimeout = DefaultReadStallTimeout+ }++ // The retry path is the only thing that logs, so a nil logger would panic exactly during an+ // incident and nowhere else.+ if spec.Logger == nil {+ spec.Logger = slog.New(slog.DiscardHandler)+ }++ if spec.IsRetryableError == nil {+ spec.IsRetryableError = func(error) bool { return true }+ }++ return &ResumingReader{+ streamCtx: spec.StreamCtx,+ logger: spec.Logger,+ fileName: spec.FileName,+ openAtOffset: spec.OpenAtOffset,+ totalBytes: spec.TotalBytes,+ stallTimeout: spec.StallTimeout,+ retryPolicy: NewReadRetryPolicy(spec.RetryPolicySpec),+ isRetryableError: spec.IsRetryableError,+ }+}++func (r *ResumingReader) Read(p []byte) (int, error) {+ for {+ if r.isClosed {+ return 0, ErrReaderClosed+ }++ if err := r.streamCtx.Err(); err != nil {+ return 0, err+ }++ if r.isFinished || r.hasReadDeclaredSize() {+ return 0, io.EOF+ }++ if r.body == nil {+ if err := r.openAtCurrentOffset(); err != nil {+ if retryErr := r.resumeAfterFailure(err); retryErr != nil {+ return 0, retryErr+ }++ continue+ }+ }++ r.stallTimer.Reset(r.stallTimeout)+ n, err := r.body.Read(p)+ r.stallTimer.Stop()++ if n > 0 {+ r.deliveredBytes += int64(n)+ r.attemptReadBytes += int64(n)+ }++ if r.totalBytes >= 0 && r.deliveredBytes > r.totalBytes {+ r.abandonAttempt()++ return n, fmt.Errorf(+ "%w: %s answered %d bytes, expected %d",+ ErrRangeNotHonoured, r.fileName, r.deliveredBytes, r.totalBytes,+ )+ }++ // A body that ends before the declared size is a severed transfer, not the end of the+ // file: net/http reports that as ErrUnexpectedEOF, but a chunked-encoded response can+ // deliver it as a plain EOF, so trust the byte count rather than the error.+ if errors.Is(err, io.EOF) {+ if r.totalBytes < 0 || r.hasReadDeclaredSize() {+ r.finish()++ if n > 0 {+ return n, nil+ }++ return 0, io.EOF+ }++ err = fmt.Errorf(+ "%w: %s ended at %d of %d bytes",+ ErrStreamTruncated, r.fileName, r.deliveredBytes, r.totalBytes,+ )+ }++ if err != nil {+ if retryErr := r.resumeAfterFailure(err); retryErr != nil {+ return n, retryErr+ }++ if n > 0 {+ return n, nil+ }++ continue+ }++ return n, nil+ }+}++func (r *ResumingReader) Close() error {+ r.isClosed = true++ r.abandonAttempt()++ return nil+}++func (r *ResumingReader) hasReadDeclaredSize() bool {+ return r.totalBytes >= 0 && r.deliveredBytes >= r.totalBytes+}++func (r *ResumingReader) finish() {+ r.isFinished = true++ r.abandonAttempt()+}++func (r *ResumingReader) openAtCurrentOffset() error {+ attemptCtx, cancelAttempt := context.WithCancel(r.streamCtx)++ body, err := r.openAtOffset(attemptCtx, r.deliveredBytes)+ if err != nil {+ cancelAttempt()++ return fmt.Errorf("open %s at offset %d: %w", r.fileName, r.deliveredBytes, err)+ }++ r.body = body+ r.attemptCancel = cancelAttempt+ r.stallTimer = time.AfterFunc(r.stallTimeout, cancelAttempt)+ r.stallTimer.Stop()++ return nil+}++func (r *ResumingReader) resumeAfterFailure(cause error) error {+ hasDeliveredBytes := r.attemptReadBytes > 0++ r.abandonAttempt()++ if err := r.streamCtx.Err(); err != nil {+ return errors.Join(cause, err)+ }++ if errors.Is(cause, ErrRangeNotHonoured) || !r.isRetryableError(cause) {+ return cause+ }++ r.retryPolicy.RegisterFailedAttempt(hasDeliveredBytes)++ if r.retryPolicy.IsExhausted() {+ return fmt.Errorf(+ "%w: %s stuck at offset %d: %w",+ ErrReadRetryBudgetExhausted, r.fileName, r.deliveredBytes, cause,+ )+ }++ r.logger.Warn(+ fmt.Sprintf(+ "storage read failed, resuming from offset %d (attempt %d of %d)",+ r.deliveredBytes,+ r.retryPolicy.GetAttemptsSinceProgress(),+ r.retryPolicy.GetMaxAttemptsWithoutProgress(),+ ),+ "file_name", r.fileName,+ "error", cause,+ )++ if hasDeliveredBytes {+ return nil+ }++ return r.retryPolicy.WaitBeforeRetry(r.streamCtx)+}++func (r *ResumingReader) abandonAttempt() {+ r.attemptReadBytes = 0++ if r.stallTimer != nil {+ r.stallTimer.Stop()+ r.stallTimer = nil+ }++ if r.body != nil {+ _ = r.body.Close()+ r.body = nil+ }++ if r.attemptCancel != nil {+ r.attemptCancel()+ r.attemptCancel = nil+ }+}diff --git a/backend/internal/util/io/resuming_reader_test.go b/backend/internal/util/io/resuming_reader_test.gonew file mode 100644index 00000000..6bfb4f79--- /dev/null+++ b/backend/internal/util/io/resuming_reader_test.go@@ -0,0 +1,336 @@+package io_utils++import (+ "context"+ "errors"+ "io"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func Test_ResumingReader_ConnectionDropsMidStream_ResumesFromOffsetAndReturnsWholeFile(t *testing.T) {+ fileBytes := generateBytes(8192)+ source := &stubSource{content: fileBytes, breakAfterBytes: severOnAttempts(1, 3000)}++ streamedBytes, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ }))++ require.NoError(t, err)+ assert.Equal(t, fileBytes, streamedBytes)+ assert.Equal(+ t,+ []int64{0, 3000},+ source.observedOffsets,+ "the retry must resume from the last byte handed out",+ )+}++// Issue #714: a multi-hour rclone restore died once the drops outnumbered the budget, even though+// every one of them resumed and made progress.+func Test_ResumingReader_DropsFarExceedTheBudgetButEachDeliversBytes_CompletesStream(t *testing.T) {+ fileBytes := generateBytes(30000)+ source := &stubSource{+ content: fileBytes,+ breakAfterBytes: func(int) int { return 1000 },+ }++ streamedBytes, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ RetryPolicySpec: ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 10},+ }))++ require.NoError(t, err)+ assert.Equal(t, fileBytes, streamedBytes)+ assert.Len(+ t,+ source.observedOffsets,+ 30,+ "a budget of 10 must not be spent by 30 drops that each made progress",+ )+}++func Test_ResumingReader_RepeatedFailuresWithoutProgress_ReturnsBudgetExhausted(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{+ content: fileBytes,+ openError: func(attempt int) error { return errors.New("connection reset") },+ }++ _, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ RetryPolicySpec: ReadRetryPolicySpec{MaxAttemptsWithoutProgress: 3, BaseDelay: time.Millisecond},+ }))++ require.ErrorIs(t, err, ErrReadRetryBudgetExhausted)+ assert.Len(t, source.observedOffsets, 4)+}++func Test_ResumingReader_BodyGoesSilent_CancelsAttemptAndResumes(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{+ content: fileBytes,+ isStalling: func(attempt int) bool { return attempt == 1 },+ }++ streamedBytes, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ StallTimeout: 50 * time.Millisecond,+ RetryPolicySpec: ReadRetryPolicySpec{BaseDelay: time.Millisecond},+ }))++ require.NoError(t, err)+ assert.Equal(t, fileBytes, streamedBytes)+ assert.Equal(t, []int64{0, 0}, source.observedOffsets)+}++func Test_ResumingReader_ConsumerPausesLongerThanStallTimeout_KeepsTheSameConnection(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{content: fileBytes}++ reader := newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ StallTimeout: 50 * time.Millisecond,+ })++ firstChunk := make([]byte, 1024)+ readBytes, err := reader.Read(firstChunk)+ require.NoError(t, err)+ require.Equal(t, 1024, readBytes)++ time.Sleep(150 * time.Millisecond)++ remainingBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, fileBytes, append(firstChunk, remainingBytes...))+ assert.Len(t, source.observedOffsets, 1, "the pause is the consumer's, not the connection's")+}++func Test_ResumingReader_BodyEndsBeforeTheDeclaredSize_ResumesInsteadOfReturningShortStream(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{content: fileBytes, truncateOnAttempts: severOnAttempts(1, 1500)}++ streamedBytes, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ RetryPolicySpec: ReadRetryPolicySpec{BaseDelay: time.Millisecond},+ }))++ require.NoError(t, err)+ assert.Equal(t, fileBytes, streamedBytes)+ assert.Equal(t, []int64{0, 1500}, source.observedOffsets)+}++func Test_ResumingReader_ResumeAnsweredFromTheStartOfTheFile_ReturnsRangeError(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{+ content: fileBytes,+ breakAfterBytes: severOnAttempts(1, 1500),+ isRangeIgnored: true,+ }++ _, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: int64(len(fileBytes)),+ }))++ require.ErrorIs(t, err, ErrRangeNotHonoured)+}++func Test_ResumingReader_SizeIsUnknown_StreamsUntilEOFWithoutCompletenessCheck(t *testing.T) {+ fileBytes := generateBytes(4096)+ source := &stubSource{content: fileBytes}++ streamedBytes, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: -1,+ }))++ require.NoError(t, err)+ assert.Equal(t, fileBytes, streamedBytes)+ assert.Len(t, source.observedOffsets, 1)+}++func Test_ResumingReader_ErrorIsNotRetryable_FailsWithoutSpendingTheBudget(t *testing.T) {+ errFileMissing := errors.New("file does not exist")+ source := &stubSource{+ content: generateBytes(1024),+ openError: func(int) error { return errFileMissing },+ }++ _, err := io.ReadAll(newTestResumingReader(t, source, ResumingReaderSpec{+ TotalBytes: 1024,+ IsRetryableError: func(err error) bool { return !errors.Is(err, errFileMissing) },+ }))++ require.ErrorIs(t, err, errFileMissing)+ assert.Len(t, source.observedOffsets, 1)+}++func Test_ResumingReader_StreamContextCancelled_ReturnsPromptlyWithoutRetrying(t *testing.T) {+ streamCtx, cancelStream := context.WithCancel(t.Context())+ source := &stubSource{+ content: generateBytes(4096),+ openError: func(int) error { return errors.New("connection reset") },+ }++ reader := NewResumingReader(ResumingReaderSpec{+ StreamCtx: streamCtx,+ FileName: "backup.dump",+ OpenAtOffset: source.OpenAtOffset,+ TotalBytes: 4096,+ RetryPolicySpec: ReadRetryPolicySpec{BaseDelay: time.Hour},+ })+ t.Cleanup(func() { _ = reader.Close() })++ go func() {+ time.Sleep(50 * time.Millisecond)+ cancelStream()+ }()++ readStart := time.Now()+ _, err := io.ReadAll(reader)++ require.ErrorIs(t, err, context.Canceled)+ assert.Less(t, time.Since(readStart), time.Second, "cancellation must not wait out the backoff")+}++func Test_ResumingReader_ReadAfterClose_ReturnsClosedError(t *testing.T) {+ source := &stubSource{content: generateBytes(1024)}+ reader := newTestResumingReader(t, source, ResumingReaderSpec{TotalBytes: 1024})++ require.NoError(t, reader.Close())++ _, err := reader.Read(make([]byte, 16))++ require.ErrorIs(t, err, ErrReaderClosed)+}++func newTestResumingReader(t *testing.T, source *stubSource, spec ResumingReaderSpec) *ResumingReader {+ t.Helper()++ spec.StreamCtx = t.Context()+ spec.FileName = "backup.dump"+ spec.OpenAtOffset = source.OpenAtOffset++ reader := NewResumingReader(spec)+ t.Cleanup(func() { _ = reader.Close() })++ return reader+}++type stubSource struct {+ content []byte+ observedOffsets []int64+ openError func(attempt int) error+ breakAfterBytes func(attempt int) int+ truncateOnAttempts func(attempt int) int+ isStalling func(attempt int) bool+ isRangeIgnored bool+}++func (s *stubSource) OpenAtOffset(ctx context.Context, offsetBytes int64) (io.ReadCloser, error) {+ attempt := len(s.observedOffsets) + 1+ s.observedOffsets = append(s.observedOffsets, offsetBytes)++ if s.openError != nil {+ if err := s.openError(attempt); err != nil {+ return nil, err+ }+ }++ servedFrom := offsetBytes+ if s.isRangeIgnored {+ servedFrom = 0+ }++ body := &stubBody{+ attemptCtx: ctx,+ remaining: s.content[servedFrom:],+ breakAfter: -1,+ truncateAt: -1,+ }++ if s.breakAfterBytes != nil {+ body.breakAfter = s.breakAfterBytes(attempt)+ }++ if s.truncateOnAttempts != nil {+ body.truncateAt = s.truncateOnAttempts(attempt)+ }++ if s.isStalling != nil {+ body.isStalling = s.isStalling(attempt)+ }++ return body, nil+}++func severOnAttempts(throughAttempt, afterBytes int) func(int) int {+ return func(attempt int) int {+ if attempt > throughAttempt {+ return -1+ }++ return afterBytes+ }+}++type stubBody struct {+ attemptCtx context.Context+ remaining []byte+ deliveredNow int+ breakAfter int+ truncateAt int+ isStalling bool+}++func (b *stubBody) Read(p []byte) (int, error) {+ if b.isStalling {+ <-b.attemptCtx.Done()++ return 0, b.attemptCtx.Err()+ }++ if b.breakAfter >= 0 && b.deliveredNow >= b.breakAfter {+ return 0, io.ErrUnexpectedEOF+ }++ if b.truncateAt >= 0 && b.deliveredNow >= b.truncateAt {+ return 0, io.EOF+ }++ if len(b.remaining) == 0 {+ return 0, io.EOF+ }++ readable := min(len(p), len(b.remaining))++ if b.breakAfter >= 0 {+ readable = min(readable, b.breakAfter-b.deliveredNow)+ }++ if b.truncateAt >= 0 {+ readable = min(readable, b.truncateAt-b.deliveredNow)+ }++ copy(p, b.remaining[:readable])+ b.remaining = b.remaining[readable:]+ b.deliveredNow += readable++ return readable, nil+}++func (b *stubBody) Close() error {+ return nil+}++func generateBytes(size int) []byte {+ generated := make([]byte, size)+ for i := range generated {+ generated[i] = byte(i % 251)+ }++ return generated+}