PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (s3): Resume interrupted restore reads with Range instead of failing (issue #712)
Rostislav Dugin1d868aa3353d
diff --git a/backend/internal/features/backups/backups/controllers/logical/controller.go b/backend/internal/features/backups/backups/controllers/logical/controller.goindex 3acf724f..0d0d107a 100644--- a/backend/internal/features/backups/backups/controllers/logical/controller.go+++ b/backend/internal/features/backups/backups/controllers/logical/controller.go@@ -275,6 +275,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) {}fileReader, backup, database, err := c.backupService.GetBackupFileWithoutAuth(+ ctx.Request.Context(),downloadToken.BackupID,)if err != nil {diff --git a/backend/internal/features/backups/backups/controllers/logical/controller_test.go b/backend/internal/features/backups/backups/controllers/logical/controller_test.goindex 6898bd87..85779a21 100644--- a/backend/internal/features/backups/backups/controllers/logical/controller_test.go+++ b/backend/internal/features/backups/backups/controllers/logical/controller_test.go@@ -1378,11 +1378,21 @@ func Test_MakeBackup_VerifyBackupAndMetadataFilesExistInStorage(t *testing.T) {encryptor := encryption.GetFieldEncryptor()- backupFile, err := backupStorage.GetFile(encryptor, backup.FileName)+ backupFile, err := backupStorage.GetFile(+ t.Context(),+ encryptor,+ slog.New(slog.NewTextHandler(io.Discard, nil)),+ backup.FileName,+ )require.NoError(t, err)backupFile.Close()- metadataFile, err := backupStorage.GetFile(encryptor, backup.FileName+".metadata")+ metadataFile, err := backupStorage.GetFile(+ t.Context(),+ encryptor,+ slog.New(slog.NewTextHandler(io.Discard, nil)),+ backup.FileName+".metadata",+ )require.NoError(t, err)metadataContent, err := io.ReadAll(metadataFile)diff --git a/backend/internal/features/backups/backups/controllers/physical/controller.go b/backend/internal/features/backups/backups/controllers/physical/controller.goindex e0ad4d50..dd349941 100644--- a/backend/internal/features/backups/backups/controllers/physical/controller.go+++ b/backend/internal/features/backups/backups/controllers/physical/controller.go@@ -374,7 +374,7 @@ func (c *PhysicalBackupController) GetRestoreStream(ctx *gin.Context) {ctx.Header("Content-Disposition",fmt.Sprintf(`attachment; filename="restore-%s.tar"`, restoreToken.DatabaseID))- if err := c.openRestoreStream(restoreToken, ctx.Writer); err != nil {+ if err := c.openRestoreStream(ctx.Request.Context(), restoreToken, ctx.Writer); err != nil {// Resolution runs before any bytes are written, so if nothing has been// flushed yet we can still answer with a proper status. Once the tar has// started, the status is already 200 and we can only log.@@ -421,12 +421,16 @@ func classifyRestoreStreamError(err error) (int, string, bool) {// openRestoreStream dispatches by what the token carries: a BackupID streams a// per-backup restore (FULL + incremental ancestors, no WAL); otherwise// TargetTime drives a point-in-time restore.-func (c *PhysicalBackupController) openRestoreStream(token *restore_token.Token, w io.Writer) error {+func (c *PhysicalBackupController) openRestoreStream(+ streamCtx context.Context,+ token *restore_token.Token,+ w io.Writer,+) error {if token.BackupID != nil {- return c.physicalBackupService.OpenRestoreStreamForBackup(token.DatabaseID, *token.BackupID, w)+ return c.physicalBackupService.OpenRestoreStreamForBackup(streamCtx, token.DatabaseID, *token.BackupID, w)}- return c.physicalBackupService.OpenRestoreStream(token.DatabaseID, token.TargetTime, w)+ return c.physicalBackupService.OpenRestoreStream(streamCtx, token.DatabaseID, token.TargetTime, w)}func (c *PhysicalBackupController) requestBackupOfType(diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.goindex cd009327..886e8fec 100644--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go@@ -73,7 +73,7 @@ func saveObject(t *testing.T, st *storages.Storage, fileName string) {func objectExists(t *testing.T, st *storages.Storage, fileName string) bool {t.Helper()- reader, err := st.GetFile(encryption.GetFieldEncryptor(), fileName)+ reader, err := st.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)if err != nil {return false}diff --git a/backend/internal/features/backups/backups/download/di.go b/backend/internal/features/backups/backups/download/di.goindex 4c31a8c2..08233d51 100644--- a/backend/internal/features/backups/backups/download/di.go+++ b/backend/internal/features/backups/backups/download/di.go@@ -37,6 +37,7 @@ func init() {restoreStreamWriter = restore_stream.NewWriter(storages.GetStorageService(),util_encryption.GetFieldEncryptor(),+ logger.GetLogger(),)}diff --git a/backend/internal/features/backups/backups/download/restore_stream/artifact.go b/backend/internal/features/backups/backups/download/restore_stream/artifact.goindex 68a7d796..60c1e24d 100644--- a/backend/internal/features/backups/backups/download/restore_stream/artifact.go+++ b/backend/internal/features/backups/backups/download/restore_stream/artifact.go@@ -2,10 +2,12 @@ package restore_streamimport ("compress/gzip"+ "context""encoding/base64""encoding/json""fmt""io"+ "log/slog""math""github.com/google/uuid"@@ -76,16 +78,17 @@ type artifactSource struct {codec physical_enums.PhysicalBackupCompression}-// openArtifact returns a plaintext reader over a stored object, layering-// decryption then decompression as needed, plus a cleanup that tears the layers-// down in order. The caller must invoke cleanup.+// The returned cleanup tears the decrypt/decompress layers down in order and must be invoked by the+// caller.func openArtifact(+ ctx context.Context,store *storages.Storage,+ logger *slog.Logger,fieldEncryptor util_encryption.FieldEncryptor,masterKey string,src artifactSource,) (io.Reader, func(), error) {- base, err := store.GetFile(fieldEncryptor, src.fileName)+ base, err := store.GetFile(ctx, fieldEncryptor, logger, src.fileName)if err != nil {return nil, nil, fmt.Errorf("open %q: %w", src.fileName, err)}@@ -179,11 +182,13 @@ func newDecryptingReader(}func readHistoryMetadata(+ ctx context.Context,store *storages.Storage,+ logger *slog.Logger,fieldEncryptor util_encryption.FieldEncryptor,historyFileName string,) (*physical_dto.PhysicalWalHistoryMetadata, error) {- reader, err := store.GetFile(fieldEncryptor, historyFileName+historyMetadataSuffix)+ reader, err := store.GetFile(ctx, fieldEncryptor, logger, historyFileName+historyMetadataSuffix)if err != nil {return nil, fmt.Errorf("open history metadata: %w", err)}diff --git a/backend/internal/features/backups/backups/download/restore_stream/writer.go b/backend/internal/features/backups/backups/download/restore_stream/writer.goindex 594973bf..28cee7b7 100644--- a/backend/internal/features/backups/backups/download/restore_stream/writer.go+++ b/backend/internal/features/backups/backups/download/restore_stream/writer.go@@ -2,8 +2,10 @@ package restore_streamimport ("archive/tar"+ "context""fmt""io"+ "log/slog""github.com/google/uuid"@@ -27,19 +29,25 @@ import (type Writer struct {storageService *storages.StorageServicefieldEncryptor util_encryption.FieldEncryptor+ logger *slog.Logger}func NewWriter(storageService *storages.StorageService,fieldEncryptor util_encryption.FieldEncryptor,+ logger *slog.Logger,) *Writer {- return &Writer{storageService, fieldEncryptor}+ return &Writer{storageService, fieldEncryptor, logger}}-// Write streams the whole restore tar into w. masterKey may be empty when no-// artifact is encrypted; an encrypted artifact with an empty key is a hard-// error rather than a silently-corrupt download.-func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey string) error {+// masterKey may be empty when no artifact is encrypted; an encrypted artifact with an empty key is+// a hard error rather than a silently-corrupt download.+func (rw *Writer) Write(+ ctx context.Context,+ w io.Writer,+ set *chain_view.RestoreSet,+ masterKey string,+) error {tarWriter := tar.NewWriter(w)storageCache := make(map[uuid.UUID]*storages.Storage)@@ -50,7 +58,7 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strinreturn fmt.Errorf("stream full backup: %w", err)}- if err := rw.writeBackupDir(tarWriter, "full", backupArtifact{+ if err := rw.writeBackupDir(ctx, tarWriter, "full", backupArtifact{FileName: deref(set.RootFull.FileName),Encryption: set.RootFull.Encryption,EncryptionSalt: set.RootFull.EncryptionSalt,@@ -72,7 +80,7 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strinreturn fmt.Errorf("stream incremental %s: %w", incremental.ID, err)}- if err := rw.writeBackupDir(tarWriter, fmt.Sprintf("incr-%d", i+1), backupArtifact{+ if err := rw.writeBackupDir(ctx, tarWriter, fmt.Sprintf("incr-%d", i+1), backupArtifact{FileName: deref(incremental.FileName),Encryption: incremental.Encryption,EncryptionSalt: incremental.EncryptionSalt,@@ -90,13 +98,13 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strin}for _, segment := range set.WalSegments {- if err := rw.writeWalSegment(tarWriter, segment, masterKey, storageCache, checksums); err != nil {+ if err := rw.writeWalSegment(ctx, tarWriter, segment, masterKey, storageCache, checksums); err != nil {return fmt.Errorf("stream wal segment %s: %w", segment.WalFilename, err)}}for _, history := range set.HistoryFiles {- if err := rw.writeHistoryFile(tarWriter, history, masterKey, storageCache, checksums); err != nil {+ if err := rw.writeHistoryFile(ctx, tarWriter, history, masterKey, storageCache, checksums); err != nil {return fmt.Errorf("stream history file %s: %w", history.HistoryFilename, err)}}@@ -108,11 +116,10 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strinreturn tarWriter.Close()}-// writeBackupDir ships one backup as its stored compressed blob under-// dirName/base.tar<ext> and drops its reconstructed backup_manifest beside it. The-// recovery script decompresses the blob into a directory and pg_combinebackup reads-// the manifest from there (one per input directory).+// The recovery script decompresses the blob into a directory and pg_combinebackup reads+// the manifest from there, one per input directory, so the manifest must ship beside the blob.func (rw *Writer) writeBackupDir(+ ctx context.Context,tarWriter *tar.Writer,dirName string,artifact backupArtifact,@@ -132,17 +139,25 @@ func (rw *Writer) writeBackupDir(return err}+ artifactLogger := rw.logger.With("backup_id", artifact.RowID)+// Ship as stored (decrypt only, still compressed); the recovery script// decompresses on the restore side. Decompressing here would inflate the blob// by its compression ratio and blow the download up by that factor.- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{- fileName: artifact.FileName,- encryption: artifact.Encryption,- salt: artifact.EncryptionSalt,- iv: artifact.EncryptionIV,- keyID: artifact.RowID,- codec: physical_enums.PhysicalBackupCompressionNone,- })+ reader, cleanup, err := openArtifact(+ ctx,+ store,+ artifactLogger,+ rw.fieldEncryptor,+ masterKey,+ artifactSource{+ fileName: artifact.FileName,+ encryption: artifact.Encryption,+ salt: artifact.EncryptionSalt,+ iv: artifact.EncryptionIV,+ keyID: artifact.RowID,+ codec: physical_enums.PhysicalBackupCompressionNone,+ })if err != nil {return err}@@ -153,11 +168,13 @@ func (rw *Writer) writeBackupDir(return err}- return rw.writeManifest(tarWriter, dirName, artifact, masterKey, storageCache, checksums)+ return rw.writeManifest(ctx, tarWriter, artifactLogger, dirName, artifact, masterKey, storageCache, checksums)}func (rw *Writer) writeManifest(+ ctx context.Context,tarWriter *tar.Writer,+ artifactLogger *slog.Logger,dirName string,artifact backupArtifact,masterKey string,@@ -171,14 +188,20 @@ func (rw *Writer) writeManifest(// The manifest sidecar is stored as raw bytes (encrypted with the backup's// row ID but a fresh salt/IV), never zstd-compressed.- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{- fileName: *artifact.ManifestFileName,- encryption: artifact.Encryption,- salt: artifact.ManifestSalt,- iv: artifact.ManifestIV,- keyID: artifact.RowID,- codec: physical_enums.PhysicalBackupCompressionNone,- })+ reader, cleanup, err := openArtifact(+ ctx,+ store,+ artifactLogger,+ rw.fieldEncryptor,+ masterKey,+ artifactSource{+ fileName: *artifact.ManifestFileName,+ encryption: artifact.Encryption,+ salt: artifact.ManifestSalt,+ iv: artifact.ManifestIV,+ keyID: artifact.RowID,+ codec: physical_enums.PhysicalBackupCompressionNone,+ })if err != nil {return err}@@ -188,6 +211,7 @@ func (rw *Writer) writeManifest(}func (rw *Writer) writeWalSegment(+ ctx context.Context,tarWriter *tar.Writer,segment *physical_models.PhysicalWalSegment,masterKey string,@@ -207,14 +231,22 @@ func (rw *Writer) writeWalSegment(// the recovery script decompresses on demand at replay time. Decompressing// here would re-inflate every near-empty segment back to a full 16 MB and// blow the download up by orders of magnitude.- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{- fileName: *segment.FileName,- encryption: segment.Encryption,- salt: segment.EncryptionSalt,- iv: segment.EncryptionIV,- keyID: segment.ID,- codec: physical_enums.PhysicalBackupCompressionNone,- })+ segmentLogger := rw.logger.With("wal_segment", segment.WalFilename)++ reader, cleanup, err := openArtifact(+ ctx,+ store,+ segmentLogger,+ rw.fieldEncryptor,+ masterKey,+ artifactSource{+ fileName: *segment.FileName,+ encryption: segment.Encryption,+ salt: segment.EncryptionSalt,+ iv: segment.EncryptionIV,+ keyID: segment.ID,+ codec: physical_enums.PhysicalBackupCompressionNone,+ })if err != nil {return err}@@ -224,6 +256,7 @@ func (rw *Writer) writeWalSegment(}func (rw *Writer) writeHistoryFile(+ ctx context.Context,tarWriter *tar.Writer,history *physical_models.PhysicalWalHistoryFile,masterKey string,@@ -235,21 +268,35 @@ func (rw *Writer) writeHistoryFile(return err}+ historyLogger := rw.logger.With("history_file", history.HistoryFilename)+// History files keep their encryption parameters only in the .metadata// sidecar, not on the catalog row — read it to learn how to decrypt.- metadata, err := readHistoryMetadata(store, rw.fieldEncryptor, history.FileName)+ metadata, err := readHistoryMetadata(+ ctx,+ store,+ historyLogger,+ rw.fieldEncryptor,+ history.FileName,+ )if err != nil {return err}- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{- fileName: history.FileName,- encryption: metadata.Encryption,- salt: emptyToNil(metadata.EncryptionSalt),- iv: emptyToNil(metadata.EncryptionIV),- keyID: history.ID,- codec: physical_enums.PhysicalBackupCompressionZstd,- })+ reader, cleanup, err := openArtifact(+ ctx,+ store,+ historyLogger,+ rw.fieldEncryptor,+ masterKey,+ artifactSource{+ fileName: history.FileName,+ encryption: metadata.Encryption,+ salt: emptyToNil(metadata.EncryptionSalt),+ iv: emptyToNil(metadata.EncryptionIV),+ keyID: history.ID,+ codec: physical_enums.PhysicalBackupCompressionZstd,+ })if err != nil {return err}diff --git a/backend/internal/features/backups/backups/download/restore_stream/writer_test.go b/backend/internal/features/backups/backups/download/restore_stream/writer_test.goindex e1f1332a..823d2d4c 100644--- a/backend/internal/features/backups/backups/download/restore_stream/writer_test.go+++ b/backend/internal/features/backups/backups/download/restore_stream/writer_test.go@@ -83,8 +83,8 @@ func Test_Writer_Write_ProducesCombineableLayoutWithIntegrityManifest(t *testing}var out bytes.Buffer- writer := restore_stream.NewWriter(storages.GetStorageService(), encryptor)- err := writer.Write(&out, set, "")+ writer := restore_stream.NewWriter(storages.GetStorageService(), encryptor, logger.GetLogger())+ err := writer.Write(t.Context(), &out, set, "")require.NoError(t, err)entries := readTar(t, &out)diff --git a/backend/internal/features/backups/backups/services/logical_service.go b/backend/internal/features/backups/backups/services/logical_service.goindex 947f57c7..85a128f1 100644--- a/backend/internal/features/backups/backups/services/logical_service.go+++ b/backend/internal/features/backups/backups/services/logical_service.go@@ -1,6 +1,7 @@package backups_servicesimport (+ "context""encoding/base64""errors""fmt"@@ -263,54 +264,12 @@ func (s *LogicalBackupService) CancelBackup(return nil}-func (s *LogicalBackupService) GetBackupFile(- user *users_models.User,+func (s *LogicalBackupService) GetBackupReader(+ ctx context.Context,backupID uuid.UUID,-) (io.ReadCloser, *backups_core_logical.LogicalBackup, *databases.Database, error) {- backup, err := s.backupRepository.FindByID(backupID)- if err != nil {- return nil, nil, nil, err- }-- database, err := s.databaseService.GetDatabaseByID(backup.DatabaseID)- if err != nil {- return nil, nil, nil, err- }-- if database.WorkspaceID == nil {- return nil, nil, nil, errors.New("cannot download backup for database without workspace")- }-- canAccess, _, err := s.workspaceService.CanUserAccessWorkspace(- *database.WorkspaceID,- user,- )- if err != nil {- return nil, nil, nil, err- }- if !canAccess {- return nil, nil, nil, errors.New(- "insufficient permissions to download backup for this database",- )- }-- s.auditLogService.WriteAuditLog(- fmt.Sprintf("Backup file downloaded for database: %s", database.Name),- &user.ID,- database.WorkspaceID,- )-- reader, err := s.GetBackupReader(backupID)- if err != nil {- return nil, nil, nil, err- }-- return reader, backup, database, nil-}+) (io.ReadCloser, error) {+ logger := s.logger.With("backup_id", backupID)-// GetBackupReader returns a reader for the backup file.-// If encrypted, wraps with DecryptionReader.-func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {backup, err := s.backupRepository.FindByID(backupID)if err != nil {return nil, fmt.Errorf("failed to find backup: %w", err)@@ -321,46 +280,42 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadClosereturn nil, fmt.Errorf("failed to get storage: %w", err)}- fileReader, err := storage.GetFile(s.fieldEncryptor, backup.FileName)+ fileReader, err := storage.GetFile(ctx, s.fieldEncryptor, logger, backup.FileName)if err != nil {return nil, fmt.Errorf("failed to get backup file: %w", err)}- // If not encrypted, return raw readerif backup.Encryption == backups_core_enums.BackupEncryptionNone {- s.logger.Info("Returning non-encrypted backup", "backupId", backupID)+ logger.Info("returning non-encrypted backup")return fileReader, nil}- // Decrypt on-the-fly for encrypted backupsif backup.Encryption != backups_core_enums.BackupEncryptionEncrypted {if err := fileReader.Close(); err != nil {- s.logger.Error("Failed to close file reader", "error", err)+ logger.Error("failed to close file reader", "error", err)}return nil, fmt.Errorf("unsupported encryption type: %s", backup.Encryption)}if backup.EncryptionSalt == nil || backup.EncryptionIV == nil {if err := fileReader.Close(); err != nil {- s.logger.Error("Failed to close file reader", "error", err)+ logger.Error("failed to close file reader", "error", err)}return nil, fmt.Errorf("backup marked as encrypted but missing encryption metadata")}- // Get master keymasterKey, err := s.secretKeyService.GetSecretKey()if err != nil {if closeErr := fileReader.Close(); closeErr != nil {- s.logger.Error("Failed to close file reader", "error", closeErr)+ logger.Error("failed to close file reader", "error", closeErr)}return nil, fmt.Errorf("failed to get master key: %w", err)}- // Decode salt and IVsalt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)if err != nil {if closeErr := fileReader.Close(); closeErr != nil {- s.logger.Error("Failed to close file reader", "error", closeErr)+ logger.Error("failed to close file reader", "error", closeErr)}return nil, fmt.Errorf("failed to decode salt: %w", err)}@@ -368,12 +323,11 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadCloseiv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV)if err != nil {if closeErr := fileReader.Close(); closeErr != nil {- s.logger.Error("Failed to close file reader", "error", closeErr)+ logger.Error("failed to close file reader", "error", closeErr)}return nil, fmt.Errorf("failed to decode IV: %w", err)}- // Wrap with decrypting readerdecryptionReader, err := encryption.NewDecryptionReader(fileReader,masterKey,@@ -383,12 +337,12 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadClose)if err != nil {if closeErr := fileReader.Close(); closeErr != nil {- s.logger.Error("Failed to close file reader", "error", closeErr)+ logger.Error("failed to close file reader", "error", closeErr)}return nil, fmt.Errorf("failed to create decrypting reader: %w", err)}- s.logger.Info("Returning encrypted backup with decryption", "backupId", backupID)+ logger.Info("returning encrypted backup with decryption")return &backups_dto_logical.DecryptionReaderCloser{DecryptionReader: decryptionReader,@@ -455,6 +409,7 @@ func (s *LogicalBackupService) GetLatestVerifiableBackup(}func (s *LogicalBackupService) GetBackupFileWithoutAuth(+ ctx context.Context,backupID uuid.UUID,) (io.ReadCloser, *backups_core_logical.LogicalBackup, *databases.Database, error) {backup, err := s.backupRepository.FindByID(backupID)@@ -467,7 +422,7 @@ func (s *LogicalBackupService) GetBackupFileWithoutAuth(return nil, nil, nil, err}- reader, err := s.GetBackupReader(backupID)+ reader, err := s.GetBackupReader(ctx, backupID)if err != nil {return nil, nil, nil, err}diff --git a/backend/internal/features/backups/backups/services/physical_service.go b/backend/internal/features/backups/backups/services/physical_service.goindex b65c150b..d9438002 100644--- a/backend/internal/features/backups/backups/services/physical_service.go+++ b/backend/internal/features/backups/backups/services/physical_service.go@@ -334,11 +334,10 @@ func (s *PhysicalBackupService) GenerateBackupRestoreToken(return s.restoreTokenService.GenerateBackupRestoreToken(databaseID, user.ID, backupID)}-// OpenRestoreStream resolves the restore set for (databaseID, targetTime) and-// streams the ready-to-pg_combinebackup tar into w. It performs NO authorization-// — callers gate access (the user path via a restore token, restore+// Performs NO authorization — callers gate access (the user path via a restore token, restore// verification via agent ownership). This is the shared seam between the two.func (s *PhysicalBackupService) OpenRestoreStream(+ ctx context.Context,databaseID uuid.UUID,targetTime *time.Time,w io.Writer,@@ -348,14 +347,13 @@ func (s *PhysicalBackupService) OpenRestoreStream(return err}- return s.writeRestoreStream(set, w)+ return s.writeRestoreStream(ctx, set, w)}-// OpenRestoreStreamForBackup resolves the per-backup restore set (FULL plus its-// incremental ancestors, no WAL) for backupID and streams it into w. Like-// OpenRestoreStream it performs no authorization — the caller's token already-// did.+// The per-backup restore set is the FULL plus its incremental ancestors, no WAL. Like+// OpenRestoreStream it performs no authorization — the caller's token already did.func (s *PhysicalBackupService) OpenRestoreStreamForBackup(+ ctx context.Context,databaseID, backupID uuid.UUID,w io.Writer,) error {@@ -364,16 +362,20 @@ func (s *PhysicalBackupService) OpenRestoreStreamForBackup(return err}- return s.writeRestoreStream(set, w)+ return s.writeRestoreStream(ctx, set, w)}-func (s *PhysicalBackupService) writeRestoreStream(set *chain_view.RestoreSet, w io.Writer) error {+func (s *PhysicalBackupService) writeRestoreStream(+ ctx context.Context,+ set *chain_view.RestoreSet,+ w io.Writer,+) error {masterKey, err := s.resolveMasterKey(set)if err != nil {return err}- return s.restoreStreamWriter.Write(w, set, masterKey)+ return s.restoreStreamWriter.Write(ctx, w, set, masterKey)}func (s *PhysicalBackupService) deleteFull(full *physical_models.PhysicalFullBackup) error {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.goindex 056ae8ba..8be570cb 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go@@ -13,6 +13,7 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql""databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/logger")func Test_FullOnly_ProducesArtifactAndManifest(t *testing.T) {@@ -44,18 +45,28 @@ func Test_FullOnly_ProducesArtifactAndManifest(t *testing.T) {encryptor := encryption.GetFieldEncryptor()- artifactReader, err := fixture.Storage.GetFile(encryptor, *finalRow.FileName)+ artifactReader, err := fixture.Storage.GetFile(t.Context(), encryptor, logger.GetLogger(), *finalRow.FileName)require.NoError(t, err, "artifact %q must be in storage", *finalRow.FileName)require.NoError(t, artifactReader.Close())- sidecarReader, err := fixture.Storage.GetFile(encryptor, *finalRow.FileName+".metadata")+ sidecarReader, err := fixture.Storage.GetFile(+ t.Context(),+ encryptor,+ logger.GetLogger(),+ *finalRow.FileName+".metadata",+ )require.NoError(t, err, "sidecar must be in storage")require.NoError(t, sidecarReader.Close())require.NotNil(t, finalRow.ManifestFileName, "ManifestFileName must be populated post-COMPLETED")assert.Equal(t, *finalRow.FileName+".manifest", *finalRow.ManifestFileName)- manifestReader, err := fixture.Storage.GetFile(encryptor, *finalRow.ManifestFileName)+ manifestReader, err := fixture.Storage.GetFile(+ t.Context(),+ encryptor,+ logger.GetLogger(),+ *finalRow.ManifestFileName,+ )require.NoError(t, err, "manifest sidecar must be in storage")require.NoError(t, manifestReader.Close())}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.goindex 45161729..cccabef4 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go@@ -61,7 +61,7 @@ func (uc *CreateIncrementalBackupUsecase) Execute(// Manifest System-Identifier from the stored row (see create_full_uc.go Execute).systemID := spec.SourceDB.SystemIdentifierUint64()- manifestPath, manifestCleanup, err := downloadParentManifest(spec)+ manifestPath, manifestCleanup, err := downloadParentManifest(ctx, spec)if err != nil {reason := physical_enums.PhysicalBackupErrorParentManifestMissing@@ -220,6 +220,7 @@ func canceledResult(}func downloadParentManifest(+ ctx context.Context,spec IncrementalBackupSpec,) (manifestPath string, cleanup func(), err error) {tmpDir, err := os.MkdirTemp(os.TempDir(), "pgincr_"+uuid.New().String())@@ -236,7 +237,7 @@ func downloadParentManifest(return "", func() {}, errors.New("storage does not support GetFile")}- reader, err := storageHandle.GetFile(spec.FieldEncryptor, spec.ParentManifest.FileName)+ reader, err := storageHandle.GetFile(ctx, spec.FieldEncryptor, spec.Logger, spec.ParentManifest.FileName)if err != nil {cleanupAll()diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.goindex 6a11f3e6..ebe7e267 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go@@ -1,7 +1,9 @@package usecases_physical_postgresqlimport (+ "context""io"+ "log/slog"util_encryption "databasus-backend/internal/util/encryption")@@ -11,5 +13,10 @@ import (// concrete storages.StorageFileSaver is asserted to this narrower seam in// downloadParentManifest so the executor depends only on what it uses.type parentManifestFetcher interface {- GetFile(encryptor util_encryption.FieldEncryptor, fileName string) (io.ReadCloser, error)+ GetFile(+ ctx context.Context,+ encryptor util_encryption.FieldEncryptor,+ logger *slog.Logger,+ fileName string,+ ) (io.ReadCloser, error)}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.goindex 9e434f04..f29b2f2f 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go@@ -92,7 +92,12 @@ func (m *mockWalStorage) DeleteFile(_ encryption.FieldEncryptor, fileName stringreturn nil}-func (m *mockWalStorage) GetFile(_ encryption.FieldEncryptor, _ string) (io.ReadCloser, error) {+func (m *mockWalStorage) GetFile(+ _ context.Context,+ _ encryption.FieldEncryptor,+ _ *slog.Logger,+ _ string,+) (io.ReadCloser, error) {return nil, errors.New("GetFile not implemented in mockWalStorage")}func (m *mockWalStorage) Validate(_ encryption.FieldEncryptor) error { return nil }diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.goindex e087a4f8..adad9b15 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go@@ -106,12 +106,22 @@ func Test_FullOverMtls_ProducesArtifactAndManifest(t *testing.T) {)require.NotEmpty(t, result.FileName)- artifact, err := fixture.Storage.GetFile(encryption.GetFieldEncryptor(), result.FileName)+ artifact, err := fixture.Storage.GetFile(+ t.Context(),+ encryption.GetFieldEncryptor(),+ logger.GetLogger(),+ result.FileName,+ )require.NoError(t, err, "artifact must be in storage")require.NoError(t, artifact.Close())require.NotEmpty(t, result.ManifestFileName)- manifest, err := fixture.Storage.GetFile(encryption.GetFieldEncryptor(), result.ManifestFileName)+ manifest, err := fixture.Storage.GetFile(+ t.Context(),+ encryption.GetFieldEncryptor(),+ logger.GetLogger(),+ result.ManifestFileName,+ )require.NoError(t, err, "reconstructed manifest must be in storage")require.NoError(t, manifest.Close())}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.goindex 60b2195b..17bea167 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go@@ -84,7 +84,12 @@ func (f *fakeStorageFileSaver) deadlineFor(fileName string) (time.Time, bool) {return deadline, hasDeadline}-func (f *fakeStorageFileSaver) GetFile(_ encryption.FieldEncryptor, fileName string) (io.ReadCloser, error) {+func (f *fakeStorageFileSaver) GetFile(+ _ context.Context,+ _ encryption.FieldEncryptor,+ _ *slog.Logger,+ fileName string,+) (io.ReadCloser, error) {f.mu.Lock()defer f.mu.Unlock()diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.goindex 90033731..8ba552ca 100644--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go@@ -139,6 +139,8 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(}}()+ logger := uc.logger.With("backup_id", backup.ID)+fieldEncryptor := util_encryption.GetFieldEncryptor()decryptedPassword, err := fieldEncryptor.Decrypt(password)if err != nil {@@ -151,14 +153,13 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(}defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()- // Stream backup directly from storage- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)if err != nil {return fmt.Errorf("failed to get backup file from storage: %w", err)}defer func() {if err := rawReader.Close(); err != nil {- uc.logger.Error("Failed to close backup reader", "error", err)+ logger.Error("failed to close backup reader", "error", err)}}()diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.goindex db0db1a5..44fb8beb 100644--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go@@ -150,15 +150,16 @@ func (uc *RestoreMongodbBackupUsecase) restoreFromStorage(}}()- // Stream backup directly from storage+ logger := uc.logger.With("backup_id", backup.ID)+fieldEncryptor := util_encryption.GetFieldEncryptor()- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)if err != nil {return fmt.Errorf("failed to get backup file from storage: %w", err)}defer func() {if err := rawReader.Close(); err != nil {- uc.logger.Error("Failed to close backup reader", "error", err)+ logger.Error("failed to close backup reader", "error", err)}}()diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.goindex ad16e0bb..f66e44e2 100644--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go@@ -128,6 +128,8 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(}}()+ logger := uc.logger.With("backup_id", backup.ID)+fieldEncryptor := util_encryption.GetFieldEncryptor()decryptedPassword, err := fieldEncryptor.Decrypt(password)if err != nil {@@ -140,14 +142,13 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(}defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()- // Stream backup directly from storage- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)if err != nil {return fmt.Errorf("failed to get backup file from storage: %w", err)}defer func() {if err := rawReader.Close(); err != nil {- uc.logger.Error("Failed to close backup reader", "error", err)+ logger.Error("failed to close backup reader", "error", err)}}()diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.goindex ad50e7ff..e833514f 100644--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go@@ -152,7 +152,6 @@ func (uc *RestorePostgresqlBackupUsecase) withTimescaleHooks(return runRestore()}-// restoreViaStdin streams backup via stdin for single CPU restorefunc (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(parentCtx context.Context,originalDB *databases.Database,@@ -161,7 +160,9 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(storage *storages.Storage,pg *pgtypes.PostgresqlLogicalDatabase,) error {- uc.logger.Info("Restoring via stdin streaming (CPU=1)", "backupId", backup.ID)+ logger := uc.logger.With("backup_id", backup.ID)++ logger.Info("restoring via stdin streaming (CPU=1)")args := []string{"-Fc", // expect custom type@@ -222,14 +223,13 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(}defer credentials.Remove()- // Get backup stream from storage- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)if err != nil {return fmt.Errorf("failed to get backup file from storage: %w", err)}defer func() {if err := rawReader.Close(); err != nil {- uc.logger.Error("Failed to close backup reader", "error", err)+ logger.Error("failed to close backup reader", "error", err)}}()@@ -270,11 +270,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(}backupReader = decryptReader- uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID)+ logger.Info("using decryption for encrypted backup")}cmd := exec.CommandContext(ctx, pgBin, args...)- uc.logger.Info("Executing PostgreSQL restore command via stdin", "command", cmd.String())+ logger.Info("executing PostgreSQL restore command via stdin", "command", cmd.String())// Setup environment variablesuc.setupPgRestoreEnvironment(cmd, credentials, pg)@@ -532,7 +532,6 @@ func (uc *RestorePostgresqlBackupUsecase) restoreFromStorage(return uc.executePgRestore(ctx, database, pgBin, args, credentials, pgConfig)}-// downloadBackupToTempFile downloads backup data from storage to a temporary filefunc (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(ctx context.Context,backup *backups_core_logical.LogicalBackup,@@ -544,25 +543,24 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(return "", nil, fmt.Errorf("failed to create temporary directory: %w", err)}+ logger := uc.logger.With("backup_id", backup.ID)+cleanupFunc := func() {_ = os.RemoveAll(tempDir)}tempBackupFile := filepath.Join(tempDir, "backup.dump")- // Get backup data from storage- uc.logger.Info(- "Downloading backup file from storage to temporary file",- "backupId",- backup.ID,- "tempFile",+ logger.Info(+ "downloading backup file from storage to temporary file",+ "temp_file",tempBackupFile,"encrypted",backup.Encryption == backups_core_enums.BackupEncryptionEncrypted,)fieldEncryptor := util_encryption.GetFieldEncryptor()- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)if err != nil {cleanupFunc()return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err)@@ -570,7 +568,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(defer func() {if err := rawReader.Close(); err != nil {- uc.logger.Error("Failed to close backup reader", "error", err)+ logger.Error("failed to close backup reader", "error", err)}}()@@ -617,7 +615,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(}backupReader = decryptReader- uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID)+ logger.Info("using decryption for encrypted backup")}// Create temporary backup file@@ -628,7 +626,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(}defer func() {if err := tempFile.Close(); err != nil {- uc.logger.Error("Failed to close temporary file", "error", err)+ logger.Error("failed to close temporary file", "error", err)}}()@@ -639,10 +637,8 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(return "", nil, fmt.Errorf("failed to write backup to temporary file: %w", err)}- // Close the temp file to ensure all data is written - this is handled by defer- // Removing explicit close to avoid double-close error+ logger.Info("backup file written to temporary location", "temp_file", tempBackupFile)- uc.logger.Info("Backup file written to temporary location", "tempFile", tempBackupFile)return tempBackupFile, cleanupFunc, nil}diff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.goindex 39fb7574..db92aed6 100644--- a/backend/internal/features/storages/interfaces.go+++ b/backend/internal/features/storages/interfaces.go@@ -19,7 +19,15 @@ type StorageFileSaver interface {file io.Reader,) error- GetFile(encryptor encryption.FieldEncryptor, fileName string) (io.ReadCloser, error)+ // The returned reader may keep fetching from the remote as it is consumed, so ctx must span the+ // lifetime of the stream, not just the call: a caller that cancels ctx on return gets a stream+ // that dies on the first read.+ GetFile(+ ctx context.Context,+ encryptor encryption.FieldEncryptor,+ logger *slog.Logger,+ fileName string,+ ) (io.ReadCloser, error)DeleteFile(encryptor encryption.FieldEncryptor, fileName string) errordiff --git a/backend/internal/features/storages/model.go b/backend/internal/features/storages/model.goindex 5deeea35..bc9b3cfa 100644--- a/backend/internal/features/storages/model.go+++ b/backend/internal/features/storages/model.go@@ -57,10 +57,12 @@ func (s *Storage) SaveFile(}func (s *Storage) GetFile(+ ctx context.Context,encryptor encryption.FieldEncryptor,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- return s.getSpecificStorage().GetFile(encryptor, fileName)+ return s.getSpecificStorage().GetFile(ctx, encryptor, logger, fileName)}func (s *Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.goindex 1e091fd6..6427d17a 100644--- a/backend/internal/features/storages/model_test.go+++ b/backend/internal/features/storages/model_test.go@@ -199,7 +199,7 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi)require.NoError(t, err, "SaveFile should succeed")- file, err := tc.storage.GetFile(encryptor, fileID.String())+ file, err := tc.storage.GetFile(t.Context(), encryptor, logger.GetLogger(), fileID.String())assert.NoError(t, err, "GetFile should succeed")defer file.Close()@@ -225,7 +225,7 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoierr = tc.storage.DeleteFile(encryptor, fileID.String())assert.NoError(t, err, "DeleteFile should succeed")- file, err := tc.storage.GetFile(encryptor, fileID.String())+ file, err := tc.storage.GetFile(t.Context(), encryptor, logger.GetLogger(), fileID.String())assert.Error(t, err, "GetFile should fail for non-existent file")if file != nil {file.Close()diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.goindex 64afbe5a..c1b890dc 100644--- a/backend/internal/features/storages/models/azure_blob/model.go+++ b/backend/internal/features/storages/models/azure_blob/model.go@@ -157,7 +157,9 @@ func (s *AzureBlobStorage) SaveFile(}func (s *AzureBlobStorage) GetFile(+ ctx context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {client, err := s.getClient(encryptor)@@ -168,7 +170,7 @@ func (s *AzureBlobStorage) GetFile(blobName := s.buildBlobName(fileName)response, err := client.DownloadStream(- context.TODO(),+ ctx,s.ContainerName,blobName,nil,diff --git a/backend/internal/features/storages/models/ftp/model.go b/backend/internal/features/storages/models/ftp/model.goindex d366116e..51d9dc36 100644--- a/backend/internal/features/storages/models/ftp/model.go+++ b/backend/internal/features/storages/models/ftp/model.go@@ -113,7 +113,9 @@ func (f *FTPStorage) SaveFile(}func (f *FTPStorage) GetFile(+ _ context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {conn, err := f.connect(encryptor, ftpConnectTimeout)diff --git a/backend/internal/features/storages/models/google_drive/model.go b/backend/internal/features/storages/models/google_drive/model.goindex 10a4b8a4..3ef42cc8 100644--- a/backend/internal/features/storages/models/google_drive/model.go+++ b/backend/internal/features/storages/models/google_drive/model.go@@ -149,12 +149,14 @@ func (r *backpressureReader) Read(p []byte) (n int, err error) {}func (s *GoogleDriveStorage) GetFile(+ ctx context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {var result io.ReadClosererr := s.withRetryOnAuth(- context.Background(),+ ctx,encryptor,func(driveService *drive.Service) error {folderID, err := s.findBackupsFolder(driveService)@@ -167,7 +169,7 @@ func (s *GoogleDriveStorage) GetFile(return err}- resp, err := driveService.Files.Get(fileIDGoogle).Download() //nolint:bodyclose+ resp, err := driveService.Files.Get(fileIDGoogle).Context(ctx).Download() //nolint:bodycloseif err != nil {return fmt.Errorf("failed to download file from Google Drive: %w", err)}diff --git a/backend/internal/features/storages/models/local/model.go b/backend/internal/features/storages/models/local/model.goindex 89271e48..a0ad7eed 100644--- a/backend/internal/features/storages/models/local/model.go+++ b/backend/internal/features/storages/models/local/model.go@@ -137,7 +137,9 @@ func (l *LocalStorage) SaveFile(}func (l *LocalStorage) GetFile(+ _ context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {filePath := filepath.Join(config.GetEnv().DataFolder, fileName)diff --git a/backend/internal/features/storages/models/nas/model.go b/backend/internal/features/storages/models/nas/model.goindex 464a3ea0..00a3da66 100644--- a/backend/internal/features/storages/models/nas/model.go+++ b/backend/internal/features/storages/models/nas/model.go@@ -156,7 +156,9 @@ func (n *NASStorage) SaveFile(}func (n *NASStorage) GetFile(+ _ context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {session, err := n.createSession(encryptor)diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.goindex ef1bd270..fbfd79e4 100644--- a/backend/internal/features/storages/models/rclone/model.go+++ b/backend/internal/features/storages/models/rclone/model.go@@ -90,11 +90,11 @@ func (r *RcloneStorage) SaveFile(}func (r *RcloneStorage) GetFile(+ ctx context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {- ctx := context.Background()-remoteFs, err := r.getFs(ctx, encryptor)if err != nil {return nil, fmt.Errorf("failed to create rclone filesystem: %w", err)diff --git a/backend/internal/features/storages/models/s3/errors.go b/backend/internal/features/storages/models/s3/errors.gonew file mode 100644index 00000000..65a260b8--- /dev/null+++ b/backend/internal/features/storages/models/s3/errors.go@@ -0,0 +1,19 @@+package s3_storage++import "errors"++// These are terminal on the read path: retrying re-reads the same wrong bytes, and passing them+// downstream would corrupt a stream the decryptor cannot resynchronise (see chunkedManifest).+var (+ ErrChunkSizeMismatch = errors.New("s3 chunk size mismatch")+ ErrChunkChecksumMismatch = errors.New("s3 chunk checksum mismatch")++ // ErrRangeNotHonoured means a resumed GET was answered with a different byte span than the+ // Range header asked for, which some S3-compatible backends do by replying 200 with the whole+ // object. Resuming from byte 0 mid-stream is what this guards against.+ ErrRangeNotHonoured = errors.New("s3 range request not honoured")++ ErrReadRetryBudgetExhausted = errors.New("s3 chunk read retries exhausted")++ ErrReaderClosed = errors.New("s3 chunk reader is closed")+)diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.goindex 7384ab06..d60e2960 100644--- a/backend/internal/features/storages/models/s3/model.go+++ b/backend/internal/features/storages/models/s3/model.go@@ -133,51 +133,50 @@ func (s *S3Storage) SaveFile(}func (s *S3Storage) GetFile(+ ctx context.Context,encryptor encryption.FieldEncryptor,+ logger *slog.Logger,fileName string,) (io.ReadCloser, error) {- client, err := s.getClient(encryptor)+ coreClient, err := s.getCoreClient(encryptor)if err != nil {return nil, err}baseKey := s.buildObjectKey(fileName)- manifest, hasManifest, err := s.readManifest(context.TODO(), client, baseKey)+ manifest, hasManifest, err := s.readManifest(ctx, coreClient.Client, baseKey)if err != nil {return nil, err}if hasManifest {- return newReassemblingReader(client, s.S3Bucket, manifest.Parts), nil+ return newReassemblingReader(reassemblingReaderSpec{+ streamCtx: ctx,+ logger: logger,+ coreClient: coreClient,+ bucket: s.S3Bucket,+ parts: manifest.Parts,+ hasManifestChecksum: true,+ }), nil}- object, err := client.GetObject(- context.TODO(),- s.S3Bucket,- baseKey,- minio.GetObjectOptions{},- )+ objectInfo, err := coreClient.StatObject(ctx, s.S3Bucket, baseKey, minio.StatObjectOptions{})if err != nil {- return nil, fmt.Errorf("failed to get file from S3: %w", err)- }-- // Check if the file actually exists by reading the first byte- buf := make([]byte, 1)- _, readErr := object.Read(buf)- if readErr != nil && !errors.Is(readErr, io.EOF) {- _ = object.Close()- return nil, fmt.Errorf("file does not exist in S3: %w", readErr)- }+ if minio.ToErrorResponse(err).StatusCode == http.StatusNotFound {+ return nil, fmt.Errorf("file does not exist in S3: %w", err)+ }- // Reset the reader to the beginning- _, seekErr := object.Seek(0, io.SeekStart)- if seekErr != nil {- _ = object.Close()- return nil, fmt.Errorf("failed to reset file reader: %w", seekErr)+ return nil, fmt.Errorf("failed to stat file in S3: %w", err)}- return object, nil+ return newReassemblingReader(reassemblingReaderSpec{+ streamCtx: ctx,+ logger: logger,+ coreClient: coreClient,+ bucket: s.S3Bucket,+ parts: []objectSpan{{Key: baseKey, Size: objectInfo.Size}},+ }), nil}func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {@@ -385,7 +384,7 @@ func (s *S3Storage) uploadChunked(maxParts := s.maxPartsPerObject()buf := make([]byte, innerPartSize)- var manifestParts []manifestPart+ var objectSpans []objectSpanvar uploadedKeys []stringvar totalSize int64@@ -483,7 +482,7 @@ func (s *S3Storage) uploadChunked(}uploadedKeys = append(uploadedKeys, objectKey)- manifestParts = append(manifestParts, manifestPart{+ objectSpans = append(objectSpans, objectSpan{Key: objectKey,Size: objectSize,SHA256: hex.EncodeToString(objectHasher.Sum(nil)),@@ -495,7 +494,7 @@ func (s *S3Storage) uploadChunked(manifest := chunkedManifest{Version: manifestVersion,TotalSize: totalSize,- Parts: manifestParts,+ Parts: objectSpans,}if err := s.writeManifest(ctx, client, baseKey, manifest); err != nil {@@ -672,12 +671,15 @@ func (s *S3Storage) getClientParams(// 3. It immunizes reads against S3 eventual consistency: a transient 404 on a middle part can no// longer be misread as the end of the stream.type chunkedManifest struct {- Version int `json:"version"`- TotalSize int64 `json:"totalSize"`- Parts []manifestPart `json:"parts"`+ Version int `json:"version"`+ TotalSize int64 `json:"totalSize"`+ Parts []objectSpan `json:"parts"`}-type manifestPart struct {+// objectSpan is one stored object with the length and sha256 a read must verify it against. The+// manifest lists them for a chunked file; a file stored before chunking is described by a single+// span with no recorded checksum.+type objectSpan struct {Key string `json:"key"`Size int64 `json:"size"`SHA256 string `json:"sha256"`@@ -773,95 +775,3 @@ func (s *S3Storage) removeObjects(ctx context.Context, client *minio.Client, keyreturn firstErr}--// reassemblingReader streams a chunked backup back as one byte stream by opening each part object-// in manifest order, reading it to EOF and only then opening the next. It holds at most one part-// connection at a time, preserving the bounded-memory streaming of the upload path (ADR-0004), and-// verifies each part's length and sha256 against the manifest so a missing, truncated, or corrupted-// part surfaces as a read error instead of a silently short stream.-type reassemblingReader struct {- client *minio.Client- bucket string- parts []manifestPart- index int- current *minio.Object- hasher hash.Hash- read int64-}--func newReassemblingReader(client *minio.Client, bucket string, parts []manifestPart) *reassemblingReader {- return &reassemblingReader{client: client, bucket: bucket, parts: parts}-}--func (r *reassemblingReader) Read(p []byte) (int, error) {- for {- if r.current == nil {- if r.index >= len(r.parts) {- return 0, io.EOF- }-- object, err := r.client.GetObject(context.TODO(), r.bucket, r.parts[r.index].Key, minio.GetObjectOptions{})- if err != nil {- return 0, fmt.Errorf("failed to open chunk %s: %w", r.parts[r.index].Key, err)- }-- r.current = object- r.hasher = sha256.New()- r.read = 0- }-- n, err := r.current.Read(p)- if n > 0 {- r.hasher.Write(p[:n])- r.read += int64(n)- }-- if errors.Is(err, io.EOF) {- if verifyErr := r.finishCurrentPart(); verifyErr != nil {- return n, verifyErr- }-- if n > 0 {- return n, nil- }-- continue- }-- if err != nil {- return n, fmt.Errorf("failed to read chunk %s: %w", r.parts[r.index].Key, err)- }-- return n, nil- }-}--func (r *reassemblingReader) Close() error {- if r.current == nil {- return nil- }-- err := r.current.Close()- r.current = nil-- return err-}--func (r *reassemblingReader) finishCurrentPart() error {- part := r.parts[r.index]-- _ = r.current.Close()- r.current = nil-- if r.read != part.Size {- return fmt.Errorf("chunk %s size mismatch: manifest %d, read %d", part.Key, part.Size, r.read)- }-- if checksum := hex.EncodeToString(r.hasher.Sum(nil)); checksum != part.SHA256 {- return fmt.Errorf("chunk %s checksum mismatch", part.Key)- }-- r.index++-- return nil-}diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.goindex 1b69c1e4..2de4fac7 100644--- a/backend/internal/features/storages/models/s3/model_test.go+++ b/backend/internal/features/storages/models/s3/model_test.go@@ -45,7 +45,7 @@ func Test_DeleteFile_LegacySingleObjectWithoutManifest_RemovesObject(t *testing.assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))- _, err := storage.GetFile(encryptor, fileName)+ _, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)assert.Error(t, err)}@@ -120,7 +120,7 @@ func Test_GetFile_ChunkedBackupMissingPart_ReturnsError(t *testing.T) {rawClient.RemoveObject(t.Context(), storage.S3Bucket, fileName+".part000002", minio.RemoveObjectOptions{}),)- reader, err := storage.GetFile(encryptor, fileName)+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)require.NoError(t, err)defer func() { _ = reader.Close() }()@@ -246,7 +246,7 @@ func Test_GetFile_ChunkedBackupCorruptedPart_ReturnsChecksumError(t *testing.T)corruptedSameSize := bytes.Repeat([]byte{0xAB}, 10*1024*1024)putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", corruptedSameSize)- reader, err := storage.GetFile(encryptor, fileName)+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)require.NoError(t, err)defer func() { _ = reader.Close() }()@@ -269,7 +269,7 @@ func Test_GetFile_ChunkedBackupTruncatedPart_ReturnsSizeError(t *testing.T) {// length check must reject it before its bytes reach the restore.putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", generateBytes(4*1024*1024))- reader, err := storage.GetFile(encryptor, fileName)+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)require.NoError(t, err)defer func() { _ = reader.Close() }()@@ -292,7 +292,7 @@ func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {// single-object path and silently read nothing.putRawObject(t, rawClient, storage.S3Bucket, fileName+manifestSuffix, []byte("not a valid manifest{"))- _, err := storage.GetFile(encryptor, fileName)+ _, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)require.Error(t, err)assert.Contains(t, err.Error(), "manifest")}@@ -356,7 +356,7 @@ func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data [func readWholeFile(t *testing.T, storage *S3Storage, encryptor encryption.FieldEncryptor, fileName string) []byte {t.Helper()- reader, err := storage.GetFile(encryptor, fileName)+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)require.NoError(t, err)defer func() { _ = reader.Close() }()diff --git a/backend/internal/features/storages/models/s3/reassembling_reader.go b/backend/internal/features/storages/models/s3/reassembling_reader.gonew file mode 100644index 00000000..93bbc9e1--- /dev/null+++ b/backend/internal/features/storages/models/s3/reassembling_reader.go@@ -0,0 +1,421 @@+package s3_storage++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "hash"+ "io"+ "log/slog"+ "net/http"+ "strconv"+ "strings"+ "time"++ "github.com/minio/minio-go/v7"+)++const (+ // Once headers have arrived the transport's ResponseHeaderTimeout no longer applies, so a backend+ // that accepts the GET and then goes silent would block until the OS TCP timeout. Armed around+ // each blocking body read only: the gap between a consumer's reads is its own business, and a+ // restore target that pauses on a slow statement must not lose its connection over it.+ s3ReadStallTimeout = 60 * time.Second++ s3MaxReadAttemptsWithoutProgress = 10+ s3ReadRetryBaseDelay = 1 * time.Second+ s3ReadRetryMaxDelay = 30 * time.Second+)++type reassemblingReaderSpec struct {+ streamCtx context.Context+ logger *slog.Logger+ coreClient *minio.Core+ bucket string+ parts []objectSpan++ // Files stored before chunking, and any file small enough to fit one object, have no manifest+ // and therefore no recorded sha256 to compare against; their length comes from StatObject.+ hasManifestChecksum bool++ stallTimeout time.Duration+ maxAttemptsWithoutProgress int+ retryBaseDelay time.Duration+}++// reassemblingReader streams a chunked backup back as one byte stream, holding at most one part+// connection at a time to preserve the bounded-memory streaming of the upload path (ADR-0004).+//+// A dropped or stalled connection is retried in place, resuming the current part from the last byte+// handed out via a Range request. It has to recover here rather than downstream: the bytes are an+// encrypted stream with no end-of-stream marker and the decryptor is stateful, so a caller that sees+// a short read cannot resynchronise.+type reassemblingReader struct {+ streamCtx context.Context+ logger *slog.Logger+ coreClient *minio.Core+ bucket string+ parts []objectSpan++ hasManifestChecksum bool+ stallTimeout time.Duration+ maxAttemptsWithoutProgress int+ retryBaseDelay time.Duration++ isClosed bool+ partIndex int+ partBody io.ReadCloser+ partHasher hash.Hash+ attemptCancel context.CancelFunc+ stallTimer *time.Timer+ partReadBytes int64+ attemptReadBytes int64+ attemptsSinceProgress int+}++func newReassemblingReader(spec reassemblingReaderSpec) *reassemblingReader {+ if spec.stallTimeout == 0 {+ spec.stallTimeout = s3ReadStallTimeout+ }++ if spec.maxAttemptsWithoutProgress == 0 {+ spec.maxAttemptsWithoutProgress = s3MaxReadAttemptsWithoutProgress+ }++ if spec.retryBaseDelay == 0 {+ spec.retryBaseDelay = s3ReadRetryBaseDelay+ }++ // The retry path is the only thing that logs, so a nil logger would panic exactly during an+ // incident and nowhere else.+ if spec.logger == nil {+ spec.logger = slog.New(slog.DiscardHandler)+ }++ return &reassemblingReader{+ streamCtx: spec.streamCtx,+ logger: spec.logger,+ coreClient: spec.coreClient,+ bucket: spec.bucket,+ parts: spec.parts,+ hasManifestChecksum: spec.hasManifestChecksum,+ stallTimeout: spec.stallTimeout,+ maxAttemptsWithoutProgress: spec.maxAttemptsWithoutProgress,+ retryBaseDelay: spec.retryBaseDelay,+ partHasher: sha256.New(),+ }+}++func (r *reassemblingReader) Read(p []byte) (int, error) {+ for {+ if r.isClosed {+ return 0, ErrReaderClosed+ }++ if err := r.streamCtx.Err(); err != nil {+ return 0, err+ }++ if r.partIndex >= len(r.parts) {+ return 0, io.EOF+ }++ if r.partBody == nil {+ if r.partReadBytes >= r.parts[r.partIndex].Size {+ if err := r.finishCurrentPart(); err != nil {+ return 0, err+ }++ continue+ }++ if err := r.openCurrentPart(); err != nil {+ if retryErr := r.resumeAfterFailure(err); retryErr != nil {+ return 0, retryErr+ }++ continue+ }+ }++ r.stallTimer.Reset(r.stallTimeout)+ n, err := r.partBody.Read(p)+ r.stallTimer.Stop()++ if n > 0 {+ if r.hasManifestChecksum {+ r.partHasher.Write(p[:n])+ }++ r.partReadBytes += int64(n)+ r.attemptReadBytes += int64(n)+ }++ // A body that ends before the length the response promised is a truncated transfer, not the+ // end of the part: net/http reports that as ErrUnexpectedEOF, but a chunked-encoded response+ // can deliver it as a plain EOF, so trust the byte count rather than the error.+ isPartComplete := r.partReadBytes >= r.parts[r.partIndex].Size+ if errors.Is(err, io.EOF) && isPartComplete {+ if finishErr := r.finishCurrentPart(); finishErr != nil {+ return n, finishErr+ }++ if n > 0 {+ return n, nil+ }++ continue+ }++ if err != nil {+ readErr := fmt.Errorf("failed to read chunk %s: %w", r.parts[r.partIndex].Key, err)+ if retryErr := r.resumeAfterFailure(readErr); retryErr != nil {+ return n, retryErr+ }++ if n > 0 {+ return n, nil+ }++ continue+ }++ return n, nil+ }+}++func (r *reassemblingReader) Close() error {+ r.isClosed = true++ r.abandonAttempt()++ return nil+}++func (r *reassemblingReader) openCurrentPart() error {+ part := r.parts[r.partIndex]++ attemptCtx, cancelAttempt := context.WithCancel(r.streamCtx)++ options := minio.GetObjectOptions{}++ if r.partReadBytes > 0 {+ if err := options.SetRange(r.partReadBytes, part.Size-1); err != nil {+ cancelAttempt()++ return fmt.Errorf("%w: chunk %s at offset %d: %w", ErrRangeNotHonoured, part.Key, r.partReadBytes, err)+ }+ }++ body, objectInfo, responseHeader, err := r.coreClient.GetObject(attemptCtx, r.bucket, part.Key, options)+ if err != nil {+ cancelAttempt()++ return fmt.Errorf("failed to open chunk %s: %w", part.Key, err)+ }++ if err := r.verifyResponseSpan(objectInfo, responseHeader); err != nil {+ cancelAttempt()+ _ = body.Close()++ return err+ }++ r.partBody = body+ r.attemptCancel = cancelAttempt+ r.stallTimer = time.AfterFunc(r.stallTimeout, cancelAttempt)+ r.stallTimer.Stop()++ return nil+}++// A resumed read must not begin before the offset already handed out: replaying bytes into a+// stateful decryptor corrupts the stream. Content-Range is the positive proof the backend applied the+// Range; failing that, a body whose length equals the remainder is accepted, and anything else is+// refused because an unverifiable resume cannot be told apart from a whole-object reply.+func (r *reassemblingReader) verifyResponseSpan(+ objectInfo minio.ObjectInfo,+ responseHeader http.Header,+) error {+ part := r.parts[r.partIndex]+ expectedBytes := part.Size - r.partReadBytes++ if r.partReadBytes > 0 {+ if rangeStart, isParsed := rangeStartOfContentRange(responseHeader.Get("Content-Range")); isParsed {+ if rangeStart != r.partReadBytes {+ return fmt.Errorf(+ "%w: chunk %s resumed at offset %d but the response starts at %d",+ ErrRangeNotHonoured, part.Key, r.partReadBytes, rangeStart,+ )+ }++ return nil+ }++ if objectInfo.Size != expectedBytes {+ return fmt.Errorf(+ "%w: chunk %s resumed at offset %d answered %d bytes, expected %d",+ ErrRangeNotHonoured, part.Key, r.partReadBytes, objectInfo.Size, expectedBytes,+ )+ }++ return nil+ }++ // A response sent with chunked transfer-encoding carries no Content-Length, which minio-go+ // reports as size -1. That is not a short object: the part's byte count and sha256 are still+ // verified when it ends, so refusing here would break backends that never declare a length.+ if objectInfo.Size >= 0 && objectInfo.Size != expectedBytes {+ return fmt.Errorf(+ "%w: chunk %s is %d bytes, expected %d",+ ErrChunkSizeMismatch, part.Key, objectInfo.Size, expectedBytes,+ )+ }++ return nil+}++func rangeStartOfContentRange(contentRange string) (int64, bool) {+ span, isBytesUnit := strings.CutPrefix(contentRange, "bytes ")+ if !isBytesUnit {+ return 0, false+ }++ start, _, isSplit := strings.Cut(span, "-")+ if !isSplit {+ return 0, false+ }++ parsed, err := strconv.ParseInt(strings.TrimSpace(start), 10, 64)+ if err != nil {+ return 0, false+ }++ return parsed, true+}++func (r *reassemblingReader) finishCurrentPart() error {+ part := r.parts[r.partIndex]++ r.abandonAttempt()++ if r.partReadBytes != part.Size {+ return fmt.Errorf(+ "%w: chunk %s read %d bytes, expected %d",+ ErrChunkSizeMismatch, part.Key, r.partReadBytes, part.Size,+ )+ }++ if r.hasManifestChecksum {+ if checksum := hex.EncodeToString(r.partHasher.Sum(nil)); checksum != part.SHA256 {+ return fmt.Errorf("%w: chunk %s", ErrChunkChecksumMismatch, part.Key)+ }+ }++ r.partIndex+++ r.partReadBytes = 0+ r.attemptsSinceProgress = 0+ r.partHasher = sha256.New()++ return nil+}++// The budget counts attempts since the last byte delivered, not attempts per part: a multi-hour+// restore with sparse drops must not exhaust it.+func (r *reassemblingReader) resumeAfterFailure(cause error) error {+ hasDeliveredBytes := r.attemptReadBytes > 0+ partKey := r.parts[r.partIndex].Key++ r.abandonAttempt()++ if err := r.streamCtx.Err(); err != nil {+ return errors.Join(cause, err)+ }++ if !isRetryableReadError(cause) {+ return cause+ }++ if hasDeliveredBytes {+ r.attemptsSinceProgress = 1+ } else {+ r.attemptsSinceProgress+++ }++ if r.attemptsSinceProgress > r.maxAttemptsWithoutProgress {+ return fmt.Errorf(+ "%w: chunk %s stuck at offset %d: %w",+ ErrReadRetryBudgetExhausted, partKey, r.partReadBytes, cause,+ )+ }++ r.logger.Warn(+ fmt.Sprintf(+ "s3 chunk read failed, resuming from offset %d (attempt %d of %d)",+ r.partReadBytes, r.attemptsSinceProgress, r.maxAttemptsWithoutProgress,+ ),+ "part_key", partKey,+ "error", cause,+ )++ if hasDeliveredBytes {+ return nil+ }++ return r.waitBeforeRetry()+}++func (r *reassemblingReader) waitBeforeRetry() error {+ delay := s3ReadRetryMaxDelay+ if shift := r.attemptsSinceProgress - 1; shift < 32 {+ delay = min(r.retryBaseDelay<<shift, s3ReadRetryMaxDelay)+ }++ retryTimer := time.NewTimer(delay)+ defer retryTimer.Stop()++ select {+ case <-r.streamCtx.Done():+ return r.streamCtx.Err()+ case <-retryTimer.C:+ return nil+ }+}++func (r *reassemblingReader) abandonAttempt() {+ r.attemptReadBytes = 0++ if r.stallTimer != nil {+ r.stallTimer.Stop()+ r.stallTimer = nil+ }++ if r.partBody != nil {+ _ = r.partBody.Close()+ r.partBody = nil+ }++ if r.attemptCancel != nil {+ r.attemptCancel()+ r.attemptCancel = nil+ }+}++func isRetryableReadError(err error) bool {+ if errors.Is(err, ErrRangeNotHonoured) ||+ errors.Is(err, ErrChunkSizeMismatch) ||+ errors.Is(err, ErrChunkChecksumMismatch) {+ return false+ }++ var errorResponse minio.ErrorResponse+ if errors.As(err, &errorResponse) {+ return errorResponse.StatusCode == http.StatusRequestTimeout ||+ errorResponse.StatusCode == http.StatusTooManyRequests ||+ errorResponse.StatusCode >= http.StatusInternalServerError+ }++ return true+}diff --git a/backend/internal/features/storages/models/s3/reassembling_reader_test.go b/backend/internal/features/storages/models/s3/reassembling_reader_test.gonew file mode 100644index 00000000..58523b51--- /dev/null+++ b/backend/internal/features/storages/models/s3/reassembling_reader_test.go@@ -0,0 +1,642 @@+package s3_storage++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "io"+ "net/http"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/minio/minio-go/v7"+ "github.com/minio/minio-go/v7/pkg/credentials"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func Test_ReadChunkedStream_ConnectionDropsMidPart_ResumesFromOffsetAndReturnsFullStream(t *testing.T) {+ firstPart := generateBytes(8192)+ secondPart := generateBytes(4096)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": firstPart, "file.part000002": secondPart},+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001", "file.part000002"),+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, append(append([]byte{}, firstPart...), secondPart...), reassembledBytes)+ assert.Equal(+ t,+ []string{"", "bytes=3000-8191", ""},+ transport.observedRanges,+ "the retry must resume the first part from the last byte handed out, then read the second part whole",+ )+}++func Test_ReadChunkedStream_ServerIgnoresRange_ReturnsErrorWithoutCorruptingStream(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ isRangeIgnored: true,+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ deliveredBytes, err := io.ReadAll(reader)++ require.ErrorIs(t, err, ErrRangeNotHonoured)+ assert.Equal(+ t,+ partBytes[:3000],+ deliveredBytes,+ "only the bytes read before the drop may reach the caller; the ignored range must not replay the part from zero",+ )+}++func Test_ReadChunkedStream_BackendDeclaresNoContentLength_StreamsAnyway(t *testing.T) {+ partBytes := generateBytes(8192)++ // A chunked transfer-encoded reply carries no Content-Length, which minio-go reports as size -1.+ // Refusing that would lock out every backend that never declares a length.+ transport := &stubS3Transport{objects: map[string][]byte{"file.part000001": partBytes}}+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, partBytes, reassembledBytes)+}++func Test_ReadChunkedStream_ResumeWithoutContentLength_AcceptsContentRangeAsProof(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isContentRangeSent: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err, "Content-Range proves the resume was honoured even with no declared length")++ assert.Equal(t, partBytes, reassembledBytes)+ assert.Equal(t, []string{"", "bytes=3000-8191"}, transport.observedRanges)+}++func Test_ReadChunkedStream_ResumeAnsweredFromTheStartOfTheObject_ReturnsError(t *testing.T) {+ partBytes := generateBytes(8192)++ // The backend reports a range it did not apply: Content-Range starts at 0, not at the offset.+ transport := &stubS3Transport{+ isContentRangeSent: true,+ isRangeIgnored: true,+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ deliveredBytes, err := io.ReadAll(reader)++ require.ErrorIs(t, err, ErrRangeNotHonoured)+ assert.Equal(t, partBytes[:3000], deliveredBytes)+}++func Test_ReadChunkedStream_PartMissing_DoesNotRetry(t *testing.T) {+ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},+ statusForAttempt: func(int) int { return http.StatusNotFound },+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ _, err := io.ReadAll(reader)++ require.Error(t, err)+ assert.Equal(t, 1, transport.attempts, "a missing chunk is permanent; retrying it only delays the failure")+}++func Test_ReadChunkedStream_BodyStalls_CancelsAttemptAndResumes(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: func(attempt int, payload []byte, request *http.Request) io.ReadCloser {+ if attempt == 1 {+ return &stallingBody{requestCtx: request.Context()}+ }++ return wholeBody(payload, request)+ },+ }++ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ stallTimeout: 50 * time.Millisecond,+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, partBytes, reassembledBytes)+ assert.Equal(t, 2, transport.attempts)+}++func Test_ReadChunkedStream_ConsumerPausesLongerThanStallTimeout_KeepsTheSameConnection(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ stallTimeout: 30 * time.Millisecond,+ })++ // A restore target blocked on a slow statement stops reading for a while; that is backpressure,+ // not a stalled connection, and must not cost the stream its part.+ reassembledBytes := make([]byte, 0, len(partBytes))+ buffer := make([]byte, 4096)++ for len(reassembledBytes) < len(partBytes) {+ time.Sleep(60 * time.Millisecond)++ n, err := reader.Read(buffer)+ require.NoError(t, err)++ reassembledBytes = append(reassembledBytes, buffer[:n]...)+ }++ assert.Equal(t, partBytes, reassembledBytes)+ assert.Equal(t, 1, transport.attempts, "a slow consumer must not trigger a re-open")+}++func Test_ReadChunkedStream_RepeatedDropsWithProgress_CompletesStream(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {+ return &connectionBody{+ payload: payload,+ breakAfterBytes: 1000,+ requestCtx: request.Context(),+ }+ },+ }++ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ maxAttemptsWithoutProgress: 2,+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(+ t,+ err,+ "every attempt delivered bytes, so a budget of 2 must never be spent even across 9 drops",+ )++ assert.Equal(t, partBytes, reassembledBytes)+ assert.Equal(t, 9, transport.attempts)+}++func Test_ReadChunkedStream_BodyEndsShortWithPlainEOF_ResumesInsteadOfFailing(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: func(attempt int, payload []byte, request *http.Request) io.ReadCloser {+ if attempt == 1 {+ return &connectionBody{+ payload: payload[:3000],+ breakAfterBytes: 3000,+ requestCtx: request.Context(),+ }+ }++ return wholeBody(payload, request)+ },+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err, "a truncated transfer must resume even when it arrives as a plain EOF")++ assert.Equal(t, partBytes, reassembledBytes)+ assert.Equal(t, []string{"", "bytes=3000-8191"}, transport.observedRanges)+}++func Test_ReadChunkedStream_RetryBudgetExhausted_ReturnsError(t *testing.T) {+ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {+ return &connectionBody{payload: payload, requestCtx: request.Context()}+ },+ }++ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ maxAttemptsWithoutProgress: 2,+ })++ _, err := io.ReadAll(reader)++ require.ErrorIs(t, err, ErrReadRetryBudgetExhausted)+ assert.Equal(t, 3, transport.attempts, "the first attempt plus the two retries the budget allows")+}++func Test_ReadChunkedStream_ReopenKeepsFailingAfterAProductiveAttempt_ExhaustsBudget(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ // 501 is retryable to us but not to minio-go, so the re-open fails without its internal retries.+ statusForAttempt: func(attempt int) int {+ if attempt == 1 {+ return 0+ }++ return http.StatusNotImplemented+ },+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 1000}),+ }++ // A regression that spins without spending the budget trips this deadline instead of hanging.+ streamCtx, cancelStream := context.WithTimeout(t.Context(), 5*time.Second)+ defer cancelStream()++ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ streamCtx: streamCtx,+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ maxAttemptsWithoutProgress: 2,+ })++ _, err := io.ReadAll(reader)++ require.ErrorIs(+ t,+ err,+ ErrReadRetryBudgetExhausted,+ "bytes delivered by an earlier attempt must not keep crediting later re-open failures with progress",+ )+}++func Test_ReadChunkedStream_StreamContextCancelled_ReturnsPromptlyWithoutRetry(t *testing.T) {+ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {+ return &connectionBody{payload: payload, requestCtx: request.Context()}+ },+ }++ streamCtx, cancelStream := context.WithCancel(t.Context())+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ streamCtx: streamCtx,+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ cancelStream()++ _, err := io.ReadAll(reader)++ require.ErrorIs(t, err, context.Canceled)+ assert.Zero(t, transport.attempts, "a cancelled restore must not open another chunk")+}++func Test_ReadChunkedStream_FileStoredWithoutAManifest_SkipsChecksumVerification(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"legacy-object": partBytes},+ }++ // A file stored before chunking has no manifest and therefore no recorded sha256, so the span+ // carries none either; the deliberately wrong checksum here must simply never be consulted.+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ parts: []objectSpan{{+ Key: "legacy-object",+ Size: int64(len(partBytes)),+ SHA256: "0000000000000000000000000000000000000000000000000000000000000000",+ }},+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, partBytes, reassembledBytes)+}++func Test_ReadChunkedStream_ReadAfterClose_ReturnsClosedError(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ }+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ })++ buffer := make([]byte, 4096)+ _, err := reader.Read(buffer)+ require.NoError(t, err)++ require.NoError(t, reader.Close())++ _, err = reader.Read(buffer)+ assert.ErrorIs(t, err, ErrReaderClosed, "a closed reader must not silently re-open the chunk")+}++func Test_ReadChunkedStream_SpecWithoutALogger_RetriesWithoutPanicking(t *testing.T) {+ partBytes := generateBytes(8192)++ transport := &stubS3Transport{+ isLengthDeclared: true,+ objects: map[string][]byte{"file.part000001": partBytes},+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),+ }++ reader := newReassemblingReader(reassemblingReaderSpec{+ streamCtx: t.Context(),+ coreClient: newStubbedCoreClient(t, transport),+ bucket: "test-bucket",+ hasManifestChecksum: true,+ parts: objectSpansOf(transport.objects, "file.part000001"),+ retryBaseDelay: time.Millisecond,+ })++ reassembledBytes, err := io.ReadAll(reader)+ require.NoError(t, err)++ assert.Equal(t, partBytes, reassembledBytes)+}++func newTestReassemblingReader(+ t *testing.T,+ transport *stubS3Transport,+ spec reassemblingReaderSpec,+) *reassemblingReader {+ t.Helper()++ if spec.streamCtx == nil {+ spec.streamCtx = t.Context()+ }++ spec.logger = discardLogger()+ spec.coreClient = newStubbedCoreClient(t, transport)+ spec.bucket = "test-bucket"+ spec.retryBaseDelay = time.Millisecond++ return newReassemblingReader(spec)+}++// Region is set explicitly so the client never issues a GetBucketLocation the stub would have to+// answer, and path lookup keeps the request URL predictable.+func newStubbedCoreClient(t *testing.T, transport *stubS3Transport) *minio.Core {+ t.Helper()++ core, err := minio.NewCore("s3.stub.invalid", &minio.Options{+ Creds: credentials.NewStaticV4("access-key", "secret-key", ""),+ Secure: false,+ Region: "us-east-1",+ BucketLookup: minio.BucketLookupPath,+ Transport: transport,+ })+ require.NoError(t, err)++ return core+}++func objectSpansOf(objects map[string][]byte, keys ...string) []objectSpan {+ parts := make([]objectSpan, 0, len(keys))++ for _, key := range keys {+ checksum := sha256.Sum256(objects[key])+ parts = append(parts, objectSpan{+ Key: key,+ Size: int64(len(objects[key])),+ SHA256: hex.EncodeToString(checksum[:]),+ })+ }++ return parts+}++// The ranges are recorded so a test can assert where a resumed read picked up.+type stubS3Transport struct {+ objects map[string][]byte+ isRangeIgnored bool+ isContentRangeSent bool+ isLengthDeclared bool+ statusForAttempt func(attempt int) int+ bodyForAttempt func(attempt int, payload []byte, request *http.Request) io.ReadCloser++ attempts int+ observedRanges []string+}++func (transport *stubS3Transport) RoundTrip(request *http.Request) (*http.Response, error) {+ transport.attempts+++ requestedRange := request.Header.Get("Range")++ transport.observedRanges = append(transport.observedRanges, requestedRange)++ header := http.Header{}+ header.Set("Last-Modified", time.Unix(0, 0).UTC().Format(http.TimeFormat))+ header.Set("ETag", `"stub-etag"`)++ respondWithStatus := func(status int) *http.Response {+ header.Set("Content-Length", "0")++ return &http.Response{+ StatusCode: status,+ Header: header,+ Body: io.NopCloser(strings.NewReader("")),+ Request: request,+ }+ }++ if failureStatus := transport.GetFailureStatus(transport.attempts); failureStatus != 0 {+ return respondWithStatus(failureStatus), nil+ }++ objectKey := strings.TrimPrefix(request.URL.Path, "/test-bucket/")+ payload := transport.objects[objectKey]++ responseStatus := http.StatusOK++ if requestedRange != "" && !transport.isRangeIgnored {+ rangeStart, isParsed := rangeStartOf(requestedRange)+ if !isParsed {+ return respondWithStatus(http.StatusRequestedRangeNotSatisfiable), nil+ }++ payload = payload[rangeStart:]+ responseStatus = http.StatusPartialContent++ if transport.isContentRangeSent {+ header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d",+ rangeStart, rangeStart+int64(len(payload))-1, len(transport.objects[objectKey])))+ }+ }++ contentLength := int64(len(payload))++ if transport.isLengthDeclared {+ header.Set("Content-Length", strconv.FormatInt(contentLength, 10))+ } else {+ contentLength = -1+ }++ body := wholeBody(payload, request)+ if transport.bodyForAttempt != nil {+ body = transport.bodyForAttempt(transport.attempts, payload, request)+ }++ return &http.Response{+ StatusCode: responseStatus,+ Header: header,+ Body: body,+ ContentLength: contentLength,+ Request: request,+ }, nil+}++func (transport *stubS3Transport) GetFailureStatus(attempt int) int {+ if transport.statusForAttempt == nil {+ return 0+ }++ return transport.statusForAttempt(attempt)+}++func rangeStartOf(requestedRange string) (int64, bool) {+ span := strings.TrimPrefix(requestedRange, "bytes=")+ start, _, _ := strings.Cut(span, "-")++ parsed, err := strconv.ParseInt(start, 10, 64)+ if err != nil {+ return 0, false+ }++ return parsed, true+}++type severSpec struct {+ throughAttempt int+ breakAfterBytes int+}++func severBodyOnAttempts(spec severSpec) func(int, []byte, *http.Request) io.ReadCloser {+ return func(attempt int, payload []byte, request *http.Request) io.ReadCloser {+ if attempt > spec.throughAttempt {+ return wholeBody(payload, request)+ }++ return &connectionBody{+ payload: payload,+ breakAfterBytes: spec.breakAfterBytes,+ requestCtx: request.Context(),+ }+ }+}++func wholeBody(payload []byte, request *http.Request) io.ReadCloser {+ return &connectionBody{+ payload: payload,+ breakAfterBytes: len(payload),+ requestCtx: request.Context(),+ }+}++type connectionBody struct {+ payload []byte+ breakAfterBytes int+ position int+ requestCtx context.Context+}++func (b *connectionBody) Read(p []byte) (int, error) {+ if err := b.requestCtx.Err(); err != nil {+ return 0, err+ }++ severAt := min(b.breakAfterBytes, len(b.payload))++ if b.position >= severAt {+ if severAt < len(b.payload) {+ return 0, io.ErrUnexpectedEOF+ }++ return 0, io.EOF+ }++ n := copy(p, b.payload[b.position:severAt])+ b.position += n++ return n, nil+}++func (b *connectionBody) Close() error { return nil }++// stallingBody never delivers a byte and unblocks only when the request context is cancelled, which+// is what a real transport does to a pending read once the request is cancelled.+type stallingBody struct {+ requestCtx context.Context+}++func (b *stallingBody) Read(_ []byte) (int, error) {+ <-b.requestCtx.Done()++ return 0, b.requestCtx.Err()+}++func (b *stallingBody) Close() error { return nil }diff --git a/backend/internal/features/storages/models/sftp/model.go b/backend/internal/features/storages/models/sftp/model.goindex 83b65f57..845a611c 100644--- a/backend/internal/features/storages/models/sftp/model.go+++ b/backend/internal/features/storages/models/sftp/model.go@@ -131,7 +131,9 @@ func (s *SFTPStorage) SaveFile(}func (s *SFTPStorage) GetFile(+ _ context.Context,encryptor encryption.FieldEncryptor,+ _ *slog.Logger,fileName string,) (io.ReadCloser, error) {client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)diff --git a/backend/internal/features/verification/runs/agent_controller.go b/backend/internal/features/verification/runs/agent_controller.goindex b4eda6d2..701d8a3f 100644--- a/backend/internal/features/verification/runs/agent_controller.go+++ b/backend/internal/features/verification/runs/agent_controller.go@@ -94,7 +94,7 @@ func (c *VerificationAgentController) BackupStream(ctx *gin.Context) {return}- reader, err := c.verificationService.GetBackupFile(agent, verificationID)+ reader, err := c.verificationService.GetBackupFile(ctx.Request.Context(), agent, verificationID)if err != nil {ctx.JSON(http.StatusGone, gin.H{"error": err.Error(), "reason": "gone"})returndiff --git a/backend/internal/features/verification/runs/service.go b/backend/internal/features/verification/runs/service.goindex c7b53195..ba65a0f0 100644--- a/backend/internal/features/verification/runs/service.go+++ b/backend/internal/features/verification/runs/service.go@@ -1,6 +1,7 @@package verification_runsimport (+ "context""errors""fmt""io"@@ -351,6 +352,7 @@ func (s *VerificationService) OnAgentHeartbeated(}func (s *VerificationService) GetBackupFile(+ ctx context.Context,agent *verification_agents.Agent,verificationID uuid.UUID,) (io.ReadCloser, error) {@@ -371,7 +373,7 @@ func (s *VerificationService) GetBackupFile(return nil, errors.New("verification is not owned by this agent")}- reader, _, _, err := s.backupService.GetBackupFileWithoutAuth(verification.BackupID)+ reader, _, _, err := s.backupService.GetBackupFileWithoutAuth(ctx, verification.BackupID)if err != nil {return nil, err}