Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (s3): Resume interrupted restore reads with Range instead of failing (issue #712)

Rostislav Dugin1d868aa3353d
diff --git a/backend/internal/features/backups/backups/controllers/logical/controller.go b/backend/internal/features/backups/backups/controllers/logical/controller.go
index 3acf724f..0d0d107a 100644
--- a/backend/internal/features/backups/backups/controllers/logical/controller.go
+++ b/backend/internal/features/backups/backups/controllers/logical/controller.go
@@ -275,6 +275,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
}
fileReader, backup, database, err := c.backupService.GetBackupFileWithoutAuth(
+ ctx.Request.Context(),
downloadToken.BackupID,
)
if err != nil {
diff --git a/backend/internal/features/backups/backups/controllers/logical/controller_test.go b/backend/internal/features/backups/backups/controllers/logical/controller_test.go
index 6898bd87..85779a21 100644
--- a/backend/internal/features/backups/backups/controllers/logical/controller_test.go
+++ b/backend/internal/features/backups/backups/controllers/logical/controller_test.go
@@ -1378,11 +1378,21 @@ func Test_MakeBackup_VerifyBackupAndMetadataFilesExistInStorage(t *testing.T) {
encryptor := encryption.GetFieldEncryptor()
- backupFile, err := backupStorage.GetFile(encryptor, backup.FileName)
+ backupFile, err := backupStorage.GetFile(
+ t.Context(),
+ encryptor,
+ slog.New(slog.NewTextHandler(io.Discard, nil)),
+ backup.FileName,
+ )
require.NoError(t, err)
backupFile.Close()
- metadataFile, err := backupStorage.GetFile(encryptor, backup.FileName+".metadata")
+ metadataFile, err := backupStorage.GetFile(
+ t.Context(),
+ encryptor,
+ slog.New(slog.NewTextHandler(io.Discard, nil)),
+ backup.FileName+".metadata",
+ )
require.NoError(t, err)
metadataContent, err := io.ReadAll(metadataFile)
diff --git a/backend/internal/features/backups/backups/controllers/physical/controller.go b/backend/internal/features/backups/backups/controllers/physical/controller.go
index e0ad4d50..dd349941 100644
--- a/backend/internal/features/backups/backups/controllers/physical/controller.go
+++ b/backend/internal/features/backups/backups/controllers/physical/controller.go
@@ -374,7 +374,7 @@ func (c *PhysicalBackupController) GetRestoreStream(ctx *gin.Context) {
ctx.Header("Content-Disposition",
fmt.Sprintf(`attachment; filename="restore-%s.tar"`, restoreToken.DatabaseID))
- if err := c.openRestoreStream(restoreToken, ctx.Writer); err != nil {
+ if err := c.openRestoreStream(ctx.Request.Context(), restoreToken, ctx.Writer); err != nil {
// Resolution runs before any bytes are written, so if nothing has been
// flushed yet we can still answer with a proper status. Once the tar has
// started, the status is already 200 and we can only log.
@@ -421,12 +421,16 @@ func classifyRestoreStreamError(err error) (int, string, bool) {
// openRestoreStream dispatches by what the token carries: a BackupID streams a
// per-backup restore (FULL + incremental ancestors, no WAL); otherwise
// TargetTime drives a point-in-time restore.
-func (c *PhysicalBackupController) openRestoreStream(token *restore_token.Token, w io.Writer) error {
+func (c *PhysicalBackupController) openRestoreStream(
+ streamCtx context.Context,
+ token *restore_token.Token,
+ w io.Writer,
+) error {
if token.BackupID != nil {
- return c.physicalBackupService.OpenRestoreStreamForBackup(token.DatabaseID, *token.BackupID, w)
+ return c.physicalBackupService.OpenRestoreStreamForBackup(streamCtx, token.DatabaseID, *token.BackupID, w)
}
- return c.physicalBackupService.OpenRestoreStream(token.DatabaseID, token.TargetTime, w)
+ return c.physicalBackupService.OpenRestoreStream(streamCtx, token.DatabaseID, token.TargetTime, w)
}
func (c *PhysicalBackupController) requestBackupOfType(
diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.go
index cd009327..886e8fec 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go
@@ -73,7 +73,7 @@ func saveObject(t *testing.T, st *storages.Storage, fileName string) {
func objectExists(t *testing.T, st *storages.Storage, fileName string) bool {
t.Helper()
- reader, err := st.GetFile(encryption.GetFieldEncryptor(), fileName)
+ reader, err := st.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
if err != nil {
return false
}
diff --git a/backend/internal/features/backups/backups/download/di.go b/backend/internal/features/backups/backups/download/di.go
index 4c31a8c2..08233d51 100644
--- a/backend/internal/features/backups/backups/download/di.go
+++ b/backend/internal/features/backups/backups/download/di.go
@@ -37,6 +37,7 @@ func init() {
restoreStreamWriter = restore_stream.NewWriter(
storages.GetStorageService(),
util_encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
)
}
diff --git a/backend/internal/features/backups/backups/download/restore_stream/artifact.go b/backend/internal/features/backups/backups/download/restore_stream/artifact.go
index 68a7d796..60c1e24d 100644
--- a/backend/internal/features/backups/backups/download/restore_stream/artifact.go
+++ b/backend/internal/features/backups/backups/download/restore_stream/artifact.go
@@ -2,10 +2,12 @@ package restore_stream
import (
"compress/gzip"
+ "context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
+ "log/slog"
"math"
"github.com/google/uuid"
@@ -76,16 +78,17 @@ type artifactSource struct {
codec physical_enums.PhysicalBackupCompression
}
-// openArtifact returns a plaintext reader over a stored object, layering
-// decryption then decompression as needed, plus a cleanup that tears the layers
-// down in order. The caller must invoke cleanup.
+// The returned cleanup tears the decrypt/decompress layers down in order and must be invoked by the
+// caller.
func openArtifact(
+ ctx context.Context,
store *storages.Storage,
+ logger *slog.Logger,
fieldEncryptor util_encryption.FieldEncryptor,
masterKey string,
src artifactSource,
) (io.Reader, func(), error) {
- base, err := store.GetFile(fieldEncryptor, src.fileName)
+ base, err := store.GetFile(ctx, fieldEncryptor, logger, src.fileName)
if err != nil {
return nil, nil, fmt.Errorf("open %q: %w", src.fileName, err)
}
@@ -179,11 +182,13 @@ func newDecryptingReader(
}
func readHistoryMetadata(
+ ctx context.Context,
store *storages.Storage,
+ logger *slog.Logger,
fieldEncryptor util_encryption.FieldEncryptor,
historyFileName string,
) (*physical_dto.PhysicalWalHistoryMetadata, error) {
- reader, err := store.GetFile(fieldEncryptor, historyFileName+historyMetadataSuffix)
+ reader, err := store.GetFile(ctx, fieldEncryptor, logger, historyFileName+historyMetadataSuffix)
if err != nil {
return nil, fmt.Errorf("open history metadata: %w", err)
}
diff --git a/backend/internal/features/backups/backups/download/restore_stream/writer.go b/backend/internal/features/backups/backups/download/restore_stream/writer.go
index 594973bf..28cee7b7 100644
--- a/backend/internal/features/backups/backups/download/restore_stream/writer.go
+++ b/backend/internal/features/backups/backups/download/restore_stream/writer.go
@@ -2,8 +2,10 @@ package restore_stream
import (
"archive/tar"
+ "context"
"fmt"
"io"
+ "log/slog"
"github.com/google/uuid"
@@ -27,19 +29,25 @@ import (
type Writer struct {
storageService *storages.StorageService
fieldEncryptor util_encryption.FieldEncryptor
+ logger *slog.Logger
}
func NewWriter(
storageService *storages.StorageService,
fieldEncryptor util_encryption.FieldEncryptor,
+ logger *slog.Logger,
) *Writer {
- return &Writer{storageService, fieldEncryptor}
+ return &Writer{storageService, fieldEncryptor, logger}
}
-// Write streams the whole restore tar into w. masterKey may be empty when no
-// artifact is encrypted; an encrypted artifact with an empty key is a hard
-// error rather than a silently-corrupt download.
-func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey string) error {
+// masterKey may be empty when no artifact is encrypted; an encrypted artifact with an empty key is
+// a hard error rather than a silently-corrupt download.
+func (rw *Writer) Write(
+ ctx context.Context,
+ w io.Writer,
+ set *chain_view.RestoreSet,
+ masterKey string,
+) error {
tarWriter := tar.NewWriter(w)
storageCache := make(map[uuid.UUID]*storages.Storage)
@@ -50,7 +58,7 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strin
return fmt.Errorf("stream full backup: %w", err)
}
- if err := rw.writeBackupDir(tarWriter, "full", backupArtifact{
+ if err := rw.writeBackupDir(ctx, tarWriter, "full", backupArtifact{
FileName: deref(set.RootFull.FileName),
Encryption: set.RootFull.Encryption,
EncryptionSalt: set.RootFull.EncryptionSalt,
@@ -72,7 +80,7 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strin
return fmt.Errorf("stream incremental %s: %w", incremental.ID, err)
}
- if err := rw.writeBackupDir(tarWriter, fmt.Sprintf("incr-%d", i+1), backupArtifact{
+ if err := rw.writeBackupDir(ctx, tarWriter, fmt.Sprintf("incr-%d", i+1), backupArtifact{
FileName: deref(incremental.FileName),
Encryption: incremental.Encryption,
EncryptionSalt: incremental.EncryptionSalt,
@@ -90,13 +98,13 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strin
}
for _, segment := range set.WalSegments {
- if err := rw.writeWalSegment(tarWriter, segment, masterKey, storageCache, checksums); err != nil {
+ if err := rw.writeWalSegment(ctx, tarWriter, segment, masterKey, storageCache, checksums); err != nil {
return fmt.Errorf("stream wal segment %s: %w", segment.WalFilename, err)
}
}
for _, history := range set.HistoryFiles {
- if err := rw.writeHistoryFile(tarWriter, history, masterKey, storageCache, checksums); err != nil {
+ if err := rw.writeHistoryFile(ctx, tarWriter, history, masterKey, storageCache, checksums); err != nil {
return fmt.Errorf("stream history file %s: %w", history.HistoryFilename, err)
}
}
@@ -108,11 +116,10 @@ func (rw *Writer) Write(w io.Writer, set *chain_view.RestoreSet, masterKey strin
return tarWriter.Close()
}
-// writeBackupDir ships one backup as its stored compressed blob under
-// dirName/base.tar<ext> and drops its reconstructed backup_manifest beside it. The
-// recovery script decompresses the blob into a directory and pg_combinebackup reads
-// the manifest from there (one per input directory).
+// The recovery script decompresses the blob into a directory and pg_combinebackup reads
+// the manifest from there, one per input directory, so the manifest must ship beside the blob.
func (rw *Writer) writeBackupDir(
+ ctx context.Context,
tarWriter *tar.Writer,
dirName string,
artifact backupArtifact,
@@ -132,17 +139,25 @@ func (rw *Writer) writeBackupDir(
return err
}
+ artifactLogger := rw.logger.With("backup_id", artifact.RowID)
+
// Ship as stored (decrypt only, still compressed); the recovery script
// decompresses on the restore side. Decompressing here would inflate the blob
// by its compression ratio and blow the download up by that factor.
- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{
- fileName: artifact.FileName,
- encryption: artifact.Encryption,
- salt: artifact.EncryptionSalt,
- iv: artifact.EncryptionIV,
- keyID: artifact.RowID,
- codec: physical_enums.PhysicalBackupCompressionNone,
- })
+ reader, cleanup, err := openArtifact(
+ ctx,
+ store,
+ artifactLogger,
+ rw.fieldEncryptor,
+ masterKey,
+ artifactSource{
+ fileName: artifact.FileName,
+ encryption: artifact.Encryption,
+ salt: artifact.EncryptionSalt,
+ iv: artifact.EncryptionIV,
+ keyID: artifact.RowID,
+ codec: physical_enums.PhysicalBackupCompressionNone,
+ })
if err != nil {
return err
}
@@ -153,11 +168,13 @@ func (rw *Writer) writeBackupDir(
return err
}
- return rw.writeManifest(tarWriter, dirName, artifact, masterKey, storageCache, checksums)
+ return rw.writeManifest(ctx, tarWriter, artifactLogger, dirName, artifact, masterKey, storageCache, checksums)
}
func (rw *Writer) writeManifest(
+ ctx context.Context,
tarWriter *tar.Writer,
+ artifactLogger *slog.Logger,
dirName string,
artifact backupArtifact,
masterKey string,
@@ -171,14 +188,20 @@ func (rw *Writer) writeManifest(
// The manifest sidecar is stored as raw bytes (encrypted with the backup's
// row ID but a fresh salt/IV), never zstd-compressed.
- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{
- fileName: *artifact.ManifestFileName,
- encryption: artifact.Encryption,
- salt: artifact.ManifestSalt,
- iv: artifact.ManifestIV,
- keyID: artifact.RowID,
- codec: physical_enums.PhysicalBackupCompressionNone,
- })
+ reader, cleanup, err := openArtifact(
+ ctx,
+ store,
+ artifactLogger,
+ rw.fieldEncryptor,
+ masterKey,
+ artifactSource{
+ fileName: *artifact.ManifestFileName,
+ encryption: artifact.Encryption,
+ salt: artifact.ManifestSalt,
+ iv: artifact.ManifestIV,
+ keyID: artifact.RowID,
+ codec: physical_enums.PhysicalBackupCompressionNone,
+ })
if err != nil {
return err
}
@@ -188,6 +211,7 @@ func (rw *Writer) writeManifest(
}
func (rw *Writer) writeWalSegment(
+ ctx context.Context,
tarWriter *tar.Writer,
segment *physical_models.PhysicalWalSegment,
masterKey string,
@@ -207,14 +231,22 @@ func (rw *Writer) writeWalSegment(
// the recovery script decompresses on demand at replay time. Decompressing
// here would re-inflate every near-empty segment back to a full 16 MB and
// blow the download up by orders of magnitude.
- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{
- fileName: *segment.FileName,
- encryption: segment.Encryption,
- salt: segment.EncryptionSalt,
- iv: segment.EncryptionIV,
- keyID: segment.ID,
- codec: physical_enums.PhysicalBackupCompressionNone,
- })
+ segmentLogger := rw.logger.With("wal_segment", segment.WalFilename)
+
+ reader, cleanup, err := openArtifact(
+ ctx,
+ store,
+ segmentLogger,
+ rw.fieldEncryptor,
+ masterKey,
+ artifactSource{
+ fileName: *segment.FileName,
+ encryption: segment.Encryption,
+ salt: segment.EncryptionSalt,
+ iv: segment.EncryptionIV,
+ keyID: segment.ID,
+ codec: physical_enums.PhysicalBackupCompressionNone,
+ })
if err != nil {
return err
}
@@ -224,6 +256,7 @@ func (rw *Writer) writeWalSegment(
}
func (rw *Writer) writeHistoryFile(
+ ctx context.Context,
tarWriter *tar.Writer,
history *physical_models.PhysicalWalHistoryFile,
masterKey string,
@@ -235,21 +268,35 @@ func (rw *Writer) writeHistoryFile(
return err
}
+ historyLogger := rw.logger.With("history_file", history.HistoryFilename)
+
// History files keep their encryption parameters only in the .metadata
// sidecar, not on the catalog row — read it to learn how to decrypt.
- metadata, err := readHistoryMetadata(store, rw.fieldEncryptor, history.FileName)
+ metadata, err := readHistoryMetadata(
+ ctx,
+ store,
+ historyLogger,
+ rw.fieldEncryptor,
+ history.FileName,
+ )
if err != nil {
return err
}
- reader, cleanup, err := openArtifact(store, rw.fieldEncryptor, masterKey, artifactSource{
- fileName: history.FileName,
- encryption: metadata.Encryption,
- salt: emptyToNil(metadata.EncryptionSalt),
- iv: emptyToNil(metadata.EncryptionIV),
- keyID: history.ID,
- codec: physical_enums.PhysicalBackupCompressionZstd,
- })
+ reader, cleanup, err := openArtifact(
+ ctx,
+ store,
+ historyLogger,
+ rw.fieldEncryptor,
+ masterKey,
+ artifactSource{
+ fileName: history.FileName,
+ encryption: metadata.Encryption,
+ salt: emptyToNil(metadata.EncryptionSalt),
+ iv: emptyToNil(metadata.EncryptionIV),
+ keyID: history.ID,
+ codec: physical_enums.PhysicalBackupCompressionZstd,
+ })
if err != nil {
return err
}
diff --git a/backend/internal/features/backups/backups/download/restore_stream/writer_test.go b/backend/internal/features/backups/backups/download/restore_stream/writer_test.go
index e1f1332a..823d2d4c 100644
--- a/backend/internal/features/backups/backups/download/restore_stream/writer_test.go
+++ b/backend/internal/features/backups/backups/download/restore_stream/writer_test.go
@@ -83,8 +83,8 @@ func Test_Writer_Write_ProducesCombineableLayoutWithIntegrityManifest(t *testing
}
var out bytes.Buffer
- writer := restore_stream.NewWriter(storages.GetStorageService(), encryptor)
- err := writer.Write(&out, set, "")
+ writer := restore_stream.NewWriter(storages.GetStorageService(), encryptor, logger.GetLogger())
+ err := writer.Write(t.Context(), &out, set, "")
require.NoError(t, err)
entries := readTar(t, &out)
diff --git a/backend/internal/features/backups/backups/services/logical_service.go b/backend/internal/features/backups/backups/services/logical_service.go
index 947f57c7..85a128f1 100644
--- a/backend/internal/features/backups/backups/services/logical_service.go
+++ b/backend/internal/features/backups/backups/services/logical_service.go
@@ -1,6 +1,7 @@
package backups_services
import (
+ "context"
"encoding/base64"
"errors"
"fmt"
@@ -263,54 +264,12 @@ func (s *LogicalBackupService) CancelBackup(
return nil
}
-func (s *LogicalBackupService) GetBackupFile(
- user *users_models.User,
+func (s *LogicalBackupService) GetBackupReader(
+ ctx context.Context,
backupID uuid.UUID,
-) (io.ReadCloser, *backups_core_logical.LogicalBackup, *databases.Database, error) {
- backup, err := s.backupRepository.FindByID(backupID)
- if err != nil {
- return nil, nil, nil, err
- }
-
- database, err := s.databaseService.GetDatabaseByID(backup.DatabaseID)
- if err != nil {
- return nil, nil, nil, err
- }
-
- if database.WorkspaceID == nil {
- return nil, nil, nil, errors.New("cannot download backup for database without workspace")
- }
-
- canAccess, _, err := s.workspaceService.CanUserAccessWorkspace(
- *database.WorkspaceID,
- user,
- )
- if err != nil {
- return nil, nil, nil, err
- }
- if !canAccess {
- return nil, nil, nil, errors.New(
- "insufficient permissions to download backup for this database",
- )
- }
-
- s.auditLogService.WriteAuditLog(
- fmt.Sprintf("Backup file downloaded for database: %s", database.Name),
- &user.ID,
- database.WorkspaceID,
- )
-
- reader, err := s.GetBackupReader(backupID)
- if err != nil {
- return nil, nil, nil, err
- }
-
- return reader, backup, database, nil
-}
+) (io.ReadCloser, error) {
+ logger := s.logger.With("backup_id", backupID)
-// GetBackupReader returns a reader for the backup file.
-// If encrypted, wraps with DecryptionReader.
-func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
backup, err := s.backupRepository.FindByID(backupID)
if err != nil {
return nil, fmt.Errorf("failed to find backup: %w", err)
@@ -321,46 +280,42 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadClose
return nil, fmt.Errorf("failed to get storage: %w", err)
}
- fileReader, err := storage.GetFile(s.fieldEncryptor, backup.FileName)
+ fileReader, err := storage.GetFile(ctx, s.fieldEncryptor, logger, backup.FileName)
if err != nil {
return nil, fmt.Errorf("failed to get backup file: %w", err)
}
- // If not encrypted, return raw reader
if backup.Encryption == backups_core_enums.BackupEncryptionNone {
- s.logger.Info("Returning non-encrypted backup", "backupId", backupID)
+ logger.Info("returning non-encrypted backup")
return fileReader, nil
}
- // Decrypt on-the-fly for encrypted backups
if backup.Encryption != backups_core_enums.BackupEncryptionEncrypted {
if err := fileReader.Close(); err != nil {
- s.logger.Error("Failed to close file reader", "error", err)
+ logger.Error("failed to close file reader", "error", err)
}
return nil, fmt.Errorf("unsupported encryption type: %s", backup.Encryption)
}
if backup.EncryptionSalt == nil || backup.EncryptionIV == nil {
if err := fileReader.Close(); err != nil {
- s.logger.Error("Failed to close file reader", "error", err)
+ logger.Error("failed to close file reader", "error", err)
}
return nil, fmt.Errorf("backup marked as encrypted but missing encryption metadata")
}
- // Get master key
masterKey, err := s.secretKeyService.GetSecretKey()
if err != nil {
if closeErr := fileReader.Close(); closeErr != nil {
- s.logger.Error("Failed to close file reader", "error", closeErr)
+ logger.Error("failed to close file reader", "error", closeErr)
}
return nil, fmt.Errorf("failed to get master key: %w", err)
}
- // Decode salt and IV
salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)
if err != nil {
if closeErr := fileReader.Close(); closeErr != nil {
- s.logger.Error("Failed to close file reader", "error", closeErr)
+ logger.Error("failed to close file reader", "error", closeErr)
}
return nil, fmt.Errorf("failed to decode salt: %w", err)
}
@@ -368,12 +323,11 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadClose
iv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV)
if err != nil {
if closeErr := fileReader.Close(); closeErr != nil {
- s.logger.Error("Failed to close file reader", "error", closeErr)
+ logger.Error("failed to close file reader", "error", closeErr)
}
return nil, fmt.Errorf("failed to decode IV: %w", err)
}
- // Wrap with decrypting reader
decryptionReader, err := encryption.NewDecryptionReader(
fileReader,
masterKey,
@@ -383,12 +337,12 @@ func (s *LogicalBackupService) GetBackupReader(backupID uuid.UUID) (io.ReadClose
)
if err != nil {
if closeErr := fileReader.Close(); closeErr != nil {
- s.logger.Error("Failed to close file reader", "error", closeErr)
+ logger.Error("failed to close file reader", "error", closeErr)
}
return nil, fmt.Errorf("failed to create decrypting reader: %w", err)
}
- s.logger.Info("Returning encrypted backup with decryption", "backupId", backupID)
+ logger.Info("returning encrypted backup with decryption")
return &backups_dto_logical.DecryptionReaderCloser{
DecryptionReader: decryptionReader,
@@ -455,6 +409,7 @@ func (s *LogicalBackupService) GetLatestVerifiableBackup(
}
func (s *LogicalBackupService) GetBackupFileWithoutAuth(
+ ctx context.Context,
backupID uuid.UUID,
) (io.ReadCloser, *backups_core_logical.LogicalBackup, *databases.Database, error) {
backup, err := s.backupRepository.FindByID(backupID)
@@ -467,7 +422,7 @@ func (s *LogicalBackupService) GetBackupFileWithoutAuth(
return nil, nil, nil, err
}
- reader, err := s.GetBackupReader(backupID)
+ reader, err := s.GetBackupReader(ctx, backupID)
if err != nil {
return nil, nil, nil, err
}
diff --git a/backend/internal/features/backups/backups/services/physical_service.go b/backend/internal/features/backups/backups/services/physical_service.go
index b65c150b..d9438002 100644
--- a/backend/internal/features/backups/backups/services/physical_service.go
+++ b/backend/internal/features/backups/backups/services/physical_service.go
@@ -334,11 +334,10 @@ func (s *PhysicalBackupService) GenerateBackupRestoreToken(
return s.restoreTokenService.GenerateBackupRestoreToken(databaseID, user.ID, backupID)
}
-// OpenRestoreStream resolves the restore set for (databaseID, targetTime) and
-// streams the ready-to-pg_combinebackup tar into w. It performs NO authorization
-// — callers gate access (the user path via a restore token, restore
+// Performs NO authorization — callers gate access (the user path via a restore token, restore
// verification via agent ownership). This is the shared seam between the two.
func (s *PhysicalBackupService) OpenRestoreStream(
+ ctx context.Context,
databaseID uuid.UUID,
targetTime *time.Time,
w io.Writer,
@@ -348,14 +347,13 @@ func (s *PhysicalBackupService) OpenRestoreStream(
return err
}
- return s.writeRestoreStream(set, w)
+ return s.writeRestoreStream(ctx, set, w)
}
-// OpenRestoreStreamForBackup resolves the per-backup restore set (FULL plus its
-// incremental ancestors, no WAL) for backupID and streams it into w. Like
-// OpenRestoreStream it performs no authorization — the caller's token already
-// did.
+// The per-backup restore set is the FULL plus its incremental ancestors, no WAL. Like
+// OpenRestoreStream it performs no authorization — the caller's token already did.
func (s *PhysicalBackupService) OpenRestoreStreamForBackup(
+ ctx context.Context,
databaseID, backupID uuid.UUID,
w io.Writer,
) error {
@@ -364,16 +362,20 @@ func (s *PhysicalBackupService) OpenRestoreStreamForBackup(
return err
}
- return s.writeRestoreStream(set, w)
+ return s.writeRestoreStream(ctx, set, w)
}
-func (s *PhysicalBackupService) writeRestoreStream(set *chain_view.RestoreSet, w io.Writer) error {
+func (s *PhysicalBackupService) writeRestoreStream(
+ ctx context.Context,
+ set *chain_view.RestoreSet,
+ w io.Writer,
+) error {
masterKey, err := s.resolveMasterKey(set)
if err != nil {
return err
}
- return s.restoreStreamWriter.Write(w, set, masterKey)
+ return s.restoreStreamWriter.Write(ctx, w, set, masterKey)
}
func (s *PhysicalBackupService) deleteFull(full *physical_models.PhysicalFullBackup) error {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go
index 056ae8ba..8be570cb 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc_test.go
@@ -13,6 +13,7 @@ import (
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
"databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
)
func Test_FullOnly_ProducesArtifactAndManifest(t *testing.T) {
@@ -44,18 +45,28 @@ func Test_FullOnly_ProducesArtifactAndManifest(t *testing.T) {
encryptor := encryption.GetFieldEncryptor()
- artifactReader, err := fixture.Storage.GetFile(encryptor, *finalRow.FileName)
+ artifactReader, err := fixture.Storage.GetFile(t.Context(), encryptor, logger.GetLogger(), *finalRow.FileName)
require.NoError(t, err, "artifact %q must be in storage", *finalRow.FileName)
require.NoError(t, artifactReader.Close())
- sidecarReader, err := fixture.Storage.GetFile(encryptor, *finalRow.FileName+".metadata")
+ sidecarReader, err := fixture.Storage.GetFile(
+ t.Context(),
+ encryptor,
+ logger.GetLogger(),
+ *finalRow.FileName+".metadata",
+ )
require.NoError(t, err, "sidecar must be in storage")
require.NoError(t, sidecarReader.Close())
require.NotNil(t, finalRow.ManifestFileName, "ManifestFileName must be populated post-COMPLETED")
assert.Equal(t, *finalRow.FileName+".manifest", *finalRow.ManifestFileName)
- manifestReader, err := fixture.Storage.GetFile(encryptor, *finalRow.ManifestFileName)
+ manifestReader, err := fixture.Storage.GetFile(
+ t.Context(),
+ encryptor,
+ logger.GetLogger(),
+ *finalRow.ManifestFileName,
+ )
require.NoError(t, err, "manifest sidecar must be in storage")
require.NoError(t, manifestReader.Close())
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
index 45161729..cccabef4 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
@@ -61,7 +61,7 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
// Manifest System-Identifier from the stored row (see create_full_uc.go Execute).
systemID := spec.SourceDB.SystemIdentifierUint64()
- manifestPath, manifestCleanup, err := downloadParentManifest(spec)
+ manifestPath, manifestCleanup, err := downloadParentManifest(ctx, spec)
if err != nil {
reason := physical_enums.PhysicalBackupErrorParentManifestMissing
@@ -220,6 +220,7 @@ func canceledResult(
}
func downloadParentManifest(
+ ctx context.Context,
spec IncrementalBackupSpec,
) (manifestPath string, cleanup func(), err error) {
tmpDir, err := os.MkdirTemp(os.TempDir(), "pgincr_"+uuid.New().String())
@@ -236,7 +237,7 @@ func downloadParentManifest(
return "", func() {}, errors.New("storage does not support GetFile")
}
- reader, err := storageHandle.GetFile(spec.FieldEncryptor, spec.ParentManifest.FileName)
+ reader, err := storageHandle.GetFile(ctx, spec.FieldEncryptor, spec.Logger, spec.ParentManifest.FileName)
if err != nil {
cleanupAll()
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go
index 6a11f3e6..ebe7e267 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/interfaces.go
@@ -1,7 +1,9 @@
package usecases_physical_postgresql
import (
+ "context"
"io"
+ "log/slog"
util_encryption "databasus-backend/internal/util/encryption"
)
@@ -11,5 +13,10 @@ import (
// concrete storages.StorageFileSaver is asserted to this narrower seam in
// downloadParentManifest so the executor depends only on what it uses.
type parentManifestFetcher interface {
- GetFile(encryptor util_encryption.FieldEncryptor, fileName string) (io.ReadCloser, error)
+ GetFile(
+ ctx context.Context,
+ encryptor util_encryption.FieldEncryptor,
+ logger *slog.Logger,
+ fileName string,
+ ) (io.ReadCloser, error)
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
index 9e434f04..f29b2f2f 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
@@ -92,7 +92,12 @@ func (m *mockWalStorage) DeleteFile(_ encryption.FieldEncryptor, fileName string
return nil
}
-func (m *mockWalStorage) GetFile(_ encryption.FieldEncryptor, _ string) (io.ReadCloser, error) {
+func (m *mockWalStorage) GetFile(
+ _ context.Context,
+ _ encryption.FieldEncryptor,
+ _ *slog.Logger,
+ _ string,
+) (io.ReadCloser, error) {
return nil, errors.New("GetFile not implemented in mockWalStorage")
}
func (m *mockWalStorage) Validate(_ encryption.FieldEncryptor) error { return nil }
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
index e087a4f8..adad9b15 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
@@ -106,12 +106,22 @@ func Test_FullOverMtls_ProducesArtifactAndManifest(t *testing.T) {
)
require.NotEmpty(t, result.FileName)
- artifact, err := fixture.Storage.GetFile(encryption.GetFieldEncryptor(), result.FileName)
+ artifact, err := fixture.Storage.GetFile(
+ t.Context(),
+ encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
+ result.FileName,
+ )
require.NoError(t, err, "artifact must be in storage")
require.NoError(t, artifact.Close())
require.NotEmpty(t, result.ManifestFileName)
- manifest, err := fixture.Storage.GetFile(encryption.GetFieldEncryptor(), result.ManifestFileName)
+ manifest, err := fixture.Storage.GetFile(
+ t.Context(),
+ encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
+ result.ManifestFileName,
+ )
require.NoError(t, err, "reconstructed manifest must be in storage")
require.NoError(t, manifest.Close())
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
index 60b2195b..17bea167 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
@@ -84,7 +84,12 @@ func (f *fakeStorageFileSaver) deadlineFor(fileName string) (time.Time, bool) {
return deadline, hasDeadline
}
-func (f *fakeStorageFileSaver) GetFile(_ encryption.FieldEncryptor, fileName string) (io.ReadCloser, error) {
+func (f *fakeStorageFileSaver) GetFile(
+ _ context.Context,
+ _ encryption.FieldEncryptor,
+ _ *slog.Logger,
+ fileName string,
+) (io.ReadCloser, error) {
f.mu.Lock()
defer f.mu.Unlock()
diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
index 90033731..8ba552ca 100644
--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go
@@ -139,6 +139,8 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(
}
}()
+ logger := uc.logger.With("backup_id", backup.ID)
+
fieldEncryptor := util_encryption.GetFieldEncryptor()
decryptedPassword, err := fieldEncryptor.Decrypt(password)
if err != nil {
@@ -151,14 +153,13 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(
}
defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()
- // Stream backup directly from storage
- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)
if err != nil {
return fmt.Errorf("failed to get backup file from storage: %w", err)
}
defer func() {
if err := rawReader.Close(); err != nil {
- uc.logger.Error("Failed to close backup reader", "error", err)
+ logger.Error("failed to close backup reader", "error", err)
}
}()
diff --git a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
index db0db1a5..44fb8beb 100644
--- a/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mongodb/restore_backup_uc.go
@@ -150,15 +150,16 @@ func (uc *RestoreMongodbBackupUsecase) restoreFromStorage(
}
}()
- // Stream backup directly from storage
+ logger := uc.logger.With("backup_id", backup.ID)
+
fieldEncryptor := util_encryption.GetFieldEncryptor()
- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)
if err != nil {
return fmt.Errorf("failed to get backup file from storage: %w", err)
}
defer func() {
if err := rawReader.Close(); err != nil {
- uc.logger.Error("Failed to close backup reader", "error", err)
+ logger.Error("failed to close backup reader", "error", err)
}
}()
diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
index ad16e0bb..f66e44e2 100644
--- a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go
@@ -128,6 +128,8 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(
}
}()
+ logger := uc.logger.With("backup_id", backup.ID)
+
fieldEncryptor := util_encryption.GetFieldEncryptor()
decryptedPassword, err := fieldEncryptor.Decrypt(password)
if err != nil {
@@ -140,14 +142,13 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(
}
defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()
- // Stream backup directly from storage
- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)
if err != nil {
return fmt.Errorf("failed to get backup file from storage: %w", err)
}
defer func() {
if err := rawReader.Close(); err != nil {
- uc.logger.Error("Failed to close backup reader", "error", err)
+ logger.Error("failed to close backup reader", "error", err)
}
}()
diff --git a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
index ad50e7ff..e833514f 100644
--- a/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
+++ b/backend/internal/features/restores/usecases/postgresql/restore_backup_uc.go
@@ -152,7 +152,6 @@ func (uc *RestorePostgresqlBackupUsecase) withTimescaleHooks(
return runRestore()
}
-// restoreViaStdin streams backup via stdin for single CPU restore
func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
parentCtx context.Context,
originalDB *databases.Database,
@@ -161,7 +160,9 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
storage *storages.Storage,
pg *pgtypes.PostgresqlLogicalDatabase,
) error {
- uc.logger.Info("Restoring via stdin streaming (CPU=1)", "backupId", backup.ID)
+ logger := uc.logger.With("backup_id", backup.ID)
+
+ logger.Info("restoring via stdin streaming (CPU=1)")
args := []string{
"-Fc", // expect custom type
@@ -222,14 +223,13 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
}
defer credentials.Remove()
- // Get backup stream from storage
- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)
if err != nil {
return fmt.Errorf("failed to get backup file from storage: %w", err)
}
defer func() {
if err := rawReader.Close(); err != nil {
- uc.logger.Error("Failed to close backup reader", "error", err)
+ logger.Error("failed to close backup reader", "error", err)
}
}()
@@ -270,11 +270,11 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
}
backupReader = decryptReader
- uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID)
+ logger.Info("using decryption for encrypted backup")
}
cmd := exec.CommandContext(ctx, pgBin, args...)
- uc.logger.Info("Executing PostgreSQL restore command via stdin", "command", cmd.String())
+ logger.Info("executing PostgreSQL restore command via stdin", "command", cmd.String())
// Setup environment variables
uc.setupPgRestoreEnvironment(cmd, credentials, pg)
@@ -532,7 +532,6 @@ func (uc *RestorePostgresqlBackupUsecase) restoreFromStorage(
return uc.executePgRestore(ctx, database, pgBin, args, credentials, pgConfig)
}
-// downloadBackupToTempFile downloads backup data from storage to a temporary file
func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
ctx context.Context,
backup *backups_core_logical.LogicalBackup,
@@ -544,25 +543,24 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
return "", nil, fmt.Errorf("failed to create temporary directory: %w", err)
}
+ logger := uc.logger.With("backup_id", backup.ID)
+
cleanupFunc := func() {
_ = os.RemoveAll(tempDir)
}
tempBackupFile := filepath.Join(tempDir, "backup.dump")
- // Get backup data from storage
- uc.logger.Info(
- "Downloading backup file from storage to temporary file",
- "backupId",
- backup.ID,
- "tempFile",
+ logger.Info(
+ "downloading backup file from storage to temporary file",
+ "temp_file",
tempBackupFile,
"encrypted",
backup.Encryption == backups_core_enums.BackupEncryptionEncrypted,
)
fieldEncryptor := util_encryption.GetFieldEncryptor()
- rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
+ rawReader, err := storage.GetFile(ctx, fieldEncryptor, logger, backup.FileName)
if err != nil {
cleanupFunc()
return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err)
@@ -570,7 +568,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
defer func() {
if err := rawReader.Close(); err != nil {
- uc.logger.Error("Failed to close backup reader", "error", err)
+ logger.Error("failed to close backup reader", "error", err)
}
}()
@@ -617,7 +615,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
}
backupReader = decryptReader
- uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID)
+ logger.Info("using decryption for encrypted backup")
}
// Create temporary backup file
@@ -628,7 +626,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
}
defer func() {
if err := tempFile.Close(); err != nil {
- uc.logger.Error("Failed to close temporary file", "error", err)
+ logger.Error("failed to close temporary file", "error", err)
}
}()
@@ -639,10 +637,8 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
return "", nil, fmt.Errorf("failed to write backup to temporary file: %w", err)
}
- // Close the temp file to ensure all data is written - this is handled by defer
- // Removing explicit close to avoid double-close error
+ logger.Info("backup file written to temporary location", "temp_file", tempBackupFile)
- uc.logger.Info("Backup file written to temporary location", "tempFile", tempBackupFile)
return tempBackupFile, cleanupFunc, nil
}
diff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.go
index 39fb7574..db92aed6 100644
--- a/backend/internal/features/storages/interfaces.go
+++ b/backend/internal/features/storages/interfaces.go
@@ -19,7 +19,15 @@ type StorageFileSaver interface {
file io.Reader,
) error
- GetFile(encryptor encryption.FieldEncryptor, fileName string) (io.ReadCloser, error)
+ // The returned reader may keep fetching from the remote as it is consumed, so ctx must span the
+ // lifetime of the stream, not just the call: a caller that cancels ctx on return gets a stream
+ // that dies on the first read.
+ GetFile(
+ ctx context.Context,
+ encryptor encryption.FieldEncryptor,
+ logger *slog.Logger,
+ fileName string,
+ ) (io.ReadCloser, error)
DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error
diff --git a/backend/internal/features/storages/model.go b/backend/internal/features/storages/model.go
index 5deeea35..bc9b3cfa 100644
--- a/backend/internal/features/storages/model.go
+++ b/backend/internal/features/storages/model.go
@@ -57,10 +57,12 @@ func (s *Storage) SaveFile(
}
func (s *Storage) GetFile(
+ ctx context.Context,
encryptor encryption.FieldEncryptor,
+ logger *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
- return s.getSpecificStorage().GetFile(encryptor, fileName)
+ return s.getSpecificStorage().GetFile(ctx, encryptor, logger, fileName)
}
func (s *Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.go
index 1e091fd6..6427d17a 100644
--- a/backend/internal/features/storages/model_test.go
+++ b/backend/internal/features/storages/model_test.go
@@ -199,7 +199,7 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi
)
require.NoError(t, err, "SaveFile should succeed")
- file, err := tc.storage.GetFile(encryptor, fileID.String())
+ file, err := tc.storage.GetFile(t.Context(), encryptor, logger.GetLogger(), fileID.String())
assert.NoError(t, err, "GetFile should succeed")
defer file.Close()
@@ -225,7 +225,7 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi
err = tc.storage.DeleteFile(encryptor, fileID.String())
assert.NoError(t, err, "DeleteFile should succeed")
- file, err := tc.storage.GetFile(encryptor, fileID.String())
+ file, err := tc.storage.GetFile(t.Context(), encryptor, logger.GetLogger(), fileID.String())
assert.Error(t, err, "GetFile should fail for non-existent file")
if file != nil {
file.Close()
diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.go
index 64afbe5a..c1b890dc 100644
--- a/backend/internal/features/storages/models/azure_blob/model.go
+++ b/backend/internal/features/storages/models/azure_blob/model.go
@@ -157,7 +157,9 @@ func (s *AzureBlobStorage) SaveFile(
}
func (s *AzureBlobStorage) GetFile(
+ ctx context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
client, err := s.getClient(encryptor)
@@ -168,7 +170,7 @@ func (s *AzureBlobStorage) GetFile(
blobName := s.buildBlobName(fileName)
response, err := client.DownloadStream(
- context.TODO(),
+ ctx,
s.ContainerName,
blobName,
nil,
diff --git a/backend/internal/features/storages/models/ftp/model.go b/backend/internal/features/storages/models/ftp/model.go
index d366116e..51d9dc36 100644
--- a/backend/internal/features/storages/models/ftp/model.go
+++ b/backend/internal/features/storages/models/ftp/model.go
@@ -113,7 +113,9 @@ func (f *FTPStorage) SaveFile(
}
func (f *FTPStorage) GetFile(
+ _ context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
conn, err := f.connect(encryptor, ftpConnectTimeout)
diff --git a/backend/internal/features/storages/models/google_drive/model.go b/backend/internal/features/storages/models/google_drive/model.go
index 10a4b8a4..3ef42cc8 100644
--- a/backend/internal/features/storages/models/google_drive/model.go
+++ b/backend/internal/features/storages/models/google_drive/model.go
@@ -149,12 +149,14 @@ func (r *backpressureReader) Read(p []byte) (n int, err error) {
}
func (s *GoogleDriveStorage) GetFile(
+ ctx context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
var result io.ReadCloser
err := s.withRetryOnAuth(
- context.Background(),
+ ctx,
encryptor,
func(driveService *drive.Service) error {
folderID, err := s.findBackupsFolder(driveService)
@@ -167,7 +169,7 @@ func (s *GoogleDriveStorage) GetFile(
return err
}
- resp, err := driveService.Files.Get(fileIDGoogle).Download() //nolint:bodyclose
+ resp, err := driveService.Files.Get(fileIDGoogle).Context(ctx).Download() //nolint:bodyclose
if err != nil {
return fmt.Errorf("failed to download file from Google Drive: %w", err)
}
diff --git a/backend/internal/features/storages/models/local/model.go b/backend/internal/features/storages/models/local/model.go
index 89271e48..a0ad7eed 100644
--- a/backend/internal/features/storages/models/local/model.go
+++ b/backend/internal/features/storages/models/local/model.go
@@ -137,7 +137,9 @@ func (l *LocalStorage) SaveFile(
}
func (l *LocalStorage) GetFile(
+ _ context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
filePath := filepath.Join(config.GetEnv().DataFolder, fileName)
diff --git a/backend/internal/features/storages/models/nas/model.go b/backend/internal/features/storages/models/nas/model.go
index 464a3ea0..00a3da66 100644
--- a/backend/internal/features/storages/models/nas/model.go
+++ b/backend/internal/features/storages/models/nas/model.go
@@ -156,7 +156,9 @@ func (n *NASStorage) SaveFile(
}
func (n *NASStorage) GetFile(
+ _ context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
session, err := n.createSession(encryptor)
diff --git a/backend/internal/features/storages/models/rclone/model.go b/backend/internal/features/storages/models/rclone/model.go
index ef1bd270..fbfd79e4 100644
--- a/backend/internal/features/storages/models/rclone/model.go
+++ b/backend/internal/features/storages/models/rclone/model.go
@@ -90,11 +90,11 @@ func (r *RcloneStorage) SaveFile(
}
func (r *RcloneStorage) GetFile(
+ ctx context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
- ctx := context.Background()
-
remoteFs, err := r.getFs(ctx, encryptor)
if err != nil {
return nil, fmt.Errorf("failed to create rclone filesystem: %w", err)
diff --git a/backend/internal/features/storages/models/s3/errors.go b/backend/internal/features/storages/models/s3/errors.go
new file mode 100644
index 00000000..65a260b8
--- /dev/null
+++ b/backend/internal/features/storages/models/s3/errors.go
@@ -0,0 +1,19 @@
+package s3_storage
+
+import "errors"
+
+// These are terminal on the read path: retrying re-reads the same wrong bytes, and passing them
+// downstream would corrupt a stream the decryptor cannot resynchronise (see chunkedManifest).
+var (
+ ErrChunkSizeMismatch = errors.New("s3 chunk size mismatch")
+ ErrChunkChecksumMismatch = errors.New("s3 chunk checksum mismatch")
+
+ // ErrRangeNotHonoured means a resumed GET was answered with a different byte span than the
+ // Range header asked for, which some S3-compatible backends do by replying 200 with the whole
+ // object. Resuming from byte 0 mid-stream is what this guards against.
+ ErrRangeNotHonoured = errors.New("s3 range request not honoured")
+
+ ErrReadRetryBudgetExhausted = errors.New("s3 chunk read retries exhausted")
+
+ ErrReaderClosed = errors.New("s3 chunk reader is closed")
+)
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.go
index 7384ab06..d60e2960 100644
--- a/backend/internal/features/storages/models/s3/model.go
+++ b/backend/internal/features/storages/models/s3/model.go
@@ -133,51 +133,50 @@ func (s *S3Storage) SaveFile(
}
func (s *S3Storage) GetFile(
+ ctx context.Context,
encryptor encryption.FieldEncryptor,
+ logger *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
- client, err := s.getClient(encryptor)
+ coreClient, err := s.getCoreClient(encryptor)
if err != nil {
return nil, err
}
baseKey := s.buildObjectKey(fileName)
- manifest, hasManifest, err := s.readManifest(context.TODO(), client, baseKey)
+ manifest, hasManifest, err := s.readManifest(ctx, coreClient.Client, baseKey)
if err != nil {
return nil, err
}
if hasManifest {
- return newReassemblingReader(client, s.S3Bucket, manifest.Parts), nil
+ return newReassemblingReader(reassemblingReaderSpec{
+ streamCtx: ctx,
+ logger: logger,
+ coreClient: coreClient,
+ bucket: s.S3Bucket,
+ parts: manifest.Parts,
+ hasManifestChecksum: true,
+ }), nil
}
- object, err := client.GetObject(
- context.TODO(),
- s.S3Bucket,
- baseKey,
- minio.GetObjectOptions{},
- )
+ objectInfo, err := coreClient.StatObject(ctx, s.S3Bucket, baseKey, minio.StatObjectOptions{})
if err != nil {
- return nil, fmt.Errorf("failed to get file from S3: %w", err)
- }
-
- // Check if the file actually exists by reading the first byte
- buf := make([]byte, 1)
- _, readErr := object.Read(buf)
- if readErr != nil && !errors.Is(readErr, io.EOF) {
- _ = object.Close()
- return nil, fmt.Errorf("file does not exist in S3: %w", readErr)
- }
+ if minio.ToErrorResponse(err).StatusCode == http.StatusNotFound {
+ return nil, fmt.Errorf("file does not exist in S3: %w", err)
+ }
- // Reset the reader to the beginning
- _, seekErr := object.Seek(0, io.SeekStart)
- if seekErr != nil {
- _ = object.Close()
- return nil, fmt.Errorf("failed to reset file reader: %w", seekErr)
+ return nil, fmt.Errorf("failed to stat file in S3: %w", err)
}
- return object, nil
+ return newReassemblingReader(reassemblingReaderSpec{
+ streamCtx: ctx,
+ logger: logger,
+ coreClient: coreClient,
+ bucket: s.S3Bucket,
+ parts: []objectSpan{{Key: baseKey, Size: objectInfo.Size}},
+ }), nil
}
func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
@@ -385,7 +384,7 @@ func (s *S3Storage) uploadChunked(
maxParts := s.maxPartsPerObject()
buf := make([]byte, innerPartSize)
- var manifestParts []manifestPart
+ var objectSpans []objectSpan
var uploadedKeys []string
var totalSize int64
@@ -483,7 +482,7 @@ func (s *S3Storage) uploadChunked(
}
uploadedKeys = append(uploadedKeys, objectKey)
- manifestParts = append(manifestParts, manifestPart{
+ objectSpans = append(objectSpans, objectSpan{
Key: objectKey,
Size: objectSize,
SHA256: hex.EncodeToString(objectHasher.Sum(nil)),
@@ -495,7 +494,7 @@ func (s *S3Storage) uploadChunked(
manifest := chunkedManifest{
Version: manifestVersion,
TotalSize: totalSize,
- Parts: manifestParts,
+ Parts: objectSpans,
}
if err := s.writeManifest(ctx, client, baseKey, manifest); err != nil {
@@ -672,12 +671,15 @@ func (s *S3Storage) getClientParams(
// 3. It immunizes reads against S3 eventual consistency: a transient 404 on a middle part can no
// longer be misread as the end of the stream.
type chunkedManifest struct {
- Version int `json:"version"`
- TotalSize int64 `json:"totalSize"`
- Parts []manifestPart `json:"parts"`
+ Version int `json:"version"`
+ TotalSize int64 `json:"totalSize"`
+ Parts []objectSpan `json:"parts"`
}
-type manifestPart struct {
+// objectSpan is one stored object with the length and sha256 a read must verify it against. The
+// manifest lists them for a chunked file; a file stored before chunking is described by a single
+// span with no recorded checksum.
+type objectSpan struct {
Key string `json:"key"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
@@ -773,95 +775,3 @@ func (s *S3Storage) removeObjects(ctx context.Context, client *minio.Client, key
return firstErr
}
-
-// reassemblingReader streams a chunked backup back as one byte stream by opening each part object
-// in manifest order, reading it to EOF and only then opening the next. It holds at most one part
-// connection at a time, preserving the bounded-memory streaming of the upload path (ADR-0004), and
-// verifies each part's length and sha256 against the manifest so a missing, truncated, or corrupted
-// part surfaces as a read error instead of a silently short stream.
-type reassemblingReader struct {
- client *minio.Client
- bucket string
- parts []manifestPart
- index int
- current *minio.Object
- hasher hash.Hash
- read int64
-}
-
-func newReassemblingReader(client *minio.Client, bucket string, parts []manifestPart) *reassemblingReader {
- return &reassemblingReader{client: client, bucket: bucket, parts: parts}
-}
-
-func (r *reassemblingReader) Read(p []byte) (int, error) {
- for {
- if r.current == nil {
- if r.index >= len(r.parts) {
- return 0, io.EOF
- }
-
- object, err := r.client.GetObject(context.TODO(), r.bucket, r.parts[r.index].Key, minio.GetObjectOptions{})
- if err != nil {
- return 0, fmt.Errorf("failed to open chunk %s: %w", r.parts[r.index].Key, err)
- }
-
- r.current = object
- r.hasher = sha256.New()
- r.read = 0
- }
-
- n, err := r.current.Read(p)
- if n > 0 {
- r.hasher.Write(p[:n])
- r.read += int64(n)
- }
-
- if errors.Is(err, io.EOF) {
- if verifyErr := r.finishCurrentPart(); verifyErr != nil {
- return n, verifyErr
- }
-
- if n > 0 {
- return n, nil
- }
-
- continue
- }
-
- if err != nil {
- return n, fmt.Errorf("failed to read chunk %s: %w", r.parts[r.index].Key, err)
- }
-
- return n, nil
- }
-}
-
-func (r *reassemblingReader) Close() error {
- if r.current == nil {
- return nil
- }
-
- err := r.current.Close()
- r.current = nil
-
- return err
-}
-
-func (r *reassemblingReader) finishCurrentPart() error {
- part := r.parts[r.index]
-
- _ = r.current.Close()
- r.current = nil
-
- if r.read != part.Size {
- return fmt.Errorf("chunk %s size mismatch: manifest %d, read %d", part.Key, part.Size, r.read)
- }
-
- if checksum := hex.EncodeToString(r.hasher.Sum(nil)); checksum != part.SHA256 {
- return fmt.Errorf("chunk %s checksum mismatch", part.Key)
- }
-
- r.index++
-
- return nil
-}
diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.go
index 1b69c1e4..2de4fac7 100644
--- a/backend/internal/features/storages/models/s3/model_test.go
+++ b/backend/internal/features/storages/models/s3/model_test.go
@@ -45,7 +45,7 @@ func Test_DeleteFile_LegacySingleObjectWithoutManifest_RemovesObject(t *testing.
assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
- _, err := storage.GetFile(encryptor, fileName)
+ _, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
assert.Error(t, err)
}
@@ -120,7 +120,7 @@ func Test_GetFile_ChunkedBackupMissingPart_ReturnsError(t *testing.T) {
rawClient.RemoveObject(t.Context(), storage.S3Bucket, fileName+".part000002", minio.RemoveObjectOptions{}),
)
- reader, err := storage.GetFile(encryptor, fileName)
+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
require.NoError(t, err)
defer func() { _ = reader.Close() }()
@@ -246,7 +246,7 @@ func Test_GetFile_ChunkedBackupCorruptedPart_ReturnsChecksumError(t *testing.T)
corruptedSameSize := bytes.Repeat([]byte{0xAB}, 10*1024*1024)
putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", corruptedSameSize)
- reader, err := storage.GetFile(encryptor, fileName)
+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
require.NoError(t, err)
defer func() { _ = reader.Close() }()
@@ -269,7 +269,7 @@ func Test_GetFile_ChunkedBackupTruncatedPart_ReturnsSizeError(t *testing.T) {
// length check must reject it before its bytes reach the restore.
putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", generateBytes(4*1024*1024))
- reader, err := storage.GetFile(encryptor, fileName)
+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
require.NoError(t, err)
defer func() { _ = reader.Close() }()
@@ -292,7 +292,7 @@ func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {
// single-object path and silently read nothing.
putRawObject(t, rawClient, storage.S3Bucket, fileName+manifestSuffix, []byte("not a valid manifest{"))
- _, err := storage.GetFile(encryptor, fileName)
+ _, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
require.Error(t, err)
assert.Contains(t, err.Error(), "manifest")
}
@@ -356,7 +356,7 @@ func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data [
func readWholeFile(t *testing.T, storage *S3Storage, encryptor encryption.FieldEncryptor, fileName string) []byte {
t.Helper()
- reader, err := storage.GetFile(encryptor, fileName)
+ reader, err := storage.GetFile(t.Context(), encryptor, discardLogger(), fileName)
require.NoError(t, err)
defer func() { _ = reader.Close() }()
diff --git a/backend/internal/features/storages/models/s3/reassembling_reader.go b/backend/internal/features/storages/models/s3/reassembling_reader.go
new file mode 100644
index 00000000..93bbc9e1
--- /dev/null
+++ b/backend/internal/features/storages/models/s3/reassembling_reader.go
@@ -0,0 +1,421 @@
+package s3_storage
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "hash"
+ "io"
+ "log/slog"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+)
+
+const (
+ // Once headers have arrived the transport's ResponseHeaderTimeout no longer applies, so a backend
+ // that accepts the GET and then goes silent would block until the OS TCP timeout. Armed around
+ // each blocking body read only: the gap between a consumer's reads is its own business, and a
+ // restore target that pauses on a slow statement must not lose its connection over it.
+ s3ReadStallTimeout = 60 * time.Second
+
+ s3MaxReadAttemptsWithoutProgress = 10
+ s3ReadRetryBaseDelay = 1 * time.Second
+ s3ReadRetryMaxDelay = 30 * time.Second
+)
+
+type reassemblingReaderSpec struct {
+ streamCtx context.Context
+ logger *slog.Logger
+ coreClient *minio.Core
+ bucket string
+ parts []objectSpan
+
+ // Files stored before chunking, and any file small enough to fit one object, have no manifest
+ // and therefore no recorded sha256 to compare against; their length comes from StatObject.
+ hasManifestChecksum bool
+
+ stallTimeout time.Duration
+ maxAttemptsWithoutProgress int
+ retryBaseDelay time.Duration
+}
+
+// reassemblingReader streams a chunked backup back as one byte stream, holding at most one part
+// connection at a time to preserve the bounded-memory streaming of the upload path (ADR-0004).
+//
+// A dropped or stalled connection is retried in place, resuming the current part from the last byte
+// handed out via a Range request. It has to recover here rather than downstream: the bytes are an
+// encrypted stream with no end-of-stream marker and the decryptor is stateful, so a caller that sees
+// a short read cannot resynchronise.
+type reassemblingReader struct {
+ streamCtx context.Context
+ logger *slog.Logger
+ coreClient *minio.Core
+ bucket string
+ parts []objectSpan
+
+ hasManifestChecksum bool
+ stallTimeout time.Duration
+ maxAttemptsWithoutProgress int
+ retryBaseDelay time.Duration
+
+ isClosed bool
+ partIndex int
+ partBody io.ReadCloser
+ partHasher hash.Hash
+ attemptCancel context.CancelFunc
+ stallTimer *time.Timer
+ partReadBytes int64
+ attemptReadBytes int64
+ attemptsSinceProgress int
+}
+
+func newReassemblingReader(spec reassemblingReaderSpec) *reassemblingReader {
+ if spec.stallTimeout == 0 {
+ spec.stallTimeout = s3ReadStallTimeout
+ }
+
+ if spec.maxAttemptsWithoutProgress == 0 {
+ spec.maxAttemptsWithoutProgress = s3MaxReadAttemptsWithoutProgress
+ }
+
+ if spec.retryBaseDelay == 0 {
+ spec.retryBaseDelay = s3ReadRetryBaseDelay
+ }
+
+ // The retry path is the only thing that logs, so a nil logger would panic exactly during an
+ // incident and nowhere else.
+ if spec.logger == nil {
+ spec.logger = slog.New(slog.DiscardHandler)
+ }
+
+ return &reassemblingReader{
+ streamCtx: spec.streamCtx,
+ logger: spec.logger,
+ coreClient: spec.coreClient,
+ bucket: spec.bucket,
+ parts: spec.parts,
+ hasManifestChecksum: spec.hasManifestChecksum,
+ stallTimeout: spec.stallTimeout,
+ maxAttemptsWithoutProgress: spec.maxAttemptsWithoutProgress,
+ retryBaseDelay: spec.retryBaseDelay,
+ partHasher: sha256.New(),
+ }
+}
+
+func (r *reassemblingReader) Read(p []byte) (int, error) {
+ for {
+ if r.isClosed {
+ return 0, ErrReaderClosed
+ }
+
+ if err := r.streamCtx.Err(); err != nil {
+ return 0, err
+ }
+
+ if r.partIndex >= len(r.parts) {
+ return 0, io.EOF
+ }
+
+ if r.partBody == nil {
+ if r.partReadBytes >= r.parts[r.partIndex].Size {
+ if err := r.finishCurrentPart(); err != nil {
+ return 0, err
+ }
+
+ continue
+ }
+
+ if err := r.openCurrentPart(); err != nil {
+ if retryErr := r.resumeAfterFailure(err); retryErr != nil {
+ return 0, retryErr
+ }
+
+ continue
+ }
+ }
+
+ r.stallTimer.Reset(r.stallTimeout)
+ n, err := r.partBody.Read(p)
+ r.stallTimer.Stop()
+
+ if n > 0 {
+ if r.hasManifestChecksum {
+ r.partHasher.Write(p[:n])
+ }
+
+ r.partReadBytes += int64(n)
+ r.attemptReadBytes += int64(n)
+ }
+
+ // A body that ends before the length the response promised is a truncated transfer, not the
+ // end of the part: net/http reports that as ErrUnexpectedEOF, but a chunked-encoded response
+ // can deliver it as a plain EOF, so trust the byte count rather than the error.
+ isPartComplete := r.partReadBytes >= r.parts[r.partIndex].Size
+ if errors.Is(err, io.EOF) && isPartComplete {
+ if finishErr := r.finishCurrentPart(); finishErr != nil {
+ return n, finishErr
+ }
+
+ if n > 0 {
+ return n, nil
+ }
+
+ continue
+ }
+
+ if err != nil {
+ readErr := fmt.Errorf("failed to read chunk %s: %w", r.parts[r.partIndex].Key, err)
+ if retryErr := r.resumeAfterFailure(readErr); retryErr != nil {
+ return n, retryErr
+ }
+
+ if n > 0 {
+ return n, nil
+ }
+
+ continue
+ }
+
+ return n, nil
+ }
+}
+
+func (r *reassemblingReader) Close() error {
+ r.isClosed = true
+
+ r.abandonAttempt()
+
+ return nil
+}
+
+func (r *reassemblingReader) openCurrentPart() error {
+ part := r.parts[r.partIndex]
+
+ attemptCtx, cancelAttempt := context.WithCancel(r.streamCtx)
+
+ options := minio.GetObjectOptions{}
+
+ if r.partReadBytes > 0 {
+ if err := options.SetRange(r.partReadBytes, part.Size-1); err != nil {
+ cancelAttempt()
+
+ return fmt.Errorf("%w: chunk %s at offset %d: %w", ErrRangeNotHonoured, part.Key, r.partReadBytes, err)
+ }
+ }
+
+ body, objectInfo, responseHeader, err := r.coreClient.GetObject(attemptCtx, r.bucket, part.Key, options)
+ if err != nil {
+ cancelAttempt()
+
+ return fmt.Errorf("failed to open chunk %s: %w", part.Key, err)
+ }
+
+ if err := r.verifyResponseSpan(objectInfo, responseHeader); err != nil {
+ cancelAttempt()
+ _ = body.Close()
+
+ return err
+ }
+
+ r.partBody = body
+ r.attemptCancel = cancelAttempt
+ r.stallTimer = time.AfterFunc(r.stallTimeout, cancelAttempt)
+ r.stallTimer.Stop()
+
+ return nil
+}
+
+// A resumed read must not begin before the offset already handed out: replaying bytes into a
+// stateful decryptor corrupts the stream. Content-Range is the positive proof the backend applied the
+// Range; failing that, a body whose length equals the remainder is accepted, and anything else is
+// refused because an unverifiable resume cannot be told apart from a whole-object reply.
+func (r *reassemblingReader) verifyResponseSpan(
+ objectInfo minio.ObjectInfo,
+ responseHeader http.Header,
+) error {
+ part := r.parts[r.partIndex]
+ expectedBytes := part.Size - r.partReadBytes
+
+ if r.partReadBytes > 0 {
+ if rangeStart, isParsed := rangeStartOfContentRange(responseHeader.Get("Content-Range")); isParsed {
+ if rangeStart != r.partReadBytes {
+ return fmt.Errorf(
+ "%w: chunk %s resumed at offset %d but the response starts at %d",
+ ErrRangeNotHonoured, part.Key, r.partReadBytes, rangeStart,
+ )
+ }
+
+ return nil
+ }
+
+ if objectInfo.Size != expectedBytes {
+ return fmt.Errorf(
+ "%w: chunk %s resumed at offset %d answered %d bytes, expected %d",
+ ErrRangeNotHonoured, part.Key, r.partReadBytes, objectInfo.Size, expectedBytes,
+ )
+ }
+
+ return nil
+ }
+
+ // A response sent with chunked transfer-encoding carries no Content-Length, which minio-go
+ // reports as size -1. That is not a short object: the part's byte count and sha256 are still
+ // verified when it ends, so refusing here would break backends that never declare a length.
+ if objectInfo.Size >= 0 && objectInfo.Size != expectedBytes {
+ return fmt.Errorf(
+ "%w: chunk %s is %d bytes, expected %d",
+ ErrChunkSizeMismatch, part.Key, objectInfo.Size, expectedBytes,
+ )
+ }
+
+ return nil
+}
+
+func rangeStartOfContentRange(contentRange string) (int64, bool) {
+ span, isBytesUnit := strings.CutPrefix(contentRange, "bytes ")
+ if !isBytesUnit {
+ return 0, false
+ }
+
+ start, _, isSplit := strings.Cut(span, "-")
+ if !isSplit {
+ return 0, false
+ }
+
+ parsed, err := strconv.ParseInt(strings.TrimSpace(start), 10, 64)
+ if err != nil {
+ return 0, false
+ }
+
+ return parsed, true
+}
+
+func (r *reassemblingReader) finishCurrentPart() error {
+ part := r.parts[r.partIndex]
+
+ r.abandonAttempt()
+
+ if r.partReadBytes != part.Size {
+ return fmt.Errorf(
+ "%w: chunk %s read %d bytes, expected %d",
+ ErrChunkSizeMismatch, part.Key, r.partReadBytes, part.Size,
+ )
+ }
+
+ if r.hasManifestChecksum {
+ if checksum := hex.EncodeToString(r.partHasher.Sum(nil)); checksum != part.SHA256 {
+ return fmt.Errorf("%w: chunk %s", ErrChunkChecksumMismatch, part.Key)
+ }
+ }
+
+ r.partIndex++
+ r.partReadBytes = 0
+ r.attemptsSinceProgress = 0
+ r.partHasher = sha256.New()
+
+ return nil
+}
+
+// The budget counts attempts since the last byte delivered, not attempts per part: a multi-hour
+// restore with sparse drops must not exhaust it.
+func (r *reassemblingReader) resumeAfterFailure(cause error) error {
+ hasDeliveredBytes := r.attemptReadBytes > 0
+ partKey := r.parts[r.partIndex].Key
+
+ r.abandonAttempt()
+
+ if err := r.streamCtx.Err(); err != nil {
+ return errors.Join(cause, err)
+ }
+
+ if !isRetryableReadError(cause) {
+ return cause
+ }
+
+ if hasDeliveredBytes {
+ r.attemptsSinceProgress = 1
+ } else {
+ r.attemptsSinceProgress++
+ }
+
+ if r.attemptsSinceProgress > r.maxAttemptsWithoutProgress {
+ return fmt.Errorf(
+ "%w: chunk %s stuck at offset %d: %w",
+ ErrReadRetryBudgetExhausted, partKey, r.partReadBytes, cause,
+ )
+ }
+
+ r.logger.Warn(
+ fmt.Sprintf(
+ "s3 chunk read failed, resuming from offset %d (attempt %d of %d)",
+ r.partReadBytes, r.attemptsSinceProgress, r.maxAttemptsWithoutProgress,
+ ),
+ "part_key", partKey,
+ "error", cause,
+ )
+
+ if hasDeliveredBytes {
+ return nil
+ }
+
+ return r.waitBeforeRetry()
+}
+
+func (r *reassemblingReader) waitBeforeRetry() error {
+ delay := s3ReadRetryMaxDelay
+ if shift := r.attemptsSinceProgress - 1; shift < 32 {
+ delay = min(r.retryBaseDelay<<shift, s3ReadRetryMaxDelay)
+ }
+
+ retryTimer := time.NewTimer(delay)
+ defer retryTimer.Stop()
+
+ select {
+ case <-r.streamCtx.Done():
+ return r.streamCtx.Err()
+ case <-retryTimer.C:
+ return nil
+ }
+}
+
+func (r *reassemblingReader) abandonAttempt() {
+ r.attemptReadBytes = 0
+
+ if r.stallTimer != nil {
+ r.stallTimer.Stop()
+ r.stallTimer = nil
+ }
+
+ if r.partBody != nil {
+ _ = r.partBody.Close()
+ r.partBody = nil
+ }
+
+ if r.attemptCancel != nil {
+ r.attemptCancel()
+ r.attemptCancel = nil
+ }
+}
+
+func isRetryableReadError(err error) bool {
+ if errors.Is(err, ErrRangeNotHonoured) ||
+ errors.Is(err, ErrChunkSizeMismatch) ||
+ errors.Is(err, ErrChunkChecksumMismatch) {
+ return false
+ }
+
+ var errorResponse minio.ErrorResponse
+ if errors.As(err, &errorResponse) {
+ return errorResponse.StatusCode == http.StatusRequestTimeout ||
+ errorResponse.StatusCode == http.StatusTooManyRequests ||
+ errorResponse.StatusCode >= http.StatusInternalServerError
+ }
+
+ return true
+}
diff --git a/backend/internal/features/storages/models/s3/reassembling_reader_test.go b/backend/internal/features/storages/models/s3/reassembling_reader_test.go
new file mode 100644
index 00000000..58523b51
--- /dev/null
+++ b/backend/internal/features/storages/models/s3/reassembling_reader_test.go
@@ -0,0 +1,642 @@
+package s3_storage
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func Test_ReadChunkedStream_ConnectionDropsMidPart_ResumesFromOffsetAndReturnsFullStream(t *testing.T) {
+ firstPart := generateBytes(8192)
+ secondPart := generateBytes(4096)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": firstPart, "file.part000002": secondPart},
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001", "file.part000002"),
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ assert.Equal(t, append(append([]byte{}, firstPart...), secondPart...), reassembledBytes)
+ assert.Equal(
+ t,
+ []string{"", "bytes=3000-8191", ""},
+ transport.observedRanges,
+ "the retry must resume the first part from the last byte handed out, then read the second part whole",
+ )
+}
+
+func Test_ReadChunkedStream_ServerIgnoresRange_ReturnsErrorWithoutCorruptingStream(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ isRangeIgnored: true,
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ deliveredBytes, err := io.ReadAll(reader)
+
+ require.ErrorIs(t, err, ErrRangeNotHonoured)
+ assert.Equal(
+ t,
+ partBytes[:3000],
+ deliveredBytes,
+ "only the bytes read before the drop may reach the caller; the ignored range must not replay the part from zero",
+ )
+}
+
+func Test_ReadChunkedStream_BackendDeclaresNoContentLength_StreamsAnyway(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ // A chunked transfer-encoded reply carries no Content-Length, which minio-go reports as size -1.
+ // Refusing that would lock out every backend that never declares a length.
+ transport := &stubS3Transport{objects: map[string][]byte{"file.part000001": partBytes}}
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ assert.Equal(t, partBytes, reassembledBytes)
+}
+
+func Test_ReadChunkedStream_ResumeWithoutContentLength_AcceptsContentRangeAsProof(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isContentRangeSent: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err, "Content-Range proves the resume was honoured even with no declared length")
+
+ assert.Equal(t, partBytes, reassembledBytes)
+ assert.Equal(t, []string{"", "bytes=3000-8191"}, transport.observedRanges)
+}
+
+func Test_ReadChunkedStream_ResumeAnsweredFromTheStartOfTheObject_ReturnsError(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ // The backend reports a range it did not apply: Content-Range starts at 0, not at the offset.
+ transport := &stubS3Transport{
+ isContentRangeSent: true,
+ isRangeIgnored: true,
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ deliveredBytes, err := io.ReadAll(reader)
+
+ require.ErrorIs(t, err, ErrRangeNotHonoured)
+ assert.Equal(t, partBytes[:3000], deliveredBytes)
+}
+
+func Test_ReadChunkedStream_PartMissing_DoesNotRetry(t *testing.T) {
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},
+ statusForAttempt: func(int) int { return http.StatusNotFound },
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ _, err := io.ReadAll(reader)
+
+ require.Error(t, err)
+ assert.Equal(t, 1, transport.attempts, "a missing chunk is permanent; retrying it only delays the failure")
+}
+
+func Test_ReadChunkedStream_BodyStalls_CancelsAttemptAndResumes(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: func(attempt int, payload []byte, request *http.Request) io.ReadCloser {
+ if attempt == 1 {
+ return &stallingBody{requestCtx: request.Context()}
+ }
+
+ return wholeBody(payload, request)
+ },
+ }
+
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ stallTimeout: 50 * time.Millisecond,
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ assert.Equal(t, partBytes, reassembledBytes)
+ assert.Equal(t, 2, transport.attempts)
+}
+
+func Test_ReadChunkedStream_ConsumerPausesLongerThanStallTimeout_KeepsTheSameConnection(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ stallTimeout: 30 * time.Millisecond,
+ })
+
+ // A restore target blocked on a slow statement stops reading for a while; that is backpressure,
+ // not a stalled connection, and must not cost the stream its part.
+ reassembledBytes := make([]byte, 0, len(partBytes))
+ buffer := make([]byte, 4096)
+
+ for len(reassembledBytes) < len(partBytes) {
+ time.Sleep(60 * time.Millisecond)
+
+ n, err := reader.Read(buffer)
+ require.NoError(t, err)
+
+ reassembledBytes = append(reassembledBytes, buffer[:n]...)
+ }
+
+ assert.Equal(t, partBytes, reassembledBytes)
+ assert.Equal(t, 1, transport.attempts, "a slow consumer must not trigger a re-open")
+}
+
+func Test_ReadChunkedStream_RepeatedDropsWithProgress_CompletesStream(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {
+ return &connectionBody{
+ payload: payload,
+ breakAfterBytes: 1000,
+ requestCtx: request.Context(),
+ }
+ },
+ }
+
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ maxAttemptsWithoutProgress: 2,
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(
+ t,
+ err,
+ "every attempt delivered bytes, so a budget of 2 must never be spent even across 9 drops",
+ )
+
+ assert.Equal(t, partBytes, reassembledBytes)
+ assert.Equal(t, 9, transport.attempts)
+}
+
+func Test_ReadChunkedStream_BodyEndsShortWithPlainEOF_ResumesInsteadOfFailing(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: func(attempt int, payload []byte, request *http.Request) io.ReadCloser {
+ if attempt == 1 {
+ return &connectionBody{
+ payload: payload[:3000],
+ breakAfterBytes: 3000,
+ requestCtx: request.Context(),
+ }
+ }
+
+ return wholeBody(payload, request)
+ },
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err, "a truncated transfer must resume even when it arrives as a plain EOF")
+
+ assert.Equal(t, partBytes, reassembledBytes)
+ assert.Equal(t, []string{"", "bytes=3000-8191"}, transport.observedRanges)
+}
+
+func Test_ReadChunkedStream_RetryBudgetExhausted_ReturnsError(t *testing.T) {
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},
+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {
+ return &connectionBody{payload: payload, requestCtx: request.Context()}
+ },
+ }
+
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ maxAttemptsWithoutProgress: 2,
+ })
+
+ _, err := io.ReadAll(reader)
+
+ require.ErrorIs(t, err, ErrReadRetryBudgetExhausted)
+ assert.Equal(t, 3, transport.attempts, "the first attempt plus the two retries the budget allows")
+}
+
+func Test_ReadChunkedStream_ReopenKeepsFailingAfterAProductiveAttempt_ExhaustsBudget(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ // 501 is retryable to us but not to minio-go, so the re-open fails without its internal retries.
+ statusForAttempt: func(attempt int) int {
+ if attempt == 1 {
+ return 0
+ }
+
+ return http.StatusNotImplemented
+ },
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 1000}),
+ }
+
+ // A regression that spins without spending the budget trips this deadline instead of hanging.
+ streamCtx, cancelStream := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancelStream()
+
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ streamCtx: streamCtx,
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ maxAttemptsWithoutProgress: 2,
+ })
+
+ _, err := io.ReadAll(reader)
+
+ require.ErrorIs(
+ t,
+ err,
+ ErrReadRetryBudgetExhausted,
+ "bytes delivered by an earlier attempt must not keep crediting later re-open failures with progress",
+ )
+}
+
+func Test_ReadChunkedStream_StreamContextCancelled_ReturnsPromptlyWithoutRetry(t *testing.T) {
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": generateBytes(8192)},
+ bodyForAttempt: func(_ int, payload []byte, request *http.Request) io.ReadCloser {
+ return &connectionBody{payload: payload, requestCtx: request.Context()}
+ },
+ }
+
+ streamCtx, cancelStream := context.WithCancel(t.Context())
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ streamCtx: streamCtx,
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ cancelStream()
+
+ _, err := io.ReadAll(reader)
+
+ require.ErrorIs(t, err, context.Canceled)
+ assert.Zero(t, transport.attempts, "a cancelled restore must not open another chunk")
+}
+
+func Test_ReadChunkedStream_FileStoredWithoutAManifest_SkipsChecksumVerification(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"legacy-object": partBytes},
+ }
+
+ // A file stored before chunking has no manifest and therefore no recorded sha256, so the span
+ // carries none either; the deliberately wrong checksum here must simply never be consulted.
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ parts: []objectSpan{{
+ Key: "legacy-object",
+ Size: int64(len(partBytes)),
+ SHA256: "0000000000000000000000000000000000000000000000000000000000000000",
+ }},
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ assert.Equal(t, partBytes, reassembledBytes)
+}
+
+func Test_ReadChunkedStream_ReadAfterClose_ReturnsClosedError(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ }
+ reader := newTestReassemblingReader(t, transport, reassemblingReaderSpec{
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ })
+
+ buffer := make([]byte, 4096)
+ _, err := reader.Read(buffer)
+ require.NoError(t, err)
+
+ require.NoError(t, reader.Close())
+
+ _, err = reader.Read(buffer)
+ assert.ErrorIs(t, err, ErrReaderClosed, "a closed reader must not silently re-open the chunk")
+}
+
+func Test_ReadChunkedStream_SpecWithoutALogger_RetriesWithoutPanicking(t *testing.T) {
+ partBytes := generateBytes(8192)
+
+ transport := &stubS3Transport{
+ isLengthDeclared: true,
+ objects: map[string][]byte{"file.part000001": partBytes},
+ bodyForAttempt: severBodyOnAttempts(severSpec{throughAttempt: 1, breakAfterBytes: 3000}),
+ }
+
+ reader := newReassemblingReader(reassemblingReaderSpec{
+ streamCtx: t.Context(),
+ coreClient: newStubbedCoreClient(t, transport),
+ bucket: "test-bucket",
+ hasManifestChecksum: true,
+ parts: objectSpansOf(transport.objects, "file.part000001"),
+ retryBaseDelay: time.Millisecond,
+ })
+
+ reassembledBytes, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ assert.Equal(t, partBytes, reassembledBytes)
+}
+
+func newTestReassemblingReader(
+ t *testing.T,
+ transport *stubS3Transport,
+ spec reassemblingReaderSpec,
+) *reassemblingReader {
+ t.Helper()
+
+ if spec.streamCtx == nil {
+ spec.streamCtx = t.Context()
+ }
+
+ spec.logger = discardLogger()
+ spec.coreClient = newStubbedCoreClient(t, transport)
+ spec.bucket = "test-bucket"
+ spec.retryBaseDelay = time.Millisecond
+
+ return newReassemblingReader(spec)
+}
+
+// Region is set explicitly so the client never issues a GetBucketLocation the stub would have to
+// answer, and path lookup keeps the request URL predictable.
+func newStubbedCoreClient(t *testing.T, transport *stubS3Transport) *minio.Core {
+ t.Helper()
+
+ core, err := minio.NewCore("s3.stub.invalid", &minio.Options{
+ Creds: credentials.NewStaticV4("access-key", "secret-key", ""),
+ Secure: false,
+ Region: "us-east-1",
+ BucketLookup: minio.BucketLookupPath,
+ Transport: transport,
+ })
+ require.NoError(t, err)
+
+ return core
+}
+
+func objectSpansOf(objects map[string][]byte, keys ...string) []objectSpan {
+ parts := make([]objectSpan, 0, len(keys))
+
+ for _, key := range keys {
+ checksum := sha256.Sum256(objects[key])
+ parts = append(parts, objectSpan{
+ Key: key,
+ Size: int64(len(objects[key])),
+ SHA256: hex.EncodeToString(checksum[:]),
+ })
+ }
+
+ return parts
+}
+
+// The ranges are recorded so a test can assert where a resumed read picked up.
+type stubS3Transport struct {
+ objects map[string][]byte
+ isRangeIgnored bool
+ isContentRangeSent bool
+ isLengthDeclared bool
+ statusForAttempt func(attempt int) int
+ bodyForAttempt func(attempt int, payload []byte, request *http.Request) io.ReadCloser
+
+ attempts int
+ observedRanges []string
+}
+
+func (transport *stubS3Transport) RoundTrip(request *http.Request) (*http.Response, error) {
+ transport.attempts++
+ requestedRange := request.Header.Get("Range")
+
+ transport.observedRanges = append(transport.observedRanges, requestedRange)
+
+ header := http.Header{}
+ header.Set("Last-Modified", time.Unix(0, 0).UTC().Format(http.TimeFormat))
+ header.Set("ETag", `"stub-etag"`)
+
+ respondWithStatus := func(status int) *http.Response {
+ header.Set("Content-Length", "0")
+
+ return &http.Response{
+ StatusCode: status,
+ Header: header,
+ Body: io.NopCloser(strings.NewReader("")),
+ Request: request,
+ }
+ }
+
+ if failureStatus := transport.GetFailureStatus(transport.attempts); failureStatus != 0 {
+ return respondWithStatus(failureStatus), nil
+ }
+
+ objectKey := strings.TrimPrefix(request.URL.Path, "/test-bucket/")
+ payload := transport.objects[objectKey]
+
+ responseStatus := http.StatusOK
+
+ if requestedRange != "" && !transport.isRangeIgnored {
+ rangeStart, isParsed := rangeStartOf(requestedRange)
+ if !isParsed {
+ return respondWithStatus(http.StatusRequestedRangeNotSatisfiable), nil
+ }
+
+ payload = payload[rangeStart:]
+ responseStatus = http.StatusPartialContent
+
+ if transport.isContentRangeSent {
+ header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d",
+ rangeStart, rangeStart+int64(len(payload))-1, len(transport.objects[objectKey])))
+ }
+ }
+
+ contentLength := int64(len(payload))
+
+ if transport.isLengthDeclared {
+ header.Set("Content-Length", strconv.FormatInt(contentLength, 10))
+ } else {
+ contentLength = -1
+ }
+
+ body := wholeBody(payload, request)
+ if transport.bodyForAttempt != nil {
+ body = transport.bodyForAttempt(transport.attempts, payload, request)
+ }
+
+ return &http.Response{
+ StatusCode: responseStatus,
+ Header: header,
+ Body: body,
+ ContentLength: contentLength,
+ Request: request,
+ }, nil
+}
+
+func (transport *stubS3Transport) GetFailureStatus(attempt int) int {
+ if transport.statusForAttempt == nil {
+ return 0
+ }
+
+ return transport.statusForAttempt(attempt)
+}
+
+func rangeStartOf(requestedRange string) (int64, bool) {
+ span := strings.TrimPrefix(requestedRange, "bytes=")
+ start, _, _ := strings.Cut(span, "-")
+
+ parsed, err := strconv.ParseInt(start, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+
+ return parsed, true
+}
+
+type severSpec struct {
+ throughAttempt int
+ breakAfterBytes int
+}
+
+func severBodyOnAttempts(spec severSpec) func(int, []byte, *http.Request) io.ReadCloser {
+ return func(attempt int, payload []byte, request *http.Request) io.ReadCloser {
+ if attempt > spec.throughAttempt {
+ return wholeBody(payload, request)
+ }
+
+ return &connectionBody{
+ payload: payload,
+ breakAfterBytes: spec.breakAfterBytes,
+ requestCtx: request.Context(),
+ }
+ }
+}
+
+func wholeBody(payload []byte, request *http.Request) io.ReadCloser {
+ return &connectionBody{
+ payload: payload,
+ breakAfterBytes: len(payload),
+ requestCtx: request.Context(),
+ }
+}
+
+type connectionBody struct {
+ payload []byte
+ breakAfterBytes int
+ position int
+ requestCtx context.Context
+}
+
+func (b *connectionBody) Read(p []byte) (int, error) {
+ if err := b.requestCtx.Err(); err != nil {
+ return 0, err
+ }
+
+ severAt := min(b.breakAfterBytes, len(b.payload))
+
+ if b.position >= severAt {
+ if severAt < len(b.payload) {
+ return 0, io.ErrUnexpectedEOF
+ }
+
+ return 0, io.EOF
+ }
+
+ n := copy(p, b.payload[b.position:severAt])
+ b.position += n
+
+ return n, nil
+}
+
+func (b *connectionBody) Close() error { return nil }
+
+// stallingBody never delivers a byte and unblocks only when the request context is cancelled, which
+// is what a real transport does to a pending read once the request is cancelled.
+type stallingBody struct {
+ requestCtx context.Context
+}
+
+func (b *stallingBody) Read(_ []byte) (int, error) {
+ <-b.requestCtx.Done()
+
+ return 0, b.requestCtx.Err()
+}
+
+func (b *stallingBody) Close() error { return nil }
diff --git a/backend/internal/features/storages/models/sftp/model.go b/backend/internal/features/storages/models/sftp/model.go
index 83b65f57..845a611c 100644
--- a/backend/internal/features/storages/models/sftp/model.go
+++ b/backend/internal/features/storages/models/sftp/model.go
@@ -131,7 +131,9 @@ func (s *SFTPStorage) SaveFile(
}
func (s *SFTPStorage) GetFile(
+ _ context.Context,
encryptor encryption.FieldEncryptor,
+ _ *slog.Logger,
fileName string,
) (io.ReadCloser, error) {
client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)
diff --git a/backend/internal/features/verification/runs/agent_controller.go b/backend/internal/features/verification/runs/agent_controller.go
index b4eda6d2..701d8a3f 100644
--- a/backend/internal/features/verification/runs/agent_controller.go
+++ b/backend/internal/features/verification/runs/agent_controller.go
@@ -94,7 +94,7 @@ func (c *VerificationAgentController) BackupStream(ctx *gin.Context) {
return
}
- reader, err := c.verificationService.GetBackupFile(agent, verificationID)
+ reader, err := c.verificationService.GetBackupFile(ctx.Request.Context(), agent, verificationID)
if err != nil {
ctx.JSON(http.StatusGone, gin.H{"error": err.Error(), "reason": "gone"})
return
diff --git a/backend/internal/features/verification/runs/service.go b/backend/internal/features/verification/runs/service.go
index c7b53195..ba65a0f0 100644
--- a/backend/internal/features/verification/runs/service.go
+++ b/backend/internal/features/verification/runs/service.go
@@ -1,6 +1,7 @@
package verification_runs
import (
+ "context"
"errors"
"fmt"
"io"
@@ -351,6 +352,7 @@ func (s *VerificationService) OnAgentHeartbeated(
}
func (s *VerificationService) GetBackupFile(
+ ctx context.Context,
agent *verification_agents.Agent,
verificationID uuid.UUID,
) (io.ReadCloser, error) {
@@ -371,7 +373,7 @@ func (s *VerificationService) GetBackupFile(
return nil, errors.New("verification is not owned by this agent")
}
- reader, _, _, err := s.backupService.GetBackupFileWithoutAuth(verification.BackupID)
+ reader, _, _, err := s.backupService.GetBackupFileWithoutAuth(ctx, verification.BackupID)
if err != nil {
return nil, err
}