Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (storages): Stream unlimited-size S3 backups via chunked objects (issue #617)

Rostislav Dugina107648beedf
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.go
index aaa69347..7384ab06 100644
--- a/backend/internal/features/storages/models/s3/model.go
+++ b/backend/internal/features/storages/models/s3/model.go
@@ -4,10 +4,14 @@ import (
"bytes"
"context"
"crypto/md5"
+ "crypto/sha256"
"crypto/tls"
"encoding/base64"
+ "encoding/hex"
+ "encoding/json"
"errors"
"fmt"
+ "hash"
"io"
"log/slog"
"net"
@@ -29,10 +33,22 @@ const (
s3TLSHandshakeTimeout = 30 * time.Second
s3DeleteTimeout = 30 * time.Second
- // Chunk size for multipart uploads - 16MB provides good balance between
- // memory usage and upload efficiency. This creates backpressure to pg_dump
- // by only reading one chunk at a time and waiting for S3 to confirm receipt.
- multipartChunkSize = 16 * 1024 * 1024
+ // defaultInnerPartSize is the multipart part we feed S3 for one object. 16 MiB balances
+ // memory against request count and creates backpressure to pg_dump: we keep one part in
+ // flight and wait for S3 to confirm it before reading the next, so pipeline memory does not
+ // grow with the database size (ADR-0004).
+ defaultInnerPartSize = 16 * 1024 * 1024
+
+ // defaultMaxPartsPerObject caps inner parts per object; once reached we roll to the next
+ // object. Effective object size = defaultMaxPartsPerObject * defaultInnerPartSize ~= 15.6 GiB.
+ // S3 multipart is bounded by part_size * part_count_limit and that limit is provider-set
+ // (AWS 10000, Scaleway and many S3-compatibles 1000). Because we stream dumps of unknown size,
+ // we write blind to the smallest common limit so one backup can span any provider without
+ // knowing its real cap; the object count, not memory, scales with the database size.
+ defaultMaxPartsPerObject = 1000
+
+ manifestSuffix = ".parts"
+ manifestVersion = 1
)
type S3Storage struct {
@@ -47,6 +63,12 @@ type S3Storage struct {
S3UseVirtualHostedStyle bool `json:"s3UseVirtualHostedStyle" gorm:"default:false;column:s3_use_virtual_hosted_style"`
SkipTLSVerify bool `json:"skipTLSVerify" gorm:"default:false;column:skip_tls_verify"`
S3StorageClass S3StorageClass `json:"s3StorageClass" gorm:"type:text;column:s3_storage_class;default:''"`
+
+ // Test seams: rolling at the production 15.6 GiB object boundary would need ~15 GiB per case,
+ // so integration tests shrink the inner part and per-object part count to exercise object
+ // rollover on a few tens of MiB. Unexported, so gorm and JSON ignore them; zero means default.
+ innerPartSizeOverride int
+ maxPartsOverride int
}
func (s *S3Storage) TableName() string {
@@ -66,123 +88,48 @@ func (s *S3Storage) SaveFile(
default:
}
- coreClient, err := s.getCoreClient(encryptor)
+ client, err := s.getClient(encryptor)
if err != nil {
return err
}
- objectKey := s.buildObjectKey(fileName)
+ baseKey := s.buildObjectKey(fileName)
+ innerPartSize := s.innerPartSize()
- uploadID, err := coreClient.NewMultipartUpload(
- ctx,
- s.S3Bucket,
- objectKey,
- s.putObjectOptions(),
- )
- if err != nil {
- return fmt.Errorf("failed to initiate multipart upload: %w", err)
+ // Look ahead by one inner part to choose the layout. A stream that fits in a single inner part
+ // (WAL segments, .metadata/.manifest sidecars, small dumps, empty files) is written as one
+ // plain object {fileName} with no manifest, exactly as legacy backups are. Anything larger is
+ // split across {fileName}.partNNNNNN objects described by a {fileName}.parts manifest.
+ firstPart := make([]byte, innerPartSize)
+ firstPartLen, readErr := io.ReadFull(file, firstPart)
+ if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
+ return fmt.Errorf("read error: %w", readErr)
}
- var parts []minio.CompletePart
- partNumber := 1
- buf := make([]byte, multipartChunkSize)
-
- for {
- select {
- case <-ctx.Done():
- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
- return fmt.Errorf("upload cancelled: %w", ctx.Err())
- default:
- }
-
- n, readErr := io.ReadFull(file, buf)
-
- if n == 0 && readErr == io.EOF {
- break
- }
-
- if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
- return fmt.Errorf("read error: %w", readErr)
- }
-
- partData := buf[:n]
- hash := md5.Sum(partData)
- md5Base64 := base64.StdEncoding.EncodeToString(hash[:])
-
- part, err := coreClient.PutObjectPart(
- ctx,
- s.S3Bucket,
- objectKey,
- uploadID,
- partNumber,
- bytes.NewReader(partData),
- int64(n),
- minio.PutObjectPartOptions{
- Md5Base64: md5Base64,
- },
- )
- if err != nil {
- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
-
- select {
- case <-ctx.Done():
- return fmt.Errorf("upload cancelled: %w", ctx.Err())
- default:
- return fmt.Errorf("failed to upload part %d: %w", partNumber, err)
- }
- }
-
- parts = append(parts, minio.CompletePart{
- PartNumber: partNumber,
- ETag: part.ETag,
- })
-
- partNumber++
-
- if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
- break
- }
+ if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
+ return s.uploadSingleObject(ctx, client, baseKey, firstPart[:firstPartLen])
}
- if len(parts) == 0 {
- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
-
- client, err := s.getClient(encryptor)
- if err != nil {
- return err
- }
- opts := s.putObjectOptions()
- opts.SendContentMd5 = true
+ // The first part filled exactly: peek one more byte to tell "stream is exactly one inner part"
+ // (still single object, e.g. a 16 MiB WAL segment) from "stream is larger" (chunked).
+ peek := make([]byte, 1)
+ peekLen, peekErr := io.ReadFull(file, peek)
+ if peekErr != nil && peekErr != io.EOF {
+ return fmt.Errorf("read error: %w", peekErr)
+ }
- _, err = client.PutObject(
- ctx,
- s.S3Bucket,
- objectKey,
- bytes.NewReader([]byte{}),
- 0,
- opts,
- )
- if err != nil {
- return fmt.Errorf("failed to upload empty file: %w", err)
- }
- return nil
+ if peekLen == 0 {
+ return s.uploadSingleObject(ctx, client, baseKey, firstPart)
}
- _, err = coreClient.CompleteMultipartUpload(
- ctx,
- s.S3Bucket,
- objectKey,
- uploadID,
- parts,
- s.putObjectOptions(),
- )
+ coreClient, err := s.getCoreClient(encryptor)
if err != nil {
- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
- return fmt.Errorf("failed to complete multipart upload: %w", err)
+ return err
}
- return nil
+ source := io.MultiReader(bytes.NewReader(firstPart), bytes.NewReader(peek[:peekLen]), file)
+
+ return s.uploadChunked(ctx, coreClient, client, baseKey, source)
}
func (s *S3Storage) GetFile(
@@ -194,12 +141,21 @@ func (s *S3Storage) GetFile(
return nil, err
}
- objectKey := s.buildObjectKey(fileName)
+ baseKey := s.buildObjectKey(fileName)
+
+ manifest, hasManifest, err := s.readManifest(context.TODO(), client, baseKey)
+ if err != nil {
+ return nil, err
+ }
+
+ if hasManifest {
+ return newReassemblingReader(client, s.S3Bucket, manifest.Parts), nil
+ }
object, err := client.GetObject(
context.TODO(),
s.S3Bucket,
- objectKey,
+ baseKey,
minio.GetObjectOptions{},
)
if err != nil {
@@ -230,15 +186,34 @@ func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName str
return err
}
- objectKey := s.buildObjectKey(fileName)
+ baseKey := s.buildObjectKey(fileName)
ctx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)
defer cancel()
+ manifest, hasManifest, err := s.readManifest(ctx, client, baseKey)
+ if err != nil {
+ return err
+ }
+
+ if hasManifest {
+ keys := make([]string, 0, len(manifest.Parts)+1)
+ for _, part := range manifest.Parts {
+ keys = append(keys, part.Key)
+ }
+ keys = append(keys, manifestObjectKey(baseKey))
+
+ if err := s.removeObjects(ctx, client, keys); err != nil {
+ return fmt.Errorf("failed to delete chunked backup from S3: %w", err)
+ }
+
+ return nil
+ }
+
err = client.RemoveObject(
ctx,
s.S3Bucket,
- objectKey,
+ baseKey,
minio.RemoveObjectOptions{},
)
if err != nil {
@@ -366,6 +341,208 @@ func (s *S3Storage) Update(incoming *S3Storage) {
// otherwise we will have to transfer all the data to the new prefix
}
+func (s *S3Storage) innerPartSize() int {
+ if s.innerPartSizeOverride > 0 {
+ return s.innerPartSizeOverride
+ }
+
+ return defaultInnerPartSize
+}
+
+func (s *S3Storage) maxPartsPerObject() int {
+ if s.maxPartsOverride > 0 {
+ return s.maxPartsOverride
+ }
+
+ return defaultMaxPartsPerObject
+}
+
+func (s *S3Storage) uploadSingleObject(
+ ctx context.Context,
+ client *minio.Client,
+ objectKey string,
+ data []byte,
+) error {
+ opts := s.putObjectOptions()
+ opts.SendContentMd5 = true
+
+ _, err := client.PutObject(ctx, s.S3Bucket, objectKey, bytes.NewReader(data), int64(len(data)), opts)
+ if err != nil {
+ return fmt.Errorf("failed to upload file to S3: %w", err)
+ }
+
+ return nil
+}
+
+func (s *S3Storage) uploadChunked(
+ ctx context.Context,
+ coreClient *minio.Core,
+ client *minio.Client,
+ baseKey string,
+ source io.Reader,
+) error {
+ innerPartSize := s.innerPartSize()
+ maxParts := s.maxPartsPerObject()
+ buf := make([]byte, innerPartSize)
+
+ var manifestParts []manifestPart
+ var uploadedKeys []string
+ var totalSize int64
+
+ // abandon undoes a failed chunked upload: it aborts the in-flight multipart (when uploadID is set)
+ // and deletes every already-completed part object. It uses a fresh context detached from ctx
+ // because the usual reason we land here is ctx cancellation, and cleanup issued on a cancelled
+ // context is a no-op — which would leave completed part objects with no manifest as dead weight in
+ // the bucket until retention or an overwrite reclaims them.
+ abandon := func(objectKey, uploadID string) {
+ cleanupCtx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)
+ defer cancel()
+
+ if uploadID != "" {
+ _ = coreClient.AbortMultipartUpload(cleanupCtx, s.S3Bucket, objectKey, uploadID)
+ }
+
+ if len(uploadedKeys) > 0 {
+ _ = s.removeObjects(cleanupCtx, client, uploadedKeys)
+ }
+ }
+
+ objectIndex := 1
+ sourceEnded := false
+
+ for !sourceEnded {
+ objectKey := partObjectKey(baseKey, objectIndex)
+
+ uploadID, err := coreClient.NewMultipartUpload(ctx, s.S3Bucket, objectKey, s.putObjectOptions())
+ if err != nil {
+ abandon("", "")
+ return fmt.Errorf("failed to initiate multipart upload for %s: %w", objectKey, err)
+ }
+
+ var parts []minio.CompletePart
+ objectHasher := sha256.New()
+ var objectSize int64
+
+ for partNumber := 1; partNumber <= maxParts; partNumber++ {
+ select {
+ case <-ctx.Done():
+ abandon(objectKey, uploadID)
+ return fmt.Errorf("upload cancelled: %w", ctx.Err())
+ default:
+ }
+
+ partLen, readErr := io.ReadFull(source, buf)
+ if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
+ abandon(objectKey, uploadID)
+ return fmt.Errorf("read error: %w", readErr)
+ }
+
+ if partLen == 0 && readErr == io.EOF {
+ sourceEnded = true
+ break
+ }
+
+ if err := s.putInnerPart(
+ ctx,
+ coreClient,
+ objectKey,
+ uploadID,
+ partNumber,
+ buf[:partLen],
+ objectHasher,
+ &parts,
+ ); err != nil {
+ abandon(objectKey, uploadID)
+ return err
+ }
+
+ objectSize += int64(partLen)
+
+ if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
+ sourceEnded = true
+ break
+ }
+ }
+
+ // The source ended exactly on the previous object's boundary, so this object got no data.
+ if len(parts) == 0 {
+ _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
+ break
+ }
+
+ if _, err := coreClient.CompleteMultipartUpload(
+ ctx,
+ s.S3Bucket,
+ objectKey,
+ uploadID,
+ parts,
+ s.putObjectOptions(),
+ ); err != nil {
+ abandon(objectKey, uploadID)
+ return fmt.Errorf("failed to complete multipart upload for %s: %w", objectKey, err)
+ }
+
+ uploadedKeys = append(uploadedKeys, objectKey)
+ manifestParts = append(manifestParts, manifestPart{
+ Key: objectKey,
+ Size: objectSize,
+ SHA256: hex.EncodeToString(objectHasher.Sum(nil)),
+ })
+ totalSize += objectSize
+ objectIndex++
+ }
+
+ manifest := chunkedManifest{
+ Version: manifestVersion,
+ TotalSize: totalSize,
+ Parts: manifestParts,
+ }
+
+ if err := s.writeManifest(ctx, client, baseKey, manifest); err != nil {
+ abandon("", "")
+ return err
+ }
+
+ return nil
+}
+
+func (s *S3Storage) putInnerPart(
+ ctx context.Context,
+ coreClient *minio.Core,
+ objectKey string,
+ uploadID string,
+ partNumber int,
+ partData []byte,
+ objectHasher hash.Hash,
+ parts *[]minio.CompletePart,
+) error {
+ partMd5 := md5.Sum(partData)
+ md5Base64 := base64.StdEncoding.EncodeToString(partMd5[:])
+
+ part, err := coreClient.PutObjectPart(
+ ctx,
+ s.S3Bucket,
+ objectKey,
+ uploadID,
+ partNumber,
+ bytes.NewReader(partData),
+ int64(len(partData)),
+ minio.PutObjectPartOptions{Md5Base64: md5Base64},
+ )
+ if err != nil {
+ if ctx.Err() != nil {
+ return fmt.Errorf("upload cancelled: %w", ctx.Err())
+ }
+
+ return fmt.Errorf("failed to upload part %d of %s: %w", partNumber, objectKey, err)
+ }
+
+ objectHasher.Write(partData)
+ *parts = append(*parts, minio.CompletePart{PartNumber: partNumber, ETag: part.ETag})
+
+ return nil
+}
+
func (s *S3Storage) putObjectOptions() minio.PutObjectOptions {
return minio.PutObjectOptions{
StorageClass: string(s.S3StorageClass),
@@ -477,3 +654,214 @@ func (s *S3Storage) getClientParams(
return endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, nil
}
+
+// chunkedManifest lists, in order, the objects a single logical file was split into, with each
+// object's size and sha256. It is written LAST, after every part object is durably committed.
+//
+// Why a manifest instead of just probing {fileName}.part1, .part2, ... until a key 404s:
+//
+// 1. The bytes we store are an encrypted stream with no end-of-stream marker (see
+// util/encryption). Probe-until-404 cannot tell a complete backup from one whose final part
+// failed to upload, was truncated, or was reaped by lifecycle rules: it would stop at the gap
+// and hand the restore a silently short, corrupt stream. TotalSize plus the per-part list and
+// sha256 let GetFile reject a missing/short/corrupted part BEFORE the restore starts, turning
+// silent data loss into a hard error.
+// 2. Because the manifest is written last, its presence is the atomic "backup is complete" marker.
+// A crash mid-upload leaves part objects but no manifest, which GetFile treats as a hard error
+// (it finds neither manifest nor single object) rather than a valid N-part backup.
+// 3. It immunizes reads against S3 eventual consistency: a transient 404 on a middle part can no
+// longer be misread as the end of the stream.
+type chunkedManifest struct {
+ Version int `json:"version"`
+ TotalSize int64 `json:"totalSize"`
+ Parts []manifestPart `json:"parts"`
+}
+
+type manifestPart struct {
+ Key string `json:"key"`
+ Size int64 `json:"size"`
+ SHA256 string `json:"sha256"`
+}
+
+func partObjectKey(baseKey string, objectIndex int) string {
+ return fmt.Sprintf("%s.part%06d", baseKey, objectIndex)
+}
+
+func manifestObjectKey(baseKey string) string {
+ return baseKey + manifestSuffix
+}
+
+func (s *S3Storage) writeManifest(
+ ctx context.Context,
+ client *minio.Client,
+ baseKey string,
+ manifest chunkedManifest,
+) error {
+ body, err := json.Marshal(manifest)
+ if err != nil {
+ return fmt.Errorf("failed to marshal chunk manifest: %w", err)
+ }
+
+ opts := s.putObjectOptions()
+ opts.SendContentMd5 = true
+ opts.ContentType = "application/json"
+
+ _, err = client.PutObject(
+ ctx,
+ s.S3Bucket,
+ manifestObjectKey(baseKey),
+ bytes.NewReader(body),
+ int64(len(body)),
+ opts,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to write chunk manifest: %w", err)
+ }
+
+ return nil
+}
+
+func (s *S3Storage) readManifest(
+ ctx context.Context,
+ client *minio.Client,
+ baseKey string,
+) (chunkedManifest, bool, error) {
+ var manifest chunkedManifest
+
+ manifestKey := manifestObjectKey(baseKey)
+
+ _, err := client.StatObject(ctx, s.S3Bucket, manifestKey, minio.StatObjectOptions{})
+ if err != nil {
+ if minio.ToErrorResponse(err).StatusCode == http.StatusNotFound {
+ return manifest, false, nil
+ }
+
+ return manifest, false, fmt.Errorf("failed to stat chunk manifest: %w", err)
+ }
+
+ object, err := client.GetObject(ctx, s.S3Bucket, manifestKey, minio.GetObjectOptions{})
+ if err != nil {
+ return manifest, false, fmt.Errorf("failed to read chunk manifest: %w", err)
+ }
+ defer func() { _ = object.Close() }()
+
+ body, err := io.ReadAll(object)
+ if err != nil {
+ return manifest, false, fmt.Errorf("failed to read chunk manifest: %w", err)
+ }
+
+ if err := json.Unmarshal(body, &manifest); err != nil {
+ return manifest, false, fmt.Errorf("failed to parse chunk manifest: %w", err)
+ }
+
+ return manifest, true, nil
+}
+
+func (s *S3Storage) removeObjects(ctx context.Context, client *minio.Client, keys []string) error {
+ objectsCh := make(chan minio.ObjectInfo, len(keys))
+ for _, key := range keys {
+ objectsCh <- minio.ObjectInfo{Key: key}
+ }
+ close(objectsCh)
+
+ var firstErr error
+ for removeResult := range client.RemoveObjects(ctx, s.S3Bucket, objectsCh, minio.RemoveObjectsOptions{}) {
+ if removeResult.Err != nil && firstErr == nil {
+ firstErr = removeResult.Err
+ }
+ }
+
+ return firstErr
+}
+
+// reassemblingReader streams a chunked backup back as one byte stream by opening each part object
+// in manifest order, reading it to EOF and only then opening the next. It holds at most one part
+// connection at a time, preserving the bounded-memory streaming of the upload path (ADR-0004), and
+// verifies each part's length and sha256 against the manifest so a missing, truncated, or corrupted
+// part surfaces as a read error instead of a silently short stream.
+type reassemblingReader struct {
+ client *minio.Client
+ bucket string
+ parts []manifestPart
+ index int
+ current *minio.Object
+ hasher hash.Hash
+ read int64
+}
+
+func newReassemblingReader(client *minio.Client, bucket string, parts []manifestPart) *reassemblingReader {
+ return &reassemblingReader{client: client, bucket: bucket, parts: parts}
+}
+
+func (r *reassemblingReader) Read(p []byte) (int, error) {
+ for {
+ if r.current == nil {
+ if r.index >= len(r.parts) {
+ return 0, io.EOF
+ }
+
+ object, err := r.client.GetObject(context.TODO(), r.bucket, r.parts[r.index].Key, minio.GetObjectOptions{})
+ if err != nil {
+ return 0, fmt.Errorf("failed to open chunk %s: %w", r.parts[r.index].Key, err)
+ }
+
+ r.current = object
+ r.hasher = sha256.New()
+ r.read = 0
+ }
+
+ n, err := r.current.Read(p)
+ if n > 0 {
+ r.hasher.Write(p[:n])
+ r.read += int64(n)
+ }
+
+ if errors.Is(err, io.EOF) {
+ if verifyErr := r.finishCurrentPart(); verifyErr != nil {
+ return n, verifyErr
+ }
+
+ if n > 0 {
+ return n, nil
+ }
+
+ continue
+ }
+
+ if err != nil {
+ return n, fmt.Errorf("failed to read chunk %s: %w", r.parts[r.index].Key, err)
+ }
+
+ return n, nil
+ }
+}
+
+func (r *reassemblingReader) Close() error {
+ if r.current == nil {
+ return nil
+ }
+
+ err := r.current.Close()
+ r.current = nil
+
+ return err
+}
+
+func (r *reassemblingReader) finishCurrentPart() error {
+ part := r.parts[r.index]
+
+ _ = r.current.Close()
+ r.current = nil
+
+ if r.read != part.Size {
+ return fmt.Errorf("chunk %s size mismatch: manifest %d, read %d", part.Key, part.Size, r.read)
+ }
+
+ if checksum := hex.EncodeToString(r.hasher.Sum(nil)); checksum != part.SHA256 {
+ return fmt.Errorf("chunk %s checksum mismatch", part.Key)
+ }
+
+ r.index++
+
+ return nil
+}
diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.go
new file mode 100644
index 00000000..1b69c1e4
--- /dev/null
+++ b/backend/internal/features/storages/models/s3/model_test.go
@@ -0,0 +1,419 @@
+package s3_storage
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "log/slog"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/testing/containers"
+)
+
+// testInnerPartSize is the S3 minimum non-final part size (5 MiB). Combined with a tiny per-object
+// part count it forces object rollover on a few tens of MiB instead of the production ~15.6 GiB.
+const testInnerPartSize = 5 * 1024 * 1024
+
+func Test_GetFile_LegacySingleObjectWithoutManifest_ReturnsContent(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(4096)
+
+ fileName := uuid.NewString()
+ putRawObject(t, rawClient, storage.S3Bucket, fileName, persistedBytes)
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+
+ assert.Equal(t, persistedBytes, reassembledBytes)
+}
+
+func Test_DeleteFile_LegacySingleObjectWithoutManifest_RemovesObject(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+
+ fileName := uuid.NewString()
+ putRawObject(t, rawClient, storage.S3Bucket, fileName, generateBytes(4096))
+
+ require.NoError(t, storage.DeleteFile(encryptor, fileName))
+
+ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ _, err := storage.GetFile(encryptor, fileName)
+ assert.Error(t, err)
+}
+
+func Test_SaveFile_StreamWithinOneInnerPart_WritesSingleObjectWithoutManifest(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+ assert.Equal(t, persistedBytes, reassembledBytes)
+}
+
+func Test_SaveFile_StreamExactlyOneInnerPart_WritesSingleObjectWithoutManifest(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(testInnerPartSize)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+ assert.Equal(t, persistedBytes, reassembledBytes)
+}
+
+func Test_SaveAndGetFile_StreamExceedsObjectSize_RoundTripsChunked(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+
+ // 23 MiB with 5 MiB parts and 2 parts/object: object1 = parts 1-2, object2 = parts 3-4,
+ // object3 = the trailing 3 MiB part, exercising multi-object rollover and a sub-part final chunk.
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ assert.ElementsMatch(t, []string{
+ fileName + ".part000001",
+ fileName + ".part000002",
+ fileName + ".part000003",
+ fileName + manifestSuffix,
+ }, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+ assert.Equal(t, persistedBytes, reassembledBytes)
+}
+
+func Test_GetFile_ChunkedBackupMissingPart_ReturnsError(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ require.NoError(
+ t,
+ rawClient.RemoveObject(t.Context(), storage.S3Bucket, fileName+".part000002", minio.RemoveObjectOptions{}),
+ )
+
+ reader, err := storage.GetFile(encryptor, fileName)
+ require.NoError(t, err)
+ defer func() { _ = reader.Close() }()
+
+ _, err = io.ReadAll(reader)
+ assert.Error(t, err, "a missing chunk must surface as a read error, never a silently truncated stream")
+}
+
+func Test_DeleteFile_ChunkedBackup_RemovesAllPartsAndManifest(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+ require.NotEmpty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ require.NoError(t, storage.DeleteFile(encryptor, fileName))
+
+ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+}
+
+func Test_SaveFile_CancelledMidChunkedUpload_LeavesNoOrphanParts(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+
+ ctx, cancel := context.WithCancel(t.Context())
+
+ // 5 MiB parts, 2 parts/object => 10 MiB objects. Deliver 30 MiB but trip cancellation after
+ // ~12 MiB: object 1 (parts 1-2) is committed, then cancellation lands while object 2 is in
+ // flight. Object 1 is the orphan that cleanup must remove even though ctx is already cancelled.
+ source := &cancelAfterReader{
+ data: generateBytes(30 * 1024 * 1024),
+ cancelAt: 12 * 1024 * 1024,
+ cancel: cancel,
+ }
+
+ fileName := uuid.NewString()
+ err := storage.SaveFile(ctx, encryptor, discardLogger(), fileName, source)
+ require.Error(t, err, "a cancelled upload must fail")
+
+ assert.Empty(
+ t,
+ listObjectKeys(t, rawClient, storage.S3Bucket, fileName),
+ "cancellation mid-upload must leave no orphan part objects behind",
+ )
+}
+
+func Test_SaveGetDeleteFile_ChunkedWithPrefix_RoundTripsAndCleansUp(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ storage.S3Prefix = "team-a/nested"
+
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+ fileName := uuid.NewString()
+
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ // Parts and manifest land under the prefix, and the manifest must reference the prefixed keys so
+ // read and delete reconstruct them correctly.
+ assert.ElementsMatch(t, []string{
+ "team-a/nested/" + fileName + ".part000001",
+ "team-a/nested/" + fileName + ".part000002",
+ "team-a/nested/" + fileName + ".part000003",
+ "team-a/nested/" + fileName + manifestSuffix,
+ }, listObjectKeys(t, rawClient, storage.S3Bucket, "team-a/"))
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+ assert.Equal(t, persistedBytes, reassembledBytes)
+
+ require.NoError(t, storage.DeleteFile(encryptor, fileName))
+ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, "team-a/"))
+}
+
+func Test_SaveAndGetFile_EmptyStream_WritesSingleEmptyObjectWithoutManifest(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+
+ fileName := uuid.NewString()
+ require.NoError(t, storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(nil)))
+
+ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ assert.Empty(t, readWholeFile(t, storage, encryptor, fileName))
+}
+
+func Test_SaveAndGetFile_StreamExactlyFillsObjects_NoTrailingEmptyObject(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+
+ // 20 MiB is exactly two 10 MiB objects (5 MiB part x 2). The boundary must not commit an empty
+ // trailing object: the bucket holds two parts plus the manifest, not three.
+ persistedBytes := generateBytes(20 * 1024 * 1024)
+ fileName := uuid.NewString()
+
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ assert.ElementsMatch(t, []string{
+ fileName + ".part000001",
+ fileName + ".part000002",
+ fileName + manifestSuffix,
+ }, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))
+
+ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)
+ assert.Equal(t, persistedBytes, reassembledBytes)
+}
+
+func Test_GetFile_ChunkedBackupCorruptedPart_ReturnsChecksumError(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ // Overwrite a committed 10 MiB part with same-length but different bytes: the size still matches
+ // the manifest, so only the per-part sha256 can catch the corruption.
+ corruptedSameSize := bytes.Repeat([]byte{0xAB}, 10*1024*1024)
+ putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", corruptedSameSize)
+
+ reader, err := storage.GetFile(encryptor, fileName)
+ require.NoError(t, err)
+ defer func() { _ = reader.Close() }()
+
+ _, err = io.ReadAll(reader)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "checksum mismatch")
+}
+
+func Test_GetFile_ChunkedBackupTruncatedPart_ReturnsSizeError(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ // Replace a committed 10 MiB part with a shorter object: the manifest expects 10 MiB, so the
+ // length check must reject it before its bytes reach the restore.
+ putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", generateBytes(4*1024*1024))
+
+ reader, err := storage.GetFile(encryptor, fileName)
+ require.NoError(t, err)
+ defer func() { _ = reader.Close() }()
+
+ _, err = io.ReadAll(reader)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "size mismatch")
+}
+
+func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {
+ rawClient, storage, encryptor := setupChunkedStorage(t)
+ persistedBytes := generateBytes(23 * 1024 * 1024)
+
+ fileName := uuid.NewString()
+ require.NoError(
+ t,
+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),
+ )
+
+ // A manifest that exists but cannot be parsed must fail loudly, not fall through to the
+ // single-object path and silently read nothing.
+ putRawObject(t, rawClient, storage.S3Bucket, fileName+manifestSuffix, []byte("not a valid manifest{"))
+
+ _, err := storage.GetFile(encryptor, fileName)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "manifest")
+}
+
+// setupChunkedStorage boots a MinIO container, creates the bucket and returns a raw client for
+// direct object assertions alongside an S3Storage configured to roll objects on small inputs.
+func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.FieldEncryptor) {
+ t.Helper()
+
+ endpoint := containers.StartMinio(t)
+ address := fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port)
+ bucketName := "test-bucket"
+
+ rawClient, err := minio.New(address, &minio.Options{
+ Creds: credentials.NewStaticV4(containers.MinioRootUser, containers.MinioRootPassword, ""),
+ Secure: false,
+ Region: containers.MinioRegion,
+ })
+ require.NoError(t, err, "failed to create raw minio client")
+
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+
+ require.NoError(t, rawClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: containers.MinioRegion}))
+
+ encryptor := encryption.GetFieldEncryptor()
+
+ accessKey, err := encryptor.Encrypt(containers.MinioRootUser)
+ require.NoError(t, err)
+ secretKey, err := encryptor.Encrypt(containers.MinioRootPassword)
+ require.NoError(t, err)
+
+ storage := &S3Storage{
+ StorageID: uuid.New(),
+ S3Bucket: bucketName,
+ S3Region: containers.MinioRegion,
+ S3AccessKey: accessKey,
+ S3SecretKey: secretKey,
+ S3Endpoint: "http://" + address,
+ innerPartSizeOverride: testInnerPartSize,
+ maxPartsOverride: 2,
+ }
+
+ return rawClient, storage, encryptor
+}
+
+func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data []byte) {
+ t.Helper()
+
+ _, err := client.PutObject(
+ t.Context(),
+ bucket,
+ key,
+ bytes.NewReader(data),
+ int64(len(data)),
+ minio.PutObjectOptions{},
+ )
+ require.NoError(t, err, "failed to write raw object %s", key)
+}
+
+func readWholeFile(t *testing.T, storage *S3Storage, encryptor encryption.FieldEncryptor, fileName string) []byte {
+ t.Helper()
+
+ reader, err := storage.GetFile(encryptor, fileName)
+ require.NoError(t, err)
+ defer func() { _ = reader.Close() }()
+
+ content, err := io.ReadAll(reader)
+ require.NoError(t, err)
+
+ return content
+}
+
+func listObjectKeys(t *testing.T, client *minio.Client, bucket, prefix string) []string {
+ t.Helper()
+
+ var keys []string
+ for object := range client.ListObjects(t.Context(), bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
+ require.NoError(t, object.Err)
+ keys = append(keys, object.Key)
+ }
+
+ return keys
+}
+
+// generateBytes returns deterministic, position-dependent bytes so a round trip can be compared
+// exactly without holding a second random source.
+func generateBytes(size int) []byte {
+ data := make([]byte, size)
+ for i := range data {
+ data[i] = byte(i*7 + 13)
+ }
+
+ return data
+}
+
+func discardLogger() *slog.Logger {
+ return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+// cancelAfterReader serves data and cancels its context once cancelAt bytes have been read, to
+// simulate a backup cancelled partway through a chunked upload.
+type cancelAfterReader struct {
+ data []byte
+ pos int
+ cancelAt int
+ cancel context.CancelFunc
+}
+
+func (r *cancelAfterReader) Read(p []byte) (int, error) {
+ if r.pos >= len(r.data) {
+ return 0, io.EOF
+ }
+
+ n := copy(p, r.data[r.pos:])
+ r.pos += n
+
+ if r.cancel != nil && r.pos >= r.cancelAt {
+ r.cancel()
+ r.cancel = nil
+ }
+
+ return n, nil
+}