PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (storages): Stream unlimited-size S3 backups via chunked objects (issue #617)
Rostislav Dugina107648beedf
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.goindex aaa69347..7384ab06 100644--- a/backend/internal/features/storages/models/s3/model.go+++ b/backend/internal/features/storages/models/s3/model.go@@ -4,10 +4,14 @@ import ("bytes""context""crypto/md5"+ "crypto/sha256""crypto/tls""encoding/base64"+ "encoding/hex"+ "encoding/json""errors""fmt"+ "hash""io""log/slog""net"@@ -29,10 +33,22 @@ const (s3TLSHandshakeTimeout = 30 * time.Seconds3DeleteTimeout = 30 * time.Second- // Chunk size for multipart uploads - 16MB provides good balance between- // memory usage and upload efficiency. This creates backpressure to pg_dump- // by only reading one chunk at a time and waiting for S3 to confirm receipt.- multipartChunkSize = 16 * 1024 * 1024+ // defaultInnerPartSize is the multipart part we feed S3 for one object. 16 MiB balances+ // memory against request count and creates backpressure to pg_dump: we keep one part in+ // flight and wait for S3 to confirm it before reading the next, so pipeline memory does not+ // grow with the database size (ADR-0004).+ defaultInnerPartSize = 16 * 1024 * 1024++ // defaultMaxPartsPerObject caps inner parts per object; once reached we roll to the next+ // object. Effective object size = defaultMaxPartsPerObject * defaultInnerPartSize ~= 15.6 GiB.+ // S3 multipart is bounded by part_size * part_count_limit and that limit is provider-set+ // (AWS 10000, Scaleway and many S3-compatibles 1000). Because we stream dumps of unknown size,+ // we write blind to the smallest common limit so one backup can span any provider without+ // knowing its real cap; the object count, not memory, scales with the database size.+ defaultMaxPartsPerObject = 1000++ manifestSuffix = ".parts"+ manifestVersion = 1)type S3Storage struct {@@ -47,6 +63,12 @@ type S3Storage struct {S3UseVirtualHostedStyle bool `json:"s3UseVirtualHostedStyle" gorm:"default:false;column:s3_use_virtual_hosted_style"`SkipTLSVerify bool `json:"skipTLSVerify" gorm:"default:false;column:skip_tls_verify"`S3StorageClass S3StorageClass `json:"s3StorageClass" gorm:"type:text;column:s3_storage_class;default:''"`++ // Test seams: rolling at the production 15.6 GiB object boundary would need ~15 GiB per case,+ // so integration tests shrink the inner part and per-object part count to exercise object+ // rollover on a few tens of MiB. Unexported, so gorm and JSON ignore them; zero means default.+ innerPartSizeOverride int+ maxPartsOverride int}func (s *S3Storage) TableName() string {@@ -66,123 +88,48 @@ func (s *S3Storage) SaveFile(default:}- coreClient, err := s.getCoreClient(encryptor)+ client, err := s.getClient(encryptor)if err != nil {return err}- objectKey := s.buildObjectKey(fileName)+ baseKey := s.buildObjectKey(fileName)+ innerPartSize := s.innerPartSize()- uploadID, err := coreClient.NewMultipartUpload(- ctx,- s.S3Bucket,- objectKey,- s.putObjectOptions(),- )- if err != nil {- return fmt.Errorf("failed to initiate multipart upload: %w", err)+ // Look ahead by one inner part to choose the layout. A stream that fits in a single inner part+ // (WAL segments, .metadata/.manifest sidecars, small dumps, empty files) is written as one+ // plain object {fileName} with no manifest, exactly as legacy backups are. Anything larger is+ // split across {fileName}.partNNNNNN objects described by a {fileName}.parts manifest.+ firstPart := make([]byte, innerPartSize)+ firstPartLen, readErr := io.ReadFull(file, firstPart)+ if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {+ return fmt.Errorf("read error: %w", readErr)}- var parts []minio.CompletePart- partNumber := 1- buf := make([]byte, multipartChunkSize)-- for {- select {- case <-ctx.Done():- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)- return fmt.Errorf("upload cancelled: %w", ctx.Err())- default:- }-- n, readErr := io.ReadFull(file, buf)-- if n == 0 && readErr == io.EOF {- break- }-- if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)- return fmt.Errorf("read error: %w", readErr)- }-- partData := buf[:n]- hash := md5.Sum(partData)- md5Base64 := base64.StdEncoding.EncodeToString(hash[:])-- part, err := coreClient.PutObjectPart(- ctx,- s.S3Bucket,- objectKey,- uploadID,- partNumber,- bytes.NewReader(partData),- int64(n),- minio.PutObjectPartOptions{- Md5Base64: md5Base64,- },- )- if err != nil {- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)-- select {- case <-ctx.Done():- return fmt.Errorf("upload cancelled: %w", ctx.Err())- default:- return fmt.Errorf("failed to upload part %d: %w", partNumber, err)- }- }-- parts = append(parts, minio.CompletePart{- PartNumber: partNumber,- ETag: part.ETag,- })-- partNumber++-- if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {- break- }+ if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {+ return s.uploadSingleObject(ctx, client, baseKey, firstPart[:firstPartLen])}- if len(parts) == 0 {- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)-- client, err := s.getClient(encryptor)- if err != nil {- return err- }- opts := s.putObjectOptions()- opts.SendContentMd5 = true+ // The first part filled exactly: peek one more byte to tell "stream is exactly one inner part"+ // (still single object, e.g. a 16 MiB WAL segment) from "stream is larger" (chunked).+ peek := make([]byte, 1)+ peekLen, peekErr := io.ReadFull(file, peek)+ if peekErr != nil && peekErr != io.EOF {+ return fmt.Errorf("read error: %w", peekErr)+ }- _, err = client.PutObject(- ctx,- s.S3Bucket,- objectKey,- bytes.NewReader([]byte{}),- 0,- opts,- )- if err != nil {- return fmt.Errorf("failed to upload empty file: %w", err)- }- return nil+ if peekLen == 0 {+ return s.uploadSingleObject(ctx, client, baseKey, firstPart)}- _, err = coreClient.CompleteMultipartUpload(- ctx,- s.S3Bucket,- objectKey,- uploadID,- parts,- s.putObjectOptions(),- )+ coreClient, err := s.getCoreClient(encryptor)if err != nil {- _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)- return fmt.Errorf("failed to complete multipart upload: %w", err)+ return err}- return nil+ source := io.MultiReader(bytes.NewReader(firstPart), bytes.NewReader(peek[:peekLen]), file)++ return s.uploadChunked(ctx, coreClient, client, baseKey, source)}func (s *S3Storage) GetFile(@@ -194,12 +141,21 @@ func (s *S3Storage) GetFile(return nil, err}- objectKey := s.buildObjectKey(fileName)+ baseKey := s.buildObjectKey(fileName)++ manifest, hasManifest, err := s.readManifest(context.TODO(), client, baseKey)+ if err != nil {+ return nil, err+ }++ if hasManifest {+ return newReassemblingReader(client, s.S3Bucket, manifest.Parts), nil+ }object, err := client.GetObject(context.TODO(),s.S3Bucket,- objectKey,+ baseKey,minio.GetObjectOptions{},)if err != nil {@@ -230,15 +186,34 @@ func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName strreturn err}- objectKey := s.buildObjectKey(fileName)+ baseKey := s.buildObjectKey(fileName)ctx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)defer cancel()+ manifest, hasManifest, err := s.readManifest(ctx, client, baseKey)+ if err != nil {+ return err+ }++ if hasManifest {+ keys := make([]string, 0, len(manifest.Parts)+1)+ for _, part := range manifest.Parts {+ keys = append(keys, part.Key)+ }+ keys = append(keys, manifestObjectKey(baseKey))++ if err := s.removeObjects(ctx, client, keys); err != nil {+ return fmt.Errorf("failed to delete chunked backup from S3: %w", err)+ }++ return nil+ }+err = client.RemoveObject(ctx,s.S3Bucket,- objectKey,+ baseKey,minio.RemoveObjectOptions{},)if err != nil {@@ -366,6 +341,208 @@ func (s *S3Storage) Update(incoming *S3Storage) {// otherwise we will have to transfer all the data to the new prefix}+func (s *S3Storage) innerPartSize() int {+ if s.innerPartSizeOverride > 0 {+ return s.innerPartSizeOverride+ }++ return defaultInnerPartSize+}++func (s *S3Storage) maxPartsPerObject() int {+ if s.maxPartsOverride > 0 {+ return s.maxPartsOverride+ }++ return defaultMaxPartsPerObject+}++func (s *S3Storage) uploadSingleObject(+ ctx context.Context,+ client *minio.Client,+ objectKey string,+ data []byte,+) error {+ opts := s.putObjectOptions()+ opts.SendContentMd5 = true++ _, err := client.PutObject(ctx, s.S3Bucket, objectKey, bytes.NewReader(data), int64(len(data)), opts)+ if err != nil {+ return fmt.Errorf("failed to upload file to S3: %w", err)+ }++ return nil+}++func (s *S3Storage) uploadChunked(+ ctx context.Context,+ coreClient *minio.Core,+ client *minio.Client,+ baseKey string,+ source io.Reader,+) error {+ innerPartSize := s.innerPartSize()+ maxParts := s.maxPartsPerObject()+ buf := make([]byte, innerPartSize)++ var manifestParts []manifestPart+ var uploadedKeys []string+ var totalSize int64++ // abandon undoes a failed chunked upload: it aborts the in-flight multipart (when uploadID is set)+ // and deletes every already-completed part object. It uses a fresh context detached from ctx+ // because the usual reason we land here is ctx cancellation, and cleanup issued on a cancelled+ // context is a no-op — which would leave completed part objects with no manifest as dead weight in+ // the bucket until retention or an overwrite reclaims them.+ abandon := func(objectKey, uploadID string) {+ cleanupCtx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)+ defer cancel()++ if uploadID != "" {+ _ = coreClient.AbortMultipartUpload(cleanupCtx, s.S3Bucket, objectKey, uploadID)+ }++ if len(uploadedKeys) > 0 {+ _ = s.removeObjects(cleanupCtx, client, uploadedKeys)+ }+ }++ objectIndex := 1+ sourceEnded := false++ for !sourceEnded {+ objectKey := partObjectKey(baseKey, objectIndex)++ uploadID, err := coreClient.NewMultipartUpload(ctx, s.S3Bucket, objectKey, s.putObjectOptions())+ if err != nil {+ abandon("", "")+ return fmt.Errorf("failed to initiate multipart upload for %s: %w", objectKey, err)+ }++ var parts []minio.CompletePart+ objectHasher := sha256.New()+ var objectSize int64++ for partNumber := 1; partNumber <= maxParts; partNumber++ {+ select {+ case <-ctx.Done():+ abandon(objectKey, uploadID)+ return fmt.Errorf("upload cancelled: %w", ctx.Err())+ default:+ }++ partLen, readErr := io.ReadFull(source, buf)+ if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {+ abandon(objectKey, uploadID)+ return fmt.Errorf("read error: %w", readErr)+ }++ if partLen == 0 && readErr == io.EOF {+ sourceEnded = true+ break+ }++ if err := s.putInnerPart(+ ctx,+ coreClient,+ objectKey,+ uploadID,+ partNumber,+ buf[:partLen],+ objectHasher,+ &parts,+ ); err != nil {+ abandon(objectKey, uploadID)+ return err+ }++ objectSize += int64(partLen)++ if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {+ sourceEnded = true+ break+ }+ }++ // The source ended exactly on the previous object's boundary, so this object got no data.+ if len(parts) == 0 {+ _ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)+ break+ }++ if _, err := coreClient.CompleteMultipartUpload(+ ctx,+ s.S3Bucket,+ objectKey,+ uploadID,+ parts,+ s.putObjectOptions(),+ ); err != nil {+ abandon(objectKey, uploadID)+ return fmt.Errorf("failed to complete multipart upload for %s: %w", objectKey, err)+ }++ uploadedKeys = append(uploadedKeys, objectKey)+ manifestParts = append(manifestParts, manifestPart{+ Key: objectKey,+ Size: objectSize,+ SHA256: hex.EncodeToString(objectHasher.Sum(nil)),+ })+ totalSize += objectSize+ objectIndex+++ }++ manifest := chunkedManifest{+ Version: manifestVersion,+ TotalSize: totalSize,+ Parts: manifestParts,+ }++ if err := s.writeManifest(ctx, client, baseKey, manifest); err != nil {+ abandon("", "")+ return err+ }++ return nil+}++func (s *S3Storage) putInnerPart(+ ctx context.Context,+ coreClient *minio.Core,+ objectKey string,+ uploadID string,+ partNumber int,+ partData []byte,+ objectHasher hash.Hash,+ parts *[]minio.CompletePart,+) error {+ partMd5 := md5.Sum(partData)+ md5Base64 := base64.StdEncoding.EncodeToString(partMd5[:])++ part, err := coreClient.PutObjectPart(+ ctx,+ s.S3Bucket,+ objectKey,+ uploadID,+ partNumber,+ bytes.NewReader(partData),+ int64(len(partData)),+ minio.PutObjectPartOptions{Md5Base64: md5Base64},+ )+ if err != nil {+ if ctx.Err() != nil {+ return fmt.Errorf("upload cancelled: %w", ctx.Err())+ }++ return fmt.Errorf("failed to upload part %d of %s: %w", partNumber, objectKey, err)+ }++ objectHasher.Write(partData)+ *parts = append(*parts, minio.CompletePart{PartNumber: partNumber, ETag: part.ETag})++ return nil+}+func (s *S3Storage) putObjectOptions() minio.PutObjectOptions {return minio.PutObjectOptions{StorageClass: string(s.S3StorageClass),@@ -477,3 +654,214 @@ func (s *S3Storage) getClientParams(return endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, nil}++// chunkedManifest lists, in order, the objects a single logical file was split into, with each+// object's size and sha256. It is written LAST, after every part object is durably committed.+//+// Why a manifest instead of just probing {fileName}.part1, .part2, ... until a key 404s:+//+// 1. The bytes we store are an encrypted stream with no end-of-stream marker (see+// util/encryption). Probe-until-404 cannot tell a complete backup from one whose final part+// failed to upload, was truncated, or was reaped by lifecycle rules: it would stop at the gap+// and hand the restore a silently short, corrupt stream. TotalSize plus the per-part list and+// sha256 let GetFile reject a missing/short/corrupted part BEFORE the restore starts, turning+// silent data loss into a hard error.+// 2. Because the manifest is written last, its presence is the atomic "backup is complete" marker.+// A crash mid-upload leaves part objects but no manifest, which GetFile treats as a hard error+// (it finds neither manifest nor single object) rather than a valid N-part backup.+// 3. It immunizes reads against S3 eventual consistency: a transient 404 on a middle part can no+// longer be misread as the end of the stream.+type chunkedManifest struct {+ Version int `json:"version"`+ TotalSize int64 `json:"totalSize"`+ Parts []manifestPart `json:"parts"`+}++type manifestPart struct {+ Key string `json:"key"`+ Size int64 `json:"size"`+ SHA256 string `json:"sha256"`+}++func partObjectKey(baseKey string, objectIndex int) string {+ return fmt.Sprintf("%s.part%06d", baseKey, objectIndex)+}++func manifestObjectKey(baseKey string) string {+ return baseKey + manifestSuffix+}++func (s *S3Storage) writeManifest(+ ctx context.Context,+ client *minio.Client,+ baseKey string,+ manifest chunkedManifest,+) error {+ body, err := json.Marshal(manifest)+ if err != nil {+ return fmt.Errorf("failed to marshal chunk manifest: %w", err)+ }++ opts := s.putObjectOptions()+ opts.SendContentMd5 = true+ opts.ContentType = "application/json"++ _, err = client.PutObject(+ ctx,+ s.S3Bucket,+ manifestObjectKey(baseKey),+ bytes.NewReader(body),+ int64(len(body)),+ opts,+ )+ if err != nil {+ return fmt.Errorf("failed to write chunk manifest: %w", err)+ }++ return nil+}++func (s *S3Storage) readManifest(+ ctx context.Context,+ client *minio.Client,+ baseKey string,+) (chunkedManifest, bool, error) {+ var manifest chunkedManifest++ manifestKey := manifestObjectKey(baseKey)++ _, err := client.StatObject(ctx, s.S3Bucket, manifestKey, minio.StatObjectOptions{})+ if err != nil {+ if minio.ToErrorResponse(err).StatusCode == http.StatusNotFound {+ return manifest, false, nil+ }++ return manifest, false, fmt.Errorf("failed to stat chunk manifest: %w", err)+ }++ object, err := client.GetObject(ctx, s.S3Bucket, manifestKey, minio.GetObjectOptions{})+ if err != nil {+ return manifest, false, fmt.Errorf("failed to read chunk manifest: %w", err)+ }+ defer func() { _ = object.Close() }()++ body, err := io.ReadAll(object)+ if err != nil {+ return manifest, false, fmt.Errorf("failed to read chunk manifest: %w", err)+ }++ if err := json.Unmarshal(body, &manifest); err != nil {+ return manifest, false, fmt.Errorf("failed to parse chunk manifest: %w", err)+ }++ return manifest, true, nil+}++func (s *S3Storage) removeObjects(ctx context.Context, client *minio.Client, keys []string) error {+ objectsCh := make(chan minio.ObjectInfo, len(keys))+ for _, key := range keys {+ objectsCh <- minio.ObjectInfo{Key: key}+ }+ close(objectsCh)++ var firstErr error+ for removeResult := range client.RemoveObjects(ctx, s.S3Bucket, objectsCh, minio.RemoveObjectsOptions{}) {+ if removeResult.Err != nil && firstErr == nil {+ firstErr = removeResult.Err+ }+ }++ return firstErr+}++// reassemblingReader streams a chunked backup back as one byte stream by opening each part object+// in manifest order, reading it to EOF and only then opening the next. It holds at most one part+// connection at a time, preserving the bounded-memory streaming of the upload path (ADR-0004), and+// verifies each part's length and sha256 against the manifest so a missing, truncated, or corrupted+// part surfaces as a read error instead of a silently short stream.+type reassemblingReader struct {+ client *minio.Client+ bucket string+ parts []manifestPart+ index int+ current *minio.Object+ hasher hash.Hash+ read int64+}++func newReassemblingReader(client *minio.Client, bucket string, parts []manifestPart) *reassemblingReader {+ return &reassemblingReader{client: client, bucket: bucket, parts: parts}+}++func (r *reassemblingReader) Read(p []byte) (int, error) {+ for {+ if r.current == nil {+ if r.index >= len(r.parts) {+ return 0, io.EOF+ }++ object, err := r.client.GetObject(context.TODO(), r.bucket, r.parts[r.index].Key, minio.GetObjectOptions{})+ if err != nil {+ return 0, fmt.Errorf("failed to open chunk %s: %w", r.parts[r.index].Key, err)+ }++ r.current = object+ r.hasher = sha256.New()+ r.read = 0+ }++ n, err := r.current.Read(p)+ if n > 0 {+ r.hasher.Write(p[:n])+ r.read += int64(n)+ }++ if errors.Is(err, io.EOF) {+ if verifyErr := r.finishCurrentPart(); verifyErr != nil {+ return n, verifyErr+ }++ if n > 0 {+ return n, nil+ }++ continue+ }++ if err != nil {+ return n, fmt.Errorf("failed to read chunk %s: %w", r.parts[r.index].Key, err)+ }++ return n, nil+ }+}++func (r *reassemblingReader) Close() error {+ if r.current == nil {+ return nil+ }++ err := r.current.Close()+ r.current = nil++ return err+}++func (r *reassemblingReader) finishCurrentPart() error {+ part := r.parts[r.index]++ _ = r.current.Close()+ r.current = nil++ if r.read != part.Size {+ return fmt.Errorf("chunk %s size mismatch: manifest %d, read %d", part.Key, part.Size, r.read)+ }++ if checksum := hex.EncodeToString(r.hasher.Sum(nil)); checksum != part.SHA256 {+ return fmt.Errorf("chunk %s checksum mismatch", part.Key)+ }++ r.index++++ return nil+}diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.gonew file mode 100644index 00000000..1b69c1e4--- /dev/null+++ b/backend/internal/features/storages/models/s3/model_test.go@@ -0,0 +1,419 @@+package s3_storage++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "log/slog"+ "testing"+ "time"++ "github.com/google/uuid"+ "github.com/minio/minio-go/v7"+ "github.com/minio/minio-go/v7/pkg/credentials"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/testing/containers"+)++// testInnerPartSize is the S3 minimum non-final part size (5 MiB). Combined with a tiny per-object+// part count it forces object rollover on a few tens of MiB instead of the production ~15.6 GiB.+const testInnerPartSize = 5 * 1024 * 1024++func Test_GetFile_LegacySingleObjectWithoutManifest_ReturnsContent(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(4096)++ fileName := uuid.NewString()+ putRawObject(t, rawClient, storage.S3Bucket, fileName, persistedBytes)++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)++ assert.Equal(t, persistedBytes, reassembledBytes)+}++func Test_DeleteFile_LegacySingleObjectWithoutManifest_RemovesObject(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)++ fileName := uuid.NewString()+ putRawObject(t, rawClient, storage.S3Bucket, fileName, generateBytes(4096))++ require.NoError(t, storage.DeleteFile(encryptor, fileName))++ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ _, err := storage.GetFile(encryptor, fileName)+ assert.Error(t, err)+}++func Test_SaveFile_StreamWithinOneInnerPart_WritesSingleObjectWithoutManifest(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)+ assert.Equal(t, persistedBytes, reassembledBytes)+}++func Test_SaveFile_StreamExactlyOneInnerPart_WritesSingleObjectWithoutManifest(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(testInnerPartSize)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)+ assert.Equal(t, persistedBytes, reassembledBytes)+}++func Test_SaveAndGetFile_StreamExceedsObjectSize_RoundTripsChunked(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)++ // 23 MiB with 5 MiB parts and 2 parts/object: object1 = parts 1-2, object2 = parts 3-4,+ // object3 = the trailing 3 MiB part, exercising multi-object rollover and a sub-part final chunk.+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ assert.ElementsMatch(t, []string{+ fileName + ".part000001",+ fileName + ".part000002",+ fileName + ".part000003",+ fileName + manifestSuffix,+ }, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)+ assert.Equal(t, persistedBytes, reassembledBytes)+}++func Test_GetFile_ChunkedBackupMissingPart_ReturnsError(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ require.NoError(+ t,+ rawClient.RemoveObject(t.Context(), storage.S3Bucket, fileName+".part000002", minio.RemoveObjectOptions{}),+ )++ reader, err := storage.GetFile(encryptor, fileName)+ require.NoError(t, err)+ defer func() { _ = reader.Close() }()++ _, err = io.ReadAll(reader)+ assert.Error(t, err, "a missing chunk must surface as a read error, never a silently truncated stream")+}++func Test_DeleteFile_ChunkedBackup_RemovesAllPartsAndManifest(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )+ require.NotEmpty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ require.NoError(t, storage.DeleteFile(encryptor, fileName))++ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))+}++func Test_SaveFile_CancelledMidChunkedUpload_LeavesNoOrphanParts(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)++ ctx, cancel := context.WithCancel(t.Context())++ // 5 MiB parts, 2 parts/object => 10 MiB objects. Deliver 30 MiB but trip cancellation after+ // ~12 MiB: object 1 (parts 1-2) is committed, then cancellation lands while object 2 is in+ // flight. Object 1 is the orphan that cleanup must remove even though ctx is already cancelled.+ source := &cancelAfterReader{+ data: generateBytes(30 * 1024 * 1024),+ cancelAt: 12 * 1024 * 1024,+ cancel: cancel,+ }++ fileName := uuid.NewString()+ err := storage.SaveFile(ctx, encryptor, discardLogger(), fileName, source)+ require.Error(t, err, "a cancelled upload must fail")++ assert.Empty(+ t,+ listObjectKeys(t, rawClient, storage.S3Bucket, fileName),+ "cancellation mid-upload must leave no orphan part objects behind",+ )+}++func Test_SaveGetDeleteFile_ChunkedWithPrefix_RoundTripsAndCleansUp(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ storage.S3Prefix = "team-a/nested"++ persistedBytes := generateBytes(23 * 1024 * 1024)+ fileName := uuid.NewString()++ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ // Parts and manifest land under the prefix, and the manifest must reference the prefixed keys so+ // read and delete reconstruct them correctly.+ assert.ElementsMatch(t, []string{+ "team-a/nested/" + fileName + ".part000001",+ "team-a/nested/" + fileName + ".part000002",+ "team-a/nested/" + fileName + ".part000003",+ "team-a/nested/" + fileName + manifestSuffix,+ }, listObjectKeys(t, rawClient, storage.S3Bucket, "team-a/"))++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)+ assert.Equal(t, persistedBytes, reassembledBytes)++ require.NoError(t, storage.DeleteFile(encryptor, fileName))+ assert.Empty(t, listObjectKeys(t, rawClient, storage.S3Bucket, "team-a/"))+}++func Test_SaveAndGetFile_EmptyStream_WritesSingleEmptyObjectWithoutManifest(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)++ fileName := uuid.NewString()+ require.NoError(t, storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(nil)))++ assert.ElementsMatch(t, []string{fileName}, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ assert.Empty(t, readWholeFile(t, storage, encryptor, fileName))+}++func Test_SaveAndGetFile_StreamExactlyFillsObjects_NoTrailingEmptyObject(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)++ // 20 MiB is exactly two 10 MiB objects (5 MiB part x 2). The boundary must not commit an empty+ // trailing object: the bucket holds two parts plus the manifest, not three.+ persistedBytes := generateBytes(20 * 1024 * 1024)+ fileName := uuid.NewString()++ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ assert.ElementsMatch(t, []string{+ fileName + ".part000001",+ fileName + ".part000002",+ fileName + manifestSuffix,+ }, listObjectKeys(t, rawClient, storage.S3Bucket, fileName))++ reassembledBytes := readWholeFile(t, storage, encryptor, fileName)+ assert.Equal(t, persistedBytes, reassembledBytes)+}++func Test_GetFile_ChunkedBackupCorruptedPart_ReturnsChecksumError(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ // Overwrite a committed 10 MiB part with same-length but different bytes: the size still matches+ // the manifest, so only the per-part sha256 can catch the corruption.+ corruptedSameSize := bytes.Repeat([]byte{0xAB}, 10*1024*1024)+ putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", corruptedSameSize)++ reader, err := storage.GetFile(encryptor, fileName)+ require.NoError(t, err)+ defer func() { _ = reader.Close() }()++ _, err = io.ReadAll(reader)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "checksum mismatch")+}++func Test_GetFile_ChunkedBackupTruncatedPart_ReturnsSizeError(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ // Replace a committed 10 MiB part with a shorter object: the manifest expects 10 MiB, so the+ // length check must reject it before its bytes reach the restore.+ putRawObject(t, rawClient, storage.S3Bucket, fileName+".part000002", generateBytes(4*1024*1024))++ reader, err := storage.GetFile(encryptor, fileName)+ require.NoError(t, err)+ defer func() { _ = reader.Close() }()++ _, err = io.ReadAll(reader)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "size mismatch")+}++func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {+ rawClient, storage, encryptor := setupChunkedStorage(t)+ persistedBytes := generateBytes(23 * 1024 * 1024)++ fileName := uuid.NewString()+ require.NoError(+ t,+ storage.SaveFile(t.Context(), encryptor, discardLogger(), fileName, bytes.NewReader(persistedBytes)),+ )++ // A manifest that exists but cannot be parsed must fail loudly, not fall through to the+ // single-object path and silently read nothing.+ putRawObject(t, rawClient, storage.S3Bucket, fileName+manifestSuffix, []byte("not a valid manifest{"))++ _, err := storage.GetFile(encryptor, fileName)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "manifest")+}++// setupChunkedStorage boots a MinIO container, creates the bucket and returns a raw client for+// direct object assertions alongside an S3Storage configured to roll objects on small inputs.+func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.FieldEncryptor) {+ t.Helper()++ endpoint := containers.StartMinio(t)+ address := fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port)+ bucketName := "test-bucket"++ rawClient, err := minio.New(address, &minio.Options{+ Creds: credentials.NewStaticV4(containers.MinioRootUser, containers.MinioRootPassword, ""),+ Secure: false,+ Region: containers.MinioRegion,+ })+ require.NoError(t, err, "failed to create raw minio client")++ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)+ defer cancel()++ require.NoError(t, rawClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: containers.MinioRegion}))++ encryptor := encryption.GetFieldEncryptor()++ accessKey, err := encryptor.Encrypt(containers.MinioRootUser)+ require.NoError(t, err)+ secretKey, err := encryptor.Encrypt(containers.MinioRootPassword)+ require.NoError(t, err)++ storage := &S3Storage{+ StorageID: uuid.New(),+ S3Bucket: bucketName,+ S3Region: containers.MinioRegion,+ S3AccessKey: accessKey,+ S3SecretKey: secretKey,+ S3Endpoint: "http://" + address,+ innerPartSizeOverride: testInnerPartSize,+ maxPartsOverride: 2,+ }++ return rawClient, storage, encryptor+}++func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data []byte) {+ t.Helper()++ _, err := client.PutObject(+ t.Context(),+ bucket,+ key,+ bytes.NewReader(data),+ int64(len(data)),+ minio.PutObjectOptions{},+ )+ require.NoError(t, err, "failed to write raw object %s", key)+}++func readWholeFile(t *testing.T, storage *S3Storage, encryptor encryption.FieldEncryptor, fileName string) []byte {+ t.Helper()++ reader, err := storage.GetFile(encryptor, fileName)+ require.NoError(t, err)+ defer func() { _ = reader.Close() }()++ content, err := io.ReadAll(reader)+ require.NoError(t, err)++ return content+}++func listObjectKeys(t *testing.T, client *minio.Client, bucket, prefix string) []string {+ t.Helper()++ var keys []string+ for object := range client.ListObjects(t.Context(), bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {+ require.NoError(t, object.Err)+ keys = append(keys, object.Key)+ }++ return keys+}++// generateBytes returns deterministic, position-dependent bytes so a round trip can be compared+// exactly without holding a second random source.+func generateBytes(size int) []byte {+ data := make([]byte, size)+ for i := range data {+ data[i] = byte(i*7 + 13)+ }++ return data+}++func discardLogger() *slog.Logger {+ return slog.New(slog.NewTextHandler(io.Discard, nil))+}++// cancelAfterReader serves data and cancels its context once cancelAt bytes have been read, to+// simulate a backup cancelled partway through a chunked upload.+type cancelAfterReader struct {+ data []byte+ pos int+ cancelAt int+ cancel context.CancelFunc+}++func (r *cancelAfterReader) Read(p []byte) (int, error) {+ if r.pos >= len(r.data) {+ return 0, io.EOF+ }++ n := copy(p, r.data[r.pos:])+ r.pos += n++ if r.cancel != nil && r.pos >= r.cancelAt {+ r.cancel()+ r.cancel = nil+ }++ return n, nil+}