PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (s3): Stop requiring bucket-wide ListBucket and DeleteObject to pass the connection test (issue #729)
Rostislav Dugin81d1cf5155a1
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.goindex d60e2960..ed82d649 100644--- a/backend/internal/features/storages/models/s3/model.go+++ b/backend/internal/features/storages/models/s3/model.go@@ -33,6 +33,11 @@ const (s3TLSHandshakeTimeout = 30 * time.Seconds3DeleteTimeout = 30 * time.Second+ s3ConnectionTestTimeout = 10 * time.Second++ connectionTestObjectName = "databasus-connection-test"+ connectionTestObjectContent = "databasus connection test"+// defaultInnerPartSize is the multipart part we feed S3 for one object. 16 MiB balances// memory against request count and creates backpressure to pg_dump: we keep one part in// flight and wait for S3 to confirm it before reading the next, so pipeline memory does not@@ -49,6 +54,9 @@ const (manifestSuffix = ".parts"manifestVersion = 1++ s3ErrCodeNoSuchBucket = "NoSuchBucket"+ s3ErrCodeAccessDenied = "AccessDenied")type S3Storage struct {@@ -236,61 +244,27 @@ func (s *S3Storage) Validate(encryptor encryption.FieldEncryptor) error {return nil}+// A backup credential may be scoped to the configured prefix and denied deletes on purpose, so the+// probe stays on a single key inside the prefix and never lets a refused delete fail the storage:+// writing and reading back is everything a backup needs from S3.func (s *S3Storage) TestConnection(encryptor encryption.FieldEncryptor) error {client, err := s.getClient(encryptor)if err != nil {return err}- // Create a context with 10 second timeout- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)- defer cancel()+ probeKey := s.buildObjectKey(connectionTestObjectName)+ probeContent := []byte(connectionTestObjectContent)- // Check if the bucket exists to verify connection- exists, err := client.BucketExists(ctx, s.S3Bucket)- if err != nil {- // Check if the error is due to context deadline exceeded- if errors.Is(err, context.DeadlineExceeded) {- return errors.New("failed to connect to the bucket. Please check params")- }- return fmt.Errorf("failed to connect to S3: %w", err)- }-- if !exists {- return fmt.Errorf("bucket '%s' does not exist", s.S3Bucket)+ if err := s.writeConnectionProbe(client, probeKey, probeContent); err != nil {+ return err}- // Test write and delete permissions by uploading and removing a small test file- testFileID := uuid.New().String() + "-test"- testObjectKey := s.buildObjectKey(testFileID)- testData := []byte("test connection")- testReader := bytes.NewReader(testData)-- // Upload test file- _, err = client.PutObject(- ctx,- s.S3Bucket,- testObjectKey,- testReader,- int64(len(testData)),- minio.PutObjectOptions{- SendContentMd5: true,- },- )- if err != nil {- return fmt.Errorf("failed to upload test file to S3: %w", err)+ if err := s.readAndVerifyConnectionProbe(client, probeKey, probeContent); err != nil {+ return err}- // Delete test file- err = client.RemoveObject(- ctx,- s.S3Bucket,- testObjectKey,- minio.RemoveObjectOptions{},- )- if err != nil {- return fmt.Errorf("failed to delete test file from S3: %w", err)- }+ _ = s.deleteConnectionProbe(client, probeKey)return nil}@@ -340,6 +314,93 @@ func (s *S3Storage) Update(incoming *S3Storage) {// otherwise we will have to transfer all the data to the new prefix}+func (s *S3Storage) writeConnectionProbe(+ client *minio.Client,+ objectKey string,+ content []byte,+) error {+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)+ defer cancel()++ // The probe carries the configured storage class so a policy that constrains+ // s3:x-amz-storage-class accepts or rejects it exactly as it would a real backup.+ options := s.putObjectOptions()+ options.SendContentMd5 = true++ _, err := client.PutObject(+ ctx,+ s.S3Bucket,+ objectKey,+ bytes.NewReader(content),+ int64(len(content)),+ options,+ )+ if err == nil {+ return nil+ }++ if errors.Is(err, context.DeadlineExceeded) {+ return errors.New("failed to connect to the bucket. Please check params")+ }++ switch minio.ToErrorResponse(err).Code {+ case s3ErrCodeNoSuchBucket:+ return fmt.Errorf("bucket '%s' does not exist", s.S3Bucket)+ case s3ErrCodeAccessDenied:+ return fmt.Errorf("credentials cannot write to '%s' in bucket '%s'", objectKey, s.S3Bucket)+ }++ return fmt.Errorf("failed to upload test file to S3: %w", err)+}++func (s *S3Storage) readAndVerifyConnectionProbe(+ client *minio.Client,+ objectKey string,+ writtenContent []byte,+) error {+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)+ defer cancel()++ object, err := client.GetObject(ctx, s.S3Bucket, objectKey, minio.GetObjectOptions{})+ if err != nil {+ return s.describeConnectionProbeReadError(err, objectKey)+ }++ defer func() {+ _ = object.Close()+ }()++ storedContent, err := io.ReadAll(object)+ if err != nil {+ return s.describeConnectionProbeReadError(err, objectKey)+ }++ if !bytes.Equal(storedContent, writtenContent) {+ return fmt.Errorf("storage returned different content than was written to '%s'", objectKey)+ }++ return nil+}++func (s *S3Storage) describeConnectionProbeReadError(err error, objectKey string) error {+ if errors.Is(err, context.DeadlineExceeded) {+ return errors.New("failed to connect to the bucket. Please check params")+ }++ if minio.ToErrorResponse(err).Code == s3ErrCodeAccessDenied {+ return fmt.Errorf("credentials cannot read back from '%s', so restores would fail", objectKey)+ }++ return fmt.Errorf("failed to read test file from S3: %w", err)+}++func (s *S3Storage) deleteConnectionProbe(client *minio.Client, objectKey string) error {+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)+ defer cancel()++ return client.RemoveObject(ctx, s.S3Bucket, objectKey, minio.RemoveObjectOptions{})+}+func (s *S3Storage) innerPartSize() int {if s.innerPartSizeOverride > 0 {return s.innerPartSizeOverridediff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.goindex 2de4fac7..72019817 100644--- a/backend/internal/features/storages/models/s3/model_test.go+++ b/backend/internal/features/storages/models/s3/model_test.go@@ -6,6 +6,9 @@ import ("fmt""io""log/slog"+ "net/http"+ "net/http/httptest"+ "strconv""testing""time"@@ -297,8 +300,48 @@ func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {assert.Contains(t, err.Error(), "manifest")}-// setupChunkedStorage boots a MinIO container, creates the bucket and returns a raw client for-// direct object assertions alongside an S3Storage configured to roll objects on small inputs.+func Test_TestConnection_WhenDeleteIsDenied_Succeeds(t *testing.T) {+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{deleteErrorCode: s3ErrCodeAccessDenied})++ assert.NoError(t, storage.TestConnection(encryptor))+}++func Test_TestConnection_WhenBucketIsMissing_ReturnsBucketDoesNotExist(t *testing.T) {+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{putErrorCode: s3ErrCodeNoSuchBucket})++ err := storage.TestConnection(encryptor)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "does not exist")+}++func Test_TestConnection_WhenWriteIsDenied_ReturnsWriteAccessError(t *testing.T) {+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{putErrorCode: s3ErrCodeAccessDenied})++ err := storage.TestConnection(encryptor)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "cannot write to")+}++func Test_TestConnection_WhenReadIsDenied_ReturnsReadAccessError(t *testing.T) {+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{getErrorCode: s3ErrCodeAccessDenied})++ err := storage.TestConnection(encryptor)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "cannot read back from")+}++func Test_TestConnection_WhenReadReturnsDifferentContent_ReturnsMismatchError(t *testing.T) {+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{servedContent: "corrupted"})++ err := storage.TestConnection(encryptor)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "different content")+}+func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.FieldEncryptor) {t.Helper()@@ -339,6 +382,103 @@ func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.Fireturn rawClient, storage, encryptor}+func setupProbeStubStorage(+ t *testing.T,+ stub s3ProbeStub,+) (*S3Storage, encryption.FieldEncryptor) {+ t.Helper()++ server := startS3ProbeServer(t, stub)++ encryptor := encryption.GetFieldEncryptor()++ accessKey, err := encryptor.Encrypt(containers.MinioRootUser)+ require.NoError(t, err)+ secretKey, err := encryptor.Encrypt(containers.MinioRootPassword)+ require.NoError(t, err)++ storage := &S3Storage{+ StorageID: uuid.New(),+ S3Bucket: "test-bucket",+ S3Region: containers.MinioRegion,+ S3AccessKey: accessKey,+ S3SecretKey: secretKey,+ S3Endpoint: server.URL,+ }++ return storage, encryptor+}++// A real MinIO container cannot express these branches: its root credentials ignore bucket+// policies, so a denied write, read or delete is not reproducible against it.+type s3ProbeStub struct {+ putErrorCode string+ getErrorCode string+ deleteErrorCode string+ servedContent string+}++func startS3ProbeServer(t *testing.T, stub s3ProbeStub) *httptest.Server {+ t.Helper()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.Method {+ case http.MethodPut:+ if stub.putErrorCode != "" {+ writeS3ErrorResponse(w, stub.putErrorCode)+ return+ }++ w.Header().Set("ETag", `"probe"`)+ case http.MethodGet:+ if stub.getErrorCode != "" {+ writeS3ErrorResponse(w, stub.getErrorCode)+ return+ }++ content := stub.servedContent+ if content == "" {+ content = connectionTestObjectContent+ }++ w.Header().Set("Content-Length", strconv.Itoa(len(content)))+ w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT")+ _, _ = w.Write([]byte(content))+ case http.MethodDelete:+ if stub.deleteErrorCode != "" {+ writeS3ErrorResponse(w, stub.deleteErrorCode)+ return+ }++ w.WriteHeader(http.StatusNoContent)+ }+ }))+ t.Cleanup(server.Close)++ return server+}++func writeS3ErrorResponse(w http.ResponseWriter, code string) {+ statusByCode := map[string]int{+ s3ErrCodeNoSuchBucket: http.StatusNotFound,+ s3ErrCodeAccessDenied: http.StatusForbidden,+ }++ status, isKnownCode := statusByCode[code]+ if !isKnownCode {+ status = http.StatusForbidden+ }++ w.Header().Set("Content-Type", "application/xml")+ w.WriteHeader(status)+ _, _ = fmt.Fprintf(+ w,+ `<?xml version="1.0" encoding="UTF-8"?><Error><Code>%s</Code><Message>%s</Message></Error>`,+ code,+ code,+ )+}+func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data []byte) {t.Helper()