Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (s3): Stop requiring bucket-wide ListBucket and DeleteObject to pass the connection test (issue #729)

Rostislav Dugin81d1cf5155a1
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.go
index d60e2960..ed82d649 100644
--- a/backend/internal/features/storages/models/s3/model.go
+++ b/backend/internal/features/storages/models/s3/model.go
@@ -33,6 +33,11 @@ const (
s3TLSHandshakeTimeout = 30 * time.Second
s3DeleteTimeout = 30 * time.Second
+ s3ConnectionTestTimeout = 10 * time.Second
+
+ connectionTestObjectName = "databasus-connection-test"
+ connectionTestObjectContent = "databasus connection test"
+
// defaultInnerPartSize is the multipart part we feed S3 for one object. 16 MiB balances
// memory against request count and creates backpressure to pg_dump: we keep one part in
// flight and wait for S3 to confirm it before reading the next, so pipeline memory does not
@@ -49,6 +54,9 @@ const (
manifestSuffix = ".parts"
manifestVersion = 1
+
+ s3ErrCodeNoSuchBucket = "NoSuchBucket"
+ s3ErrCodeAccessDenied = "AccessDenied"
)
type S3Storage struct {
@@ -236,61 +244,27 @@ func (s *S3Storage) Validate(encryptor encryption.FieldEncryptor) error {
return nil
}
+// A backup credential may be scoped to the configured prefix and denied deletes on purpose, so the
+// probe stays on a single key inside the prefix and never lets a refused delete fail the storage:
+// writing and reading back is everything a backup needs from S3.
func (s *S3Storage) TestConnection(encryptor encryption.FieldEncryptor) error {
client, err := s.getClient(encryptor)
if err != nil {
return err
}
- // Create a context with 10 second timeout
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
+ probeKey := s.buildObjectKey(connectionTestObjectName)
+ probeContent := []byte(connectionTestObjectContent)
- // Check if the bucket exists to verify connection
- exists, err := client.BucketExists(ctx, s.S3Bucket)
- if err != nil {
- // Check if the error is due to context deadline exceeded
- if errors.Is(err, context.DeadlineExceeded) {
- return errors.New("failed to connect to the bucket. Please check params")
- }
- return fmt.Errorf("failed to connect to S3: %w", err)
- }
-
- if !exists {
- return fmt.Errorf("bucket '%s' does not exist", s.S3Bucket)
+ if err := s.writeConnectionProbe(client, probeKey, probeContent); err != nil {
+ return err
}
- // Test write and delete permissions by uploading and removing a small test file
- testFileID := uuid.New().String() + "-test"
- testObjectKey := s.buildObjectKey(testFileID)
- testData := []byte("test connection")
- testReader := bytes.NewReader(testData)
-
- // Upload test file
- _, err = client.PutObject(
- ctx,
- s.S3Bucket,
- testObjectKey,
- testReader,
- int64(len(testData)),
- minio.PutObjectOptions{
- SendContentMd5: true,
- },
- )
- if err != nil {
- return fmt.Errorf("failed to upload test file to S3: %w", err)
+ if err := s.readAndVerifyConnectionProbe(client, probeKey, probeContent); err != nil {
+ return err
}
- // Delete test file
- err = client.RemoveObject(
- ctx,
- s.S3Bucket,
- testObjectKey,
- minio.RemoveObjectOptions{},
- )
- if err != nil {
- return fmt.Errorf("failed to delete test file from S3: %w", err)
- }
+ _ = s.deleteConnectionProbe(client, probeKey)
return nil
}
@@ -340,6 +314,93 @@ func (s *S3Storage) Update(incoming *S3Storage) {
// otherwise we will have to transfer all the data to the new prefix
}
+func (s *S3Storage) writeConnectionProbe(
+ client *minio.Client,
+ objectKey string,
+ content []byte,
+) error {
+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)
+ defer cancel()
+
+ // The probe carries the configured storage class so a policy that constrains
+ // s3:x-amz-storage-class accepts or rejects it exactly as it would a real backup.
+ options := s.putObjectOptions()
+ options.SendContentMd5 = true
+
+ _, err := client.PutObject(
+ ctx,
+ s.S3Bucket,
+ objectKey,
+ bytes.NewReader(content),
+ int64(len(content)),
+ options,
+ )
+ if err == nil {
+ return nil
+ }
+
+ if errors.Is(err, context.DeadlineExceeded) {
+ return errors.New("failed to connect to the bucket. Please check params")
+ }
+
+ switch minio.ToErrorResponse(err).Code {
+ case s3ErrCodeNoSuchBucket:
+ return fmt.Errorf("bucket '%s' does not exist", s.S3Bucket)
+ case s3ErrCodeAccessDenied:
+ return fmt.Errorf("credentials cannot write to '%s' in bucket '%s'", objectKey, s.S3Bucket)
+ }
+
+ return fmt.Errorf("failed to upload test file to S3: %w", err)
+}
+
+func (s *S3Storage) readAndVerifyConnectionProbe(
+ client *minio.Client,
+ objectKey string,
+ writtenContent []byte,
+) error {
+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)
+ defer cancel()
+
+ object, err := client.GetObject(ctx, s.S3Bucket, objectKey, minio.GetObjectOptions{})
+ if err != nil {
+ return s.describeConnectionProbeReadError(err, objectKey)
+ }
+
+ defer func() {
+ _ = object.Close()
+ }()
+
+ storedContent, err := io.ReadAll(object)
+ if err != nil {
+ return s.describeConnectionProbeReadError(err, objectKey)
+ }
+
+ if !bytes.Equal(storedContent, writtenContent) {
+ return fmt.Errorf("storage returned different content than was written to '%s'", objectKey)
+ }
+
+ return nil
+}
+
+func (s *S3Storage) describeConnectionProbeReadError(err error, objectKey string) error {
+ if errors.Is(err, context.DeadlineExceeded) {
+ return errors.New("failed to connect to the bucket. Please check params")
+ }
+
+ if minio.ToErrorResponse(err).Code == s3ErrCodeAccessDenied {
+ return fmt.Errorf("credentials cannot read back from '%s', so restores would fail", objectKey)
+ }
+
+ return fmt.Errorf("failed to read test file from S3: %w", err)
+}
+
+func (s *S3Storage) deleteConnectionProbe(client *minio.Client, objectKey string) error {
+ ctx, cancel := context.WithTimeout(context.Background(), s3ConnectionTestTimeout)
+ defer cancel()
+
+ return client.RemoveObject(ctx, s.S3Bucket, objectKey, minio.RemoveObjectOptions{})
+}
+
func (s *S3Storage) innerPartSize() int {
if s.innerPartSizeOverride > 0 {
return s.innerPartSizeOverride
diff --git a/backend/internal/features/storages/models/s3/model_test.go b/backend/internal/features/storages/models/s3/model_test.go
index 2de4fac7..72019817 100644
--- a/backend/internal/features/storages/models/s3/model_test.go
+++ b/backend/internal/features/storages/models/s3/model_test.go
@@ -6,6 +6,9 @@ import (
"fmt"
"io"
"log/slog"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
"testing"
"time"
@@ -297,8 +300,48 @@ func Test_GetFile_CorruptedManifest_ReturnsError(t *testing.T) {
assert.Contains(t, err.Error(), "manifest")
}
-// setupChunkedStorage boots a MinIO container, creates the bucket and returns a raw client for
-// direct object assertions alongside an S3Storage configured to roll objects on small inputs.
+func Test_TestConnection_WhenDeleteIsDenied_Succeeds(t *testing.T) {
+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{deleteErrorCode: s3ErrCodeAccessDenied})
+
+ assert.NoError(t, storage.TestConnection(encryptor))
+}
+
+func Test_TestConnection_WhenBucketIsMissing_ReturnsBucketDoesNotExist(t *testing.T) {
+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{putErrorCode: s3ErrCodeNoSuchBucket})
+
+ err := storage.TestConnection(encryptor)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "does not exist")
+}
+
+func Test_TestConnection_WhenWriteIsDenied_ReturnsWriteAccessError(t *testing.T) {
+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{putErrorCode: s3ErrCodeAccessDenied})
+
+ err := storage.TestConnection(encryptor)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "cannot write to")
+}
+
+func Test_TestConnection_WhenReadIsDenied_ReturnsReadAccessError(t *testing.T) {
+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{getErrorCode: s3ErrCodeAccessDenied})
+
+ err := storage.TestConnection(encryptor)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "cannot read back from")
+}
+
+func Test_TestConnection_WhenReadReturnsDifferentContent_ReturnsMismatchError(t *testing.T) {
+ storage, encryptor := setupProbeStubStorage(t, s3ProbeStub{servedContent: "corrupted"})
+
+ err := storage.TestConnection(encryptor)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "different content")
+}
+
func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.FieldEncryptor) {
t.Helper()
@@ -339,6 +382,103 @@ func setupChunkedStorage(t *testing.T) (*minio.Client, *S3Storage, encryption.Fi
return rawClient, storage, encryptor
}
+func setupProbeStubStorage(
+ t *testing.T,
+ stub s3ProbeStub,
+) (*S3Storage, encryption.FieldEncryptor) {
+ t.Helper()
+
+ server := startS3ProbeServer(t, stub)
+
+ encryptor := encryption.GetFieldEncryptor()
+
+ accessKey, err := encryptor.Encrypt(containers.MinioRootUser)
+ require.NoError(t, err)
+ secretKey, err := encryptor.Encrypt(containers.MinioRootPassword)
+ require.NoError(t, err)
+
+ storage := &S3Storage{
+ StorageID: uuid.New(),
+ S3Bucket: "test-bucket",
+ S3Region: containers.MinioRegion,
+ S3AccessKey: accessKey,
+ S3SecretKey: secretKey,
+ S3Endpoint: server.URL,
+ }
+
+ return storage, encryptor
+}
+
+// A real MinIO container cannot express these branches: its root credentials ignore bucket
+// policies, so a denied write, read or delete is not reproducible against it.
+type s3ProbeStub struct {
+ putErrorCode string
+ getErrorCode string
+ deleteErrorCode string
+ servedContent string
+}
+
+func startS3ProbeServer(t *testing.T, stub s3ProbeStub) *httptest.Server {
+ t.Helper()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodPut:
+ if stub.putErrorCode != "" {
+ writeS3ErrorResponse(w, stub.putErrorCode)
+ return
+ }
+
+ w.Header().Set("ETag", `"probe"`)
+ case http.MethodGet:
+ if stub.getErrorCode != "" {
+ writeS3ErrorResponse(w, stub.getErrorCode)
+ return
+ }
+
+ content := stub.servedContent
+ if content == "" {
+ content = connectionTestObjectContent
+ }
+
+ w.Header().Set("Content-Length", strconv.Itoa(len(content)))
+ w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT")
+ _, _ = w.Write([]byte(content))
+ case http.MethodDelete:
+ if stub.deleteErrorCode != "" {
+ writeS3ErrorResponse(w, stub.deleteErrorCode)
+ return
+ }
+
+ w.WriteHeader(http.StatusNoContent)
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ return server
+}
+
+func writeS3ErrorResponse(w http.ResponseWriter, code string) {
+ statusByCode := map[string]int{
+ s3ErrCodeNoSuchBucket: http.StatusNotFound,
+ s3ErrCodeAccessDenied: http.StatusForbidden,
+ }
+
+ status, isKnownCode := statusByCode[code]
+ if !isKnownCode {
+ status = http.StatusForbidden
+ }
+
+ w.Header().Set("Content-Type", "application/xml")
+ w.WriteHeader(status)
+ _, _ = fmt.Fprintf(
+ w,
+ `<?xml version="1.0" encoding="UTF-8"?><Error><Code>%s</Code><Message>%s</Message></Error>`,
+ code,
+ code,
+ )
+}
+
func putRawObject(t *testing.T, client *minio.Client, bucket, key string, data []byte) {
t.Helper()