Skip to content

Modern Docker Management, Designed for Everyone

12.8 KB
Raw Blame
// Package s3 provides reusable S3 configuration and connectivity helpers. It
// deals in infrastructure configuration only; mapping from API DTOs belongs to
// the backup destination domain.
package s3

import (
	"bytes"
	"cmp"
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path"
	"slices"
	"strings"
	"time"
	"uuid"

	"github.com/aws/aws-sdk-go-v2/aws"
	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
	awsconfig "github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/credentials"
	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/smithy-go"
	backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
	kit "go.getarcane.app/kit/pkg"
)

const (
	RepositoryReasonMissingBucket     = "missing_bucket"
	RepositoryReasonMissingRepository = "missing_repository"
	connectionTestPayload             = "arcane-s3-connection-test"
)

// Configuration contains the credentials and addressing options for an S3 destination.
type Configuration struct {
	ID              string
	Name            string
	Endpoint        string
	Bucket          string
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	Prefix          string
	UseSSL          bool
	ForcePathStyle  bool
}

// Normalized returns a copy with surrounding whitespace and prefix separators removed.
func (c Configuration) Normalized() Configuration {
	c.ID = strings.TrimSpace(c.ID)
	c.Name = strings.TrimSpace(c.Name)
	c.Endpoint = strings.TrimSpace(c.Endpoint)
	c.Bucket = strings.TrimSpace(c.Bucket)
	c.Region = strings.TrimSpace(c.Region)
	c.AccessKeyID = strings.TrimSpace(c.AccessKeyID)
	c.SecretAccessKey = strings.TrimSpace(c.SecretAccessKey)
	c.Prefix = strings.Trim(strings.TrimSpace(c.Prefix), "/")
	return c
}

// ValidateEndpoint checks a custom endpoint: only HTTP or HTTPS with a host,
// no userinfo, query, or fragment. Private and self-hosted hosts are allowed.
func ValidateEndpoint(endpoint string) error {
	endpoint = strings.TrimSpace(endpoint)
	if endpoint == "" {
		return nil
	}
	candidate := endpoint
	if !strings.Contains(candidate, "://") {
		candidate = "https://" + candidate
	}
	parsed, err := url.Parse(candidate)
	if err != nil {
		return fmt.Errorf("invalid S3 endpoint: %w", err)
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return errors.New("invalid S3 endpoint: only http and https are supported")
	}
	if parsed.Host == "" {
		return errors.New("invalid S3 endpoint: host is required")
	}
	if parsed.User != nil {
		return errors.New("invalid S3 endpoint: credentials in the URL are not allowed")
	}
	if parsed.RawQuery != "" || parsed.Fragment != "" || parsed.RawFragment != "" {
		return errors.New("invalid S3 endpoint: query strings and fragments are not allowed")
	}
	return nil
}

// Validate verifies the fields required to connect to the configured destination.
func (c Configuration) Validate(requireSecret bool) error {
	c = c.Normalized()
	if c.Name == "" {
		return errors.New("name is required")
	}
	if c.Bucket == "" {
		return errors.New("bucket is required")
	}
	if c.Endpoint == "" && c.Region == "" {
		return errors.New("region is required for AWS S3")
	}
	if err := ValidateEndpoint(c.Endpoint); err != nil {
		return err
	}
	if c.AccessKeyID == "" {
		return errors.New("access key ID is required")
	}
	if requireSecret && c.SecretAccessKey == "" {
		return errors.New("secret access key is required")
	}
	return nil
}

// ConnectionFieldsEqual reports whether two configurations address the same
// service the same way. A change in any of these fields means a stored secret
// may no longer be combined with the configuration.
func ConnectionFieldsEqual(a, b Configuration) bool {
	a, b = a.Normalized(), b.Normalized()
	return a.Endpoint == b.Endpoint &&
		a.Bucket == b.Bucket &&
		a.Region == b.Region &&
		a.AccessKeyID == b.AccessKeyID &&
		a.UseSSL == b.UseSSL &&
		a.ForcePathStyle == b.ForcePathStyle
}

// EndpointURL returns the normalized endpoint URL, applying UseSSL to custom endpoints.
func (c Configuration) EndpointURL() string {
	endpoint := strings.TrimSpace(c.Endpoint)
	if endpoint == "" {
		return ""
	}
	endpoint = strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
	scheme := kit.Ternary(c.UseSSL, "https://", "http://")
	return scheme + strings.TrimRight(endpoint, "/")
}

// RusticEnvironment builds the environment used by Rustic's OpenDAL S3 backend.
func (c Configuration) RusticEnvironment(rootParts ...string) []string {
	c = c.Normalized()
	repositoryParts := append([]string{"/", c.Prefix}, rootParts...)
	region := cmp.Or(c.Region, "us-east-1")
	environment := []string{
		"RUSTIC_REPOSITORY=opendal:s3",
		"RUSTIC_REPO_OPT_BUCKET=" + c.Bucket,
		"RUSTIC_REPO_OPT_ROOT=" + path.Join(repositoryParts...),
		"AWS_ACCESS_KEY_ID=" + c.AccessKeyID,
		"AWS_SECRET_ACCESS_KEY=" + c.SecretAccessKey,
		"AWS_EC2_METADATA_DISABLED=true",
	}
	if endpoint := c.EndpointURL(); endpoint != "" {
		environment = append(environment, "RUSTIC_REPO_OPT_ENDPOINT="+endpoint)
	}
	environment = append(
		environment,
		"RUSTIC_REPO_OPT_REGION="+region,
		"AWS_REGION="+region,
	)
	if !c.ForcePathStyle {
		environment = append(environment, "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true")
	}
	return environment
}

// TestConnection verifies upload, download, and delete access for a destination.
func TestConnection(ctx context.Context, configuration Configuration) (err error) {
	configuration = configuration.Normalized()
	if validateErr := configuration.Validate(true); validateErr != nil {
		return validateErr
	}
	client, err := newClientInternal(ctx, configuration)
	if err != nil {
		return fmt.Errorf("failed to upload S3 connection test object: %w", err)
	}
	payload := []byte(connectionTestPayload)
	remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.New().String())
	if putObjectErr := putObjectInternal(ctx, client, configuration.Bucket, remoteKey, payload); putObjectErr != nil {
		return fmt.Errorf("failed to upload S3 connection test object: %w", putObjectErr)
	}

	deleted := false
	defer func() {
		if deleted {
			return
		}
		if cleanupErr := deleteObjectInternal(context.WithoutCancel(ctx), client, configuration.Bucket, remoteKey); cleanupErr != nil {
			err = errors.Join(err, fmt.Errorf("failed to clean up S3 connection test object: %w", cleanupErr))
		}
	}()

	downloaded, err := getObjectInternal(ctx, client, configuration.Bucket, remoteKey)
	if err != nil {
		return fmt.Errorf("failed to download S3 connection test object: %w", err)
	}
	if !bytes.Equal(downloaded, payload) {
		return errors.New("S3 connection test object contents did not match")
	}
	if deleteObjectErr := deleteObjectInternal(ctx, client, configuration.Bucket, remoteKey); deleteObjectErr != nil {
		return fmt.Errorf("failed to delete S3 connection test object: %w", deleteObjectErr)
	}
	deleted = true
	return nil
}

// CheckRepository reads only the config and, when supplied, one snapshot object.
func CheckRepository(ctx context.Context, configuration Configuration, root, snapshotID string) (backuptypes.RepositoryObservation, error) {
	configuration = configuration.Normalized()
	if err := configuration.Validate(true); err != nil {
		return backuptypes.RepositoryObservation{}, err
	}
	client, err := newClientInternal(ctx, configuration)
	if err != nil {
		return backuptypes.RepositoryObservation{}, fmt.Errorf("failed to configure S3 repository check: %w", err)
	}
	configKey := path.Join(configuration.Prefix, root, "config")
	if checkBackupObjectErr := checkBackupObjectInternal(ctx, client, configuration.Bucket, configKey); checkBackupObjectErr != nil {
		if isMissingResourceInternal(checkBackupObjectErr, "NoSuchBucket") {
			return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
		}
		if isMissingResourceInternal(checkBackupObjectErr, "NoSuchKey") {
			return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingRepository}, nil
		}
		return backuptypes.RepositoryObservation{}, checkBackupObjectErr
	}
	result := backuptypes.RepositoryObservation{Available: true}
	if snapshotID == "" {
		return result, nil
	}
	snapshotKey := path.Join(configuration.Prefix, root, "snapshots", snapshotID)
	if checkBackupObjectErr2 := checkBackupObjectInternal(ctx, client, configuration.Bucket, snapshotKey); checkBackupObjectErr2 != nil {
		if isMissingResourceInternal(checkBackupObjectErr2, "NoSuchKey") {
			return result, nil
		}
		if isMissingResourceInternal(checkBackupObjectErr2, "NoSuchBucket") {
			return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
		}
		return backuptypes.RepositoryObservation{}, checkBackupObjectErr2
	}
	result.SnapshotAvailable = true
	return result, nil
}

// A one-byte GET preserves the named S3 errors that generic HEAD responses omit.
func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
	object, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Range: aws.String("bytes=0-0")})
	if err != nil {
		return err
	}
	return object.Body.Close()
}

// ListRepositoryRoots returns the first-level prefixes below root, one per Arcane instance.
func ListRepositoryRoots(ctx context.Context, configuration Configuration, root string) ([]string, error) {
	configuration = configuration.Normalized()
	if err := configuration.Validate(true); err != nil {
		return nil, err
	}
	client, err := newClientInternal(ctx, configuration)
	if err != nil {
		return nil, fmt.Errorf("failed to configure S3 repository listing: %w", err)
	}
	prefix := path.Join(configuration.Prefix, strings.Trim(root, "/")) + "/"
	var roots []string
	paginator := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{
		Bucket:    aws.String(configuration.Bucket),
		Prefix:    aws.String(prefix),
		Delimiter: aws.String("/"),
	})
	for paginator.HasMorePages() {
		page, nextPageErr := paginator.NextPage(ctx)
		if nextPageErr != nil {
			return nil, fmt.Errorf("failed to list S3 repository roots: %w", nextPageErr)
		}
		for _, commonPrefix := range page.CommonPrefixes {
			if commonPrefix.Prefix == nil {
				continue
			}
			name := strings.Trim(strings.TrimPrefix(*commonPrefix.Prefix, prefix), "/")
			if name != "" {
				roots = append(roots, name)
			}
		}
	}
	return roots, nil
}

func isMissingResourceInternal(err error, codes ...string) bool {
	var apiErr smithy.APIError
	if !errors.As(err, &apiErr) {
		return false
	}
	return slices.Contains(codes, apiErr.ErrorCode())
}

type gcsHttpSigner struct {
	wrapped awss3.HTTPSignerV4
}

func (s *gcsHttpSigner) SignHTTP(ctx context.Context, creds aws.Credentials, req *http.Request, payloadHash, service, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {
	req.Header.Del("Accept-Encoding")
	return s.wrapped.SignHTTP(ctx, creds, req, payloadHash, service, region, signingTime, optFns...)
}

func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {
	awsConfiguration, err := awsconfig.LoadDefaultConfig(
		ctx,
		awsconfig.WithRegion(cmp.Or(configuration.Region, "us-east-1")),
		awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(configuration.AccessKeyID, configuration.SecretAccessKey, "")),
	)
	if err != nil {
		return nil, err
	}
	endpoint := configuration.EndpointURL()
	return awss3.NewFromConfig(awsConfiguration, func(options *awss3.Options) {
		options.UsePathStyle = configuration.ForcePathStyle
		// S3-compatible stores such as GCS reject the SDK's default flexible checksums.
		options.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
		options.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
		if endpoint != "" {
			options.BaseEndpoint = aws.String(endpoint)
		}
		if endpoint == "https://storage.googleapis.com" {
			options.HTTPSignerV4 = &gcsHttpSigner{wrapped: options.HTTPSignerV4}
		}
	}), nil
}

func putObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string, payload []byte) error {
	_, err := client.PutObject(ctx, &awss3.PutObjectInput{
		Bucket:        aws.String(bucket),
		Key:           aws.String(key),
		Body:          bytes.NewReader(payload),
		ContentLength: new(int64(len(payload))),
		ContentType:   aws.String("application/gzip"),
	})
	return err
}

func getObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) ([]byte, error) {
	output, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
	if err != nil {
		return nil, err
	}
	payload, readErr := io.ReadAll(output.Body)
	closeErr := output.Body.Close()
	if readErr != nil {
		return nil, errors.Join(readErr, closeErr)
	}
	if closeErr != nil {
		return nil, closeErr
	}
	return payload, nil
}

func deleteObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
	_, err := client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
	return err
}