Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
// Package s3 provides reusable S3 configuration and connectivity helpers. It
// deals in infrastructure configuration only; mapping from API DTOs belongs to
// the backup destination domain.
package s3
import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"slices"
"strings"
"time"
"uuid"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
kit "go.getarcane.app/kit/pkg"
)
const (
RepositoryReasonMissingBucket = "missing_bucket"
RepositoryReasonMissingRepository = "missing_repository"
connectionTestPayload = "arcane-s3-connection-test"
)
// Configuration contains the credentials and addressing options for an S3 destination.
type Configuration struct {
ID string
Name string
Endpoint string
Bucket string
Region string
AccessKeyID string
SecretAccessKey string
Prefix string
UseSSL bool
ForcePathStyle bool
}
// Normalized returns a copy with surrounding whitespace and prefix separators removed.
func (c Configuration) Normalized() Configuration {
c.ID = strings.TrimSpace(c.ID)
c.Name = strings.TrimSpace(c.Name)
c.Endpoint = strings.TrimSpace(c.Endpoint)
c.Bucket = strings.TrimSpace(c.Bucket)
c.Region = strings.TrimSpace(c.Region)
c.AccessKeyID = strings.TrimSpace(c.AccessKeyID)
c.SecretAccessKey = strings.TrimSpace(c.SecretAccessKey)
c.Prefix = strings.Trim(strings.TrimSpace(c.Prefix), "/")
return c
}
// ValidateEndpoint checks a custom endpoint: only HTTP or HTTPS with a host,
// no userinfo, query, or fragment. Private and self-hosted hosts are allowed.
func ValidateEndpoint(endpoint string) error {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return nil
}
candidate := endpoint
if !strings.Contains(candidate, "://") {
candidate = "https://" + candidate
}
parsed, err := url.Parse(candidate)
if err != nil {
return fmt.Errorf("invalid S3 endpoint: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return errors.New("invalid S3 endpoint: only http and https are supported")
}
if parsed.Host == "" {
return errors.New("invalid S3 endpoint: host is required")
}
if parsed.User != nil {
return errors.New("invalid S3 endpoint: credentials in the URL are not allowed")
}
if parsed.RawQuery != "" || parsed.Fragment != "" || parsed.RawFragment != "" {
return errors.New("invalid S3 endpoint: query strings and fragments are not allowed")
}
return nil
}
// Validate verifies the fields required to connect to the configured destination.
func (c Configuration) Validate(requireSecret bool) error {
c = c.Normalized()
if c.Name == "" {
return errors.New("name is required")
}
if c.Bucket == "" {
return errors.New("bucket is required")
}
if c.Endpoint == "" && c.Region == "" {
return errors.New("region is required for AWS S3")
}
if err := ValidateEndpoint(c.Endpoint); err != nil {
return err
}
if c.AccessKeyID == "" {
return errors.New("access key ID is required")
}
if requireSecret && c.SecretAccessKey == "" {
return errors.New("secret access key is required")
}
return nil
}
// ConnectionFieldsEqual reports whether two configurations address the same
// service the same way. A change in any of these fields means a stored secret
// may no longer be combined with the configuration.
func ConnectionFieldsEqual(a, b Configuration) bool {
a, b = a.Normalized(), b.Normalized()
return a.Endpoint == b.Endpoint &&
a.Bucket == b.Bucket &&
a.Region == b.Region &&
a.AccessKeyID == b.AccessKeyID &&
a.UseSSL == b.UseSSL &&
a.ForcePathStyle == b.ForcePathStyle
}
// EndpointURL returns the normalized endpoint URL, applying UseSSL to custom endpoints.
func (c Configuration) EndpointURL() string {
endpoint := strings.TrimSpace(c.Endpoint)
if endpoint == "" {
return ""
}
endpoint = strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
scheme := kit.Ternary(c.UseSSL, "https://", "http://")
return scheme + strings.TrimRight(endpoint, "/")
}
// RusticEnvironment builds the environment used by Rustic's OpenDAL S3 backend.
func (c Configuration) RusticEnvironment(rootParts ...string) []string {
c = c.Normalized()
repositoryParts := append([]string{"/", c.Prefix}, rootParts...)
region := cmp.Or(c.Region, "us-east-1")
environment := []string{
"RUSTIC_REPOSITORY=opendal:s3",
"RUSTIC_REPO_OPT_BUCKET=" + c.Bucket,
"RUSTIC_REPO_OPT_ROOT=" + path.Join(repositoryParts...),
"AWS_ACCESS_KEY_ID=" + c.AccessKeyID,
"AWS_SECRET_ACCESS_KEY=" + c.SecretAccessKey,
"AWS_EC2_METADATA_DISABLED=true",
}
if endpoint := c.EndpointURL(); endpoint != "" {
environment = append(environment, "RUSTIC_REPO_OPT_ENDPOINT="+endpoint)
}
environment = append(
environment,
"RUSTIC_REPO_OPT_REGION="+region,
"AWS_REGION="+region,
)
if !c.ForcePathStyle {
environment = append(environment, "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true")
}
return environment
}
// TestConnection verifies upload, download, and delete access for a destination.
func TestConnection(ctx context.Context, configuration Configuration) (err error) {
configuration = configuration.Normalized()
if validateErr := configuration.Validate(true); validateErr != nil {
return validateErr
}
client, err := newClientInternal(ctx, configuration)
if err != nil {
return fmt.Errorf("failed to upload S3 connection test object: %w", err)
}
payload := []byte(connectionTestPayload)
remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.New().String())
if putObjectErr := putObjectInternal(ctx, client, configuration.Bucket, remoteKey, payload); putObjectErr != nil {
return fmt.Errorf("failed to upload S3 connection test object: %w", putObjectErr)
}
deleted := false
defer func() {
if deleted {
return
}
if cleanupErr := deleteObjectInternal(context.WithoutCancel(ctx), client, configuration.Bucket, remoteKey); cleanupErr != nil {
err = errors.Join(err, fmt.Errorf("failed to clean up S3 connection test object: %w", cleanupErr))
}
}()
downloaded, err := getObjectInternal(ctx, client, configuration.Bucket, remoteKey)
if err != nil {
return fmt.Errorf("failed to download S3 connection test object: %w", err)
}
if !bytes.Equal(downloaded, payload) {
return errors.New("S3 connection test object contents did not match")
}
if deleteObjectErr := deleteObjectInternal(ctx, client, configuration.Bucket, remoteKey); deleteObjectErr != nil {
return fmt.Errorf("failed to delete S3 connection test object: %w", deleteObjectErr)
}
deleted = true
return nil
}
// CheckRepository reads only the config and, when supplied, one snapshot object.
func CheckRepository(ctx context.Context, configuration Configuration, root, snapshotID string) (backuptypes.RepositoryObservation, error) {
configuration = configuration.Normalized()
if err := configuration.Validate(true); err != nil {
return backuptypes.RepositoryObservation{}, err
}
client, err := newClientInternal(ctx, configuration)
if err != nil {
return backuptypes.RepositoryObservation{}, fmt.Errorf("failed to configure S3 repository check: %w", err)
}
configKey := path.Join(configuration.Prefix, root, "config")
if checkBackupObjectErr := checkBackupObjectInternal(ctx, client, configuration.Bucket, configKey); checkBackupObjectErr != nil {
if isMissingResourceInternal(checkBackupObjectErr, "NoSuchBucket") {
return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
}
if isMissingResourceInternal(checkBackupObjectErr, "NoSuchKey") {
return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingRepository}, nil
}
return backuptypes.RepositoryObservation{}, checkBackupObjectErr
}
result := backuptypes.RepositoryObservation{Available: true}
if snapshotID == "" {
return result, nil
}
snapshotKey := path.Join(configuration.Prefix, root, "snapshots", snapshotID)
if checkBackupObjectErr2 := checkBackupObjectInternal(ctx, client, configuration.Bucket, snapshotKey); checkBackupObjectErr2 != nil {
if isMissingResourceInternal(checkBackupObjectErr2, "NoSuchKey") {
return result, nil
}
if isMissingResourceInternal(checkBackupObjectErr2, "NoSuchBucket") {
return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
}
return backuptypes.RepositoryObservation{}, checkBackupObjectErr2
}
result.SnapshotAvailable = true
return result, nil
}
// A one-byte GET preserves the named S3 errors that generic HEAD responses omit.
func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
object, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Range: aws.String("bytes=0-0")})
if err != nil {
return err
}
return object.Body.Close()
}
// ListRepositoryRoots returns the first-level prefixes below root, one per Arcane instance.
func ListRepositoryRoots(ctx context.Context, configuration Configuration, root string) ([]string, error) {
configuration = configuration.Normalized()
if err := configuration.Validate(true); err != nil {
return nil, err
}
client, err := newClientInternal(ctx, configuration)
if err != nil {
return nil, fmt.Errorf("failed to configure S3 repository listing: %w", err)
}
prefix := path.Join(configuration.Prefix, strings.Trim(root, "/")) + "/"
var roots []string
paginator := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{
Bucket: aws.String(configuration.Bucket),
Prefix: aws.String(prefix),
Delimiter: aws.String("/"),
})
for paginator.HasMorePages() {
page, nextPageErr := paginator.NextPage(ctx)
if nextPageErr != nil {
return nil, fmt.Errorf("failed to list S3 repository roots: %w", nextPageErr)
}
for _, commonPrefix := range page.CommonPrefixes {
if commonPrefix.Prefix == nil {
continue
}
name := strings.Trim(strings.TrimPrefix(*commonPrefix.Prefix, prefix), "/")
if name != "" {
roots = append(roots, name)
}
}
}
return roots, nil
}
func isMissingResourceInternal(err error, codes ...string) bool {
var apiErr smithy.APIError
if !errors.As(err, &apiErr) {
return false
}
return slices.Contains(codes, apiErr.ErrorCode())
}
type gcsHttpSigner struct {
wrapped awss3.HTTPSignerV4
}
func (s *gcsHttpSigner) SignHTTP(ctx context.Context, creds aws.Credentials, req *http.Request, payloadHash, service, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {
req.Header.Del("Accept-Encoding")
return s.wrapped.SignHTTP(ctx, creds, req, payloadHash, service, region, signingTime, optFns...)
}
func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {
awsConfiguration, err := awsconfig.LoadDefaultConfig(
ctx,
awsconfig.WithRegion(cmp.Or(configuration.Region, "us-east-1")),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(configuration.AccessKeyID, configuration.SecretAccessKey, "")),
)
if err != nil {
return nil, err
}
endpoint := configuration.EndpointURL()
return awss3.NewFromConfig(awsConfiguration, func(options *awss3.Options) {
options.UsePathStyle = configuration.ForcePathStyle
// S3-compatible stores such as GCS reject the SDK's default flexible checksums.
options.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
options.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
if endpoint != "" {
options.BaseEndpoint = aws.String(endpoint)
}
if endpoint == "https://storage.googleapis.com" {
options.HTTPSignerV4 = &gcsHttpSigner{wrapped: options.HTTPSignerV4}
}
}), nil
}
func putObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string, payload []byte) error {
_, err := client.PutObject(ctx, &awss3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(payload),
ContentLength: new(int64(len(payload))),
ContentType: aws.String("application/gzip"),
})
return err
}
func getObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) ([]byte, error) {
output, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
if err != nil {
return nil, err
}
payload, readErr := io.ReadAll(output.Body)
closeErr := output.Body.Close()
if readErr != nil {
return nil, errors.Join(readErr, closeErr)
}
if closeErr != nil {
return nil, closeErr
}
return payload, nil
}
func deleteObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
_, err := client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
return err
}