Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
// Package backup owns the shared Rustic backup engine used by volume and
// system backups: typed repository operations, per-repository serialization
// and run admission.
package backup
import (
"cmp"
"context"
"crypto/rand"
"encoding/base32"
"encoding/hex"
"encoding/json/jsontext"
"encoding/json/v2"
"errors"
"fmt"
"io"
"log/slog"
"path"
"regexp"
"slices"
"strings"
"sync"
"time"
"uuid"
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
s3util "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
)
const (
VolumeAdmissionScope = "volume-backup"
SystemAdmissionScope = "system-backup"
// SystemAdmissionID is the one admission key system and system-managed volume backups share.
SystemAdmissionID = "system"
RecoveryKeyConfigID = "system-recovery"
// VolumeRoot is the S3 prefix under which each instance keeps its volume backup repository.
VolumeRoot = "arcane-volume-backups"
// rusticRepositoryMissingMessage is how Rustic reports an uninitialized repository.
rusticRepositoryMissingMessage = "No repository config file found"
)
type Repository struct {
ID string
Environment []string
Mounts []mount.Mount
}
type Snapshot struct {
ID string `json:"id"`
Size int64 `json:"-"`
}
type rusticSnapshotOutput struct {
ID string `json:"id"`
Summary struct {
TotalBytesProcessed int64 `json:"total_bytes_processed"`
} `json:"summary"`
}
// RestoreOptions selects what a snapshot restore writes and where.
type RestoreOptions struct {
// DeleteExtra removes files in the target that are absent from the snapshot.
DeleteExtra bool
// SourcePath restores only this path inside the snapshot when non-empty.
SourcePath string
// DestinationPath overrides the restore target path inside the helper
// container; the target mount's path is used when empty.
DestinationPath string
// ExtraMounts are attached alongside the target, e.g. mounts nested inside it.
ExtraMounts []mount.Mount
}
// CreateSnapshotInput describes what one snapshot captures: the mounts to
// attach to the helper and the helper paths to back up. AsPath rewrites the
// snapshot path of a single source and cannot combine with several.
type CreateSnapshotInput struct {
Mounts []mount.Mount
Sources []string
AsPath string
Tags []string
Globs []string
}
// RootSnapshotInput captures one mount with its contents at the snapshot root.
func RootSnapshotInput(source mount.Mount, tags ...string) CreateSnapshotInput {
return CreateSnapshotInput{Mounts: []mount.Mount{source}, Sources: []string{source.Target}, AsPath: "/", Tags: tags}
}
func snapshotCommand(label string, input CreateSnapshotInput) ([]string, error) {
if len(input.Sources) == 0 {
return nil, errors.New("at least one snapshot source is required")
}
if input.AsPath != "" && len(input.Sources) > 1 {
return nil, errors.New("a snapshot path rewrite requires a single source")
}
command := []string{"backup", "--init", "--json", "--host", "arcane", "--label", label}
for _, tag := range input.Tags {
command = append(command, "--tag", tag)
}
for _, glob := range input.Globs {
command = append(command, "--glob", glob)
}
if input.AsPath != "" {
command = append(command, "--as-path", input.AsPath)
}
command = append(command, "--")
return append(command, input.Sources...), nil
}
// Engine owns repository serialization and backup admission.
type Engine struct {
imageService *image.ImageService
admission *runs.Admission
mu sync.Mutex
repositories map[string]*sync.Mutex
// held parks admission a manual start won until the run that won it takes it over.
held map[scheduler.AdmissionKey]heldLease
authorize func(context.Context, backup.Requester) error
}
// heldLease is a parked lease and the run entitled to take it.
type heldLease struct {
run string
lease *runs.Lease
}
// NewRequester captures the user, and API key if any, behind a manual backup request.
func NewRequester(ctx context.Context, requestedBy user.Actor, environmentID, permission string) backup.Requester {
keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
return backup.Requester{UserID: requestedBy.ID, APIKeyID: keyID, EnvironmentID: environmentID, Permission: permission}
}
// NewEngine builds the backup engine on the shared admission gate.
func NewEngine(admission *runs.Admission, imageService *image.ImageService) *Engine {
return &Engine{
imageService: imageService,
admission: admission,
repositories: make(map[string]*sync.Mutex),
held: make(map[scheduler.AdmissionKey]heldLease),
}
}
// TryAcquireRun takes the admission lease for one backup resource, reporting false while another run holds it.
func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {
return e.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: scope, ID: id})
}
// Hold parks a lease won by a manual start for run until that run's task takes it over with AcquireRun.
// The returned func releases the lease if the run never took it, such as a start that failed to submit.
func (e *Engine) Hold(scope, id, run string, lease *runs.Lease) func(context.Context) {
key := scheduler.AdmissionKey{Scope: scope, ID: id}
e.mu.Lock()
e.held[key] = heldLease{run: run, lease: lease}
e.mu.Unlock()
return func(ctx context.Context) {
e.mu.Lock()
untaken := e.held[key].lease == lease
if untaken {
delete(e.held, key)
}
e.mu.Unlock()
if untaken {
lease.Release(ctx)
}
}
}
// AcquireRun takes over the lease held for run, or acquires admission otherwise, such as after a restart.
// A lease parked for another run stays parked, so that run's task still finds its admission.
func (e *Engine) AcquireRun(ctx context.Context, scope, id, run string) (*runs.Lease, bool, error) {
key := scheduler.AdmissionKey{Scope: scope, ID: id}
e.mu.Lock()
held, ok := e.held[key]
if ok = ok && run != "" && held.run == run; ok {
delete(e.held, key)
}
e.mu.Unlock()
if ok {
return held.lease, true, nil
}
return e.TryAcquireRun(ctx, scope, id)
}
// SetAuthorize sets the check Authorize applies to manual backup requesters.
func (e *Engine) SetAuthorize(authorize func(context.Context, backup.Requester) error) {
e.authorize = authorize
}
// Authorize checks that the requester may still run the backup.
func (e *Engine) Authorize(ctx context.Context, requester backup.Requester) error {
if e.authorize == nil {
return nil
}
return e.authorize(ctx, requester)
}
// Stop releases leases that no workflow task took over.
func (e *Engine) Stop(ctx context.Context) error {
e.mu.Lock()
held := e.held
e.held = make(map[scheduler.AdmissionKey]heldLease)
e.mu.Unlock()
for _, held := range held {
held.lease.Release(ctx)
}
return nil
}
// CreateSnapshot backs the input's sources up into the repository as one
// snapshot. The repository is initialized on first use.
func (e *Engine) CreateSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, label string, input CreateSnapshotInput) (Snapshot, error) {
command, err := snapshotCommand(label, input)
if err != nil {
return Snapshot{}, err
}
output, err := e.run(ctx, dockerClient, repository, password, command, input.Mounts...)
if err != nil {
return Snapshot{}, err
}
var decoded rusticSnapshotOutput
if unmarshalErr := json.Unmarshal([]byte(output), &decoded); unmarshalErr != nil {
return Snapshot{}, fmt.Errorf("failed to decode Rustic snapshot: %w", unmarshalErr)
}
if decoded.ID == "" {
return Snapshot{}, errors.New("rustic did not return a snapshot ID")
}
return Snapshot{ID: decoded.ID, Size: decoded.Summary.TotalBytesProcessed}, nil
}
// RestoreSnapshot restores a snapshot (or one path inside it) onto the target mount.
func (e *Engine) RestoreSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string, target mount.Mount, options RestoreOptions) error {
command := []string{"restore", "--verify-existing"}
if options.DeleteExtra {
command = append(command, "--delete")
}
source := snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")
command = append(command, "--", source, cmp.Or(options.DestinationPath, target.Target))
mounts := append([]mount.Mount{target}, options.ExtraMounts...)
_, err := e.run(ctx, dockerClient, repository, password, command, mounts...)
return err
}
// ListSnapshotFiles lists a snapshot path and returns paths relative to the
// snapshot root. A supplied path is nonrecursive unless recursive is true.
func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID, filePath string, recursive bool) ([]string, error) {
command := []string{"ls", "--json"}
if recursive {
command = append(command, "--recursive")
}
filePath = strings.TrimSpace(filePath)
cleanedPath := strings.Trim(filePath, "/")
source := snapshotID + ":/" + cleanedPath
if cleanedPath != "" && strings.HasSuffix(filePath, "/") {
source += "/"
}
command = append(command, "--", source)
output, err := e.run(ctx, dockerClient, repository, password, command)
if err != nil {
return nil, fmt.Errorf("failed to list Rustic snapshot: %w", err)
}
var files []string
if unmarshalErr := json.Unmarshal([]byte(output), &files); unmarshalErr != nil {
return nil, fmt.Errorf("failed to decode Rustic file list: %w", unmarshalErr)
}
if !recursive && len(files) > 0 {
// Rustic's JSON listing omits node types; the stable long listing restores them.
longCommand := slices.Clone(command)
longCommand[1] = "--long"
longOutput, runErr := e.run(ctx, dockerClient, repository, password, longCommand)
if runErr != nil {
return nil, fmt.Errorf("failed to list Rustic snapshot metadata: %w", runErr)
}
files, runErr = markSnapshotDirectories(files, longOutput)
if runErr != nil {
return nil, runErr
}
}
return qualifySnapshotListing(files, cleanedPath), nil
}
// qualifySnapshotListing prefixes listed files with the cleaned snapshot path they were listed under.
func qualifySnapshotListing(files []string, prefix string) []string {
qualified := slices.Clone(files)
if prefix == "" {
return qualified
}
for index, file := range qualified {
qualified[index] = path.Join(prefix, strings.TrimPrefix(strings.TrimPrefix(file, "./"), "/")) + kit.Ternary(strings.HasSuffix(file, "/"), "/", "")
}
return qualified
}
func markSnapshotDirectories(files []string, longOutput string) ([]string, error) {
if len(files) == 0 {
if strings.TrimSpace(longOutput) != "" {
return nil, errors.New("rustic file and metadata listings have different lengths")
}
return []string{}, nil
}
lines := strings.Split(strings.ReplaceAll(longOutput, "\r\n", "\n"), "\n")
if len(lines) != len(files) {
return nil, errors.New("rustic file and metadata listings have different lengths")
}
marked := slices.Clone(files)
for index, line := range lines {
if line == "" {
return nil, errors.New("rustic returned empty file metadata")
}
if line[0] == 'd' && !strings.HasSuffix(marked[index], "/") {
marked[index] += "/"
}
}
return marked, nil
}
// ReadSnapshotTextFile returns one text file from a snapshot.
func (e *Engine) ReadSnapshotTextFile(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID, filePath string) (string, error) {
output, err := e.run(ctx, dockerClient, repository, password, []string{
"dump", "--archive", "content", "--", snapshotID + ":/" + strings.TrimPrefix(filePath, "/"),
})
if err != nil {
return "", fmt.Errorf("failed to read Rustic snapshot file: %w", err)
}
return output, nil
}
// DiscoveredSnapshot describes one snapshot found in a repository.
type DiscoveredSnapshot struct {
ID string `json:"id"`
Time time.Time `json:"time"`
Label string `json:"label"`
Tags []string `json:"tags"`
Summary struct {
TotalBytesProcessed int64 `json:"total_bytes_processed"`
} `json:"summary"`
}
// RunSnapshotTag links committed snapshots to their durable backup record.
func RunSnapshotTag(runID string) string { return "arcane-run:" + runID }
// ConfirmRunSnapshot makes a validated snapshot discoverable by run ID.
func (e *Engine) ConfirmRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, error) {
if !fullSnapshotID(snapshotID) {
return Snapshot{}, errors.New("a full snapshot ID is required")
}
command := []string{"tag", "--add", RunSnapshotTag(runID), "--", snapshotID}
if _, err := e.run(ctx, dockerClient, repository, password, command); err != nil {
return Snapshot{}, err
}
// Tagging rewrites the snapshot and changes its ID.
snapshot, found, err := e.FindRunSnapshot(ctx, dockerClient, repository, password, runID, "")
if err != nil {
return Snapshot{}, err
}
if !found {
return Snapshot{}, errors.New("confirmed backup snapshot is unavailable")
}
return snapshot, nil
}
// FindRunSnapshot verifies a known snapshot or recovers its committed response by run tag.
func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, bool, error) {
snapshots, err := e.ListSnapshots(ctx, dockerClient, repository, password)
if err != nil {
return Snapshot{}, false, err
}
var found Snapshot
for _, snapshot := range snapshots {
if !kit.Ternary(snapshotID == "", slices.Contains(snapshot.Tags, RunSnapshotTag(runID)), snapshot.ID == snapshotID) {
continue
}
if found.ID != "" {
return Snapshot{}, false, errors.New("multiple snapshots match the backup run")
}
found = Snapshot{ID: snapshot.ID, Size: snapshot.Summary.TotalBytesProcessed}
}
if snapshotID != "" && found.ID == "" {
return Snapshot{}, false, errors.New("recorded backup snapshot is unavailable")
}
return found, found.ID != "", nil
}
// ListSnapshots enumerates every snapshot in the repository.
func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {
output, err := e.run(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
if err != nil {
return nil, err
}
return decodeSnapshots(output)
}
func decodeSnapshots(output string) ([]DiscoveredSnapshot, error) {
trimmedOutput := strings.TrimSpace(output)
if trimmedOutput == "" || trimmedOutput[0] != '[' {
return nil, errors.New("invalid Rustic snapshot listing: expected an array")
}
var entries []jsontext.Value
if err := json.Unmarshal([]byte(output), &entries); err != nil {
return nil, fmt.Errorf("failed to decode Rustic snapshots: %w", err)
}
snapshots := make([]DiscoveredSnapshot, 0, len(entries))
grouped := false
for index, entry := range entries {
if entry.Kind() != '{' {
return nil, fmt.Errorf("invalid Rustic snapshot listing entry %d: expected an object", index)
}
var fields map[string]jsontext.Value
if err := json.Unmarshal(entry, &fields); err != nil {
return nil, fmt.Errorf("failed to decode Rustic snapshot listing entry %d: %w", index, err)
}
_, hasID := fields["id"]
groupEntries, hasSnapshots := fields["snapshots"]
if hasID == hasSnapshots {
return nil, fmt.Errorf("invalid Rustic snapshot listing entry %d: expected an ID or snapshot group", index)
}
if index > 0 && grouped != hasSnapshots {
return nil, errors.New("invalid Rustic snapshot listing: mixed flat and grouped entries")
}
grouped = hasSnapshots
members := []jsontext.Value{entry}
if hasSnapshots {
if groupEntries.Kind() != '[' {
return nil, fmt.Errorf("invalid Rustic snapshot group %d: expected a snapshots array", index)
}
members = nil
if err := json.Unmarshal(groupEntries, &members); err != nil {
return nil, fmt.Errorf("failed to decode Rustic snapshot group %d: %w", index, err)
}
}
for memberIndex, member := range members {
location := kit.Ternary(hasSnapshots, fmt.Sprintf("group %d entry %d", index, memberIndex), fmt.Sprintf("listing entry %d", index))
var memberFields map[string]jsontext.Value
if member.Kind() != '{' {
return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a snapshot object", location)
}
if err := json.Unmarshal(member, &memberFields); err != nil {
return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot fields: %w", location, err)
}
if _, nested := memberFields["snapshots"]; nested {
return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a flat snapshot object", location)
}
var snapshot DiscoveredSnapshot
if err := json.Unmarshal(member, &snapshot); err != nil {
return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot payload: %w", location, err)
}
if !fullSnapshotID(snapshot.ID) {
return nil, fmt.Errorf("invalid Rustic snapshot %s: invalid full snapshot ID", location)
}
snapshots = append(snapshots, snapshot)
}
}
return snapshots, nil
}
func fullSnapshotID(id string) bool {
if len(id) != 64 {
return false
}
_, err := hex.DecodeString(id)
return err == nil
}
// ForgetSnapshots removes the snapshots and prunes their data in a single pass;
// with no IDs it only prunes, skipping an uninitialized repository.
func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string, snapshotIDs []string) error {
requested := make([]string, 0, len(snapshotIDs))
for _, id := range snapshotIDs {
if !fullSnapshotID(id) {
return errors.New("a full snapshot ID is required")
}
requested = append(requested, strings.ToLower(id))
}
requested = kit.Unique(requested)
unlock, err := e.lockRepository(ctx, repository.ID)
if err != nil {
return err
}
defer unlock()
output, err := e.runContainer(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
if err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
return nil
}
if err != nil {
return err
}
snapshots, err := decodeSnapshots(output)
if err != nil {
return err
}
existing := make(map[string]string, len(snapshots))
for _, snapshot := range snapshots {
existing[strings.ToLower(snapshot.ID)] = snapshot.ID
}
command := []string{"forget", "--prune"}
for _, id := range requested {
if listedID, found := existing[id]; found {
if len(command) == 2 {
command = append(command, "--")
}
command = append(command, listedID)
}
}
if len(command) == 2 && len(requested) > 0 {
slog.DebugContext(ctx, "No requested snapshots found in repository; skipping forget", "repositoryId", repository.ID, "requested", requested)
return nil
}
if len(command) == 2 {
command = []string{"prune"}
}
_, err = e.runContainer(ctx, dockerClient, repository, password, command)
return err
}
// ChangeRepositoryPassword re-keys the repository; the scratch Rustic image has no shell, so the new password travels as an argument.
func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *client.Client, repository Repository, currentPassword, newPassword string) error {
if strings.TrimSpace(newPassword) == "" {
return errors.New("new repository password is required")
}
_, err := e.run(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})
return err
}
func (e *Engine) RekeyRepository(ctx context.Context, dockerClient *client.Client, repository Repository, newPassword string, currentPasswords ...string) (string, error) {
_, err := e.ListSnapshots(ctx, dockerClient, repository, newPassword)
if err == nil || strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
return "", nil
}
for _, candidate := range currentPasswords {
if candidate == "" || candidate == newPassword {
continue
}
if changeErr := e.ChangeRepositoryPassword(ctx, dockerClient, repository, candidate, newPassword); changeErr == nil {
return candidate, nil
}
}
return "", fmt.Errorf("backup repository %s does not open with the recovery key: %w", repository.ID, err)
}
func (e *Engine) RekeyRepositories(ctx context.Context, dockerClient *client.Client, repositories []Repository, newPassword string, currentPasswords ...string) error {
type rekeyed struct {
repository Repository
previous string
}
changed := make([]rekeyed, 0, len(repositories))
for _, repository := range repositories {
previous, err := e.RekeyRepository(ctx, dockerClient, repository, newPassword, currentPasswords...)
if err != nil {
errs := []error{err}
var failedRollbackErrs []error
for _, candidate := range currentPasswords {
if candidate == "" || candidate == newPassword {
continue
}
_, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, repository, candidate, newPassword)
if rollbackErr == nil {
failedRollbackErrs = nil
break
}
failedRollbackErrs = append(failedRollbackErrs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", repository.ID, rollbackErr))
}
errs = append(errs, failedRollbackErrs...)
for _, done := range changed {
if _, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, done.repository, done.previous, newPassword); rollbackErr != nil {
errs = append(errs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", done.repository.ID, rollbackErr))
}
}
return errors.Join(errs...)
}
if previous != "" {
changed = append(changed, rekeyed{
repository: repository,
previous: previous,
})
}
}
return nil
}
// Replicate copies one snapshot into another repository through a temporary volume, never the live data.
// Rustic's native `copy` needs a TOML profile for the target, which the env-only Repository cannot express.
func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {
temporaryVolume := "arcane-rustic-copy-" + uuid.New().String()
if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {
return Snapshot{}, fmt.Errorf("failed to create temporary Rustic copy volume: %w", err)
}
defer func() {
_, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), temporaryVolume, client.VolumeRemoveOptions{Force: true})
}()
if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, mount.Mount{
Type: mount.TypeVolume,
Source: temporaryVolume,
Target: "/volume",
}, RestoreOptions{DeleteExtra: true}); err != nil {
return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)
}
snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(mount.Mount{
Type: mount.TypeVolume,
Source: temporaryVolume,
Target: "/volume",
ReadOnly: true,
}, tags...))
if err != nil {
return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)
}
return snapshot, nil
}
func (e *Engine) run(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
unlock, err := e.lockRepository(ctx, repository.ID)
if err != nil {
return "", err
}
defer unlock()
return e.runContainer(ctx, dockerClient, repository, password, command, extraMounts...)
}
// runContainer runs one Rustic command; callers hold the repository lock.
func (e *Engine) runContainer(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err != nil {
if pullErr := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); pullErr != nil {
return "", fmt.Errorf("failed to pull official Rustic image: %w", pullErr)
}
}
return rustic.Run(
ctx,
dockerClient,
password,
command,
repository.Environment,
append(slices.Clone(repository.Mounts), extraMounts...),
func() container.NetworkMode {
arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
if err != nil || kit.FromPtr(arcane).ID == "" {
slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)
return ""
}
return container.NetworkMode("container:" + arcane.ID)
}(),
)
}
var (
ErrRecoveryKeyNotConfigured = errors.New("recovery key is not configured")
// ErrBackupSettled reports a checkpointed backup whose own attempt already failed it, so there is nothing to
// resume; a retry starts a new backup instead.
ErrBackupSettled = errors.New("the checkpointed backup already failed")
// RecoveryKeyFormat is 8 hyphenated groups of 6 base32 characters, used verbatim as the Rustic password.
RecoveryKeyFormat = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)
)
// ValidateRecoveryKey checks that a key has the generated recovery key format.
func ValidateRecoveryKey(key string) error {
if !RecoveryKeyFormat.MatchString(strings.TrimSpace(key)) {
return errors.New("enter the generated recovery key (8 groups of 6 characters)")
}
return nil
}
// GenerateRecoveryKey creates a random recovery key in RecoveryKeyFormat.
func GenerateRecoveryKey() (string, error) {
raw := make([]byte, 30)
if _, err := rand.Read(raw); err != nil {
return "", fmt.Errorf("failed to generate recovery key: %w", err)
}
encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)
groups := make([]string, 0, len(encoded)/6)
for i := 0; i < len(encoded); i += 6 {
groups = append(groups, encoded[i:i+6])
}
return strings.Join(groups, "-"), nil
}
// RecoveryKeyStore persists the recovery key shared by the system and volume backup domains.
type RecoveryKeyStore struct {
db *database.DB
}
// NewRecoveryKeyStore builds the recovery key store.
func NewRecoveryKeyStore(db *database.DB) *RecoveryKeyStore {
return &RecoveryKeyStore{db: db}
}
// Configured reports whether a recovery key is stored.
func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {
var count int64
if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).
Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).Count(&count).Error; err != nil {
return false, fmt.Errorf("failed to load recovery key status: %w", err)
}
return count > 0, nil
}
// Get returns the decrypted recovery key or ErrRecoveryKeyNotConfigured.
func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {
return s.load(ctx, RecoveryKeyConfigID)
}
// Set validates and stores the recovery key, encrypted.
func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
return s.save(ctx, RecoveryKeyConfigID, recoveryKey)
}
// HoldRunKey keeps a recovery key entered for one backup run, encrypted, until ReleaseRunKey, so the run can
// resume after a restart without it ever entering the run's workflow payload.
func (s *RecoveryKeyStore) HoldRunKey(ctx context.Context, runID, recoveryKey string) error {
return s.save(ctx, "run:"+runID, recoveryKey)
}
// RunKey returns the key held for a run, or "" when none is.
func (s *RecoveryKeyStore) RunKey(ctx context.Context, runID string) (string, error) {
key, err := s.load(ctx, "run:"+runID)
if errors.Is(err, ErrRecoveryKeyNotConfigured) {
return "", nil
}
return key, err
}
// ReleaseRunKey forgets the key held for a run.
func (s *RecoveryKeyStore) ReleaseRunKey(ctx context.Context, runID string) error {
if err := s.db.WithContext(ctx).Where("id = ?", "run:"+runID).Delete(&SystemBackupRecoveryConfig{}).Error; err != nil {
return fmt.Errorf("failed to release backup run recovery key: %w", err)
}
return nil
}
func (s *RecoveryKeyStore) load(ctx context.Context, id string) (string, error) {
var config SystemBackupRecoveryConfig
if err := s.db.WithContext(ctx).Where("id = ? AND encrypted_recovery_key <> ''", id).First(&config).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return "", ErrRecoveryKeyNotConfigured
}
return "", fmt.Errorf("failed to load recovery key: %w", err)
}
key, err := crypto.Decrypt(config.EncryptedRecoveryKey)
if err != nil {
return "", fmt.Errorf("failed to decrypt recovery key: %w", err)
}
return key, nil
}
func (s *RecoveryKeyStore) save(ctx context.Context, id, recoveryKey string) error {
if err := ValidateRecoveryKey(recoveryKey); err != nil {
return err
}
encrypted, err := crypto.Encrypt(recoveryKey)
if err != nil {
return fmt.Errorf("failed to encrypt recovery key: %w", err)
}
if saveErr := s.db.WithContext(ctx).Save(&SystemBackupRecoveryConfig{BaseModel: database.BaseModel{ID: id}, EncryptedRecoveryKey: encrypted}).Error; saveErr != nil {
return fmt.Errorf("failed to save recovery key: %w", saveErr)
}
return nil
}
// AcceptedTarget is the progress a manual backup's workflow starts with, so startup recovery protects
// its record from the moment the backup is accepted.
func AcceptedTarget(backupID string) scheduler.TargetOutcome {
data, _ := json.Marshal(struct {
BackupID string `json:"backupId"`
}{BackupID: backupID})
return scheduler.TargetOutcome{
ResourceType: "backup",
ID: "accepted:" + backupID,
Status: scheduler.Queued,
RecoveryData: data,
}
}
// AcceptedBackupID returns the backup a manual backup workflow's run owns, from its AcceptedTarget.
func AcceptedBackupID(run scheduler.Run) string {
index := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
return strings.HasPrefix(target.ID, "accepted:")
})
if index < 0 {
return ""
}
return strings.TrimPrefix(run.Outcome.Targets[index].ID, "accepted:")
}
// lockRepository serializes commands against one repository and returns its unlock func.
func (e *Engine) lockRepository(ctx context.Context, id string) (func(), error) {
if strings.TrimSpace(id) == "" {
return nil, errors.New("backup repository ID is required")
}
e.mu.Lock()
lock := e.repositories[id]
if lock == nil {
lock = &sync.Mutex{}
e.repositories[id] = lock
}
e.mu.Unlock()
lock.Lock()
if err := ctx.Err(); err != nil {
lock.Unlock()
return nil, err
}
return lock.Unlock, nil
}
// PolicyReconciliation reconciles a full set of policy updates against the
// existing policies of one backup domain, then swaps the scheduled jobs.
type PolicyReconciliation[P, U any] struct {
// Domain names the backup flavor in validation errors ("volume", "system").
Domain string
DB *database.DB
// Existing is every policy currently persisted for the reconciled scope.
Existing []P
ID func(*P) string
UpdateID func(U) string
// New returns the blank policy for created entries, pre-scoped by the
// caller (e.g. with the volume name set).
New func() P
// Build validates an update and applies it to an existing or new policy.
Build func(ctx context.Context, policy *P, update U) error
// Persist overrides the default GORM transaction for settings-backed policies.
Persist func(ctx context.Context, policies []P) error
Unregister func(ctx context.Context, policyID string)
Reschedule func(ctx context.Context, policy *P)
}
// Run applies updates as the full policy set: it saves them, removes the rest, and reschedules their jobs.
func (r PolicyReconciliation[P, U]) Run(ctx context.Context, updates []U) error {
byID := make(map[string]P, len(r.Existing))
for i := range r.Existing {
byID[r.ID(&r.Existing[i])] = r.Existing[i]
}
policies := make([]P, 0, len(updates))
kept := make(map[string]struct{}, len(updates))
for _, update := range updates {
policy := r.New()
if updateID := r.UpdateID(update); updateID != "" {
var ok bool
policy, ok = byID[updateID]
if !ok {
return fmt.Errorf("%s backup policy not found", r.Domain)
}
if _, duplicate := kept[updateID]; duplicate {
return fmt.Errorf("duplicate %s backup policy", r.Domain)
}
}
if err := r.Build(ctx, &policy, update); err != nil {
return err
}
if policyID := r.ID(&policy); policyID != "" {
kept[policyID] = struct{}{}
}
policies = append(policies, policy)
}
removed := slices.DeleteFunc(slices.Clone(r.Existing), func(policy P) bool {
_, ok := kept[r.ID(&policy)]
return ok
})
var persistErr error
switch {
case r.Persist != nil:
persistErr = r.Persist(ctx, policies)
default:
persistErr = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for i := range policies {
if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {
return saveErr
}
}
if len(removed) == 0 {
return nil
}
return tx.Delete(&removed).Error
})
}
if persistErr != nil {
return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, persistErr)
}
for i := range removed {
r.Unregister(ctx, r.ID(&removed[i]))
}
for i := range policies {
r.Reschedule(ctx, &policies[i])
}
return nil
}
// ValidatePolicyUpdate applies the shared cron, retention, and destination rules used by backup policies.
func ValidatePolicyUpdate(ctx context.Context, domain string, update backup.UpdateBackupPolicy, s3Service *s3.S3DestinationService) (backup.UpdateBackupPolicy, error) {
normalized, err := schedule.NormalizeSixField(update.Schedule, domain+" backup")
if err != nil {
return update, err
}
if update.RetentionCount < 0 || update.RetentionCount > 3650 {
return update, errors.New("retentionCount must be between 0 and 3650")
}
if !update.LocalEnabled && !update.S3Enabled {
return update, fmt.Errorf("select at least one %s backup destination", domain)
}
if update.S3Enabled && strings.TrimSpace(update.S3DestinationID) == "" {
return update, fmt.Errorf("select an S3 destination for %s backups", domain)
}
if update.S3Enabled {
if _, configurationErr := s3Service.Configuration(ctx, update.S3DestinationID); configurationErr != nil {
return update, fmt.Errorf("select a valid S3 destination for %s backups", domain)
}
}
return backup.UpdateBackupPolicy{
ID: update.ID,
Enabled: update.Enabled,
Schedule: normalized,
RetentionCount: update.RetentionCount,
StopContainers: update.StopContainers,
LocalEnabled: update.LocalEnabled,
S3Enabled: update.S3Enabled,
S3DestinationID: kit.Ternary(update.S3Enabled, update.S3DestinationID, ""),
}, nil
}
var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")
const RemoteDisabledMessage = "S3 backup storage is missing. Remote backups were disabled; edit the policy to resume."
// RemoteSnapshotChecker memoizes targeted checks within one response budget; each lookup names the
// repository root that holds the snapshot. A nil result means the remote copy could not be verified.
func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationService) func(destinationID, root, snapshotID string) *bool {
checked := make(map[string]*backup.RepositoryObservation)
deadline := time.Now().Add(10 * time.Second)
return func(destinationID, root, snapshotID string) *bool {
if root == "" || destinationID == "" || snapshotID == "" {
return nil
}
key := destinationID + ":" + root + ":" + snapshotID
observation, seen := checked[key]
if !seen {
checked[key] = nil
checkCtx, cancel := context.WithDeadline(ctx, deadline)
defer cancel()
configuration, err := destinations.Configuration(checkCtx, destinationID)
if err != nil {
return nil
}
result, err := s3util.CheckRepository(checkCtx, configuration, root, snapshotID)
if err != nil {
return nil
}
observation = &result
checked[key] = observation
}
if observation == nil {
return nil
}
return new(observation.Available && observation.SnapshotAvailable)
}
}
// CheckRemoteRepository probes the destination's repository config within a bounded budget.
// Transport and credential failures are returned as errors so callers never forget snapshots blindly.
func CheckRemoteRepository(ctx context.Context, destinations *s3.S3DestinationService, destinationID, root string) (backup.RepositoryObservation, error) {
configuration, err := destinations.Configuration(ctx, destinationID)
if err != nil {
return backup.RepositoryObservation{}, err
}
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
return s3util.CheckRepository(checkCtx, configuration, root, "")
}
// DisableMissingRemote stops a scheduled policy from using S3 storage that went missing: a policy that also backs up
// locally loses S3, an S3-only one is disabled. save clears that field on the stored policy unless it changed since it
// was loaded, and reports whether it did; the loaded flags then follow. It reports whether S3 was turned off.
func DisableMissingRemote(remoteErr error, localEnabled bool, s3Enabled, enabled *bool, save func(field string) (bool, error)) (bool, error) {
if !errors.Is(remoteErr, ErrRemoteRepositoryMissing) {
return false, nil
}
saved, err := save(kit.Ternary(localEnabled, "s3_enabled", "enabled"))
if err != nil || !saved {
return false, err
}
*kit.Ternary(localEnabled, s3Enabled, enabled) = false
return true, nil
}
// DisableStoredRemote is DisableMissingRemote's save for a policy table row, which it changes only while the row still
// matches the loaded policy.
func DisableStoredRemote(ctx context.Context, db *database.DB, model any, id, destinationID string, localEnabled bool, field string) (bool, error) {
result := db.WithContext(ctx).Model(model).
Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", id, destinationID, true, true, localEnabled).
Update(field, false)
return result.Error == nil && result.RowsAffected > 0, result.Error
}
// CheckScheduledRemote permits first-use initialization, but detects lost repositories.
func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3.S3DestinationService, table, destinationID, root string) error {
result, err := CheckRemoteRepository(ctx, destinations, destinationID, root)
if err != nil || result.Available {
return err
}
if result.Reason == s3util.RepositoryReasonMissingRepository {
var retained int64
if countRetainedSnapshotsErr := db.WithContext(ctx).Table(table).
Where("s3_destination_id = ? AND COALESCE(remote_snapshot_id, '') <> ''", destinationID).
Count(&retained).Error; countRetainedSnapshotsErr != nil {
return countRetainedSnapshotsErr
}
if retained == 0 {
return nil
}
}
return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)
}
// ExpiredRunIDs returns the IDs of succeeded and failed runs with snapshots that
// fall outside the newest keep entries of their status for one policy, oldest
// last within each status. Callers delete each run through their own delete
// path so snapshots are forgotten too.
func ExpiredRunIDs(ctx context.Context, db *database.DB, table, policyID string, keep int) ([]string, error) {
var expired []string
for _, status := range []string{"succeeded", "failed"} {
var ids []string
if err := db.WithContext(ctx).
Table(table).
Where(
"policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",
policyID,
status,
).
Order("created_at DESC").
Offset(keep).
Pluck("id", &ids).Error; err != nil {
return nil, err
}
expired = append(expired, ids...)
}
return expired, nil
}