Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package cloudcp
import (
"archive/tar"
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
_ "modernc.org/sqlite"
)
const (
ProviderMSPBackupManifestVersion = "provider-msp-backup/v1"
providerMSPBackupManifestName = "manifest.json"
providerMSPBackupControlPlaneDir = "control-plane"
providerMSPBackupTenantsDir = "tenants"
providerMSPBackupLicenseDir = "license"
providerMSPBackupLicenseName = "provider-msp-license.jwt"
maxProviderMSPBackupManifestSize = 1024 * 1024
// Restore writes archive-supplied data straight to disk, so a single entry
// and the restore as a whole are both bounded. Without this a gzip bomb, a
// sparse entry declaring a huge logical size, or a corrupt stream can fill
// the target volume. The limits are far above any plausible provider MSP
// backup; they exist to make the write bounded, not to size-check backups.
maxProviderMSPBackupRestoreEntryBytes = 8 << 30 // 8 GiB per restored entry
maxProviderMSPBackupRestoreTotalBytes = 64 << 30 // 64 GiB per restore
)
// ProviderMSPBackupManifest is the recovery contract stored inside every
// provider-hosted MSP backup archive.
type ProviderMSPBackupManifest struct {
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
ControlPlaneMode string `json:"control_plane_mode"`
Environment string `json:"environment"`
BaseURL string `json:"base_url"`
PlanVersion string `json:"plan_version"`
PlanSource string `json:"plan_source"`
LicenseID string `json:"license_id,omitempty"`
LicenseEmail string `json:"license_email,omitempty"`
LicenseIncluded bool `json:"license_included"`
WorkspaceLimit int `json:"workspace_limit"`
ControlPlaneDir string `json:"control_plane_dir"`
TenantsDir string `json:"tenants_dir"`
RegistryAccountCount int `json:"registry_account_count"`
RegistryTenantCount int `json:"registry_tenant_count"`
RuntimeTenantCount int `json:"runtime_tenant_count"`
RuntimeTenantIDs []string `json:"runtime_tenant_ids"`
RegistryTenantIDs []string `json:"registry_tenant_ids"`
RegistryStateCounts map[string]int `json:"registry_state_counts"`
ControlPlaneDBBackups []string `json:"control_plane_db_backups"`
}
// ProviderMSPBackupCreateResult describes a newly written provider MSP backup.
type ProviderMSPBackupCreateResult struct {
ArchivePath string
Manifest ProviderMSPBackupManifest
ControlPlaneEntries int
TenantEntries int
LicenseEntries int
BytesWritten int64
}
// ProviderMSPBackupVerifyResult describes a verified provider MSP backup.
type ProviderMSPBackupVerifyResult struct {
ArchivePath string
Manifest ProviderMSPBackupManifest
ControlPlaneEntries int
TenantEntries int
LicenseEntries int
HasTenantRegistryDB bool
HasLicenseFile bool
RuntimeTenantDirs []string
ControlPlaneDBFiles []string
VerifiedArchiveBytes int64
}
// ProviderMSPBackupRestoreOptions controls restoring a provider MSP recovery
// archive into a target control-plane data directory.
type ProviderMSPBackupRestoreOptions struct {
ArchivePath string
TargetDataDir string
LicenseOutputPath string
ReplaceExisting bool
DryRun bool
}
// ProviderMSPBackupRestoreResult describes a completed or dry-run provider MSP
// recovery operation.
type ProviderMSPBackupRestoreResult struct {
ArchivePath string
TargetDataDir string
ControlPlaneDir string
TenantsDir string
LicenseOutputPath string
DryRun bool
ReplaceExisting bool
Manifest ProviderMSPBackupManifest
VerifiedArchiveBytes int64
ControlPlaneEntriesRestored int
TenantEntriesRestored int
LicenseEntriesRestored int
RestoredRegistryTenantCount int
RestoredRuntimeTenantIDs []string
}
// DefaultProviderMSPBackupPath returns the compose-friendly default backup
// location under CP_DATA_DIR without placing the archive inside the source trees.
func DefaultProviderMSPBackupPath(cfg *CPConfig, now time.Time) string {
if now.IsZero() {
now = time.Now().UTC()
}
name := "provider-msp-backup-" + now.UTC().Format("20060102T150405Z") + ".tar.gz"
if cfg == nil || strings.TrimSpace(cfg.DataDir) == "" {
return name
}
return filepath.Join(cfg.DataDir, "backups", "provider-msp", name)
}
// CreateProviderMSPBackup creates a recovery archive for a provider-hosted MSP
// control plane. It backs up the control-plane registry through SQLite's online
// VACUUM INTO path instead of copying live WAL files directly.
func CreateProviderMSPBackup(ctx context.Context, cfg *CPConfig, outputPath string) (*ProviderMSPBackupCreateResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if cfg == nil {
return nil, fmt.Errorf("control plane config is required")
}
if !cfg.IsMSPControlPlane() {
return nil, fmt.Errorf("provider MSP backup requires CP_CONTROL_PLANE_MODE=%s or %s", ControlPlaneModeProviderHostedMSP, ControlPlaneModePulseHostedMSP)
}
if cfg.UsesStripeBilling() {
return nil, fmt.Errorf("provider MSP backup is unavailable for Stripe-backed control planes")
}
if strings.TrimSpace(outputPath) == "" {
outputPath = DefaultProviderMSPBackupPath(cfg, time.Now().UTC())
}
archivePath, err := filepath.Abs(filepath.Clean(outputPath))
if err != nil {
return nil, fmt.Errorf("resolve backup archive path: %w", err)
}
controlPlaneDir, err := filepath.Abs(filepath.Clean(cfg.ControlPlaneDir()))
if err != nil {
return nil, fmt.Errorf("resolve control-plane dir: %w", err)
}
tenantsDir, err := filepath.Abs(filepath.Clean(cfg.TenantsDir()))
if err != nil {
return nil, fmt.Errorf("resolve tenants dir: %w", err)
}
if pathIsInside(archivePath, controlPlaneDir) || pathIsInside(archivePath, tenantsDir) {
return nil, fmt.Errorf("backup archive must not be written inside %s or %s", cfg.ControlPlaneDir(), cfg.TenantsDir())
}
if err := requireDirectory(controlPlaneDir, "control-plane dir"); err != nil {
return nil, err
}
registryDB := filepath.Join(controlPlaneDir, "tenants.db")
if err := requireRegularFile(registryDB, "tenant registry database"); err != nil {
return nil, err
}
reg, err := registry.NewTenantRegistry(controlPlaneDir)
if err != nil {
return nil, fmt.Errorf("open tenant registry: %w", err)
}
tenants, listErr := reg.List()
accounts, accountsErr := reg.ListAccounts()
closeErr := reg.Close()
if listErr != nil {
return nil, fmt.Errorf("list tenant registry rows: %w", listErr)
}
if accountsErr != nil {
return nil, fmt.Errorf("list registry accounts: %w", accountsErr)
}
if closeErr != nil {
return nil, fmt.Errorf("close tenant registry before backup: %w", closeErr)
}
manifest, err := buildProviderMSPBackupManifest(cfg, tenants, len(accounts))
if err != nil {
return nil, err
}
if err := requireProviderMSPRuntimeTenantDirs(tenantsDir, manifest.RuntimeTenantIDs); err != nil {
return nil, err
}
licensePath, licenseIncluded, err := resolveProviderMSPBackupLicensePath(cfg)
if err != nil {
return nil, err
}
manifest.LicenseIncluded = licenseIncluded
if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil {
return nil, fmt.Errorf("create backup archive parent: %w", err)
}
tempDir, err := os.MkdirTemp(filepath.Dir(archivePath), ".provider-msp-backup-*")
if err != nil {
return nil, fmt.Errorf("create backup staging dir: %w", err)
}
defer os.RemoveAll(tempDir)
tmpArchive := archivePath + ".tmp"
_ = os.Remove(tmpArchive)
file, err := os.OpenFile(tmpArchive, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return nil, fmt.Errorf("create backup archive: %w", err)
}
result, err := writeProviderMSPBackupArchive(ctx, file, tempDir, controlPlaneDir, tenantsDir, licensePath, manifest)
closeArchiveErr := file.Close()
if err != nil {
_ = os.Remove(tmpArchive)
return nil, err
}
if closeArchiveErr != nil {
_ = os.Remove(tmpArchive)
return nil, fmt.Errorf("close backup archive: %w", closeArchiveErr)
}
if err := os.Rename(tmpArchive, archivePath); err != nil {
_ = os.Remove(tmpArchive)
return nil, fmt.Errorf("publish backup archive: %w", err)
}
info, err := os.Stat(archivePath)
if err != nil {
return nil, fmt.Errorf("stat backup archive: %w", err)
}
result.ArchivePath = archivePath
result.BytesWritten = info.Size()
return result, nil
}
// RestoreProviderMSPBackup verifies and restores a provider MSP recovery
// archive. It intentionally does not require a fully validated CPConfig because
// restore may need to recover the MSP license before normal provider-hosted
// config validation can pass.
func RestoreProviderMSPBackup(ctx context.Context, opts ProviderMSPBackupRestoreOptions) (*ProviderMSPBackupRestoreResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
opts.ArchivePath = strings.TrimSpace(opts.ArchivePath)
if opts.ArchivePath == "" {
return nil, fmt.Errorf("backup archive path is required")
}
targetDataDir, err := resolveProviderMSPRestoreTargetDataDir(opts.TargetDataDir)
if err != nil {
return nil, err
}
licenseOutputPath, err := resolveProviderMSPRestoreLicenseOutputPath(opts.LicenseOutputPath, targetDataDir)
if err != nil {
return nil, err
}
controlPlaneDir := filepath.Join(targetDataDir, providerMSPBackupControlPlaneDir)
tenantsDir := filepath.Join(targetDataDir, providerMSPBackupTenantsDir)
if pathIsInside(licenseOutputPath, controlPlaneDir) || pathIsInside(licenseOutputPath, tenantsDir) {
return nil, fmt.Errorf("license output path must not be inside restored %s or %s", providerMSPBackupControlPlaneDir, providerMSPBackupTenantsDir)
}
verified, err := VerifyProviderMSPBackup(ctx, opts.ArchivePath)
if err != nil {
return nil, err
}
conflicts, err := providerMSPRestoreConflicts(controlPlaneDir, tenantsDir, licenseOutputPath, verified.Manifest.LicenseIncluded)
if err != nil {
return nil, err
}
if len(conflicts) > 0 && !opts.ReplaceExisting {
return nil, fmt.Errorf("restore target already contains provider MSP state (%s); rerun with replace enabled only after stopping the control plane", strings.Join(conflicts, ", "))
}
result := &ProviderMSPBackupRestoreResult{
ArchivePath: verified.ArchivePath,
TargetDataDir: targetDataDir,
ControlPlaneDir: controlPlaneDir,
TenantsDir: tenantsDir,
LicenseOutputPath: licenseOutputPath,
DryRun: opts.DryRun,
ReplaceExisting: opts.ReplaceExisting,
Manifest: verified.Manifest,
VerifiedArchiveBytes: verified.VerifiedArchiveBytes,
}
if opts.DryRun {
result.ControlPlaneEntriesRestored = verified.ControlPlaneEntries
result.TenantEntriesRestored = verified.TenantEntries
result.LicenseEntriesRestored = verified.LicenseEntries
result.RestoredRegistryTenantCount = verified.Manifest.RegistryTenantCount
result.RestoredRuntimeTenantIDs = append([]string(nil), verified.Manifest.RuntimeTenantIDs...)
return result, nil
}
if opts.ReplaceExisting {
if err := removeProviderMSPRestoreTargets(controlPlaneDir, tenantsDir, licenseOutputPath, verified.Manifest.LicenseIncluded); err != nil {
return nil, err
}
}
if err := os.MkdirAll(targetDataDir, 0o755); err != nil {
return nil, fmt.Errorf("create restore target data dir: %w", err)
}
if err := extractProviderMSPBackupArchive(ctx, verified.ArchivePath, targetDataDir, licenseOutputPath, result); err != nil {
return nil, failProviderMSPRestore(result, err)
}
if err := validateRestoredProviderMSPBackup(result); err != nil {
return nil, failProviderMSPRestore(result, err)
}
return result, nil
}
// failProviderMSPRestore rolls a half-finished restore back to an empty target.
// Everything under the restore paths was written by this call: the conflict
// check refuses to overwrite pre-existing state, and replace clears it up
// front. So removing them is safe, and it leaves a clean slate that a retry can
// use without needing replace, rather than a partial control plane that looks
// bootable. It cannot bring back state that replace already deleted, so the
// error says so.
func failProviderMSPRestore(result *ProviderMSPBackupRestoreResult, cause error) error {
if err := removeProviderMSPPartialRestore(result); err != nil {
return fmt.Errorf("%w (rolling back the partial restore also failed: %v)", cause, err)
}
if result.ReplaceExisting {
return fmt.Errorf("%w (the partial restore was rolled back, and the state it replaced was already deleted)", cause)
}
return fmt.Errorf("%w (the partial restore was rolled back)", cause)
}
func removeProviderMSPPartialRestore(result *ProviderMSPBackupRestoreResult) error {
for _, dir := range []string{result.ControlPlaneDir, result.TenantsDir} {
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("remove partial restore target %s: %w", dir, err)
}
}
// Only the license this restore wrote is ours to remove.
if result.LicenseEntriesRestored > 0 {
if err := os.Remove(result.LicenseOutputPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove partial restored provider MSP license: %w", err)
}
}
return nil
}
// VerifyProviderMSPBackup validates that a provider MSP backup contains the
// manifest, tenant registry snapshot, required tenant directories, and license
// artifact declared by the manifest.
func VerifyProviderMSPBackup(ctx context.Context, archivePath string) (*ProviderMSPBackupVerifyResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
archivePath = strings.TrimSpace(archivePath)
if archivePath == "" {
return nil, fmt.Errorf("backup archive path is required")
}
resolvedArchivePath, err := filepath.Abs(filepath.Clean(archivePath))
if err != nil {
return nil, fmt.Errorf("resolve backup archive path: %w", err)
}
file, err := os.Open(resolvedArchivePath)
if err != nil {
return nil, fmt.Errorf("open backup archive: %w", err)
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("stat backup archive: %w", err)
}
gz, err := gzip.NewReader(file)
if err != nil {
return nil, fmt.Errorf("open backup gzip stream: %w", err)
}
defer gz.Close()
result := &ProviderMSPBackupVerifyResult{
ArchivePath: resolvedArchivePath,
VerifiedArchiveBytes: info.Size(),
}
tenantDirs := map[string]bool{}
dbFiles := map[string]bool{}
var manifestBytes []byte
tr := tar.NewReader(gz)
for {
if err := ctx.Err(); err != nil {
return nil, err
}
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read backup archive: %w", err)
}
name, err := cleanProviderMSPArchiveName(header.Name)
if err != nil {
return nil, err
}
switch {
case name == providerMSPBackupManifestName:
manifestBytes, err = readProviderMSPBackupManifestBytes(tr, header.Size)
if err != nil {
return nil, err
}
case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"):
result.ControlPlaneEntries++
if strings.HasSuffix(name, ".db") {
dbFiles[name] = true
}
if name == providerMSPBackupControlPlaneDir+"/tenants.db" {
result.HasTenantRegistryDB = true
}
case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"):
result.TenantEntries++
if tenantID := archiveTenantID(name); tenantID != "" {
tenantDirs[tenantID] = true
}
case name == providerMSPBackupLicenseDir || strings.HasPrefix(name, providerMSPBackupLicenseDir+"/"):
result.LicenseEntries++
if name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName {
result.HasLicenseFile = true
}
default:
return nil, fmt.Errorf("unexpected top-level backup entry %q", name)
}
}
if len(manifestBytes) == 0 {
return nil, fmt.Errorf("backup manifest is missing")
}
if err := json.Unmarshal(manifestBytes, &result.Manifest); err != nil {
return nil, fmt.Errorf("parse backup manifest: %w", err)
}
if err := validateProviderMSPBackupManifest(result.Manifest); err != nil {
return nil, err
}
if !result.HasTenantRegistryDB {
return nil, fmt.Errorf("backup is missing control-plane tenant registry snapshot")
}
if result.ControlPlaneEntries == 0 {
return nil, fmt.Errorf("backup is missing control-plane data")
}
if result.TenantEntries == 0 {
return nil, fmt.Errorf("backup is missing tenants directory")
}
if result.Manifest.LicenseIncluded && !result.HasLicenseFile {
return nil, fmt.Errorf("backup manifest declares included MSP license but license file is missing")
}
for _, dbFile := range result.Manifest.ControlPlaneDBBackups {
if !dbFiles[dbFile] {
return nil, fmt.Errorf("backup manifest declares control-plane DB snapshot %q but it is missing", dbFile)
}
}
for _, tenantID := range result.Manifest.RuntimeTenantIDs {
if !tenantDirs[tenantID] {
return nil, fmt.Errorf("backup is missing runtime tenant directory %q", tenantID)
}
}
for tenantID := range tenantDirs {
result.RuntimeTenantDirs = append(result.RuntimeTenantDirs, tenantID)
}
sort.Strings(result.RuntimeTenantDirs)
for name := range dbFiles {
result.ControlPlaneDBFiles = append(result.ControlPlaneDBFiles, name)
}
sort.Strings(result.ControlPlaneDBFiles)
return result, nil
}
func resolveProviderMSPRestoreTargetDataDir(raw string) (string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return "", fmt.Errorf("restore target data dir is required")
}
resolved, err := filepath.Abs(filepath.Clean(raw))
if err != nil {
return "", fmt.Errorf("resolve restore target data dir: %w", err)
}
return resolved, nil
}
func resolveProviderMSPRestoreLicenseOutputPath(raw, targetDataDir string) (string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
raw = filepath.Join(targetDataDir, providerMSPBackupLicenseName)
}
resolved, err := filepath.Abs(filepath.Clean(raw))
if err != nil {
return "", fmt.Errorf("resolve provider MSP license output path: %w", err)
}
return resolved, nil
}
func providerMSPRestoreConflicts(controlPlaneDir, tenantsDir, licenseOutputPath string, requireLicense bool) ([]string, error) {
var conflicts []string
for _, item := range []struct {
label string
path string
}{
{label: providerMSPBackupControlPlaneDir, path: controlPlaneDir},
{label: providerMSPBackupTenantsDir, path: tenantsDir},
} {
exists, err := providerMSPRestorePathExists(item.path)
if err != nil {
return nil, err
}
if exists {
conflicts = append(conflicts, item.label)
}
}
if requireLicense {
exists, err := providerMSPRestorePathExists(licenseOutputPath)
if err != nil {
return nil, err
}
if exists {
conflicts = append(conflicts, "provider-msp-license")
}
}
return conflicts, nil
}
func providerMSPRestorePathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("stat restore target %s: %w", path, err)
}
func removeProviderMSPRestoreTargets(controlPlaneDir, tenantsDir, licenseOutputPath string, removeLicense bool) error {
for _, dir := range []string{controlPlaneDir, tenantsDir} {
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("remove existing restore target %s: %w", dir, err)
}
}
if removeLicense {
if err := os.Remove(licenseOutputPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove existing provider MSP license output: %w", err)
}
}
return nil
}
func extractProviderMSPBackupArchive(ctx context.Context, archivePath, targetDataDir, licenseOutputPath string, result *ProviderMSPBackupRestoreResult) error {
file, err := os.Open(archivePath)
if err != nil {
return fmt.Errorf("open backup archive for restore: %w", err)
}
defer file.Close()
gz, err := gzip.NewReader(file)
if err != nil {
return fmt.Errorf("open backup gzip stream for restore: %w", err)
}
defer gz.Close()
budget := newProviderMSPRestoreBudget()
tr := tar.NewReader(gz)
for {
if err := ctx.Err(); err != nil {
return err
}
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("read backup archive for restore: %w", err)
}
name, err := cleanProviderMSPArchiveName(header.Name)
if err != nil {
return err
}
switch {
case name == providerMSPBackupManifestName:
continue
case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"):
if err := restoreProviderMSPArchiveEntry(tr, header, targetDataDir, name, budget); err != nil {
return err
}
result.ControlPlaneEntriesRestored++
case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"):
if err := restoreProviderMSPArchiveEntry(tr, header, targetDataDir, name, budget); err != nil {
return err
}
result.TenantEntriesRestored++
case name == providerMSPBackupLicenseDir:
continue
case name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName:
if err := restoreProviderMSPArchiveFile(tr, header, licenseOutputPath, budget); err != nil {
return err
}
result.LicenseEntriesRestored++
default:
return fmt.Errorf("unexpected backup entry during restore %q", name)
}
}
return nil
}
// providerMSPRestoreBudget bounds how many bytes a restore may write to disk,
// per entry and across the whole archive. Both bounds are enforced against the
// bytes actually copied rather than the size the tar header declares, because
// that header comes from the archive and is therefore attacker-influenced.
type providerMSPRestoreBudget struct {
entryCap int64
totalCap int64
remaining int64
}
func newProviderMSPRestoreBudget() *providerMSPRestoreBudget {
return &providerMSPRestoreBudget{
entryCap: maxProviderMSPBackupRestoreEntryBytes,
totalCap: maxProviderMSPBackupRestoreTotalBytes,
remaining: maxProviderMSPBackupRestoreTotalBytes,
}
}
// entryLimit is the most the next entry may write: the per-entry cap, or
// whatever is left of the whole-restore budget when that is smaller.
func (b *providerMSPRestoreBudget) entryLimit() int64 {
if b.remaining < b.entryCap {
return b.remaining
}
return b.entryCap
}
func (b *providerMSPRestoreBudget) consume(n int64) {
b.remaining -= n
}
// overrunError names whichever bound the entry just crossed.
func (b *providerMSPRestoreBudget) overrunError() error {
if b.remaining < b.entryCap {
return fmt.Errorf("backup restore exceeds the %d byte total restore limit", b.totalCap)
}
return fmt.Errorf("backup restore entry exceeds the %d byte per-entry limit", b.entryCap)
}
func restoreProviderMSPArchiveEntry(reader io.Reader, header *tar.Header, targetDataDir, archiveName string, budget *providerMSPRestoreBudget) error {
destPath := filepath.Join(targetDataDir, filepath.FromSlash(archiveName))
if !pathIsInside(destPath, targetDataDir) {
return fmt.Errorf("backup restore entry escapes target data dir: %s", archiveName)
}
switch header.Typeflag {
case tar.TypeDir:
return restoreProviderMSPArchiveDir(header, destPath)
case tar.TypeReg, tar.TypeRegA:
return restoreProviderMSPArchiveFile(reader, header, destPath, budget)
default:
return fmt.Errorf("unsupported backup restore entry type for %s", archiveName)
}
}
func restoreProviderMSPArchiveDir(header *tar.Header, destPath string) error {
mode := os.FileMode(header.Mode) & 0o777
if mode == 0 {
mode = 0o755
}
if err := os.MkdirAll(destPath, mode); err != nil {
return fmt.Errorf("create restored directory %s: %w", destPath, err)
}
return nil
}
func restoreProviderMSPArchiveFile(reader io.Reader, header *tar.Header, destPath string, budget *providerMSPRestoreBudget) error {
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA {
return fmt.Errorf("backup restore expected regular file at %s", header.Name)
}
// The declared size is a hint, not a bound: it comes from the archive, so
// it can lie in either direction. Reject an absurd declaration up front to
// avoid touching the disk at all, then bound the copy itself below.
if header.Size < 0 || header.Size > budget.entryCap {
return fmt.Errorf("backup restore entry %s declares %d bytes, above the %d byte per-entry limit", header.Name, header.Size, budget.entryCap)
}
mode := os.FileMode(header.Mode) & 0o777
if mode == 0 {
mode = 0o600
}
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
return fmt.Errorf("create restored file parent %s: %w", destPath, err)
}
file, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
if err != nil {
return fmt.Errorf("create restored file %s: %w", destPath, err)
}
// Read one byte past the limit so an overrun is detected rather than
// silently truncated into a file that looks complete.
limit := budget.entryLimit()
written, writeErr := io.Copy(file, io.LimitReader(reader, limit+1))
closeErr := file.Close()
if writeErr == nil && written > limit {
writeErr = budget.overrunError()
}
if writeErr == nil {
writeErr = closeErr
}
if writeErr != nil {
// Fail the restore instead of leaving a partial file behind.
if removeErr := os.Remove(destPath); removeErr != nil && !os.IsNotExist(removeErr) {
return fmt.Errorf("restore file %s: %w (removing the partial file also failed: %v)", destPath, writeErr, removeErr)
}
return fmt.Errorf("restore file %s: %w", destPath, writeErr)
}
budget.consume(written)
return nil
}
func validateRestoredProviderMSPBackup(result *ProviderMSPBackupRestoreResult) error {
if result == nil {
return fmt.Errorf("restore result is required")
}
if err := requireRegularFile(filepath.Join(result.ControlPlaneDir, "tenants.db"), "restored tenant registry database"); err != nil {
return err
}
if result.Manifest.LicenseIncluded {
if err := requireRegularFile(result.LicenseOutputPath, "restored provider MSP license file"); err != nil {
return err
}
}
reg, err := registry.NewTenantRegistry(result.ControlPlaneDir)
if err != nil {
return fmt.Errorf("open restored tenant registry: %w", err)
}
tenants, listErr := reg.List()
closeErr := reg.Close()
if listErr != nil {
return fmt.Errorf("list restored tenant registry rows: %w", listErr)
}
if closeErr != nil {
return fmt.Errorf("close restored tenant registry: %w", closeErr)
}
if len(tenants) != result.Manifest.RegistryTenantCount {
return fmt.Errorf("restored tenant registry count = %d, want %d", len(tenants), result.Manifest.RegistryTenantCount)
}
if err := requireProviderMSPRuntimeTenantDirs(result.TenantsDir, result.Manifest.RuntimeTenantIDs); err != nil {
return fmt.Errorf("validate restored tenant runtime dirs: %w", err)
}
result.RestoredRegistryTenantCount = len(tenants)
result.RestoredRuntimeTenantIDs = append([]string(nil), result.Manifest.RuntimeTenantIDs...)
return nil
}
func buildProviderMSPBackupManifest(cfg *CPConfig, tenants []*registry.Tenant, accountCount int) (ProviderMSPBackupManifest, error) {
workspaceLimit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
if !known {
return ProviderMSPBackupManifest{}, fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion)
}
manifest := ProviderMSPBackupManifest{
Version: ProviderMSPBackupManifestVersion,
CreatedAt: time.Now().UTC(),
ControlPlaneMode: string(cfg.ControlPlaneMode),
Environment: strings.TrimSpace(cfg.Environment),
BaseURL: strings.TrimSpace(cfg.BaseURL),
PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion),
PlanSource: providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource),
LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID),
LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
WorkspaceLimit: workspaceLimit,
ControlPlaneDir: providerMSPBackupControlPlaneDir,
TenantsDir: providerMSPBackupTenantsDir,
RegistryAccountCount: accountCount,
RegistryStateCounts: map[string]int{},
ControlPlaneDBBackups: []string{},
}
for _, tenant := range tenants {
if tenant == nil {
continue
}
tenantID := strings.TrimSpace(tenant.ID)
if tenantID == "" {
continue
}
manifest.RegistryTenantIDs = append(manifest.RegistryTenantIDs, tenantID)
manifest.RegistryStateCounts[string(tenant.State)]++
if providerMSPBackupRequiresRuntimeTenantDir(tenant.State) {
manifest.RuntimeTenantIDs = append(manifest.RuntimeTenantIDs, tenantID)
}
}
sort.Strings(manifest.RegistryTenantIDs)
sort.Strings(manifest.RuntimeTenantIDs)
manifest.RegistryTenantCount = len(manifest.RegistryTenantIDs)
manifest.RuntimeTenantCount = len(manifest.RuntimeTenantIDs)
return manifest, nil
}
func providerMSPBackupRequiresRuntimeTenantDir(state registry.TenantState) bool {
switch state {
case registry.TenantStateCanceled, registry.TenantStateDeleting, registry.TenantStateDeleted:
return false
default:
return true
}
}
func resolveProviderMSPBackupLicensePath(cfg *CPConfig) (string, bool, error) {
if !ProviderMSPPlanSourceIsSignedLicense(providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource)) {
return "", false, nil
}
licensePath := strings.TrimSpace(cfg.ProviderMSPLicenseFile)
if licensePath == "" {
return "", false, fmt.Errorf("provider MSP backup requires CP_PROVIDER_MSP_LICENSE_FILE when plan_source=%s", ProviderMSPPlanSourceLicenseFile)
}
if err := requireRegularFile(licensePath, "provider MSP license file"); err != nil {
return "", false, err
}
return licensePath, true, nil
}
func requireProviderMSPRuntimeTenantDirs(tenantsDir string, tenantIDs []string) error {
if len(tenantIDs) == 0 {
return nil
}
if err := requireDirectory(tenantsDir, "tenants dir"); err != nil {
return err
}
for _, tenantID := range tenantIDs {
dir := filepath.Join(tenantsDir, tenantID)
if err := requireDirectory(dir, "runtime tenant dir "+tenantID); err != nil {
return err
}
}
return nil
}
func writeProviderMSPBackupArchive(ctx context.Context, output io.Writer, tempDir, controlPlaneDir, tenantsDir, licensePath string, manifest ProviderMSPBackupManifest) (*ProviderMSPBackupCreateResult, error) {
gz := gzip.NewWriter(output)
tw := tar.NewWriter(gz)
result := &ProviderMSPBackupCreateResult{
Manifest: manifest,
}
controlPlaneEntries, dbBackups, err := addProviderMSPBackupDir(ctx, tw, tempDir, controlPlaneDir, providerMSPBackupControlPlaneDir, true, manifest.CreatedAt)
if err != nil {
return nil, err
}
result.ControlPlaneEntries = controlPlaneEntries
manifest.ControlPlaneDBBackups = dbBackups
tenantEntries, _, err := addProviderMSPBackupDir(ctx, tw, tempDir, tenantsDir, providerMSPBackupTenantsDir, false, manifest.CreatedAt)
if err != nil {
return nil, err
}
result.TenantEntries = tenantEntries
if strings.TrimSpace(licensePath) != "" {
if err := writeProviderMSPBackupRootDir(tw, providerMSPBackupLicenseDir, manifest.CreatedAt); err != nil {
return nil, err
}
result.LicenseEntries++
if err := addProviderMSPBackupFile(tw, licensePath, providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName, 0); err != nil {
return nil, err
}
result.LicenseEntries++
}
result.Manifest = manifest
manifestBytes, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return nil, fmt.Errorf("encode backup manifest: %w", err)
}
manifestBytes = append(manifestBytes, '\n')
if err := writeProviderMSPBackupBytes(tw, providerMSPBackupManifestName, manifestBytes, 0o600, manifest.CreatedAt); err != nil {
return nil, err
}
if err := tw.Close(); err != nil {
return nil, fmt.Errorf("close backup tar stream: %w", err)
}
if err := gz.Close(); err != nil {
return nil, fmt.Errorf("close backup gzip stream: %w", err)
}
return result, nil
}
func addProviderMSPBackupDir(ctx context.Context, tw *tar.Writer, tempDir, sourceDir, archiveRoot string, snapshotSQLite bool, createdAt time.Time) (int, []string, error) {
if strings.TrimSpace(sourceDir) == "" {
return 0, nil, fmt.Errorf("source dir is required for %s", archiveRoot)
}
entries := 0
dbBackups := []string{}
if err := writeProviderMSPBackupRootDir(tw, archiveRoot, createdAt); err != nil {
return 0, nil, err
}
entries++
if _, err := os.Stat(sourceDir); os.IsNotExist(err) {
return entries, dbBackups, nil
} else if err != nil {
return 0, nil, fmt.Errorf("stat %s: %w", sourceDir, err)
}
err := filepath.WalkDir(sourceDir, func(filePath string, entry os.DirEntry, walkErr error) error {
if err := ctx.Err(); err != nil {
return err
}
if walkErr != nil {
return fmt.Errorf("walk %s: %w", filePath, walkErr)
}
rel, err := filepath.Rel(sourceDir, filePath)
if err != nil {
return fmt.Errorf("resolve backup relative path: %w", err)
}
if rel == "." {
return nil
}
archiveName := filepath.ToSlash(filepath.Join(archiveRoot, rel))
info, err := entry.Info()
if err != nil {
return fmt.Errorf("stat backup source %s: %w", filePath, err)
}
if entry.Type()&os.ModeSymlink != 0 {
if err := addProviderMSPBackupSymlink(tw, filePath, archiveName, info); err != nil {
return err
}
entries++
return nil
}
if entry.IsDir() {
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
return err
}
entries++
return nil
}
if !entry.Type().IsRegular() {
return fmt.Errorf("unsupported backup source file type at %s", filePath)
}
if snapshotSQLite && isProviderMSPSQLiteSidecar(rel) {
return nil
}
if snapshotSQLite && strings.EqualFold(filepath.Ext(filePath), ".db") {
snapshotPath := filepath.Join(tempDir, archiveRoot, rel)
if err := snapshotSQLiteDatabase(filePath, snapshotPath); err != nil {
return err
}
if err := addProviderMSPBackupFile(tw, snapshotPath, archiveName, info.Mode().Perm()); err != nil {
return err
}
dbBackups = append(dbBackups, archiveName)
entries++
return nil
}
if err := addProviderMSPBackupFile(tw, filePath, archiveName, 0); err != nil {
return err
}
entries++
return nil
})
sort.Strings(dbBackups)
if err != nil {
return 0, nil, err
}
return entries, dbBackups, nil
}
func snapshotSQLiteDatabase(sourcePath, destPath string) error {
if err := requireRegularFile(sourcePath, "sqlite database"); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
return fmt.Errorf("create sqlite snapshot parent: %w", err)
}
_ = os.Remove(destPath)
dsn := sourcePath + "?" + url.Values{
"_pragma": []string{
"busy_timeout(30000)",
"foreign_keys(ON)",
},
}.Encode()
db, err := sql.Open("sqlite", dsn)
if err != nil {
return fmt.Errorf("open sqlite database for backup: %w", err)
}
defer db.Close()
if _, err := db.Exec("VACUUM INTO ?", destPath); err != nil {
return fmt.Errorf("snapshot sqlite database %s: %w", filepath.Base(sourcePath), err)
}
return nil
}
func addProviderMSPBackupFile(tw *tar.Writer, sourcePath, archiveName string, modeOverride os.FileMode) error {
info, err := os.Lstat(sourcePath)
if err != nil {
return fmt.Errorf("stat backup file %s: %w", sourcePath, err)
}
if modeOverride != 0 {
info = providerMSPFileInfoWithMode{FileInfo: info, mode: modeOverride}
}
file, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("open backup file %s: %w", sourcePath, err)
}
defer file.Close()
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
return err
}
if _, err := io.Copy(tw, file); err != nil {
return fmt.Errorf("write backup file %s: %w", archiveName, err)
}
return nil
}
func addProviderMSPBackupSymlink(tw *tar.Writer, sourcePath, archiveName string, info os.FileInfo) error {
target, err := os.Readlink(sourcePath)
if err != nil {
return fmt.Errorf("read backup symlink %s: %w", sourcePath, err)
}
return addProviderMSPBackupHeader(tw, info, archiveName, target)
}
func addProviderMSPBackupHeader(tw *tar.Writer, info os.FileInfo, archiveName, linkTarget string) error {
header, err := tar.FileInfoHeader(info, linkTarget)
if err != nil {
return fmt.Errorf("create backup tar header %s: %w", archiveName, err)
}
header.Name = archiveName
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if info.IsDir() && !strings.HasSuffix(header.Name, "/") {
header.Name += "/"
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
return nil
}
func writeProviderMSPBackupRootDir(tw *tar.Writer, archiveName string, modTime time.Time) error {
name := strings.TrimSuffix(archiveName, "/") + "/"
header := &tar.Header{
Name: name,
Mode: 0o755,
Typeflag: tar.TypeDir,
ModTime: modTime,
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
return nil
}
func writeProviderMSPBackupBytes(tw *tar.Writer, archiveName string, content []byte, mode int64, modTime time.Time) error {
header := &tar.Header{
Name: archiveName,
Mode: mode,
Size: int64(len(content)),
Typeflag: tar.TypeReg,
ModTime: modTime,
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
if _, err := tw.Write(content); err != nil {
return fmt.Errorf("write backup entry %s: %w", archiveName, err)
}
return nil
}
func validateProviderMSPBackupManifest(manifest ProviderMSPBackupManifest) error {
if manifest.Version != ProviderMSPBackupManifestVersion {
return fmt.Errorf("unsupported provider MSP backup manifest version %q", manifest.Version)
}
if !isMSPControlPlaneMode(ControlPlaneMode(manifest.ControlPlaneMode)) {
return fmt.Errorf("backup control_plane_mode = %q, want %q or %q", manifest.ControlPlaneMode, ControlPlaneModeProviderHostedMSP, ControlPlaneModePulseHostedMSP)
}
if strings.TrimSpace(manifest.PlanVersion) == "" {
return fmt.Errorf("backup manifest is missing plan_version")
}
if strings.TrimSpace(manifest.PlanSource) == "" {
return fmt.Errorf("backup manifest is missing plan_source")
}
if manifest.ControlPlaneDir != providerMSPBackupControlPlaneDir {
return fmt.Errorf("backup control_plane_dir = %q, want %q", manifest.ControlPlaneDir, providerMSPBackupControlPlaneDir)
}
if manifest.TenantsDir != providerMSPBackupTenantsDir {
return fmt.Errorf("backup tenants_dir = %q, want %q", manifest.TenantsDir, providerMSPBackupTenantsDir)
}
if manifest.RegistryTenantCount != len(manifest.RegistryTenantIDs) {
return fmt.Errorf("backup registry_tenant_count = %d, want %d ids", manifest.RegistryTenantCount, len(manifest.RegistryTenantIDs))
}
if manifest.RuntimeTenantCount != len(manifest.RuntimeTenantIDs) {
return fmt.Errorf("backup runtime_tenant_count = %d, want %d ids", manifest.RuntimeTenantCount, len(manifest.RuntimeTenantIDs))
}
return nil
}
func readProviderMSPBackupManifestBytes(reader io.Reader, size int64) ([]byte, error) {
if size < 0 || size > maxProviderMSPBackupManifestSize {
return nil, fmt.Errorf("backup manifest size %d is outside allowed range", size)
}
limited := io.LimitReader(reader, maxProviderMSPBackupManifestSize+1)
content, err := io.ReadAll(limited)
if err != nil {
return nil, fmt.Errorf("read backup manifest: %w", err)
}
if len(content) > maxProviderMSPBackupManifestSize {
return nil, fmt.Errorf("backup manifest exceeds %d bytes", maxProviderMSPBackupManifestSize)
}
return content, nil
}
func cleanProviderMSPArchiveName(raw string) (string, error) {
name := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/"))
if name == "" {
return "", fmt.Errorf("backup archive contains empty entry name")
}
name = strings.TrimPrefix(name, "./")
cleaned := path.Clean(name)
// ".." must be rejected on its own, not just as a "../" prefix: an entry
// named ".." or "a/../.." cleans to exactly ".." and would otherwise pass
// this gate and resolve to the parent of the restore target.
if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") || path.IsAbs(cleaned) {
return "", fmt.Errorf("backup archive contains unsafe entry name %q", raw)
}
return cleaned, nil
}
func archiveTenantID(name string) string {
if name == providerMSPBackupTenantsDir || !strings.HasPrefix(name, providerMSPBackupTenantsDir+"/") {
return ""
}
rest := strings.TrimPrefix(name, providerMSPBackupTenantsDir+"/")
parts := strings.Split(rest, "/")
if len(parts) == 0 {
return ""
}
return strings.TrimSpace(parts[0])
}
func isProviderMSPSQLiteSidecar(rel string) bool {
name := strings.ToLower(filepath.Base(rel))
return strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
}
func pathIsInside(candidate, parent string) bool {
candidate = filepath.Clean(candidate)
parent = filepath.Clean(parent)
rel, err := filepath.Rel(parent, candidate)
if err != nil {
return false
}
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
}
func requireDirectory(dir, label string) error {
info, err := os.Stat(dir)
if err != nil {
return fmt.Errorf("%s unavailable: %w", label, err)
}
if !info.IsDir() {
return fmt.Errorf("%s %q is not a directory", label, dir)
}
return nil
}
func requireRegularFile(filePath, label string) error {
info, err := os.Stat(filePath)
if err != nil {
return fmt.Errorf("%s unavailable: %w", label, err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("%s %q is not a regular file", label, filePath)
}
return nil
}
type providerMSPFileInfoWithMode struct {
os.FileInfo
mode os.FileMode
}
func (i providerMSPFileInfoWithMode) Mode() os.FileMode {
return i.mode
}