Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

56.3 KB
Raw Blame
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/api"
	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/account"
	cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker"
	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/entitlements"
	cphandoff "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/handoff"
	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/portal"
	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
	cpstripe "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/stripe"
	runtimeconfig "github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
	pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
	"github.com/rcourtman/pulse-go-rewrite/pkg/securityutil"
	"github.com/spf13/cobra"
)

const defaultProviderMSPProofWorkspacePrefix = "Provider MSP Proof"

var handoffTokenInputPattern = regexp.MustCompile(`name="token"\s+value="([^"]+)"`)

type providerMSPProofOptions struct {
	AccountName                string
	OwnerEmail                 string
	WorkspacePrefix            string
	WorkspaceCount             int
	InstallType                string
	TargetPath                 string
	Cleanup                    bool
	AllowEnvPlan               bool
	AllowNonProofWorkspaceName bool
}

type providerMSPProofRuntime struct {
	cfg         *cloudcp.CPConfig
	registry    *registry.TenantRegistry
	docker      *cpDocker.Manager
	provisioner *cpstripe.Provisioner
}

type providerMSPProofWorkspace struct {
	TenantID                    string
	DisplayName                 string
	State                       string
	PlanVersion                 string
	ContainerID                 string
	PublicURL                   string
	InstallType                 string
	InstallToken                string
	InstallTokenID              string
	InstallCommandGenerated     bool
	AgentTokenAuthVerified      bool
	SetupFactsTokenUseVisible   bool
	AgentReportIngestVerified   bool
	AgentReportAgentID          string
	AgentReportHostname         string
	TokenRotationVerified       bool
	RotatedInstallToken         string
	RotatedInstallTokenID       string
	OldInstallTokenRejected     bool
	RotatedAgentReportVerified  bool
	HandoffExchangeVerified     bool
	HandoffTargetPath           string
	EntitlementLeaseChecked     bool
	EntitlementLeaseVerified    bool
	EntitlementWhiteLabel       bool
	EntitlementSkippedReason    string
	ReportScheduleCreated       bool
	ReportScheduleID            string
	ReportScheduleVisible       bool
	ReportScheduleCount         int
	DisabledReportScheduleCount int
	ActiveAlertPersisted        bool
	ActiveAlertRollupVisible    bool
	CriticalAlertCount          int
	WarningAlertCount           int
}

type providerMSPProofReport struct {
	AccountID                 string
	AccountName               string
	OwnerUserID               string
	OwnerEmail                string
	PlanVersion               string
	PlanSource                string
	LicenseID                 string
	LicenseEmail              string
	WorkspaceLimit            int
	WorkspaceCount            int
	Workspaces                []providerMSPProofWorkspace
	DockerlessProvisioning    bool
	RuntimeContainerVerified  bool
	HandoffExchangeVerified   bool
	InstallTokenBoundaryOK    bool
	SetupFactsTokenUseVisible bool
	AgentReportIngestVerified bool
	TokenRotationVerified     bool
	ReportScheduleVisible     bool
	ActiveAlertRollupVisible  bool
	Cleanup                   bool
}

func newProviderMSPProofCmd() *cobra.Command {
	opts := providerMSPProofOptions{
		WorkspacePrefix: defaultProviderMSPProofWorkspacePrefix,
		WorkspaceCount:  2,
		InstallType:     "pve",
		TargetPath:      "/settings/infrastructure?add=linux-host",
	}

	cmd := &cobra.Command{
		Use:   "proof",
		Short: "Prove the provider-hosted MSP workspace and handoff path",
		RunE: func(cmd *cobra.Command, args []string) error {
			rt, err := newProviderMSPProofRuntime(cmd.Context())
			if err != nil {
				return err
			}
			defer rt.close()

			report, err := rt.runProviderMSPProof(cmd.Context(), opts)
			if err != nil {
				return err
			}
			printProviderMSPProofReport(report)
			return nil
		},
	}

	cmd.Flags().StringVar(&opts.AccountName, "account-name", "", "Provider MSP account display name")
	cmd.Flags().StringVar(&opts.OwnerEmail, "owner-email", "", "Provider owner email address")
	cmd.Flags().StringVar(&opts.WorkspacePrefix, "workspace-prefix", opts.WorkspacePrefix, "Proof workspace display-name prefix")
	cmd.Flags().IntVar(&opts.WorkspaceCount, "workspace-count", opts.WorkspaceCount, "Number of proof client workspaces to create; minimum 2")
	cmd.Flags().StringVar(&opts.InstallType, "install-type", opts.InstallType, "Hosted tenant install command type: pve or pbs")
	cmd.Flags().StringVar(&opts.TargetPath, "target-path", opts.TargetPath, "Tenant-local target path to verify during handoff exchange")
	cmd.Flags().BoolVar(&opts.Cleanup, "cleanup", false, "Delete proof workspaces after the proof completes")
	cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development")
	cmd.Flags().BoolVar(&opts.AllowNonProofWorkspaceName, "allow-non-proof-workspace-name", false, "Allow proof workspace names without proof/canary/rehearsal markers")
	_ = cmd.MarkFlagRequired("account-name")
	_ = cmd.MarkFlagRequired("owner-email")
	return cmd
}

func newProviderMSPProofRuntime(ctx context.Context) (*providerMSPProofRuntime, error) {
	cfg, err := cloudcp.LoadConfig()
	if err != nil {
		return nil, fmt.Errorf("load control plane config: %w", err)
	}
	return newProviderMSPProofRuntimeFromConfig(cfg)
}

func newProviderMSPProofRuntimeFromConfig(cfg *cloudcp.CPConfig) (*providerMSPProofRuntime, error) {
	if cfg == nil {
		return nil, fmt.Errorf("control plane config is required")
	}
	if !cfg.IsMSPControlPlane() {
		return nil, fmt.Errorf("provider MSP proof requires CP_CONTROL_PLANE_MODE=%s or %s", cloudcp.ControlPlaneModeProviderHostedMSP, cloudcp.ControlPlaneModePulseHostedMSP)
	}
	if err := os.MkdirAll(cfg.TenantsDir(), 0o755); err != nil {
		return nil, fmt.Errorf("create tenants dir: %w", err)
	}
	if err := os.MkdirAll(cfg.ControlPlaneDir(), 0o755); err != nil {
		return nil, fmt.Errorf("create control-plane dir: %w", err)
	}

	reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
	if err != nil {
		return nil, fmt.Errorf("open tenant registry: %w", err)
	}

	var dockerMgr *cpDocker.Manager
	dockerMgr, err = cpDocker.NewManager(providerMSPDockerManagerConfig(cfg))
	if err != nil {
		if !cfg.AllowDockerlessProvisioning {
			reg.Close()
			return nil, fmt.Errorf("create docker manager: %w", err)
		}
		dockerMgr = nil
	}

	hostedEntitlements := entitlements.NewService(reg, cfg.BaseURL, cfg.TrialActivationPrivateKey)
	if cfg.IsProviderHostedMSP() {
		hostedEntitlements.SetProviderHosted(true)
		hostedEntitlements.SetProviderLicense(cfg.ProviderMSPLicenseKey)
	}
	provisioner := cpstripe.NewProvisioner(
		reg,
		cfg.TenantsDir(),
		dockerMgr,
		nil,
		cfg.BaseURL,
		nil,
		cfg.EmailFrom,
		cfg.AllowDockerlessProvisioning,
		cpstripe.WithHostedEntitlementService(hostedEntitlements),
		cpstripe.WithTrialActivationPrivateKey(cfg.TrialActivationPrivateKey),
		cpstripe.WithAdmissionCheck(func(checkCtx context.Context) error {
			return cloudcp.EnforceStorageAdmission(checkCtx, cfg, dockerMgr)
		}),
		cpstripe.WithDefaultMSPPlanVersion(providerMSPProofPlanVersion(cfg)),
	)

	return &providerMSPProofRuntime{
		cfg:         cfg,
		registry:    reg,
		docker:      dockerMgr,
		provisioner: provisioner,
	}, nil
}

func (rt *providerMSPProofRuntime) close() {
	if rt == nil {
		return
	}
	if rt.docker != nil {
		rt.docker.Close()
	}
	if rt.registry != nil {
		rt.registry.Close()
	}
}

func (rt *providerMSPProofRuntime) runProviderMSPProof(ctx context.Context, opts providerMSPProofOptions) (*providerMSPProofReport, error) {
	if rt == nil || rt.cfg == nil || rt.registry == nil || rt.provisioner == nil {
		return nil, fmt.Errorf("provider MSP proof runtime is not initialized")
	}
	opts, err := normalizeProviderMSPProofOptions(opts)
	if err != nil {
		return nil, err
	}
	if !opts.AllowEnvPlan && !cloudcp.ProviderMSPPlanSourceIsSignedLicense(rt.cfg.ProviderMSPPlanSource) {
		return nil, fmt.Errorf("provider MSP proof requires %s plan source; rerun with --allow-env-plan only for local development", cloudcp.ProviderMSPPlanSourceLicenseFile)
	}

	bootstrap, err := cloudcp.BootstrapProviderMSP(ctx, rt.cfg, cloudcp.ProviderMSPBootstrapOptions{
		AccountName:       opts.AccountName,
		OwnerEmail:        opts.OwnerEmail,
		GenerateMagicLink: false,
	})
	if err != nil {
		return nil, fmt.Errorf("bootstrap provider MSP account: %w", err)
	}

	// The proof creates its workspaces on the provider's own account, so they
	// count against the same client limit real clients do. Check for room
	// before creating anything: on a two-client evaluation that already has a
	// client, the second proof workspace would otherwise be refused after the
	// first was created, leaving it behind in one of the evaluation's slots.
	if bootstrap.WorkspaceLimit > 0 {
		inUse, err := rt.registry.CountActiveByAccountID(bootstrap.AccountID)
		if err != nil {
			return nil, fmt.Errorf("count client workspaces: %w", err)
		}
		if free := bootstrap.WorkspaceLimit - inUse; free < opts.WorkspaceCount {
			return nil, fmt.Errorf("the proof creates %d temporary client workspaces but this account has %d free (%d of %d in use); run it on a fresh platform, before adding clients",
				opts.WorkspaceCount, max(free, 0), inUse, bootstrap.WorkspaceLimit)
		}
	}

	createdTenants := make([]*registry.Tenant, 0, opts.WorkspaceCount)
	partial := func(err error) error {
		return &providerMSPProofPartialError{err: err, CreatedTenantIDs: providerMSPProofTenantIDs(createdTenants)}
	}
	if opts.Cleanup {
		defer func() {
			_ = rt.cleanupProviderMSPProofTenants(context.Background(), createdTenants)
		}()
	}

	report := &providerMSPProofReport{
		AccountID:                 bootstrap.AccountID,
		AccountName:               bootstrap.AccountName,
		OwnerUserID:               bootstrap.OwnerUserID,
		OwnerEmail:                bootstrap.OwnerEmail,
		PlanVersion:               bootstrap.PlanVersion,
		PlanSource:                bootstrap.PlanSource,
		LicenseID:                 bootstrap.LicenseID,
		LicenseEmail:              bootstrap.LicenseEmail,
		WorkspaceLimit:            bootstrap.WorkspaceLimit,
		RuntimeContainerVerified:  true,
		HandoffExchangeVerified:   true,
		InstallTokenBoundaryOK:    true,
		SetupFactsTokenUseVisible: true,
		AgentReportIngestVerified: true,
		TokenRotationVerified:     true,
		ReportScheduleVisible:     true,
		ActiveAlertRollupVisible:  true,
		Cleanup:                   opts.Cleanup,
	}

	for idx := 0; idx < opts.WorkspaceCount; idx++ {
		displayName := fmt.Sprintf("%s %02d", opts.WorkspacePrefix, idx+1)
		tenant, err := rt.createProviderMSPProofWorkspace(ctx, bootstrap.AccountID, displayName, bootstrap.OwnerEmail)
		if err != nil {
			return nil, partial(fmt.Errorf("create proof workspace %d: %w", idx+1, err))
		}
		createdTenants = append(createdTenants, tenant)

		workspace, err := rt.proveProviderMSPWorkspace(ctx, tenant, bootstrap.OwnerUserID, opts.InstallType, opts.TargetPath)
		if err != nil {
			return nil, partial(fmt.Errorf("prove workspace %s: %w", tenant.ID, err))
		}
		if workspace.ContainerID == "" {
			report.RuntimeContainerVerified = false
			report.DockerlessProvisioning = true
		}
		if !workspace.HandoffExchangeVerified {
			report.HandoffExchangeVerified = false
		}
		if !workspace.SetupFactsTokenUseVisible {
			report.SetupFactsTokenUseVisible = false
		}
		if !workspace.AgentReportIngestVerified {
			report.AgentReportIngestVerified = false
		}
		if !workspace.TokenRotationVerified {
			report.TokenRotationVerified = false
		}
		if !workspace.ReportScheduleVisible {
			report.ReportScheduleVisible = false
		}
		if !workspace.ActiveAlertRollupVisible {
			report.ActiveAlertRollupVisible = false
		}
		report.Workspaces = append(report.Workspaces, workspace)
	}

	boundaryOK, err := rt.verifyProviderMSPInstallTokenIsolation(report.Workspaces)
	if err != nil {
		return nil, partial(err)
	}
	report.InstallTokenBoundaryOK = boundaryOK
	report.WorkspaceCount = len(report.Workspaces)
	return report, nil
}

// providerMSPProofPartialError is a proof that failed after creating some of
// its workspaces. It carries their IDs so the caller can remove them: without
// that, a failure part-way through left proof workspaces occupying client
// slots, because no report (and so no workspace list) comes back on error.
type providerMSPProofPartialError struct {
	err              error
	CreatedTenantIDs []string
}

func (e *providerMSPProofPartialError) Error() string { return e.err.Error() }
func (e *providerMSPProofPartialError) Unwrap() error { return e.err }

func providerMSPProofTenantIDs(tenants []*registry.Tenant) []string {
	ids := make([]string, 0, len(tenants))
	for _, tenant := range tenants {
		if tenant != nil && strings.TrimSpace(tenant.ID) != "" {
			ids = append(ids, tenant.ID)
		}
	}
	return ids
}

func normalizeProviderMSPProofOptions(opts providerMSPProofOptions) (providerMSPProofOptions, error) {
	opts.AccountName = strings.TrimSpace(opts.AccountName)
	opts.OwnerEmail = strings.ToLower(strings.TrimSpace(opts.OwnerEmail))
	opts.WorkspacePrefix = strings.TrimSpace(opts.WorkspacePrefix)
	opts.InstallType = strings.ToLower(strings.TrimSpace(opts.InstallType))
	opts.TargetPath = strings.TrimSpace(opts.TargetPath)
	if opts.AccountName == "" {
		return opts, fmt.Errorf("--account-name is required")
	}
	if opts.OwnerEmail == "" {
		return opts, fmt.Errorf("--owner-email is required")
	}
	if opts.WorkspacePrefix == "" {
		opts.WorkspacePrefix = defaultProviderMSPProofWorkspacePrefix
	}
	if opts.WorkspaceCount < 2 {
		return opts, fmt.Errorf("--workspace-count must be at least 2 so tenant isolation can be proven")
	}
	if _, err := apiInstallTypeForProviderMSPProof(opts.InstallType); err != nil {
		return opts, err
	}
	if !opts.AllowNonProofWorkspaceName && !containsProviderMSPProofMarker(opts.WorkspacePrefix) {
		return opts, fmt.Errorf("--workspace-prefix must contain proof, canary, rehearsal, or seed")
	}
	if opts.TargetPath == "" {
		opts.TargetPath = "/settings/infrastructure?add=linux-host"
	}
	normalizedTargetPath, err := securityutil.NormalizeLocalRedirectPath(opts.TargetPath)
	if err != nil {
		return opts, fmt.Errorf("--target-path must be a local absolute path")
	}
	opts.TargetPath = normalizedTargetPath
	return opts, nil
}

func apiInstallTypeForProviderMSPProof(raw string) (string, error) {
	switch strings.ToLower(strings.TrimSpace(raw)) {
	case "pve", "pbs":
		return strings.ToLower(strings.TrimSpace(raw)), nil
	default:
		return "", fmt.Errorf("--install-type must be pve or pbs")
	}
}

func containsProviderMSPProofMarker(values ...string) bool {
	for _, value := range values {
		normalized := strings.ToLower(strings.TrimSpace(value))
		for _, marker := range []string{"proof", "canary", "rehearsal", "seed"} {
			if strings.Contains(normalized, marker) {
				return true
			}
		}
	}
	return false
}

func (rt *providerMSPProofRuntime) createProviderMSPProofWorkspace(ctx context.Context, accountID, displayName, ownerEmail string) (*registry.Tenant, error) {
	handler := account.HandleCreateTenantWithWorkspaceLimitPolicy(
		rt.registry,
		rt.provisioner,
		account.WorkspaceLimitPolicy{
			ProviderHostedMSP:        true,
			ProviderMSPPlanVersion:   providerMSPProofPlanVersion(rt.cfg),
			ProviderMSPLicenseLapsed: func() bool { return rt.cfg.ProviderMSPLicenseLapsed(time.Now()) },
		},
	)
	mux := http.NewServeMux()
	mux.Handle("/api/accounts/{account_id}/tenants", handler)

	payload, err := json.Marshal(map[string]string{"display_name": displayName})
	if err != nil {
		return nil, fmt.Errorf("marshal create workspace request: %w", err)
	}
	path := "/api/accounts/" + url.PathEscape(accountID) + "/tenants"
	req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(payload))
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-User-Email", ownerEmail)
	rec := httptest.NewRecorder()
	mux.ServeHTTP(rec, req)
	if rec.Code != http.StatusCreated {
		return nil, fmt.Errorf("workspace create status=%d body=%s", rec.Code, strings.TrimSpace(rec.Body.String()))
	}

	var tenant registry.Tenant
	if err := json.Unmarshal(rec.Body.Bytes(), &tenant); err != nil {
		return nil, fmt.Errorf("decode workspace create response: %w", err)
	}
	if strings.TrimSpace(tenant.ID) == "" {
		return nil, fmt.Errorf("workspace create response missing tenant id")
	}
	return &tenant, nil
}

func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context, tenant *registry.Tenant, ownerUserID, installType, targetPath string) (providerMSPProofWorkspace, error) {
	if tenant == nil {
		return providerMSPProofWorkspace{}, fmt.Errorf("tenant is required")
	}
	tenantDataDir, err := safeProviderMSPProofTenantDataDir(rt.cfg.TenantsDir(), tenant.ID)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}
	publicURL := rt.providerMSPProofTenantPublicURL(tenant.ID)
	install, err := api.GenerateHostedTenantAgentInstallCommand(api.HostedTenantAgentInstallCommandOptions{
		Config: &runtimeconfig.Config{
			DataPath:   tenantDataDir,
			ConfigPath: tenantDataDir,
			PublicURL:  publicURL,
		},
		Persistence: configPersistenceForProviderMSPProof(tenantDataDir),
		MultiTenant: runtimeconfig.NewMultiTenantPersistence(tenantDataDir),
		HostedMode:  true,
		OrgID:       tenant.ID,
		InstallType: installType,
		OwnerUserID: ownerUserID,
		BaseURL:     publicURL,
	})
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token generation", err)
	if err != nil {
		return providerMSPProofWorkspace{}, fmt.Errorf("generate hosted tenant install command: %w", err)
	}
	if install == nil || strings.TrimSpace(install.Command) == "" || strings.TrimSpace(install.Token) == "" {
		return providerMSPProofWorkspace{}, fmt.Errorf("hosted tenant install command result is incomplete")
	}

	tokenAuthVerified, err := markProviderMSPProofAgentTokenUsed(tenantDataDir, tenant.ID, install.Token)
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token use", err)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}
	facts := portal.NewTenantDirWorkspaceSetupFactReader(rt.cfg.TenantsDir()).FactsForWorkspace(tenant.ID)
	setupFactsVisible := facts.AgentCount != nil && *facts.AgentCount > 0 &&
		facts.AgentTokenCount != nil && *facts.AgentTokenCount > 0 &&
		facts.LastAgentSeenAt != nil

	agentReport, err := rt.verifyProviderMSPProofAgentReportIngest(ctx, tenant, tenantDataDir, install.Token, install.TokenID)
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "agent report ingest", err)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}

	rotation, err := rt.verifyProviderMSPProofInstallTokenRotation(ctx, tenant, tenantDataDir, install.Token, install.TokenID)
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token rotation", err)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}

	exchangedTargetPath, err := rt.verifyProviderMSPProofHandoff(ctx, tenant, ownerUserID, targetPath)
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "handoff exchange", err)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}

	entitlement, err := rt.verifyProviderMSPProofEntitlementLease(tenant, tenantDataDir)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}

	portalRollup, err := rt.verifyProviderMSPProofPortalRollup(ctx, tenant, tenantDataDir, agentReport)
	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "portal rollup", err)
	if err != nil {
		return providerMSPProofWorkspace{}, err
	}

	return providerMSPProofWorkspace{
		TenantID:                    tenant.ID,
		DisplayName:                 tenant.DisplayName,
		State:                       string(tenant.State),
		PlanVersion:                 tenant.PlanVersion,
		ContainerID:                 tenant.ContainerID,
		PublicURL:                   publicURL,
		InstallType:                 install.InstallType,
		InstallToken:                install.Token,
		InstallTokenID:              install.TokenID,
		InstallCommandGenerated:     true,
		AgentTokenAuthVerified:      tokenAuthVerified,
		SetupFactsTokenUseVisible:   setupFactsVisible,
		AgentReportIngestVerified:   agentReport.Verified,
		AgentReportAgentID:          agentReport.AgentID,
		AgentReportHostname:         agentReport.Hostname,
		TokenRotationVerified:       rotation.Verified,
		RotatedInstallToken:         rotation.RotatedToken,
		RotatedInstallTokenID:       rotation.RotatedTokenID,
		OldInstallTokenRejected:     rotation.OldTokenRejected,
		RotatedAgentReportVerified:  rotation.RotatedAgentReportVerified,
		HandoffExchangeVerified:     exchangedTargetPath == targetPath,
		HandoffTargetPath:           exchangedTargetPath,
		EntitlementLeaseChecked:     entitlement.Checked,
		EntitlementLeaseVerified:    entitlement.Verified,
		EntitlementWhiteLabel:       entitlement.WhiteLabel,
		EntitlementSkippedReason:    entitlement.SkippedReason,
		ReportScheduleCreated:       portalRollup.ReportScheduleCreated,
		ReportScheduleID:            portalRollup.ReportScheduleID,
		ReportScheduleVisible:       portalRollup.ReportScheduleVisible,
		ReportScheduleCount:         portalRollup.ReportScheduleCount,
		DisabledReportScheduleCount: portalRollup.DisabledReportScheduleCount,
		ActiveAlertPersisted:        portalRollup.ActiveAlertPersisted,
		ActiveAlertRollupVisible:    portalRollup.ActiveAlertRollupVisible,
		CriticalAlertCount:          portalRollup.CriticalAlertCount,
		WarningAlertCount:           portalRollup.WarningAlertCount,
	}, nil
}

func (rt *providerMSPProofRuntime) reconcileProviderMSPProofRuntimeMutation(tenant *registry.Tenant, tenantDataDir, stage string, mutationErr error) error {
	if rt == nil || rt.docker == nil || tenant == nil || strings.TrimSpace(tenant.ContainerID) == "" {
		return mutationErr
	}
	reconcileErr := rt.docker.ReconcileTenantRuntimeMountSources(tenantDataDir)
	if reconcileErr != nil {
		reconcileErr = fmt.Errorf("reconcile tenant runtime ownership after %s: %w", stage, reconcileErr)
	}
	if mutationErr != nil && reconcileErr != nil {
		return errors.Join(mutationErr, reconcileErr)
	}
	if mutationErr != nil {
		return mutationErr
	}
	return reconcileErr
}

type providerMSPProofPortalRollup struct {
	ReportScheduleCreated       bool
	ReportScheduleID            string
	ReportScheduleVisible       bool
	ReportScheduleCount         int
	DisabledReportScheduleCount int
	ActiveAlertPersisted        bool
	ActiveAlertRollupVisible    bool
	CriticalAlertCount          int
	WarningAlertCount           int
}

func (rt *providerMSPProofRuntime) verifyProviderMSPProofPortalRollup(ctx context.Context, tenant *registry.Tenant, tenantDataDir string, agentReport providerMSPProofAgentReportIngest) (providerMSPProofPortalRollup, error) {
	result := providerMSPProofPortalRollup{}
	if tenant == nil {
		return result, fmt.Errorf("tenant is required")
	}
	tenantID := strings.TrimSpace(tenant.ID)
	if tenantID == "" {
		return result, fmt.Errorf("tenant id is required")
	}
	if strings.TrimSpace(agentReport.AgentID) == "" {
		return result, fmt.Errorf("tenant %s agent report id is required before portal rollup proof", tenantID)
	}

	scheduleID, err := rt.createProviderMSPProofReportSchedule(ctx, tenant, tenantDataDir, agentReport.AgentID, agentReport.Hostname)
	if err != nil {
		return result, err
	}
	result.ReportScheduleCreated = true
	result.ReportScheduleID = scheduleID

	orgDir := filepath.Join(tenantDataDir, "orgs", tenantID)
	if err := writeProviderMSPProofActiveAlerts(orgDir, agentReport.AgentID, agentReport.Hostname); err != nil {
		return result, fmt.Errorf("write provider MSP proof active alerts for tenant %s: %w", tenantID, err)
	}
	result.ActiveAlertPersisted = true

	facts := portal.NewTenantDirWorkspaceSetupFactReader(rt.cfg.TenantsDir()).FactsForWorkspace(tenantID)
	result.ReportScheduleCount = providerMSPProofFactInt(facts.ReportScheduleCount)
	result.DisabledReportScheduleCount = providerMSPProofFactInt(facts.DisabledReportScheduleCount)
	result.CriticalAlertCount = providerMSPProofFactInt(facts.ActiveCriticalAlertCount)
	result.WarningAlertCount = providerMSPProofFactInt(facts.ActiveWarningAlertCount)
	result.ReportScheduleVisible = result.ReportScheduleCount == 1 && result.DisabledReportScheduleCount == 0
	result.ActiveAlertRollupVisible = result.CriticalAlertCount == 1 && result.WarningAlertCount == 1 && facts.ActiveAlertsUpdatedAt != nil
	if !result.ReportScheduleVisible {
		return result, fmt.Errorf("portal setup facts for tenant %s report schedules = enabled:%d disabled:%d, want enabled:1 disabled:0", tenantID, result.ReportScheduleCount, result.DisabledReportScheduleCount)
	}
	if !result.ActiveAlertRollupVisible {
		return result, fmt.Errorf("portal setup facts for tenant %s active alerts = critical:%d warning:%d updated:%t, want critical:1 warning:1 updated:true", tenantID, result.CriticalAlertCount, result.WarningAlertCount, facts.ActiveAlertsUpdatedAt != nil)
	}
	return result, nil
}

func (rt *providerMSPProofRuntime) createProviderMSPProofReportSchedule(ctx context.Context, tenant *registry.Tenant, tenantDataDir, resourceID, resourceName string) (string, error) {
	if tenant == nil {
		return "", fmt.Errorf("tenant is required")
	}
	tenantID := strings.TrimSpace(tenant.ID)
	if tenantID == "" {
		return "", fmt.Errorf("tenant id is required")
	}
	if strings.TrimSpace(resourceName) == "" {
		resourceName = strings.TrimSpace(resourceID)
	}
	tenantCfg := &runtimeconfig.Config{
		DataPath:   tenantDataDir,
		ConfigPath: tenantDataDir,
		PublicURL:  rt.providerMSPProofTenantPublicURL(tenantID),
	}
	tenantPersistence := runtimeconfig.NewMultiTenantPersistence(tenantDataDir)
	if !tenantPersistence.OrgExists(tenantID) {
		return "", fmt.Errorf("tenant %s organization metadata is missing before report schedule proof", tenantID)
	}
	tenantMonitor := monitoring.NewMultiTenantMonitor(tenantCfg, tenantPersistence, nil)
	defer tenantMonitor.Stop()

	handler := api.NewReportingHandlers(tenantMonitor, nil)
	payload := runtimeconfig.ReportSchedule{
		Name:    "Provider MSP proof monthly report",
		Enabled: true,
		Cadence: runtimeconfig.ReportScheduleCadence{
			Type:       runtimeconfig.ReportScheduleCadenceMonthly,
			DayOfMonth: 1,
			Time:       "09:00",
			Timezone:   "UTC",
		},
		Scope: runtimeconfig.ReportScheduleScope{
			Resources: []runtimeconfig.ReportScheduleResource{
				{
					ResourceType: "agent",
					ResourceID:   strings.TrimSpace(resourceID),
					Name:         strings.TrimSpace(resourceName),
				},
			},
		},
		Format: runtimeconfig.ReportScheduleFormatPDF,
		Delivery: runtimeconfig.ReportScheduleDelivery{
			Method:     runtimeconfig.ReportScheduleDeliveryDisk,
			Attach:     true,
			SaveToDisk: true,
		},
	}
	body, err := json.Marshal(payload)
	if err != nil {
		return "", fmt.Errorf("marshal provider MSP proof report schedule: %w", err)
	}
	req := httptest.NewRequest(http.MethodPost, "/api/reports/schedules", bytes.NewReader(body))
	req = req.WithContext(context.WithValue(ctx, api.OrgIDContextKey, tenantID))
	req.Header.Set("Content-Type", "application/json")
	rec := httptest.NewRecorder()
	handler.HandleCreateReportSchedule(rec, req)
	if rec.Code != http.StatusCreated {
		return "", fmt.Errorf("tenant %s report schedule create status=%d body=%s", tenantID, rec.Code, strings.TrimSpace(rec.Body.String()))
	}

	var created runtimeconfig.ReportSchedule
	if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil {
		return "", fmt.Errorf("decode provider MSP proof report schedule: %w", err)
	}
	if strings.TrimSpace(created.ID) == "" {
		return "", fmt.Errorf("tenant %s report schedule create response missing id", tenantID)
	}
	return created.ID, nil
}

func writeProviderMSPProofActiveAlerts(orgDir, resourceID, resourceName string) error {
	if strings.TrimSpace(orgDir) == "" {
		return fmt.Errorf("org dir is required")
	}
	if strings.TrimSpace(resourceName) == "" {
		resourceName = strings.TrimSpace(resourceID)
	}
	now := time.Now().UTC()
	active := []alerts.Alert{
		{
			ID:           "provider-msp-proof-critical",
			Type:         "cpu",
			Level:        alerts.AlertLevelCritical,
			ResourceID:   strings.TrimSpace(resourceID),
			ResourceName: strings.TrimSpace(resourceName),
			Node:         strings.TrimSpace(resourceName),
			Instance:     "provider-msp-proof",
			Message:      "Provider MSP proof critical alert",
			Value:        96,
			Threshold:    80,
			StartTime:    now,
			LastSeen:     now,
		},
		{
			ID:           "provider-msp-proof-warning",
			Type:         "memory",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   strings.TrimSpace(resourceID),
			ResourceName: strings.TrimSpace(resourceName),
			Node:         strings.TrimSpace(resourceName),
			Instance:     "provider-msp-proof",
			Message:      "Provider MSP proof warning alert",
			Value:        88,
			Threshold:    85,
			StartTime:    now,
			LastSeen:     now,
		},
	}
	data, err := json.Marshal(active)
	if err != nil {
		return fmt.Errorf("marshal active alerts: %w", err)
	}
	alertsDir := filepath.Join(orgDir, "alerts")
	if err := os.MkdirAll(alertsDir, 0o700); err != nil {
		return fmt.Errorf("create active alerts dir: %w", err)
	}
	path := filepath.Join(alertsDir, "active-alerts.json")
	if err := os.WriteFile(path, data, 0o600); err != nil {
		return fmt.Errorf("write active alerts file: %w", err)
	}
	return nil
}

func providerMSPProofFactInt(value *int) int {
	if value == nil {
		return 0
	}
	return *value
}

type providerMSPProofEntitlementLease struct {
	Checked       bool
	Verified      bool
	WhiteLabel    bool
	SkippedReason string
}

// verifyProviderMSPProofEntitlementLease verifies the provisioned workspace's
// hosted entitlement lease exactly the way a release-build client runtime
// will: against the embedded entitlement trust root, through the provider
// MSP license chain. The proof previously passed while every workspace ran
// unlicensed; a present license with an unverifiable lease must fail here,
// not in the client's first branded report.
func (rt *providerMSPProofRuntime) verifyProviderMSPProofEntitlementLease(tenant *registry.Tenant, tenantDataDir string) (providerMSPProofEntitlementLease, error) {
	result := providerMSPProofEntitlementLease{}
	if tenant == nil {
		return result, fmt.Errorf("tenant is required")
	}
	if !rt.cfg.IsProviderHostedMSP() || strings.TrimSpace(rt.cfg.ProviderMSPLicenseKey) == "" {
		result.SkippedReason = "no_provider_msp_license"
		return result, nil
	}
	result.Checked = true

	state, err := runtimeconfig.NewFileBillingStore(tenantDataDir).GetBillingState("default")
	if err != nil {
		return result, fmt.Errorf("read provisioned billing state for workspace %s: %w", tenant.ID, err)
	}
	if state == nil || strings.TrimSpace(state.EntitlementJWT) == "" {
		return result, fmt.Errorf("workspace %s was provisioned without a hosted entitlement lease", tenant.ID)
	}
	if strings.TrimSpace(state.EntitlementRefreshToken) == "" {
		return result, fmt.Errorf("workspace %s was provisioned without an entitlement refresh token", tenant.ID)
	}

	root, err := pkglicensing.HostedEntitlementPublicKey()
	if err != nil {
		return result, fmt.Errorf("resolve the entitlement verification root a client runtime would use: %w", err)
	}
	claims, err := pkglicensing.VerifyEntitlementLeaseToken(state.EntitlementJWT, root, "", time.Now().UTC())
	if err != nil {
		return result, fmt.Errorf("workspace %s entitlement lease does not verify the way a client runtime would: %w", tenant.ID, err)
	}
	if strings.TrimSpace(claims.ProviderLicense) == "" {
		return result, fmt.Errorf("workspace %s entitlement lease does not chain the provider MSP license; release-build client runtimes would reject it", tenant.ID)
	}
	result.Verified = true
	for _, capability := range claims.Capabilities {
		if capability == pkglicensing.FeatureWhiteLabel {
			result.WhiteLabel = true
			break
		}
	}
	if !result.WhiteLabel {
		return result, fmt.Errorf("workspace %s entitlement lease is missing white_label; branded client reports would be locked", tenant.ID)
	}
	return result, nil
}

type providerMSPProofAgentReportIngest struct {
	Verified bool
	AgentID  string
	Hostname string
}

type providerMSPProofInstallTokenRotation struct {
	Verified                   bool
	RotatedToken               string
	RotatedTokenID             string
	OldTokenRejected           bool
	RotatedAgentReportVerified bool
}

func (rt *providerMSPProofRuntime) verifyProviderMSPProofAgentReportIngest(ctx context.Context, tenant *registry.Tenant, tenantDataDir, rawToken, tokenID string) (providerMSPProofAgentReportIngest, error) {
	if tenant == nil {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant is required")
	}
	tenantID := strings.TrimSpace(tenant.ID)
	if tenantID == "" {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant id is required")
	}

	tokens, err := configPersistenceForProviderMSPProof(tenantDataDir).LoadAPITokens()
	if err != nil {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("load tenant api tokens before report ingest: %w", err)
	}
	tenantCfg := &runtimeconfig.Config{
		DataPath:   tenantDataDir,
		ConfigPath: tenantDataDir,
		PublicURL:  rt.providerMSPProofTenantPublicURL(tenantID),
		APITokens:  tokens,
	}
	tenantPersistence := runtimeconfig.NewMultiTenantPersistence(tenantDataDir)
	if !tenantPersistence.OrgExists(tenantID) {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s organization metadata is missing from runtime data dir", tenantID)
	}

	tenantMonitor := monitoring.NewMultiTenantMonitor(tenantCfg, tenantPersistence, nil)
	defer tenantMonitor.Stop()

	handler := api.NewUnifiedAgentHandlers(tenantMonitor, nil, nil)
	report := providerMSPProofAgentReport()
	body, err := json.Marshal(report)
	if err != nil {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("marshal provider MSP proof agent report: %w", err)
	}

	req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(body))
	req = req.WithContext(context.WithValue(ctx, api.OrgIDContextKey, tenantID))
	req.Header.Set("Authorization", "Bearer "+rawToken)
	rec := httptest.NewRecorder()
	api.RequireAuth(tenantCfg, api.RequireScope(runtimeconfig.ScopeAgentReport, handler.HandleReport))(rec, req)
	if rec.Code != http.StatusOK {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s agent report status=%d body=%s", tenantID, rec.Code, strings.TrimSpace(rec.Body.String()))
	}

	var payload struct {
		AgentID string `json:"agentId"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("decode provider MSP proof agent report response: %w", err)
	}
	if strings.TrimSpace(payload.AgentID) == "" {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s agent report response omitted agent id", tenantID)
	}

	monitor, ok := tenantMonitor.PeekMonitor(tenantID)
	if !ok || monitor == nil {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s monitor was not initialized by agent report", tenantID)
	}
	hosts := monitor.GetLiveHostsSnapshot()
	if len(hosts) != 1 {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s host count after agent report = %d, want 1", tenantID, len(hosts))
	}
	host := hosts[0]
	if host.ID != payload.AgentID {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s ingested host id=%q, response agent id=%q", tenantID, host.ID, payload.AgentID)
	}
	if host.Hostname != "pve1" {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s ingested hostname=%q, want pve1", tenantID, host.Hostname)
	}
	if strings.TrimSpace(tokenID) != "" && strings.TrimSpace(host.TokenID) != strings.TrimSpace(tokenID) {
		return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s ingested token id=%q, want %q", tenantID, host.TokenID, tokenID)
	}

	if defaultMonitor, ok := tenantMonitor.PeekMonitor("default"); ok && defaultMonitor != nil {
		if defaultHosts := defaultMonitor.GetLiveHostsSnapshot(); len(defaultHosts) != 0 {
			return providerMSPProofAgentReportIngest{}, fmt.Errorf("tenant %s report leaked into default runtime: %#v", tenantID, defaultHosts)
		}
	}

	return providerMSPProofAgentReportIngest{
		Verified: true,
		AgentID:  payload.AgentID,
		Hostname: host.Hostname,
	}, nil
}

func providerMSPProofAgentReport() agentshost.Report {
	return agentshost.Report{
		Agent: agentshost.AgentInfo{
			ID:              "provider-msp-proof-agent",
			Version:         "6.0.0-provider-msp-proof",
			Type:            "unified",
			IntervalSeconds: 30,
		},
		Host: agentshost.HostInfo{
			ID:            "provider-msp-proof-machine",
			MachineID:     "provider-msp-proof-machine",
			Hostname:      "pve1",
			DisplayName:   "pve1",
			Platform:      "linux",
			OSName:        "Debian",
			OSVersion:     "12",
			Architecture:  "amd64",
			CPUCount:      4,
			UptimeSeconds: 3600,
			KernelVersion: "6.1.0-provider-proof",
		},
		Metrics: agentshost.Metrics{
			CPUUsagePercent: 12.5,
			Memory: agentshost.MemoryMetric{
				TotalBytes: 8 * 1024 * 1024 * 1024,
				UsedBytes:  2 * 1024 * 1024 * 1024,
				FreeBytes:  6 * 1024 * 1024 * 1024,
				Usage:      25,
			},
		},
		Timestamp:  time.Now().UTC(),
		SequenceID: "provider-msp-proof-report",
	}
}

func configPersistenceForProviderMSPProof(tenantDataDir string) *runtimeconfig.ConfigPersistence {
	return runtimeconfig.NewConfigPersistence(tenantDataDir)
}

func (rt *providerMSPProofRuntime) verifyProviderMSPProofInstallTokenRotation(ctx context.Context, tenant *registry.Tenant, tenantDataDir, oldRawToken, oldTokenID string) (providerMSPProofInstallTokenRotation, error) {
	if tenant == nil {
		return providerMSPProofInstallTokenRotation{}, fmt.Errorf("tenant is required")
	}
	tenantID := strings.TrimSpace(tenant.ID)
	if tenantID == "" {
		return providerMSPProofInstallTokenRotation{}, fmt.Errorf("tenant id is required")
	}

	newRawToken, newTokenID, err := rotateProviderMSPProofInstallToken(tenantDataDir, tenantID, oldRawToken, oldTokenID)
	if err != nil {
		return providerMSPProofInstallTokenRotation{}, err
	}
	oldRejected, err := rt.verifyProviderMSPProofAgentReportRejected(ctx, tenant, tenantDataDir, oldRawToken)
	if err != nil {
		return providerMSPProofInstallTokenRotation{}, err
	}
	if !oldRejected {
		return providerMSPProofInstallTokenRotation{}, fmt.Errorf("tenant %s accepted rotated-out install token %s", tenantID, oldTokenID)
	}

	rotatedReport, err := rt.verifyProviderMSPProofAgentReportIngest(ctx, tenant, tenantDataDir, newRawToken, newTokenID)
	if err != nil {
		return providerMSPProofInstallTokenRotation{}, fmt.Errorf("verify rotated install token report ingest: %w", err)
	}

	return providerMSPProofInstallTokenRotation{
		Verified:                   oldRejected && rotatedReport.Verified,
		RotatedToken:               newRawToken,
		RotatedTokenID:             newTokenID,
		OldTokenRejected:           oldRejected,
		RotatedAgentReportVerified: rotatedReport.Verified,
	}, nil
}

func rotateProviderMSPProofInstallToken(tenantDataDir, tenantID, oldRawToken, oldTokenID string) (string, string, error) {
	persistence := configPersistenceForProviderMSPProof(tenantDataDir)
	tokens, err := persistence.LoadAPITokens()
	if err != nil {
		return "", "", fmt.Errorf("load tenant api tokens before rotation: %w", err)
	}
	cfg := &runtimeconfig.Config{APITokens: tokens}
	oldRecord, ok := cfg.ValidateAPIToken(oldRawToken)
	if !ok || oldRecord == nil {
		return "", "", fmt.Errorf("tenant %s old install token did not validate before rotation", tenantID)
	}
	if strings.TrimSpace(oldRecord.ID) != strings.TrimSpace(oldTokenID) {
		return "", "", fmt.Errorf("tenant %s old install token id=%q, want %q", tenantID, oldRecord.ID, oldTokenID)
	}
	if strings.TrimSpace(oldRecord.OrgID) != tenantID {
		return "", "", fmt.Errorf("tenant %s old install token org id=%q", tenantID, oldRecord.OrgID)
	}

	newRawToken, err := internalauth.GenerateAPIToken()
	if err != nil {
		return "", "", fmt.Errorf("generate rotated install token: %w", err)
	}
	newRecord, err := runtimeconfig.NewAPITokenRecord(newRawToken, oldRecord.Name, oldRecord.Scopes)
	if err != nil {
		return "", "", fmt.Errorf("create rotated install token record: %w", err)
	}
	newRecord.OrgID = oldRecord.OrgID
	newRecord.OrgIDs = append([]string{}, oldRecord.OrgIDs...)
	newRecord.Metadata = cloneProviderMSPProofTokenMetadata(oldRecord.Metadata)
	if oldRecord.ExpiresAt != nil {
		expiresAt := *oldRecord.ExpiresAt
		newRecord.ExpiresAt = &expiresAt
	}

	nextTokens := make([]runtimeconfig.APITokenRecord, 0, len(cfg.APITokens))
	removedOld := false
	for _, token := range cfg.APITokens {
		if strings.TrimSpace(token.ID) == strings.TrimSpace(oldTokenID) {
			removedOld = true
			continue
		}
		nextTokens = append(nextTokens, token)
	}
	if !removedOld {
		return "", "", fmt.Errorf("tenant %s old install token id %s disappeared before rotation", tenantID, oldTokenID)
	}
	nextTokens = append(nextTokens, *newRecord)
	rotatedCfg := &runtimeconfig.Config{APITokens: nextTokens}
	rotatedCfg.SortAPITokens()
	if err := persistence.SaveAPITokens(rotatedCfg.APITokens); err != nil {
		return "", "", fmt.Errorf("persist rotated tenant install token: %w", err)
	}

	loaded, err := persistence.LoadAPITokens()
	if err != nil {
		return "", "", fmt.Errorf("reload rotated tenant api tokens: %w", err)
	}
	loadedCfg := &runtimeconfig.Config{APITokens: loaded}
	if _, ok := loadedCfg.ValidateAPIToken(oldRawToken); ok {
		return "", "", fmt.Errorf("tenant %s old install token still validates after rotation", tenantID)
	}
	replacement, ok := loadedCfg.ValidateAPIToken(newRawToken)
	if !ok || replacement == nil {
		return "", "", fmt.Errorf("tenant %s rotated install token does not validate", tenantID)
	}
	if strings.TrimSpace(replacement.OrgID) != tenantID {
		return "", "", fmt.Errorf("tenant %s rotated install token org id=%q", tenantID, replacement.OrgID)
	}
	if replacement.Metadata["issued_via"] != "hosted_agent_install_command" {
		return "", "", fmt.Errorf("tenant %s rotated install token lost install metadata", tenantID)
	}
	if err := persistence.SaveAPITokens(loadedCfg.APITokens); err != nil {
		return "", "", fmt.Errorf("persist rotated tenant install token use: %w", err)
	}
	return newRawToken, replacement.ID, nil
}

func cloneProviderMSPProofTokenMetadata(metadata map[string]string) map[string]string {
	if len(metadata) == 0 {
		return nil
	}
	clone := make(map[string]string, len(metadata))
	for key, value := range metadata {
		clone[key] = value
	}
	return clone
}

func (rt *providerMSPProofRuntime) verifyProviderMSPProofAgentReportRejected(ctx context.Context, tenant *registry.Tenant, tenantDataDir, rawToken string) (bool, error) {
	if tenant == nil {
		return false, fmt.Errorf("tenant is required")
	}
	tenantID := strings.TrimSpace(tenant.ID)
	tokens, err := configPersistenceForProviderMSPProof(tenantDataDir).LoadAPITokens()
	if err != nil {
		return false, fmt.Errorf("load tenant api tokens before old-token rejection proof: %w", err)
	}
	tenantCfg := &runtimeconfig.Config{
		DataPath:   tenantDataDir,
		ConfigPath: tenantDataDir,
		PublicURL:  rt.providerMSPProofTenantPublicURL(tenantID),
		APITokens:  tokens,
	}
	tenantPersistence := runtimeconfig.NewMultiTenantPersistence(tenantDataDir)
	if !tenantPersistence.OrgExists(tenantID) {
		return false, fmt.Errorf("tenant %s organization metadata is missing from runtime data dir", tenantID)
	}
	tenantMonitor := monitoring.NewMultiTenantMonitor(tenantCfg, tenantPersistence, nil)
	defer tenantMonitor.Stop()

	handler := api.NewUnifiedAgentHandlers(tenantMonitor, nil, nil)
	body, err := json.Marshal(providerMSPProofAgentReport())
	if err != nil {
		return false, fmt.Errorf("marshal provider MSP proof rejected agent report: %w", err)
	}
	req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(body))
	req = req.WithContext(context.WithValue(ctx, api.OrgIDContextKey, tenantID))
	req.Header.Set("Authorization", "Bearer "+rawToken)
	rec := httptest.NewRecorder()
	api.RequireAuth(tenantCfg, api.RequireScope(runtimeconfig.ScopeAgentReport, handler.HandleReport))(rec, req)
	if rec.Code == http.StatusOK {
		return false, nil
	}
	if rec.Code != http.StatusUnauthorized {
		return false, fmt.Errorf("tenant %s rotated-out token report status=%d body=%s, want %d", tenantID, rec.Code, strings.TrimSpace(rec.Body.String()), http.StatusUnauthorized)
	}
	return true, nil
}

func markProviderMSPProofAgentTokenUsed(tenantDataDir, tenantID, rawToken string) (bool, error) {
	persistence := configPersistenceForProviderMSPProof(tenantDataDir)
	tokens, err := persistence.LoadAPITokens()
	if err != nil {
		return false, fmt.Errorf("load tenant api tokens: %w", err)
	}
	cfg := &runtimeconfig.Config{APITokens: tokens}
	record, ok := cfg.ValidateAPIToken(rawToken)
	if !ok || record == nil {
		return false, fmt.Errorf("generated install token did not validate in tenant %s", tenantID)
	}
	if strings.TrimSpace(record.OrgID) != tenantID {
		return false, fmt.Errorf("install token org id = %q, want %q", record.OrgID, tenantID)
	}
	if err := persistence.SaveAPITokens(cfg.APITokens); err != nil {
		return false, fmt.Errorf("persist used tenant api token: %w", err)
	}
	return true, nil
}

func (rt *providerMSPProofRuntime) verifyProviderMSPProofHandoff(ctx context.Context, tenant *registry.Tenant, ownerUserID, targetPath string) (string, error) {
	baseDomain := providerMSPBaseDomainFromURL(rt.cfg.BaseURL)
	if baseDomain == "" {
		return "", fmt.Errorf("could not derive provider base domain from %q", rt.cfg.BaseURL)
	}

	mux := http.NewServeMux()
	mux.Handle("/api/accounts/{account_id}/tenants/{tenant_id}/handoff", cphandoff.HandleHandoff(rt.registry, rt.cfg.TenantsDir()))
	handoffURL := "https://" + baseDomain + "/api/accounts/" + url.PathEscape(tenant.AccountID) + "/tenants/" + url.PathEscape(tenant.ID) + "/handoff?target_path=" + url.QueryEscape(targetPath)
	req := httptest.NewRequest(http.MethodPost, handoffURL, nil)
	req = req.WithContext(ctx)
	req.RemoteAddr = "127.0.0.1:12345"
	req.Host = baseDomain
	req.Header.Set("X-User-ID", ownerUserID)
	rec := httptest.NewRecorder()
	mux.ServeHTTP(rec, req)
	if rec.Code != http.StatusOK {
		return "", fmt.Errorf("handoff status=%d body=%s", rec.Code, strings.TrimSpace(rec.Body.String()))
	}
	matches := handoffTokenInputPattern.FindStringSubmatch(rec.Body.String())
	if len(matches) != 2 || strings.TrimSpace(matches[1]) == "" {
		return "", fmt.Errorf("handoff response did not include a token input")
	}

	tenantDataDir, err := safeProviderMSPProofTenantDataDir(rt.cfg.TenantsDir(), tenant.ID)
	if err != nil {
		return "", err
	}
	form := url.Values{"token": []string{matches[1]}}
	exchangeURL := rt.providerMSPProofTenantPublicURL(tenant.ID) + "/api/cloud/handoff/exchange?format=json"
	exchangeReq := httptest.NewRequest(http.MethodPost, exchangeURL, strings.NewReader(form.Encode()))
	exchangeReq = exchangeReq.WithContext(ctx)
	exchangeReq.RemoteAddr = "127.0.0.1:12345"
	exchangeReq.Header.Set("Accept", "application/json")
	exchangeReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	exchangeRec := httptest.NewRecorder()
	api.HandleHandoffExchange(tenantDataDir).ServeHTTP(exchangeRec, exchangeReq)
	if exchangeRec.Code != http.StatusOK {
		return "", fmt.Errorf("handoff exchange status=%d body=%s", exchangeRec.Code, strings.TrimSpace(exchangeRec.Body.String()))
	}

	var payload struct {
		TenantID   string `json:"tenant_id"`
		UserID     string `json:"user_id"`
		TargetPath string `json:"target_path"`
	}
	if err := json.Unmarshal(exchangeRec.Body.Bytes(), &payload); err != nil {
		return "", fmt.Errorf("decode handoff exchange response: %w", err)
	}
	if payload.TenantID != tenant.ID {
		return "", fmt.Errorf("handoff exchange tenant_id=%q, want %q", payload.TenantID, tenant.ID)
	}
	if payload.UserID != ownerUserID {
		return "", fmt.Errorf("handoff exchange user_id=%q, want %q", payload.UserID, ownerUserID)
	}
	return payload.TargetPath, nil
}

func (rt *providerMSPProofRuntime) verifyProviderMSPInstallTokenIsolation(workspaces []providerMSPProofWorkspace) (bool, error) {
	for _, workspace := range workspaces {
		activeTokenID := strings.TrimSpace(workspace.RotatedInstallTokenID)
		if activeTokenID == "" {
			activeTokenID = strings.TrimSpace(workspace.InstallTokenID)
		}
		tenantDataDir, err := safeProviderMSPProofTenantDataDir(rt.cfg.TenantsDir(), workspace.TenantID)
		if err != nil {
			return false, err
		}
		tokens, err := configPersistenceForProviderMSPProof(tenantDataDir).LoadAPITokens()
		if err != nil {
			return false, fmt.Errorf("load tenant %s api tokens: %w", workspace.TenantID, err)
		}
		found := false
		for _, token := range tokens {
			if strings.TrimSpace(token.ID) == activeTokenID {
				found = true
				break
			}
		}
		if !found {
			return false, fmt.Errorf("tenant %s missing active proof install token id %s", workspace.TenantID, activeTokenID)
		}
	}
	for _, left := range workspaces {
		leftTokenID := strings.TrimSpace(left.RotatedInstallTokenID)
		if leftTokenID == "" {
			leftTokenID = strings.TrimSpace(left.InstallTokenID)
		}
		leftRawToken := strings.TrimSpace(left.RotatedInstallToken)
		if leftRawToken == "" {
			leftRawToken = strings.TrimSpace(left.InstallToken)
		}
		for _, right := range workspaces {
			if left.TenantID == right.TenantID {
				continue
			}
			rightTokenID := strings.TrimSpace(right.RotatedInstallTokenID)
			if rightTokenID == "" {
				rightTokenID = strings.TrimSpace(right.InstallTokenID)
			}
			if leftTokenID == rightTokenID {
				return false, fmt.Errorf("tenant %s and %s reused install token id %s", left.TenantID, right.TenantID, leftTokenID)
			}
			rightTenantDataDir, err := safeProviderMSPProofTenantDataDir(rt.cfg.TenantsDir(), right.TenantID)
			if err != nil {
				return false, err
			}
			accepted, err := providerMSPProofTenantAcceptsToken(rightTenantDataDir, leftRawToken)
			if err != nil {
				return false, err
			}
			if accepted {
				return false, fmt.Errorf("tenant %s accepted install token minted for tenant %s", right.TenantID, left.TenantID)
			}
		}
	}
	return true, nil
}

func providerMSPProofTenantAcceptsToken(tenantDataDir, rawToken string) (bool, error) {
	tokens, err := configPersistenceForProviderMSPProof(tenantDataDir).LoadAPITokens()
	if err != nil {
		return false, fmt.Errorf("load tenant api tokens: %w", err)
	}
	cfg := &runtimeconfig.Config{APITokens: tokens}
	_, ok := cfg.ValidateAPIToken(rawToken)
	return ok, nil
}

func (rt *providerMSPProofRuntime) cleanupProviderMSPProofTenants(ctx context.Context, tenants []*registry.Tenant) error {
	var failures []string
	for i := len(tenants) - 1; i >= 0; i-- {
		tenant := tenants[i]
		if tenant == nil {
			continue
		}
		if err := rt.provisioner.DeprovisionWorkspaceContainer(ctx, tenant); err != nil {
			failures = append(failures, fmt.Sprintf("%s deprovision: %v", tenant.ID, err))
		}
		tenantDataDir, err := safeProviderMSPProofTenantDataDir(rt.cfg.TenantsDir(), tenant.ID)
		if err != nil {
			failures = append(failures, fmt.Sprintf("%s dir: %v", tenant.ID, err))
			continue
		}
		if err := os.RemoveAll(tenantDataDir); err != nil {
			failures = append(failures, fmt.Sprintf("%s remove data: %v", tenant.ID, err))
		}
		if err := rt.registry.Delete(tenant.ID); err != nil {
			failures = append(failures, fmt.Sprintf("%s delete registry: %v", tenant.ID, err))
		}
	}
	if len(failures) > 0 {
		return fmt.Errorf("cleanup failures: %s", strings.Join(failures, "; "))
	}
	return nil
}

func safeProviderMSPProofTenantDataDir(tenantsDir, tenantID string) (string, error) {
	tenantsDir = strings.TrimSpace(tenantsDir)
	tenantID = strings.TrimSpace(tenantID)
	if tenantsDir == "" {
		return "", fmt.Errorf("tenants dir is required")
	}
	if tenantID == "" || strings.ContainsAny(tenantID, `/\`) {
		return "", fmt.Errorf("unsafe tenant id %q", tenantID)
	}
	cleanTenantsDir, err := filepath.Abs(tenantsDir)
	if err != nil {
		return "", fmt.Errorf("resolve tenants dir: %w", err)
	}
	candidate := filepath.Join(cleanTenantsDir, tenantID)
	if !strings.HasPrefix(candidate, cleanTenantsDir+string(os.PathSeparator)) {
		return "", fmt.Errorf("tenant data dir escaped tenants dir")
	}
	return candidate, nil
}

func providerMSPProofPlanVersion(cfg *cloudcp.CPConfig) string {
	if cfg == nil || strings.TrimSpace(cfg.ProviderMSPPlanVersion) == "" {
		return "msp_starter"
	}
	return strings.TrimSpace(cfg.ProviderMSPPlanVersion)
}

func providerMSPBaseDomainFromURL(baseURL string) string {
	domain := strings.TrimSpace(baseURL)
	for _, prefix := range []string{"https://", "http://"} {
		if strings.HasPrefix(domain, prefix) {
			domain = strings.TrimPrefix(domain, prefix)
			break
		}
	}
	if idx := strings.IndexAny(domain, ":/"); idx >= 0 {
		domain = domain[:idx]
	}
	return domain
}

func (rt *providerMSPProofRuntime) providerMSPProofTenantPublicURL(tenantID string) string {
	baseDomain := providerMSPBaseDomainFromURL(rt.cfg.BaseURL)
	if baseDomain == "" {
		return ""
	}
	return "https://" + strings.TrimSpace(tenantID) + "." + baseDomain
}

func printProviderMSPProofReport(report *providerMSPProofReport) {
	if report == nil {
		fmt.Println("provider_msp_control_plane_proof_ok=false")
		return
	}
	fmt.Println("provider_msp_control_plane_proof_ok=true")
	fmt.Printf("account_id=%s\n", report.AccountID)
	fmt.Printf("account_name=%s\n", report.AccountName)
	fmt.Printf("owner_user_id=%s\n", report.OwnerUserID)
	fmt.Printf("owner_email=%s\n", report.OwnerEmail)
	fmt.Printf("plan_version=%s\n", report.PlanVersion)
	fmt.Printf("plan_source=%s\n", report.PlanSource)
	fmt.Printf("license_id=%s\n", report.LicenseID)
	fmt.Printf("license_email=%s\n", report.LicenseEmail)
	fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit)
	fmt.Printf("workspace_count=%d\n", report.WorkspaceCount)
	fmt.Printf("dockerless_provisioning=%t\n", report.DockerlessProvisioning)
	fmt.Printf("runtime_container_verified=%t\n", report.RuntimeContainerVerified)
	fmt.Printf("handoff_exchange_verified=%t\n", report.HandoffExchangeVerified)
	fmt.Printf("install_token_boundary_verified=%t\n", report.InstallTokenBoundaryOK)
	fmt.Printf("setup_facts_token_use_visible=%t\n", report.SetupFactsTokenUseVisible)
	fmt.Printf("agent_report_ingest_verified=%t\n", report.AgentReportIngestVerified)
	fmt.Printf("token_rotation_verified=%t\n", report.TokenRotationVerified)
	fmt.Printf("report_schedule_visible=%t\n", report.ReportScheduleVisible)
	fmt.Printf("active_alert_rollup_visible=%t\n", report.ActiveAlertRollupVisible)
	fmt.Printf("cleanup=%t\n", report.Cleanup)
	for _, workspace := range report.Workspaces {
		fmt.Printf("workspace=%s display_name=%q state=%s plan_version=%s container_id=%s public_url=%s install_type=%s install_token_id=%s install_command_generated=%t agent_token_auth_verified=%t setup_facts_token_use_visible=%t agent_report_ingest_verified=%t agent_report_agent_id=%s agent_report_hostname=%s token_rotation_verified=%t rotated_install_token_id=%s old_install_token_rejected=%t rotated_agent_report_verified=%t handoff_exchange_verified=%t handoff_target_path=%s entitlement_lease_checked=%t entitlement_lease_verified=%t entitlement_white_label=%t entitlement_skipped_reason=%s report_schedule_created=%t report_schedule_id=%s report_schedule_visible=%t report_schedule_count=%d disabled_report_schedule_count=%d active_alert_persisted=%t active_alert_rollup_visible=%t critical_alert_count=%d warning_alert_count=%d\n",
			workspace.TenantID,
			workspace.DisplayName,
			workspace.State,
			workspace.PlanVersion,
			workspace.ContainerID,
			workspace.PublicURL,
			workspace.InstallType,
			workspace.InstallTokenID,
			workspace.InstallCommandGenerated,
			workspace.AgentTokenAuthVerified,
			workspace.SetupFactsTokenUseVisible,
			workspace.AgentReportIngestVerified,
			workspace.AgentReportAgentID,
			workspace.AgentReportHostname,
			workspace.TokenRotationVerified,
			workspace.RotatedInstallTokenID,
			workspace.OldInstallTokenRejected,
			workspace.RotatedAgentReportVerified,
			workspace.HandoffExchangeVerified,
			workspace.HandoffTargetPath,
			workspace.EntitlementLeaseChecked,
			workspace.EntitlementLeaseVerified,
			workspace.EntitlementWhiteLabel,
			workspace.EntitlementSkippedReason,
			workspace.ReportScheduleCreated,
			workspace.ReportScheduleID,
			workspace.ReportScheduleVisible,
			workspace.ReportScheduleCount,
			workspace.DisabledReportScheduleCount,
			workspace.ActiveAlertPersisted,
			workspace.ActiveAlertRollupVisible,
			workspace.CriticalAlertCount,
			workspace.WarningAlertCount,
		)
	}
}