Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Repair unlicensed provider-MSP production startup
pulse-triage[bot]bf40374241d8
diff --git a/.github/workflows/provider-msp-startup.yml b/.github/workflows/provider-msp-startup.ymlnew file mode 100644index 0000000000..3de7097251--- /dev/null+++ b/.github/workflows/provider-msp-startup.yml@@ -0,0 +1,81 @@+name: Provider MSP startup++on:+ push:+ branches: [main]+ paths:+ - 'internal/cloudcp/**'+ - 'cmd/pulse-control-plane/**'+ - 'pkg/licensing/**'+ - 'deploy/provider-msp/**'+ - 'scripts/installtests/provider_msp*'+ - 'frontend-modern/**'+ - 'go.mod'+ - 'go.sum'+ - '.github/workflows/provider-msp-startup.yml'+ pull_request:+ branches: [main]+ paths:+ - 'internal/cloudcp/**'+ - 'cmd/pulse-control-plane/**'+ - 'pkg/licensing/**'+ - 'deploy/provider-msp/**'+ - 'scripts/installtests/provider_msp*'+ - 'frontend-modern/**'+ - 'go.mod'+ - 'go.sum'+ - '.github/workflows/provider-msp-startup.yml'++permissions:+ contents: read++concurrency:+ group: provider-msp-startup-${{ github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++jobs:+ startup:+ name: Provider MSP production startup+ runs-on: ubuntu-24.04+ timeout-minutes: 20+ steps:+ - name: Checkout exact source+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ github.sha }}+ persist-credentials: false++ - name: Verify source identity+ env:+ EXPECTED_SOURCE_SHA: ${{ github.sha }}+ run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"++ - name: Set up Go+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0+ with:+ go-version-file: go.mod+ cache: true++ - name: Set up Node+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version: '24'+ cache: npm+ cache-dependency-path: frontend-modern/package-lock.json++ - name: Build real embedded frontend+ working-directory: frontend-modern+ run: npm ci --no-audit && npm run build++ # The test generates an ephemeral issuer, embeds only its public key in+ # a release daemon and signs synthetic MSP fixtures. No project secrets,+ # customer keys, provider account, edge ingress or publication is used.+ - name: Prove mounted-secret startup, persistence and refusal+ env:+ PULSE_TEST_PROVIDER_MSP_STARTUP: compose+ run: |+ set -euo pipefail+ go test -count=1 -timeout 12m ./scripts/installtests \+ -run '^TestProviderMSPProductionStartup$' -v \+ | tee "$RUNNER_TEMP/provider-msp-startup.log"+ grep -q '^--- PASS: TestProviderMSPProductionStartup ' "$RUNNER_TEMP/provider-msp-startup.log"diff --git a/deploy/provider-msp/docker-compose.yml b/deploy/provider-msp/docker-compose.ymlindex f35718eaf5..6f2d9ba70f 100644--- a/deploy/provider-msp/docker-compose.yml+++ b/deploy/provider-msp/docker-compose.yml@@ -68,7 +68,8 @@ services:- DOCKER_HOST=tcp://docker-socket-proxy:2375- CP_DOCKER_NETWORK=${PULSE_PROVIDER_MSP_DOCKER_NETWORK:-pulse-provider-msp}- CP_TRUSTED_PROXY_CIDRS=${CP_TRUSTED_PROXY_CIDRS}- - CP_PROVIDER_MSP_LICENSE_FILE=/run/secrets/provider_msp_license+ # A blank host path selects evaluation, not an explicitly empty licence.+ - CP_PROVIDER_MSP_LICENSE_FILE=${CP_PROVIDER_MSP_LICENSE_FILE:+/run/secrets/provider_msp_license}- CP_TENANT_MEMORY_LIMIT=${CP_TENANT_MEMORY_LIMIT:-536870912}- CP_ALLOW_DOCKERLESS_PROVISIONING=${CP_ALLOW_DOCKERLESS_PROVISIONING:-false}- CP_STORAGE_GUARDRAILS_ENABLED=${CP_STORAGE_GUARDRAILS_ENABLED:-true}@@ -106,7 +107,8 @@ services:secrets:provider_msp_license:- file: ${CP_PROVIDER_MSP_LICENSE_FILE:?CP_PROVIDER_MSP_LICENSE_FILE must point to the signed provider MSP license file}+ # Compose requires a secret source even when the daemon will not read it.+ file: ${CP_PROVIDER_MSP_LICENSE_FILE:-/dev/null}networks:pulse-provider-msp:diff --git a/deploy/provider-msp/setup.sh b/deploy/provider-msp/setup.shindex ef91f0ad4f..d4b0c4a6a9 100755--- a/deploy/provider-msp/setup.sh+++ b/deploy/provider-msp/setup.sh@@ -704,15 +704,8 @@ the license must bind that key or the control plane will refuse to start."validate_compose_config() {log "validating compose config"- local license_file- license_file="$(env_value CP_PROVIDER_MSP_LICENSE_FILE "${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env")"- if [[ "${1:-}" == "--allow-missing-evaluation-license" && -z "${license_file}" ]]; then- # Compose requires a non-empty secret source even for a config-only- # parse. Use a non-secret placeholder for this pre-issuance validation;- # the normal validation after ensure_eval_license uses the real file.- (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet)- return 0- fi+ # The same config supports both an installed licence and unlicensed+ # evaluation, before and after a best-effort evaluation request.(cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && docker compose config --quiet)}@@ -843,7 +836,7 @@ main() {create_data_dirsensure_docker_networkblock_container_metadata_service- validate_compose_config --allow-missing-evaluation-license+ validate_compose_configpull_provider_images# Issue the evaluation only after the host is configured and the immutable# images are reachable. This makes an issued evaluation a useful activationdiff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.mdindex c2288a78b5..676bdc2302 100644--- a/docs/release-control/v6/internal/subsystems/cloud-paid.md+++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md@@ -150,6 +150,31 @@ contract: an agent outside the monitored site performs the check and reportsits result to Pulse, while Pulse remains the alerting authority. Commercialcopy must not imply that the external agent independently sends alerts.+### Provider-hosted production evaluation and renewal startup++A provider-hosted MSP control plane may start in production without a host+licence only on `msp_eval` (two client workspaces). Unsigned environment settings+cannot select paid limits. Pulse-hosted MSP production still requires its licence+path. Explicit empty, truncated, oversized or invalid licence files are not an+invitation to evaluate. Existing valid-renewal precedence, signature/MSP-tier and+lease-key binding checks, lapse behaviour, Stripe exclusions and tenant isolation+are unchanged.++A provider that first ran unlicensed must read its persisted signed renewal on+restart even when the original host secret remains unused. An invalid persisted+renewal with no host licence refuses startup rather than silently dropping to+evaluation. This is the existing refresh/startup path, not a new purchase or+entitlement surface.++`TestLoadConfig_ProviderHostedMSPUnlicensedProduction`, the explicit-invalid and+renewal controls, plus `TestProviderMSPProductionStartup` cover these boundaries.+The opt-in daemon test builds the actual release command with an ephemeral test+issuer and checks private status, plan/limit readback, account/owner persistence,+readiness across restarts and graceful stop. Hosted Compose execution additionally+checks the real `/dev/null` placeholder and signed-file mounts, refused startup,+and owned container/network/volume cleanup. Process-only VM execution does not+establish Compose mounts, tenant provisioning or installed provider recovery.+## Canonical Files1. `pkg/licensing/features.go`diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex d4748a2b8c..5b0d05577b 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -1481,6 +1481,29 @@ dedicated worker when combined; concurrent execution can stretch otherwisepassing monitoring tests into load-induced failures and must not be used as arelease-latency optimization.+### Provider-MSP unlicensed startup uses one Compose configuration++The provider bundle leaves `CP_PROVIDER_MSP_LICENSE_FILE` blank when evaluation+issuance is deliberately skipped or unavailable. Both pre-issuance and final+setup validation use the same shipped Compose file: a blank host path mounts+`/dev/null` as the required unused secret source and leaves the daemon licence+path unset; a supplied path maps to `/run/secrets/provider_msp_license` and must+validate. Setup no longer substitutes an explicit empty licence path only for+its first configuration parse. Image pins, Docker proxy, storage guardrails,+least-privilege mounts and all Stripe/Relay exclusions are unchanged.++The `Provider MSP startup` PR/main workflow uses exact-source credential-free+checkout and the declared Go/Node graphs on a disposable hosted Linux VM. It+builds real API assets and the release daemon using only synthetic test licences,+then runs the opt-in `TestProviderMSPProductionStartup` Compose transport. Missing+Docker/Compose fails; a skipped test cannot satisfy the job. The test checks actual+mounted secrets, licensed/unlicensed readiness and authenticated plan limits,+account/owner persistence across clean restarts, explicit licence/configuration+refusals, and owned container/network/volume teardown. It does not use an edge+router, customer data, production Docker socket, licence service or release+publisher. The offline process transport verifies the daemon but is not proof of+the native Compose mount or a full provider installation/tenant lifecycle.+## Canonical Files1. `internal/updates/`diff --git a/internal/cloudcp/config.go b/internal/cloudcp/config.goindex 0fce5e538e..975af40054 100644--- a/internal/cloudcp/config.go+++ b/internal/cloudcp/config.go@@ -222,24 +222,29 @@ func LoadConfig() (*CPConfig, error) {providerMSPLicenseKey := ""providerMSPLeaseSigningPublicKey := ""providerMSPLicenseExpiresAt := time.Time{}- if isMSPControlPlaneMode(controlPlaneMode) && providerMSPLicenseFile != "" {+ // A provider that started without a host licence can later persist a paid+ // renewal. Read that same signed source on restart even while the original+ // Compose secret remains the unused /dev/null placeholder.+ if isMSPControlPlaneMode(controlPlaneMode) && (providerMSPLicenseFile != "" || controlPlaneMode == ControlPlaneModeProviderHostedMSP) {resolved, source, err := resolveProviderMSPLicense(providerMSPLicenseFile, dataDir)if err != nil {return nil, fmt.Errorf("resolve provider MSP license: %w", err)}- providerMSPPlanVersion = resolved.PlanVersion- providerMSPPlanSource = source- providerMSPLicenseExpiresAt = resolved.ExpiresAt- providerMSPLicenseID = resolved.LicenseID- providerMSPLicenseEmail = resolved.LicenseEmail- providerMSPLicenseKey = resolved.LicenseKey- providerMSPLeaseSigningPublicKey = resolved.LeaseSigningPublicKey- if resolved.lapsed(time.Now()) {- log.Warn().- Str("license_id", resolved.LicenseID).- Str("plan_version", resolved.PlanVersion).- Time("expired_at", resolved.ExpiresAt).- Msg("Provider MSP license has lapsed: the portal stays up to sell the renewal, no new clients can be added, and client workspaces drop MSP capabilities")+ if resolved != nil {+ providerMSPPlanVersion = resolved.PlanVersion+ providerMSPPlanSource = source+ providerMSPLicenseExpiresAt = resolved.ExpiresAt+ providerMSPLicenseID = resolved.LicenseID+ providerMSPLicenseEmail = resolved.LicenseEmail+ providerMSPLicenseKey = resolved.LicenseKey+ providerMSPLeaseSigningPublicKey = resolved.LeaseSigningPublicKey+ if resolved.lapsed(time.Now()) {+ log.Warn().+ Str("license_id", resolved.LicenseID).+ Str("plan_version", resolved.PlanVersion).+ Time("expired_at", resolved.ExpiresAt).+ Msg("Provider MSP license has lapsed: the portal stays up to sell the renewal, no new clients can be added, and client workspaces drop MSP capabilities")+ }}}@@ -336,7 +341,7 @@ func LoadConfig() (*CPConfig, error) {func (c *CPConfig) validateProviderMSPLeaseSigningBinding() error {if c == nil || !c.IsProviderHostedMSP() || strings.TrimSpace(c.ProviderMSPLicenseKey) == "" {// Pulse-hosted modes sign with Pulse's own key, and the env-fallback- // plan path (dev/install-proof without a license file) has nothing+ // plan path (unlicensed evaluation or dev/install-proof) has nothing// to bind against.return nil}@@ -455,7 +460,10 @@ func (c *CPConfig) validate() error {if strings.TrimSpace(c.TrialActivationPrivateKey) == "" {return fmt.Errorf("CP_ENTITLEMENT_SIGNING_PRIVATE_KEY is required when CP_CONTROL_PLANE_MODE=%s", c.ControlPlaneMode)}- if c.Environment == "production" && strings.TrimSpace(c.ProviderMSPLicenseFile) == "" {+ // Provider-operated portals must remain reachable when setup deliberately+ // skips evaluation issuance or cannot reach the licence server. This is+ // not an unlicensed production route for Pulse-hosted MSP planes.+ if c.Environment == "production" && !c.IsProviderHostedMSP() && strings.TrimSpace(c.ProviderMSPLicenseFile) == "" {return fmt.Errorf("CP_PROVIDER_MSP_LICENSE_FILE is required in production when CP_CONTROL_PLANE_MODE=%s", c.ControlPlaneMode)}if !strings.HasPrefix(strings.ToLower(c.ProviderMSPPlanVersion), "msp_") {@@ -464,6 +472,9 @@ func (c *CPConfig) validate() error {if _, known := pkglicensing.WorkspaceLimitForPlan(c.ProviderMSPPlanVersion); !known {return fmt.Errorf("CP_PROVIDER_MSP_PLAN_VERSION must have a known workspace limit, got %q", c.ProviderMSPPlanVersion)}+ if c.Environment == "production" && c.IsProviderHostedMSP() && !ProviderMSPPlanSourceIsSignedLicense(c.ProviderMSPPlanSource) && c.ProviderMSPPlanVersion != defaultProviderHostedMSPPlanVersion {+ return fmt.Errorf("CP_PROVIDER_MSP_PLAN_VERSION must be %s without a signed provider MSP license in production", defaultProviderHostedMSPPlanVersion)+ }}if c.StorageGuardrailsEnabled {if strings.TrimSpace(c.StorageRootPath) == "" {@@ -689,7 +700,7 @@ func resolveProviderMSPLicenseFile(path string, allowLapsed bool) (*providerMSPL// ProviderMSPPlanSourceIsSignedLicense reports whether the plan came from a// Pulse-signed licence, either the host licence file or a renewed licence,-// rather than the development-only environment fallback.+// rather than unlicensed evaluation or the development environment fallback.func ProviderMSPPlanSourceIsSignedLicense(source string) bool {switch strings.TrimSpace(source) {case ProviderMSPPlanSourceLicenseFile, ProviderMSPPlanSourceRenewedLicense:@@ -738,20 +749,29 @@ func (c *CPConfig) ProviderMSPLicenseLapsed(now time.Time) bool {// candidates must be authentic Pulse licences, and validate still ties the// chosen one to this control plane's lease signing key.func resolveProviderMSPLicense(hostFile, dataDir string) (*providerMSPLicenseResolution, string, error) {+ hostFile = strings.TrimSpace(hostFile)now := time.Now()var renewed *providerMSPLicenseResolutionrenewedPath := ProviderMSPRenewedLicensePath(dataDir)if _, err := os.Stat(renewedPath); err == nil {candidate, err := resolveProviderMSPLicenseFile(renewedPath, true)if err != nil {+ if hostFile == "" {+ return nil, "", fmt.Errorf("read renewed provider MSP license: %w", err)+ }log.Warn().Err(err).Str("path", renewedPath).Msg("Renewed provider MSP license is unusable; falling back to CP_PROVIDER_MSP_LICENSE_FILE")} else {renewed = candidate}+ } else if hostFile == "" && !os.IsNotExist(err) {+ return nil, "", fmt.Errorf("stat renewed provider MSP license: %w", err)}- if renewed != nil && !renewed.lapsed(now) {+ if renewed != nil && (!renewed.lapsed(now) || hostFile == "") {return renewed, ProviderMSPPlanSourceRenewedLicense, nil}+ if hostFile == "" {+ return nil, ProviderMSPPlanSourceEnvFallback, nil+ }host, err := resolveProviderMSPLicenseFile(hostFile, true)if err != nil {if renewed != nil {diff --git a/internal/cloudcp/config_test.go b/internal/cloudcp/config_test.goindex a68f8a5ef9..35f8bf2624 100644--- a/internal/cloudcp/config_test.go+++ b/internal/cloudcp/config_test.go@@ -5,6 +5,7 @@ import ("crypto/rand""encoding/base64""encoding/json"+ "fmt""os""path/filepath""strings"@@ -458,6 +459,9 @@ func setProviderHostedMSPEnv(t *testing.T) {t.Setenv("CP_BASE_URL", "https://msp.example.com")t.Setenv("CP_ENV", "development")t.Setenv("CP_CONTROL_PLANE_MODE", "provider_hosted_msp")+ t.Setenv("CP_DATA_DIR", t.TempDir())+ t.Setenv("CP_PROVIDER_MSP_LICENSE_FILE", "")+ t.Setenv("CP_PROVIDER_MSP_PLAN_VERSION", "")t.Setenv("CP_REQUIRE_EMAIL_PROVIDER", "false")t.Setenv("STRIPE_WEBHOOK_SECRET", "")t.Setenv("STRIPE_API_KEY", "")@@ -794,9 +798,92 @@ func TestLoadConfig_PulseHostedMSPDoesNotRequireLeaseSigningBinding(t *testing.T}}-func TestLoadConfig_ProviderHostedMSPRequiresLicenseFileInProduction(t *testing.T) {+// Both deliberate skip and failed best-effort issuance leave the path blank.+// Production may evaluate, but local configuration must not grant paid caps.+func TestLoadConfig_ProviderHostedMSPUnlicensedProduction(t *testing.T) {+ for _, plan := range []string{"", "msp_eval", "msp_starter", "msp_growth", "msp_scale"} {+ t.Run("plan="+plan, func(t *testing.T) {+ setProviderHostedMSPEnv(t)+ t.Setenv("CP_ENV", "production")+ t.Setenv("CP_PROVIDER_MSP_PLAN_VERSION", plan)+ cfg, err := LoadConfig()+ if plan != "" && plan != "msp_eval" {+ if err == nil || !strings.Contains(err.Error(), "must be msp_eval without a signed provider MSP license") {+ t.Fatalf("unsigned production plan %q: cfg=%v err=%v", plan, cfg != nil, err)+ }+ return+ }+ if err != nil {+ t.Fatal(err)+ }+ limit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)+ if cfg.ProviderMSPPlanVersion != pkglicensing.PlanVersionMSPEval || cfg.ProviderMSPPlanSource != ProviderMSPPlanSourceEnvFallback || !known || limit != 2 || cfg.ProviderMSPLicenseKey != "" {+ t.Fatalf("unlicensed plan=%q source=%q limit=%d known=%v signed=%v", cfg.ProviderMSPPlanVersion, cfg.ProviderMSPPlanSource, limit, known, cfg.ProviderMSPLicenseKey != "")+ }+ })+ }+}++func TestLoadConfig_ProviderHostedMSPExplicitInvalidLicenseIsNotEvaluation(t *testing.T) {+ for _, tc := range []struct{ name, content, want string }{+ {"empty", "", "is empty"},+ {"whitespace", " \n", "is empty"},+ {"truncated", "eyJhbGciOiJFZERTQSJ9.", "validate CP_PROVIDER_MSP_LICENSE_FILE"},+ {"oversized", strings.Repeat("x", maxProviderMSPLicenseFileBytes+1), "exceeds"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ setProviderHostedMSPEnv(t)+ t.Setenv("CP_ENV", "production")+ path := filepath.Join(t.TempDir(), "explicit.jwt")+ if err := os.WriteFile(path, []byte(tc.content), 0o600); err != nil {+ t.Fatal(err)+ }+ t.Setenv("CP_PROVIDER_MSP_LICENSE_FILE", path)+ if _, err := LoadConfig(); err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("explicit %s licence must refuse, got %v", tc.name, err)+ }+ })+ }+}++// A refresh persists a licence and restarts the daemon without changing the+// operator's read-only Compose secret. This must also work after blank-path+// evaluation, including keeping the portal up on an authentic lapsed licence.+func TestLoadConfig_ProviderHostedMSPRenewalWithoutHostLicense(t *testing.T) {+ for _, lapsed := range []bool{false, true} {+ t.Run(fmt.Sprintf("lapsed=%t", lapsed), func(t *testing.T) {+ setProviderHostedMSPEnv(t)+ t.Setenv("CP_ENV", "production")+ issuer := newProviderMSPTestIssuer(t)+ expires := time.Now().Add(24 * time.Hour)+ if lapsed {+ expires = time.Now().Add(-60 * 24 * time.Hour)+ }+ writeProviderMSPTestFile(t, ProviderMSPRenewedLicensePath(os.Getenv("CP_DATA_DIR")),+ issuer.sign(t, "lic_msp_renewed", "msp_growth", expires, trialSigningEnvPublicKey(t)))+ cfg, err := LoadConfig()+ if err != nil {+ t.Fatal(err)+ }+ if cfg.ProviderMSPPlanVersion != "msp_growth" || cfg.ProviderMSPPlanSource != ProviderMSPPlanSourceRenewedLicense || cfg.ProviderMSPLicenseLapsed(time.Now()) != lapsed {+ t.Fatalf("renewed plan=%q source=%q lapsed=%v", cfg.ProviderMSPPlanVersion, cfg.ProviderMSPPlanSource, cfg.ProviderMSPLicenseLapsed(time.Now()))+ }+ })+ }+}++func TestLoadConfig_ProviderHostedMSPInvalidRenewalWithoutHostLicenseRefuses(t *testing.T) {setProviderHostedMSPEnv(t)t.Setenv("CP_ENV", "production")+ writeProviderMSPTestFile(t, ProviderMSPRenewedLicensePath(os.Getenv("CP_DATA_DIR")), "truncated")+ if _, err := LoadConfig(); err == nil || !strings.Contains(err.Error(), "read renewed provider MSP license") {+ t.Fatalf("unusable persisted renewal must not silently become evaluation: %v", err)+ }+}++func TestLoadConfig_PulseHostedMSPRequiresLicenseFileInProduction(t *testing.T) {+ setPulseHostedMSPEnv(t)+ t.Setenv("CP_ENV", "production")_, err := LoadConfig()if err == nil {diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex 6a3c248530..32ddf1349e 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -5407,3 +5407,82 @@ func TestDockerModuleDownloadKeepsPinnedGraphAndBoundedRecovery(t *testing.T) {t.Fatalf("missing actual-shell module-download proof: %v", err)}}++func TestProviderMSPStartupWorkflowRequiresMountedDaemonAcceptance(t *testing.T) {+ content, err := os.ReadFile(repoFile(".github", "workflows", "provider-msp-startup.yml"))+ if err != nil {+ t.Fatal(err)+ }+ var workflow struct {+ Permissions map[string]string `yaml:"permissions"`+ On map[string]struct {+ Branches []string `yaml:"branches"`+ Paths []string `yaml:"paths"`+ } `yaml:"on"`+ Jobs map[string]struct {+ RunsOn string `yaml:"runs-on"`+ Timeout int `yaml:"timeout-minutes"`+ Steps []struct {+ Name string `yaml:"name"`+ Uses string `yaml:"uses"`+ Run string `yaml:"run"`+ Dir string `yaml:"working-directory"`+ Env map[string]string `yaml:"env"`+ With map[string]string `yaml:"with"`+ Tolerant any `yaml:"continue-on-error"`+ Conditional any `yaml:"if"`+ } `yaml:"steps"`+ } `yaml:"jobs"`+ }+ if err := yaml.Unmarshal(content, &workflow); err != nil {+ t.Fatal(err)+ }+ if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" || len(workflow.Jobs) != 1 || len(workflow.On) != 2 {+ t.Fatal("startup CI must remain one read-only PR/main job")+ }+ for _, event := range []string{"push", "pull_request"} {+ trigger, ok := workflow.On[event]+ if !ok || len(trigger.Branches) != 1 || trigger.Branches[0] != "main" {+ t.Fatalf("missing main admission for %s", event)+ }+ for _, path := range []string{"internal/cloudcp/**", "cmd/pulse-control-plane/**", "deploy/provider-msp/**", "pkg/licensing/**", "scripts/installtests/provider_msp*", "frontend-modern/**", "go.mod", "go.sum", ".github/workflows/provider-msp-startup.yml"} {+ found := false+ for _, actual := range trigger.Paths {+ found = found || path == actual+ }+ if !found {+ t.Fatalf("%s cannot admit changes to %s", event, path)+ }+ }+ }+ job := workflow.Jobs["startup"]+ if job.RunsOn != "ubuntu-24.04" || job.Timeout <= 0 || job.Timeout > 20 {+ t.Fatal("startup CI must use a bounded disposable hosted runner")+ }+ built, accepted, exact := false, false, false+ for _, step := range job.Steps {+ if step.Tolerant != nil || step.Conditional != nil {+ t.Fatalf("startup proof step must not be optional/tolerant: %s", step.Name)+ }+ if strings.HasPrefix(step.Uses, "actions/checkout@") {+ if step.With["persist-credentials"] != "false" || step.With["ref"] != "${{ github.sha }}" {+ t.Fatal("checkout must be credential-free and exact-source")+ }+ }+ if step.Env["EXPECTED_SOURCE_SHA"] == "${{ github.sha }}" && strings.Contains(step.Run, `test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"`) {+ exact = true+ }+ if step.Dir == "frontend-modern" && strings.Contains(step.Run, "npm ci --no-audit && npm run build") {+ built = true+ }+ if step.Env["PULSE_TEST_PROVIDER_MSP_STARTUP"] == "compose" {+ if !built || !exact || !strings.Contains(step.Run, "set -euo pipefail") || !strings.Contains(step.Run, "go test -count=1 -timeout 12m ./scripts/installtests") || !strings.Contains(step.Run, "-run '^TestProviderMSPProductionStartup$' -v") || !strings.Contains(step.Run, "grep -q '^--- PASS: TestProviderMSPProductionStartup '") {+ t.Fatal("native acceptance must require real embeds, exact source, Compose execution and an actual PASS (not skip)")+ }+ accepted = true+ }+ }+ if !accepted || strings.Contains(string(content), "secrets.") || strings.Contains(string(content), "pull_request_target") {+ t.Fatal("missing mounted-daemon acceptance or unsafe credential/event scope")+ }+}diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.goindex 54f0659c93..89cf6e57a2 100644--- a/scripts/installtests/provider_msp_deploy_test.go+++ b/scripts/installtests/provider_msp_deploy_test.go@@ -78,7 +78,7 @@ func TestProviderMSPDeployComposeIsProviderModeAndStripeFree(t *testing.T) {)assertContainsAll(t, text,"CP_DATA_DIR=${PULSE_PROVIDER_MSP_DATA_DIR:-/data}",- "CP_PROVIDER_MSP_LICENSE_FILE=/run/secrets/provider_msp_license",+ "CP_PROVIDER_MSP_LICENSE_FILE=${CP_PROVIDER_MSP_LICENSE_FILE:+/run/secrets/provider_msp_license}","CP_DOCKER_NETWORK=${PULSE_PROVIDER_MSP_DOCKER_NETWORK:-pulse-provider-msp}","CP_TRUSTED_PROXY_CIDRS=${CP_TRUSTED_PROXY_CIDRS}","DOCKER_HOST=tcp://docker-socket-proxy:2375",@@ -411,8 +411,7 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {`--format '{{json .Manifest}}'`,`jq -r 'if type == "object" then .digest // empty else empty end'`,`awk '$1 == "Digest:" {print $2; exit}'`,- "validate_compose_config --allow-missing-evaluation-license",- `CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet`,+ "validate_compose_config\n pull_provider_images",`docker pull "${image_ref}"`,`if [[ "${current}" == *@sha256:*`,`ref="${current}"`,@@ -434,7 +433,7 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {if strings.LastIndex(script, "pull_provider_images\n") > strings.LastIndex(script, "ensure_eval_license\n") {t.Fatal("evaluation must be issued only after pinned provider images are reachable")}- setupSequence := "validate_compose_config --allow-missing-evaluation-license\n pull_provider_images\n # Issue the evaluation only after the host is configured and the immutable\n # images are reachable. This makes an issued evaluation a useful activation\n # signal rather than a record created before setup can succeed.\n ensure_eval_license\n validate_compose_config"+ setupSequence := "validate_compose_config\n pull_provider_images\n # Issue the evaluation only after the host is configured and the immutable\n # images are reachable. This makes an issued evaluation a useful activation\n # signal rather than a record created before setup can succeed.\n ensure_eval_license\n validate_compose_config"if !strings.Contains(script, setupSequence) {t.Fatal("setup must validate compose, pull pinned images, issue the evaluation, then validate compose with the installed licence")}@@ -829,3 +828,56 @@ func TestProviderMSPUpgradeFromBundleRepinsToTheBundleRelease(t *testing.T) {}})}++func TestProviderMSPEvaluationFallbackUsesTheSameFinalComposeConfig(t *testing.T) {+ scriptBytes, err := os.ReadFile(repoFile("deploy", "provider-msp", "setup.sh"))+ if err != nil {+ t.Fatal(err)+ }+ script := string(scriptBytes)+ const entry = "\nmain \"$@\"\n"+ if !strings.HasSuffix(script, entry) {+ t.Fatal("cannot safely remove the installer entrypoint from the shell fixture")+ }+ for _, skip := range []string{"1", "0"} {+ t.Run("skip="+skip, func(t *testing.T) {+ dir := t.TempDir()+ env := "CP_PROVIDER_MSP_LICENSE_FILE=\n"+ if err := os.WriteFile(filepath.Join(dir, ".env"), []byte(env), 0o600); err != nil {+ t.Fatal(err)+ }+ // No installer, package manager, Docker daemon or licence service.+ // Only the actual best-effort and validation functions execute.+ fixture := strings.TrimSuffix(script, entry) + `++derive_lease_signing_public_key() { printf 'synthetic-public-key'; }+jq() { printf '{}'; }+curl() { return 7; }+docker() {+ [[ "$*" == 'compose config --quiet' ]] || exit 81+ [[ ! -v CP_PROVIDER_MSP_LICENSE_FILE ]] || exit 82+ printf 'same-compose-config\n'+}+validate_compose_config+ensure_eval_license+validate_compose_config+`+ cmd := exec.Command("bash", "-c", fixture)+ cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "PULSE_PROVIDER_MSP_INSTALL_DIR=" + dir, "PULSE_PROVIDER_MSP_SKIP_EVAL_LICENSE=" + skip}+ output, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("best-effort evaluation must reach final validation: %v\n%s", err, output)+ }+ if strings.Count(string(output), "same-compose-config\n") != 2 {+ t.Fatalf("pre/final validation did not both use the same config: %s", output)+ }+ if skip == "0" && !strings.Contains(string(output), "could not reach the license server") {+ t.Fatal("fixture did not exercise unavailable licence issuance")+ }+ after, err := os.ReadFile(filepath.Join(dir, ".env"))+ if err != nil || string(after) != env {+ t.Fatal("fallback changed the blank operator licence selection")+ }+ })+ }+}diff --git a/scripts/installtests/provider_msp_startup_test.go b/scripts/installtests/provider_msp_startup_test.gonew file mode 100644index 0000000000..47ece0c5c5--- /dev/null+++ b/scripts/installtests/provider_msp_startup_test.go@@ -0,0 +1,386 @@+package installtests++import (+ "bytes"+ "context"+ "crypto/ed25519"+ "crypto/rand"+ "crypto/sha256"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "runtime"+ "strconv"+ "strings"+ "syscall"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"+ pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"+)++// Explicitly opt in: ordinary unit runs neither build another daemon nor use+// Docker. The hosted job requires Compose, not a skip or a parser-only pass.+// "process" also permits the same real release daemon in an offline proof VM;+// it does NOT establish Docker secret-mount or installed provider acceptance.+func TestProviderMSPProductionStartup(t *testing.T) {+ transport := os.Getenv("PULSE_TEST_PROVIDER_MSP_STARTUP")+ if transport == "" {+ t.Skip("set PULSE_TEST_PROVIDER_MSP_STARTUP=process or compose for real daemon acceptance")+ }+ if runtime.GOOS != "linux" || (transport != "process" && transport != "compose") {+ t.Fatalf("unsupported startup acceptance transport/platform: %q/%s", transport, runtime.GOOS)+ }+ root := repoFile()+ if assets, err := filepath.Glob(filepath.Join(root, "internal/api/frontend-modern/dist/assets/*.js")); err != nil || len(assets) == 0 {+ t.Fatal("startup acceptance requires the real frontend build, not the unit-test embed stub")+ }+ issuerPublic, issuerPrivate, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ leasePublic, leasePrivate, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ binaryDir := t.TempDir()+ binary := filepath.Join(binaryDir, "binaries", "pulse-control-plane-linux-amd64")+ if err := os.MkdirAll(filepath.Dir(binary), 0o700); err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)+ defer cancel()+ build := exec.CommandContext(ctx, "go", "build", "-tags=release", "-trimpath", "-buildvcs=false",+ "-ldflags=-X main.Version=msp-startup-integration -X github.com/rcourtman/pulse-go-rewrite/pkg/licensing.EmbeddedPublicKey="+base64.StdEncoding.EncodeToString(issuerPublic),+ "-o", binary, "./cmd/pulse-control-plane")+ build.Dir = root+ build.Env = append(os.Environ(), "CGO_ENABLED=0")+ if output, err := build.CombinedOutput(); err != nil {+ t.Fatalf("build exact-source release daemon: %v\n%s", err, output)+ }+ payload, err := os.ReadFile(binary)+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("release_daemon_sha256=%x transport=%s", sha256.Sum256(payload), transport)++ dockerEnv := []string{"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "DOCKER_CONFIG=" + t.TempDir()}+ nonce := fmt.Sprintf("%x", sha256.Sum256(issuerPublic))[:16]+ image := "pulse-msp-startup-test:" + nonce+ if transport == "compose" {+ startupCommand(t, dockerEnv, root, "docker", "compose", "version")+ startupCommand(t, dockerEnv, root, "docker", "version")+ startupCommand(t, dockerEnv, root, "docker", "build", "--build-context", "compiled_payload="+binaryDir,+ "--target", "control_plane_prebuilt", "-f", "deploy/provider-msp/Dockerfile.control-plane", "-t", image, ".")+ t.Cleanup(func() { startupCommand(t, dockerEnv, root, "docker", "image", "rm", image) })+ }+ claims := pkglicensing.Claims{+ LicenseID: "lic_synthetic_startup", Email: "[email protected]", Tier: pkglicensing.TierMSP,+ PlanVersion: "msp_growth", IssuedAt: time.Now().Add(-time.Minute).Unix(), ExpiresAt: time.Now().Add(24 * time.Hour).Unix(),+ EntitlementSigningPublicKey: base64.StdEncoding.EncodeToString(leasePublic),+ }+ payloadJSON, err := json.Marshal(claims)+ if err != nil {+ t.Fatal(err)+ }+ signingInput := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"EdDSA","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(payloadJSON)+ license := signingInput + "." + base64.RawURLEncoding.EncodeToString(ed25519.Sign(issuerPrivate, []byte(signingInput)))++ for _, kind := range []string{"unlicensed", "licensed", "renewed", "empty", "truncated", "unsigned_paid", "pulse_hosted"} {+ t.Run(kind, func(t *testing.T) {+ dir := t.TempDir()+ dataDir := filepath.Join(dir, "data")+ if err := os.MkdirAll(filepath.Join(dataDir, "control-plane"), 0o700); err != nil {+ t.Fatal(err)+ }+ licensePath := ""+ plan, source, limit := "msp_eval", "environment_fallback", 2+ mode, configuredPlan, refusal := "provider_hosted_msp", "", ""+ switch kind {+ case "licensed", "empty", "truncated":+ licensePath = filepath.Join(dir, "license.jwt")+ content := license+ if kind == "empty" {+ content, refusal = "", "CP_PROVIDER_MSP_LICENSE_FILE is empty"+ } else if kind == "truncated" {+ content, refusal = "truncated.jwt", "validate CP_PROVIDER_MSP_LICENSE_FILE"+ } else {+ plan, source, limit = "msp_growth", "license_file", 15+ }+ startupWrite(t, licensePath, content)+ case "renewed":+ startupWrite(t, cloudcp.ProviderMSPRenewedLicensePath(dataDir), license)+ plan, source, limit = "msp_growth", "renewed_license", 15+ case "unsigned_paid":+ configuredPlan, refusal = "msp_scale", "must be msp_eval without a signed provider MSP license"+ case "pulse_hosted":+ mode, refusal = "pulse_hosted_msp", "CP_PROVIDER_MSP_LICENSE_FILE is required in production"+ }+ env := []string{+ "PATH=" + os.Getenv("PATH"), "CP_ENV=production", "CP_DATA_DIR=" + dataDir,+ "CP_CONTROL_PLANE_MODE=" + mode, "CP_ADMIN_KEY=" + strings.Repeat("synthetic-startup-", 3),+ "CP_BASE_URL=https://msp.example.invalid", "CP_REQUIRE_EMAIL_PROVIDER=false",+ "CP_PROVIDER_MSP_LICENSE_FILE=" + licensePath, "CP_PROVIDER_MSP_PLAN_VERSION=" + configuredPlan,+ "CP_ENTITLEMENT_SIGNING_PRIVATE_KEY=" + base64.StdEncoding.EncodeToString(leasePrivate),+ "PULSE_LICENSE_SERVER_URL=http://127.0.0.1:1", "CP_ALLOW_DOCKERLESS_PROVISIONING=false",+ "CP_STORAGE_GUARDRAILS_ENABLED=true", "CP_STORAGE_ROOT_PATH=" + dir, "CP_STORAGE_DATA_PATH=" + dataDir,+ "CP_STORAGE_DOCKER_PATH=" + dir, "CP_STORAGE_MIN_ROOT_AVAILABLE=1MiB",+ "CP_STORAGE_MIN_DATA_AVAILABLE=1MiB", "CP_STORAGE_MIN_DOCKER_AVAILABLE=1MiB",+ }+ var run func(...string) string+ var start func() (string, func())+ if transport == "process" {+ listener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ port := listener.Addr().(*net.TCPAddr).Port+ if err := listener.Close(); err != nil {+ t.Fatal(err)+ }+ env = append(env, "CP_BIND_ADDRESS=127.0.0.1", "CP_PORT="+strconv.Itoa(port), "DOCKER_HOST=tcp://127.0.0.1:1")+ run = func(args ...string) string { return startupCommand(t, env, dir, binary, args...) }+ if refusal != "" {+ startupRefusal(t, env, dir, refusal, binary)+ return+ }+ start = func() (string, func()) {+ return startupProcess(t, binary, dir, env, "http://127.0.0.1:"+strconv.Itoa(port))+ }+ } else {+ composeBytes, err := os.ReadFile(repoFile("deploy", "provider-msp", "docker-compose.yml"))+ if err != nil {+ t.Fatal(err)+ }+ startupWrite(t, filepath.Join(dir, "docker-compose.yml"), string(composeBytes))+ startupWrite(t, filepath.Join(dir, "startup.yml"), "services:\n control-plane:\n ports: ['127.0.0.1::8443']\n restart: 'no'\n")+ env = append(env,+ "DOMAIN=msp.example.invalid", "[email protected]", "CP_TRUSTED_PROXY_CIDRS=172.30.0.0/24",+ "CONTROL_PLANE_IMAGE="+image, "CP_PULSE_IMAGE="+image, "TRAEFIK_IMAGE="+image, "DOCKER_SOCKET_PROXY_IMAGE="+image,+ "PULSE_PROVIDER_MSP_DATA_DIR="+dataDir, "PULSE_PROVIDER_MSP_ROOT_SPACECHECK_DIR="+dir,+ "PULSE_PROVIDER_MSP_DOCKER_SPACECHECK_DIR="+dir, "PULSE_PROVIDER_MSP_DOCKER_NETWORK=msp-startup-"+nonce+"-"+kind,+ )+ startupWrite(t, filepath.Join(dir, ".env"), strings.Join(env, "\n")+"\n")+ project := "msp-startup-" + nonce + "-" + kind+ prefix := []string{"compose", "--project-name", project, "-f", "docker-compose.yml", "-f", "startup.yml"}+ compose := func(args ...string) string {+ return startupCommand(t, dockerEnv, dir, "docker", append(append([]string{}, prefix...), args...)...)+ }+ t.Cleanup(func() {+ compose("down", "--volumes", "--remove-orphans")+ for _, resource := range []string{"container", "network", "volume"} {+ args := []string{resource, "ls", "-q", "--filter", "label=com.docker.compose.project=" + project}+ if resource == "container" {+ args = append(args, "-a")+ }+ if remaining := startupCommand(t, dockerEnv, dir, "docker", args...); strings.TrimSpace(remaining) != "" {+ t.Errorf("owned %s remain after cleanup: %s", resource, remaining)+ }+ }+ })+ run = func(args ...string) string {+ return compose(append([]string{"run", "--rm", "--no-deps", "control-plane"}, args...)...)+ }+ compose("config", "--quiet")+ if refusal != "" {+ startupRefusal(t, dockerEnv, dir, refusal, "docker", append(prefix, "run", "--rm", "--no-deps", "control-plane")...)+ return+ }+ start = func() (string, func()) {+ compose("up", "--no-deps", "-d", "control-plane")+ id := strings.TrimSpace(compose("ps", "-q", "control-plane"))+ if id == "" {+ t.Fatal("Compose did not return the daemon container identity")+ }+ if licensePath == "" {+ // This is the actual /dev/null secret bind mount, not config text.+ startupCommand(t, dockerEnv, dir, "docker", "exec", id, "sh", "-c", "test -c /run/secrets/provider_msp_license && test $(wc -c < /run/secrets/provider_msp_license) -eq 0")+ } else {+ mountHash := startupCommand(t, dockerEnv, dir, "docker", "exec", id, "sha256sum", "/run/secrets/provider_msp_license")+ if !strings.HasPrefix(mountHash, fmt.Sprintf("%x", sha256.Sum256([]byte(license)))) {+ t.Fatal("mounted synthetic licence does not match the host file")+ }+ }+ address := strings.TrimSpace(compose("port", "control-plane", "8443"))+ stopped := false+ return "http://" + address, func() {+ if stopped {+ return+ }+ stopped = true+ compose("stop", "--timeout", "10", "control-plane")+ state := startupCommand(t, dockerEnv, dir, "docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}}", id)+ if strings.TrimSpace(state) != "false 0" {+ t.Errorf("daemon stop must be clean, got %s", state)+ }+ }+ }+ }+ accountID, ownerID := "", ""+ for restart := 0; restart < 2; restart++ {+ address, stop := start()+ // Cleanup is retained even when readiness or readback fails.+ t.Cleanup(stop)+ startupReady(t, address)+ startupStatus(t, address, plan, source, limit)+ args := []string{"provider-msp", "bootstrap", "--account-name=Startup acceptance", "[email protected]", "--magic-link=false"}+ if accountID != "" {+ args = append(args, "--account-id="+accountID)+ }+ facts := startupFacts(run(args...))+ if facts["provider_msp_bootstrap_ok"] != "true" || facts["plan_version"] != plan || facts["plan_source"] != source || facts["workspace_limit"] != strconv.Itoa(limit) || facts["account_id"] == "" || facts["owner_user_id"] == "" {+ t.Fatalf("bootstrap readback differs: %v", facts)+ }+ if restart == 1 && (facts["account_id"] != accountID || facts["owner_user_id"] != ownerID) {+ t.Fatal("account/owner identity did not persist across daemon restart")+ }+ accountID, ownerID = facts["account_id"], facts["owner_user_id"]+ for observation := 0; observation < 3; observation++ {+ startupReady(t, address)+ startupStatus(t, address, plan, source, limit)+ }+ stop()+ }+ })+ }+}++func startupWrite(t *testing.T, path, content string) {+ t.Helper()+ if err := os.WriteFile(path, []byte(content), 0o600); err != nil {+ t.Fatal(err)+ }+}++func startupCommand(t *testing.T, env []string, dir, program string, args ...string) string {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)+ defer cancel()+ cmd := exec.CommandContext(ctx, program, args...)+ cmd.Dir, cmd.Env = dir, env+ output, err := cmd.CombinedOutput()+ if err != nil {+ // Environment and licence contents are never diagnostic output.+ t.Fatalf("%s command failed: %v\n%s", filepath.Base(program), err, output)+ }+ return string(output)+}++func startupRefusal(t *testing.T, env []string, dir, want, program string, args ...string) {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, program, args...)+ cmd.Dir, cmd.Env = dir, env+ output, err := cmd.CombinedOutput()+ if ctx.Err() != nil || err == nil || !strings.Contains(string(output), want) {+ t.Fatalf("expected bounded daemon refusal containing %q, err=%v timeout=%v\n%s", want, err, ctx.Err(), output)+ }+}++func startupProcess(t *testing.T, binary, dir string, env []string, address string) (string, func()) {+ t.Helper()+ cmd := exec.Command(binary)+ cmd.Dir, cmd.Env = dir, env+ var output bytes.Buffer+ cmd.Stdout, cmd.Stderr = &output, &output+ if err := cmd.Start(); err != nil {+ t.Fatal(err)+ }+ done := make(chan error, 1)+ go func() { done <- cmd.Wait() }()+ stopped := false+ return address, func() {+ if stopped {+ return+ }+ stopped = true+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {+ t.Errorf("signal owned daemon: %v", err)+ }+ select {+ case err := <-done:+ if err != nil {+ t.Errorf("daemon exit: %v\n%s", err, output.String())+ }+ case <-time.After(10 * time.Second):+ _ = cmd.Process.Kill()+ <-done+ t.Error("owned daemon did not terminate gracefully within 10 seconds")+ }+ }+}++func startupReady(t *testing.T, address string) {+ t.Helper()+ client := &http.Client{Timeout: time.Second}+ deadline := time.Now().Add(20 * time.Second)+ for time.Now().Before(deadline) {+ resp, err := client.Get(address + "/readyz")+ if err == nil {+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1024))+ resp.Body.Close()+ if readErr == nil && resp.StatusCode == http.StatusOK && string(body) == "ready" {+ return+ }+ }+ time.Sleep(100 * time.Millisecond)+ }+ t.Fatal("daemon did not become persistently ready within 20 seconds")+}++func startupStatus(t *testing.T, address, plan, source string, limit int) {+ t.Helper()+ client := &http.Client{Timeout: time.Second}+ for _, authenticated := range []bool{false, true} {+ req, err := http.NewRequest(http.MethodGet, address+"/status", nil)+ if err != nil {+ t.Fatal(err)+ }+ if authenticated {+ req.Header.Set("Authorization", "Bearer "+strings.Repeat("synthetic-startup-", 3))+ }+ resp, err := client.Do(req)+ if err != nil {+ t.Fatal(err)+ }+ body, err := io.ReadAll(io.LimitReader(resp.Body, 16*1024))+ resp.Body.Close()+ if err != nil {+ t.Fatal(err)+ }+ if !authenticated {+ if resp.StatusCode != http.StatusUnauthorized {+ t.Fatalf("operator status must remain private, got %d", resp.StatusCode)+ }+ continue+ }+ var status struct {+ Version string `json:"version"`+ Mode string `json:"control_plane_mode"`+ Plan string `json:"provider_msp_plan_version"`+ Source string `json:"provider_msp_plan_source"`+ Limit int `json:"provider_msp_workspace_limit"`+ }+ if err := json.Unmarshal(body, &status); err != nil || resp.StatusCode != http.StatusOK || status.Version != "msp-startup-integration" || status.Mode != "provider_hosted_msp" || status.Plan != plan || status.Source != source || status.Limit != limit {+ t.Fatalf("daemon status=%+v HTTP=%d err=%v", status, resp.StatusCode, err)+ }+ }+}++func startupFacts(output string) map[string]string {+ facts := make(map[string]string)+ for _, line := range strings.Split(output, "\n") {+ if key, value, ok := strings.Cut(line, "="); ok {+ facts[key] = value+ }+ }+ return facts+}