Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Distinguish MSP evaluation rejection from transport failure
pulse-triage[bot]65d29ec8f351
diff --git a/deploy/provider-msp/setup.sh b/deploy/provider-msp/setup.shindex ffc6bbad28..062974b447 100755--- a/deploy/provider-msp/setup.sh+++ b/deploy/provider-msp/setup.sh@@ -366,9 +366,9 @@ Set ${key} in ${env_path} by hand, or check this host can reach the registry."# by email put a human round-trip in front of the first screen; this asks the# license server directly with the public half of the key generated above.#-# The private key never leaves this host. Failure is a warning, not a stop: the-# portal, provisioning and client isolation all work regardless, and an-# air-gapped operator can skip it outright.+# The private key never leaves this host. Failure is a warning, not a stop:+# the portal can start unlicensed, but a running portal is not proof that client+# runtimes have MSP capabilities. An air-gapped operator can skip the request.ensure_eval_license() {local env_path="${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env"[[ -f "${env_path}" ]] || die "missing ${env_path}"@@ -386,8 +386,7 @@ ensure_eval_license() {if truthy "${PULSE_PROVIDER_MSP_SKIP_EVAL_LICENSE}"; thenlog "PULSE_PROVIDER_MSP_SKIP_EVAL_LICENSE set: staying unlicensed"- log " the portal and client isolation work, but client workspaces will not"- log " carry MSP capabilities until a license is installed"+ log " client MSP capabilities are not active until a license is installed"return 0fi@@ -409,8 +408,8 @@ ensure_eval_license() {return 0fi- log "requesting a 2-client evaluation license from ${PULSE_PROVIDER_MSP_LICENSE_URL}"- local body response token+ log "requesting a 2-client evaluation license"+ local body response http_status curl_exit token=""body="$(jq -cn \--arg public_key "${public_key}" \--arg email "${PULSE_PROVIDER_MSP_EVAL_EMAIL}" \@@ -418,22 +417,59 @@ ensure_eval_license() {'{entitlement_signing_public_key: $public_key, setup_stage: "images_ready"}+ (if $email == "" then {} else {email: $email} end)+ (if $signup_source == "" then {} else {signup_source: $signup_source} end)')"- response="$(curl -fsS --max-time 20 \- -H 'Content-Type: application/json' \- -d "${body}" \- "${PULSE_PROVIDER_MSP_LICENSE_URL%/}/v1/provider-msp/eval-license" 2>/dev/null || true)"-- if [[ -z "${response}" ]]; then- log "warning: could not reach the license server"- log " continuing unlicensed. The portal and client isolation work, but client"- log " workspaces will not carry MSP capabilities. Re-run setup.sh to retry,"- log " or set CP_PROVIDER_MSP_LICENSE_FILE if you already hold a license."- return 0+ # Keep transport exit, HTTP status and body separate. --fail discards the+ # distinction between a request rejection and a connection failure. The+ # final curl-owned line is metadata, even when the JSON body has newlines.+ # Neither arbitrary response/error text nor the configured URL is logged.+ if response="$(curl -sS --max-time 20 --write-out $'\n%{http_code}' \+ -H 'Content-Type: application/json' \+ -d "${body}" \+ "${PULSE_PROVIDER_MSP_LICENSE_URL%/}/v1/provider-msp/eval-license" 2>/dev/null)"; then+ curl_exit=0+ else+ curl_exit=$?+ fi+ http_status="${response##*$'\n'}"+ response="${response%$'\n'*}"++ if [[ "${curl_exit}" != 0 ]]; then+ log "warning: evaluation license request failed (curl exit ${curl_exit})"+ case "${curl_exit}" in+ 6) log " check this host's DNS resolution before retrying" ;;+ 7|28) log " check this host's connection to the license server before retrying" ;;+ 35|51|60) log " check the license endpoint and certificate trust before retrying; keep TLS verification enabled" ;;+ *) log " check the configured license endpoint and host connectivity before retrying" ;;+ esac+ elif [[ ! "${http_status}" =~ ^[1-5][0-9]{2}$ ]]; then+ log "warning: license server returned an unexpected HTTP status"+ elif [[ "${http_status}" == 429 ]]; then+ log "warning: evaluation license request was rate limited (HTTP 429)"+ log " wait before a deliberate retry"+ elif [[ "${http_status}" == 4* ]]; then+ log "warning: license server rejected the evaluation request (HTTP ${http_status})"+ log " Check the request details before retrying, including the configured endpoint and public signing key."+ elif [[ "${http_status}" == 5* ]]; then+ log "warning: evaluation license service is temporarily unavailable (HTTP ${http_status})"+ log " try again later; no automatic retry was attempted"+ elif [[ "${http_status}" != 2* ]]; then+ log "warning: license server returned an unexpected HTTP status (HTTP ${http_status})"+ log " check the configured license endpoint before retrying"+ else+ # A successful HTTP exchange is not an issued licence. Admit exactly one+ # JSON object and a nonempty string, never a number/object or extra document.+ token="$(printf '%s' "${response}" | jq -ers '+ if length == 1 and (.[0] | type == "object") then+ .[0].license | select(type == "string" and length > 0 and (test("[[:space:][:cntrl:]]") | not))+ else empty end' 2>/dev/null || true)"+ if [[ -z "${token}" ]]; then+ log "warning: license server response contained no valid license"+ fifi-- token="$(printf '%s' "${response}" | jq -r '.license // empty' 2>/dev/null || true)"if [[ -z "${token}" ]]; then- log "warning: license server response contained no license; continuing unlicensed"+ log " continuing unlicensed; client MSP capabilities are not active"+ log " resolve the failure before deliberately re-running setup.sh, or set"+ log " CP_PROVIDER_MSP_LICENSE_FILE to an existing license bound to this host's signing key."+ log " Keep saved clients and signing keys; do not recreate them to retry issuance."return 0fi@@ -442,7 +478,11 @@ ensure_eval_license() {set_env_value CP_PROVIDER_MSP_LICENSE_FILE "./provider-msp-eval-license.jwt" "${env_path}"local expires- expires="$(printf '%s' "${response}" | jq -r '.expires_at // empty' 2>/dev/null || true)"+ expires="$(printf '%s' "${response}" | jq -r '.expires_at | select(type == "string")' 2>/dev/null || true)"+ # Preserve useful expiry information without echoing arbitrary server data.+ if [[ ! "${expires}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$ ]]; then+ expires=""+ filog "evaluation license installed: 2 client workspaces${expires:+, expires ${expires}}"log " Plan in your provider portal shows whether a paid upgrade is available"log " keep within two clients until Plan confirms a higher active limit"@@ -955,10 +995,11 @@ Prove the platform before the first real client:Portal (after bootstrap):https://${domain}/portal-Plan: the evaluation covers two clients. Open Plan in the portal to see whether-a paid upgrade is available. Keep within two clients until Plan confirms a-higher active limit. A paid plan exposes Manage billing for changes or-cancellation.+Plan: check Plan in the portal for this deployment's licence status and whether+a paid upgrade is available. A successfully issued evaluation covers two clients;+a running portal alone does not confirm active client MSP capabilities. Keep+within two clients until Plan confirms a higher active limit. A paid plan exposes+Manage billing for changes or cancellation.Day 2: portal sessions last 7 days. Re-run the bootstrap command above anytime to print a fresh owner sign-in link, or usediff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.goindex 04533e0c9a..f6b5b7bc35 100644--- a/scripts/installtests/provider_msp_deploy_test.go+++ b/scripts/installtests/provider_msp_deploy_test.go@@ -471,7 +471,7 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {"/v1/provider-msp/eval-license","PULSE_PROVIDER_MSP_SKIP_EVAL_LICENSE","reusing existing evaluation license",- "could not reach the license server",+ "license server rejected the evaluation request (HTTP ${http_status})","PULSE_PROVIDER_MSP_EVAL_EMAIL","PULSE_PROVIDER_MSP_SIGNUP_SOURCE",`setup_stage: "images_ready"`,@@ -1041,7 +1041,7 @@ validate_compose_configif strings.Count(string(output), "same-compose-config\n") != 2 {t.Fatalf("pre/final validation did not both use the same config: %s", output)}- if skip == "0" && !strings.Contains(string(output), "could not reach the license server") {+ if skip == "0" && !strings.Contains(string(output), "evaluation license request failed (curl exit 7)") {t.Fatal("fixture did not exercise unavailable licence issuance")}after, err := os.ReadFile(filepath.Join(dir, ".env"))diff --git a/scripts/tests/test_provider_msp_eval_license.py b/scripts/tests/test_provider_msp_eval_license.pynew file mode 100644index 0000000000..2e49f3594c--- /dev/null+++ b/scripts/tests/test_provider_msp_eval_license.py@@ -0,0 +1,252 @@+#!/usr/bin/env python3+"""Exercise best-effort evaluation setup with inert licence transport.++The real setup functions, JSON parser and private-file writes run; installer+main, network, Docker, live licensing and client runtimes do not. The normal+smoke inventory discovers this suite.+"""++import json+import os+from pathlib import Path+import stat+import subprocess+import tempfile+import unittest+++ROOT = Path(__file__).resolve().parents[2]+SETUP = Path(os.environ.get("PULSE_MSP_SETUP_UNDER_TEST", ROOT / "deploy/provider-msp/setup.sh"))+ENTRY = '\nmain "$@"\n'+LICENSE = "synthetic.signed.license"+PRIVATE = "synthetic-private-key-never-send"+ENV = f"CP_PROVIDER_MSP_LICENSE_FILE=\nCP_ENTITLEMENT_SIGNING_PRIVATE_KEY={PRIVATE}\nDOMAIN=provider.invalid\n"++HARNESS = r'''+derive_lease_signing_public_key() {+ printf 'derive\n' >> "$FIXTURE/derivations"+ if [[ "$DERIVE" == fail ]]; then exit 1; fi+ printf 'synthetic-public-key'+}+curl() {+ printf '%s\0' "$@" >> "$FIXTURE/curl-args"+ printf 'request\n' >> "$FIXTURE/requests"+ local fail=0 write_out=0 arg+ for arg in "$@"; do+ case "$arg" in -fsS|--fail) fail=1 ;; --write-out|-w) write_out=1 ;; esac+ done+ printf 'synthetic transport stderr must stay private\n' >&2+ # Model --fail faithfully so the parent control observes the original+ # HTTP-rejection/transport conflation instead of a double's artefact.+ if [[ "$TRANSPORT_EXIT" == 0 && "$fail" == 1 && "$HTTP_STATUS" == [45]* ]]; then+ return 22+ fi+ printf '%s' "$RESPONSE"+ if [[ "$write_out" == 1 ]]; then printf '\n%s' "$HTTP_STATUS"; fi+ return "$TRANSPORT_EXIT"+}+ensure_eval_license+if [[ "$SHOW_SUMMARY" == 1 ]]; then+ PORTAL_READINESS_REPORT="synthetic portal reachability, not client acceptance"+ print_summary+fi+printf 'FUNCTION_COMPLETED\n'+'''+++class ProviderMSPEvalLicenseTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ source = SETUP.read_text()+ if not source.endswith(ENTRY) or source.count(ENTRY) != 1:+ raise AssertionError("cannot safely remove the exact installer entrypoint")+ cls.functions = source[:-len(ENTRY)]++ def setUp(self):+ self.temp = tempfile.TemporaryDirectory(prefix="pulse-msp-eval-")+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name)+ self.env_file = self.root / ".env"+ self.env_file.write_text(ENV)+ self.env_file.chmod(0o600)+ # Saved client/identity data is deliberately outside the licence path.+ (self.root / "saved-client").write_text("preserve client identity and state\n")+ self.eval_file = self.root / "provider-msp-eval-license.jwt"++ def run_setup(self, *, status="200", transport=0, response=None, skip="0", derive="ok", email="", summary=False):+ if response is None:+ response = json.dumps({"license": LICENSE, "expires_at": "2030-01-02T03:04:05Z"})+ env = {+ "PATH": os.environ["PATH"], "FIXTURE": str(self.root),+ "PULSE_PROVIDER_MSP_INSTALL_DIR": str(self.root),+ "PULSE_PROVIDER_MSP_LICENSE_URL": "https://license.invalid/",+ "PULSE_PROVIDER_MSP_SKIP_EVAL_LICENSE": skip,+ "PULSE_PROVIDER_MSP_EVAL_EMAIL": email,+ "PULSE_PROVIDER_MSP_SIGNUP_SOURCE": "synthetic-setup",+ "HTTP_STATUS": status, "TRANSPORT_EXIT": str(transport),+ "RESPONSE": response, "DERIVE": derive, "SHOW_SUMMARY": "1" if summary else "0",+ }+ result = subprocess.run(["bash", "-c", self.functions + HARNESS], env=env,+ capture_output=True, text=True, timeout=10)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertIn("FUNCTION_COMPLETED", result.stdout)+ self.assertEqual(result.stderr, "")+ self.assertNotIn(PRIVATE, result.stdout)+ self.assertNotIn(LICENSE, result.stdout)+ self.assertNotIn("synthetic transport stderr", result.stdout)+ self.assertEqual((self.root / "saved-client").read_text(), "preserve client identity and state\n")+ return result.stdout++ def requests(self):+ path = self.root / "requests"+ return path.read_text().splitlines() if path.exists() else []++ def assert_unlicensed(self, output):+ self.assertFalse(self.eval_file.exists())+ self.assertEqual(self.env_file.read_text(), ENV)+ self.assertEqual(stat.S_IMODE(self.env_file.stat().st_mode), 0o600)+ self.assertNotIn("evaluation license installed", output)+ self.assertIn("continuing unlicensed", output)+ self.assertIn("CP_PROVIDER_MSP_LICENSE_FILE", output)+ self.assertNotIn("portal and client isolation work", output)+ self.assertEqual(len(self.requests()), 1, "no automatic retries")++ def test_http_rejection_is_not_a_connection_failure(self):+ for status in ("400", "401", "403", "409", "422"):+ with self.subTest(status=status):+ self.reset_fixture()+ output = self.run_setup(status=status, response='{"error":"private response must not be logged"}')+ self.assertIn(f"rejected the evaluation request (HTTP {status})", output)+ self.assertIn("Check the request details before retrying", output)+ self.assertNotIn("could not reach", output)+ self.assertNotIn("private response", output)+ self.assert_unlicensed(output)++ def reset_fixture(self):+ # Each subcase has its own complete request identity and saved state.+ for name in ("requests", "curl-args", "derivations", "provider-msp-eval-license.jwt"):+ (self.root / name).unlink(missing_ok=True)+ self.env_file.write_text(ENV)++ def test_rate_limit_and_server_failure_have_distinct_recovery(self):+ for status, message in (("429", "rate limited"), ("500", "temporarily unavailable"), ("503", "temporarily unavailable")):+ with self.subTest(status=status):+ self.reset_fixture()+ output = self.run_setup(status=status)+ self.assertIn(message, output)+ self.assertIn(f"HTTP {status}", output)+ self.assertNotIn("could not reach", output)+ self.assert_unlicensed(output)++ def test_transport_failure_retains_exit_even_with_partial_response(self):+ for code in (6, 7, 23, 28, 35, 51, 60):+ with self.subTest(code=code):+ self.reset_fixture()+ output = self.run_setup(status="200", transport=code)+ self.assertIn(f"request failed (curl exit {code})", output)+ if code in (35, 51, 60):+ self.assertIn("certificate trust", output)+ self.assertNotIn("--insecure", output)+ self.assert_unlicensed(output)++ def test_unexpected_status_never_accepts_even_a_license_body(self):+ for status in ("000", "199", "301", "307", "600", "private-status"):+ with self.subTest(status=status):+ self.reset_fixture()+ output = self.run_setup(status=status)+ self.assertIn("unexpected HTTP status", output)+ self.assertNotIn("private-status", output)+ self.assert_unlicensed(output)++ def test_success_requires_one_object_with_nonempty_string_license(self):+ for response in ("", "not JSON", "{}", '"string"', "[]", '{"license":null}',+ '{"license":42}', '{"license":true}', '{"license":{}}',+ '{"license":[]}', '{"license":""}', '{"license":"bad\\nvalue"}',+ '{"license":"bad value"}', '{"license":"bad\\u0001value"}',+ '{"license":"one"}\n{"license":"two"}'):+ with self.subTest(response=response):+ self.reset_fixture()+ output = self.run_setup(response=response)+ self.assertIn("no valid license", output)+ self.assert_unlicensed(output)++ def test_success_writes_private_license_and_preserves_other_configuration(self):+ for status in ("200", "201", "299"):+ with self.subTest(status=status):+ self.reset_fixture()+ output = self.run_setup(status=status, email="[email protected]")+ self.assertIn("evaluation license installed: 2 client workspaces", output)+ self.assertIn("expires 2030-01-02T03:04:05Z", output)+ self.assertEqual(self.eval_file.read_text(), LICENSE)+ self.assertEqual(stat.S_IMODE(self.eval_file.stat().st_mode), 0o600)+ self.assertEqual(self.env_file.read_text(), ENV.replace("CP_PROVIDER_MSP_LICENSE_FILE=\n",+ "CP_PROVIDER_MSP_LICENSE_FILE=./provider-msp-eval-license.jwt\n"))+ self.assertEqual(len(self.requests()), 1)+ args = (self.root / "curl-args").read_text().split("\0")[:-1]+ self.assertEqual(args[-1], "https://license.invalid/v1/provider-msp/eval-license")+ self.assertEqual(args[args.index("--max-time") + 1], "20")+ self.assertEqual(args[args.index("--write-out") + 1], "\n%{http_code}")+ for forbidden in ("--insecure", "-k", "--location", "-L", "--retry", "--fail", "-fsS"):+ self.assertNotIn(forbidden, args)+ body = json.loads(args[args.index("-d") + 1])+ self.assertEqual(body, {"entitlement_signing_public_key": "synthetic-public-key",+ "setup_stage": "images_ready", "email": "[email protected]", "signup_source": "synthetic-setup"})+ self.assertNotIn(PRIVATE, str(args))++ def test_response_metadata_is_not_arbitrary_log_output(self):+ output = self.run_setup(response=json.dumps({"license": LICENSE,+ "expires_at": "private response metadata\nforged success"}))+ self.assertNotIn("private response metadata", output)+ self.assertNotIn("forged success", output)+ self.assertEqual(self.eval_file.read_text(), LICENSE)++ def test_omits_email_without_changing_setup_marker(self):+ self.run_setup()+ args = (self.root / "curl-args").read_text().split("\0")+ body = json.loads(args[args.index("-d") + 1])+ self.assertNotIn("email", body)+ self.assertEqual(body["setup_stage"], "images_ready")++ def test_configured_license_and_existing_eval_need_no_request(self):+ for kind in ("configured", "existing"):+ with self.subTest(kind=kind):+ self.reset_fixture()+ if kind == "configured":+ self.env_file.write_text(ENV.replace("CP_PROVIDER_MSP_LICENSE_FILE=\n", "CP_PROVIDER_MSP_LICENSE_FILE=./operator.jwt\n"))+ else:+ self.eval_file.write_text(LICENSE)+ self.eval_file.chmod(0o600)+ self.run_setup()+ self.assertEqual(self.requests(), [])+ self.assertFalse((self.root / "derivations").exists())+ self.assertIn("CP_ENTITLEMENT_SIGNING_PRIVATE_KEY=" + PRIVATE, self.env_file.read_text())++ def test_skip_and_derivation_failure_still_do_not_issue(self):+ for skip, derive in (("1", "ok"), ("0", "fail")):+ with self.subTest(skip=skip, derive=derive):+ self.reset_fixture()+ output = self.run_setup(skip=skip, derive=derive)+ self.assertNotIn("evaluation license installed", output)+ self.assertEqual(self.requests(), [])+ self.assertFalse(self.eval_file.exists())+ self.assertEqual(self.env_file.read_text(), ENV)++ def test_explicit_retry_can_install_without_resetting_saved_state(self):+ output = self.run_setup(status="403")+ self.assert_unlicensed(output)+ self.run_setup()+ self.assertEqual(len(self.requests()), 2, "one request per deliberate invocation")+ self.assertEqual(self.eval_file.read_text(), LICENSE)+ self.run_setup()+ self.assertEqual(len(self.requests()), 2, "installed licence must not be reissued")++ def test_final_summary_does_not_turn_rejection_into_capability_success(self):+ output = self.run_setup(status="403", summary=True)+ self.assert_unlicensed(output)+ self.assertIn("running portal alone does not confirm active client MSP capabilities", output)+ self.assertIn("check Plan in the portal for this deployment's licence status", output)+ self.assertNotIn("Plan: the evaluation covers two clients", output)+++if __name__ == "__main__":+ unittest.main()