Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stabilize private bootstrap and guest-agent HTTP fixtures
rcourtmand87c98cfabab
diff --git a/internal/api/configapi/install_command_test.go b/internal/api/configapi/install_command_test.goindex e619f7983c..849ec78e80 100644--- a/internal/api/configapi/install_command_test.go+++ b/internal/api/configapi/install_command_test.go@@ -85,24 +85,28 @@ while time.monotonic() < deadline:exit_sent = Trueif b"\x1ePULSE_COMMAND_READY\x1f" in output:command_ready = True- # The interactive shell also echoes the copied command, which contains- # this prompt literal. A PTY read may end there before the command has- # even run. Never deliver a credential while terminal echo is enabled:- # the real bootstrap disables it before printing the input prompt.- silent = not (termios.tcgetattr(tty_watch)[3] & termios.ECHO)- if output.endswith(b"(paste at this prompt, not in the command): ") and command_ready and silent and not sent:- if p.get("signal_on_prompt"):- # Keep the controlling terminal/session alive. The selected- # tool records the privileged reader that owns EXIT cleanup;- # signalling the entire session kills that observation owner.- with open(os.path.join(env["FAKE_ROOT"], "reader-pid")) as f:- reader = int(f.read())+ # Readline also changes terminal modes while echoing a copied command.+ # Wait for the real prompt and Bash's noncanonical read, then poll even+ # without new output so that a running reader can become blocked.+ mode = termios.tcgetattr(tty_watch)[3]+ input_ready = not (mode & (termios.ECHO | termios.ICANON))+ if output.endswith(b"(paste at this prompt, not in the command): ") and command_ready and input_ready and not sent:+ if p.get("signal_on_prompt"):+ # Keep the controlling terminal/session alive. The selected tool+ # records the privileged reader that owns EXIT cleanup. A visible+ # prompt precedes Bash completing its read setup, so on Linux also+ # observe the reader sleeping before delivering the real signal.+ with open(os.path.join(env["FAKE_ROOT"], "reader-pid")) as f:+ reader = int(f.read())+ state = reader_signal_state(reader)+ if not sys.platform.startswith("linux") or state.get("State", "").startswith("S "):signal_observation = {"signal": p["signal_on_prompt"], "sent": False,- "before": reader_signal_state(reader)}+ "before": state}os.kill(reader, getattr(signal, "SIG" + p["signal_on_prompt"]))signal_observation["sent"] = True- else:- os.write(master, (p["input"] + "\n").encode())+ sent = True+ else:+ os.write(master, (p["input"] + "\n").encode())sent = Trueif child.poll() is not None and not ready: breakelse:diff --git a/internal/api/configapi/portable_bootstrap_test.go b/internal/api/configapi/portable_bootstrap_test.goindex beb6b52a9d..da128abaf9 100644--- a/internal/api/configapi/portable_bootstrap_test.go+++ b/internal/api/configapi/portable_bootstrap_test.go@@ -7,6 +7,7 @@ import ("os""os/exec""path/filepath"+ "runtime""strings""testing")@@ -242,6 +243,51 @@ func TestPrivateBootstrapRealBusyBoxInput(t *testing.T) {}}+func TestPrivateBootstrapSignalWaitsForBlockedReader(t *testing.T) {+ if runtime.GOOS != "linux" {+ t.Skip("blocked-reader observation uses Linux process state")+ }+ root := t.TempDir()+ python, err := exec.LookPath("python3")+ if err != nil {+ t.Fatal(err)+ }+ // Keep the prompt's owner running briefly after printing. Prompt visibility+ // alone must not dispatch a signal before the reader is actually blocked.+ program := `+import os, pathlib, signal, sys, termios, time+root = pathlib.Path(os.environ["FAKE_ROOT"])+fd = os.open("/dev/tty", os.O_RDWR)+original = termios.tcgetattr(fd)+mode = termios.tcgetattr(fd)+mode[3] &= ~(termios.ECHO | termios.ICANON)+termios.tcsetattr(fd, termios.TCSANOW, mode)+phase = "setting-up-reader"+def terminate(signum, frame):+ (root / "signal-phase").write_text(phase)+ termios.tcsetattr(fd, termios.TCSANOW, original)+ sys.exit(143)+signal.signal(signal.SIGTERM, terminate)+(root / "reader-pid").write_text(str(os.getpid()))+os.write(fd, b"Pulse agent token (paste at this prompt, not in the command): ")+until = time.monotonic() + 1+while time.monotonic() < until:+ pass+phase = "blocked-reader"+os.read(fd, 1)+sys.exit("reader unexpectedly received input")+`+ command := posixShellQuote(python) + " -c " + posixShellQuote(program)+ out, err := runPortableBootstrap(t, command, "", []string{"FAKE_ROOT=" + root}, "", "TERM")+ if err == nil || strings.Contains(string(out), "PTY fixture deadline expired") || strings.Contains(string(out), "Bootstrap did not restore") {+ t.Fatalf("controlled reader did not terminate with its terminal restored: %v\n%s", err, out)+ }+ phase, err := os.ReadFile(filepath.Join(root, "signal-phase"))+ if err != nil || string(phase) != "blocked-reader" {+ t.Fatalf("signal reached %q instead of a blocked reader: %v", phase, err)+ }+}+func TestPrivateBootstrapStandalonePreferredAndBusyBoxHistory(t *testing.T) {for _, caller := range portableBootstrapCallers() {for _, mode := range []string{"prefer-standalone", "busybox"} {diff --git a/pkg/proxmox/cluster_client_endpoint_ownership_test.go b/pkg/proxmox/cluster_client_endpoint_ownership_test.goindex 5ad31ac18e..f8e447668e 100644--- a/pkg/proxmox/cluster_client_endpoint_ownership_test.go+++ b/pkg/proxmox/cluster_client_endpoint_ownership_test.go@@ -68,7 +68,7 @@ func testClusterClientEndpointOwnershipRoutingAndTrust(t *testing.T) {}func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)entered, finish := make(chan struct{}), make(chan struct{})var firstCommand atomic.Boolvar originalCommands, otherRequests atomic.Int32diff --git a/pkg/proxmox/guest_agent_capacity_test.go b/pkg/proxmox/guest_agent_capacity_test.goindex fc09a1bdc7..e4a90f123f 100644--- a/pkg/proxmox/guest_agent_capacity_test.go+++ b/pkg/proxmox/guest_agent_capacity_test.go@@ -12,23 +12,6 @@ import ("time")-// These tests are serial, as are the other guest-guard tests. Restore the-// process-wide state before the package's parallel HTTP tests can start.-func isolateGuestAgentCapacity(t *testing.T) {- t.Helper()- guestAgentGuards.Lock()- aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup- guestAgentGuards.aliases = make(map[string][]string)- guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)- guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)- guestAgentGuards.Unlock()- t.Cleanup(func() {- guestAgentGuards.Lock()- guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next- guestAgentGuards.Unlock()- })-}-// Half the protected entries are busy; half retain uncertain completion.func seedProtectedGuestAgentCapacity(count int) map[guestAgentGuardKey]guestAgentGuardEntry {protected := make(map[guestAgentGuardKey]guestAgentGuardEntry, count)@@ -61,7 +44,7 @@ func assertProtectedGuestAgentCapacity(t *testing.T, protected map[guestAgentGuafunc TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {for _, expiredAliases := range []int{1, 2} {t.Run(fmt.Sprintf("expired-%d", expiredAliases), func(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)first, second := "https://first.capacity.invalid/api2/json", "https://second.capacity.invalid/api2/json"registerGuestAgentEndpoints("https://first.capacity.invalid", []string{"https://second.capacity.invalid"})protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 1 - expiredAliases)@@ -96,7 +79,7 @@ func TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {for name, read := range backupAgentReads() {t.Run(name, func(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)var commands, configs atomic.Int32var locked atomic.Boolserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -148,7 +131,7 @@ func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {func TestGuestAgentCapacityNeverEvictsProtectedWork(t *testing.T) {for _, slotsLeft := range []int{0, 1} {t.Run(fmt.Sprintf("slots-%d", slotsLeft), func(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)var requests atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {requests.Add(1)diff --git a/pkg/proxmox/guest_agent_fixture_test.go b/pkg/proxmox/guest_agent_fixture_test.gonew file mode 100644index 0000000000..76e30cb8b1--- /dev/null+++ b/pkg/proxmox/guest_agent_fixture_test.go@@ -0,0 +1,24 @@+package proxmox++import (+ "testing"+ "time"+)++// Serial guest-agent fixtures own one registry for their whole scenario.+// Keep admission shared across clients and aliases within that fixture, then+// restore prior state before the package's parallel HTTP tests can start.+func isolateGuestAgentRegistry(t *testing.T) {+ t.Helper()+ guestAgentGuards.Lock()+ aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup+ guestAgentGuards.aliases = make(map[string][]string)+ guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)+ guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)+ guestAgentGuards.Unlock()+ t.Cleanup(func() {+ guestAgentGuards.Lock()+ guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next+ guestAgentGuards.Unlock()+ })+}diff --git a/pkg/proxmox/guest_agent_http_status_test.go b/pkg/proxmox/guest_agent_http_status_test.goindex eda4066a06..18c8961d66 100644--- a/pkg/proxmox/guest_agent_http_status_test.go+++ b/pkg/proxmox/guest_agent_http_status_test.go@@ -39,6 +39,7 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {t.Run(tc.name, func(t *testing.T) {for name, first := range backupAgentReads() {t.Run(name, func(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch {@@ -86,6 +87,7 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {for name, first := range backupAgentReads() {t.Run(name, func(t *testing.T) {+ isolateGuestAgentRegistry(t)var commands, configs atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch {@@ -129,9 +131,51 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {}}+func TestGuestAgentHTTPFixtureDoesNotInheritPriorAdmission(t *testing.T) {+ isolateGuestAgentRegistry(t)+ var commands atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ commands.Add(1)+ w.WriteHeader(http.StatusUpgradeRequired)+ fmt.Fprint(w, "upstream unavailable")+ }))+ defer server.Close()+ client := backupTestClient(t, server.URL)+ // A closed fixture can leave this same origin in the process-wide guard+ // until its one-minute uncertainty interval expires. Kernel port reuse must+ // not turn the first request of a new HTTP case into an old cooldown.+ release, err := acquireGuestAgent(client.baseURL, 105)+ if err != nil {+ t.Fatal(err)+ }+ release(true)+ t.Run("new-http-fixture", func(t *testing.T) {+ isolateGuestAgentRegistry(t)+ var response *apiResponseError+ _, err := backupTestClient(t, server.URL).GetVMAgentVersion(context.Background(), "node", 105)+ if GuestAgentDeferredReason(err) != "agent-completion-unverified" || !errors.As(err, &response) || response.statusCode != http.StatusUpgradeRequired {+ t.Errorf("new fixture inherited an earlier origin's admission: %v", err)+ }+ if _, err := backupTestClient(t, server.URL).GetVMAgentVersion(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("clients inside one fixture no longer share uncertainty: %v", err)+ }+ if commands.Load() != 1 {+ t.Errorf("new fixture dispatched %d commands, want one", commands.Load())+ }+ })+ if _, err := acquireGuestAgent(client.baseURL, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("fixture isolation discarded the outer uncertainty fence: %v", err)+ }+}+func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {for _, status := range []int{409, 425, 499} {t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ isolateGuestAgentRegistry(t)var commands atomic.Int32var healthy atomic.Boolhandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -206,6 +250,7 @@ func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {}func TestGuestAgentHTTPFailureDoesNotFailOver(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch {@@ -263,6 +308,7 @@ func TestGuestAgentHTTPExplicitRefusalsRemainErrors(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if strings.HasSuffix(r.URL.Path, "/config") {@@ -311,6 +357,7 @@ func TestGuestAgentHTTPAmbiguousFailuresDoNotClearAdmission(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if strings.HasSuffix(r.URL.Path, "/config") {@@ -347,6 +394,7 @@ func TestGuestAgentHTTPTerminalFailuresRemainCommandBound(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if strings.HasSuffix(r.URL.Path, "/config") {@@ -376,6 +424,7 @@ func TestGuestAgentHTTPTerminalFailuresRemainCommandBound(t *testing.T) {}func TestGuestAgentHTTPOrdinaryRequestPreservesStatusAndError(t *testing.T) {+ isolateGuestAgentRegistry(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.WriteHeader(502)fmt.Fprint(w, "API error 403: permission denied")@@ -393,6 +442,7 @@ func TestGuestAgentHTTPOrdinaryRequestPreservesStatusAndError(t *testing.T) {}func TestGuestAgentHTTPUncertaintyExpiresAndKeepsOtherVMsIndependent(t *testing.T) {+ isolateGuestAgentRegistry(t)var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if strings.HasSuffix(r.URL.Path, "/config") {diff --git a/pkg/proxmox/guest_agent_ipv4mapped_test.go b/pkg/proxmox/guest_agent_ipv4mapped_test.goindex 400b05b685..d760fe2631 100644--- a/pkg/proxmox/guest_agent_ipv4mapped_test.go+++ b/pkg/proxmox/guest_agent_ipv4mapped_test.go@@ -89,7 +89,7 @@ func mappedEndpointReadNames() []string {func TestGuestAgentIPv4MappedInflight(t *testing.T) {for _, reverse := range []bool{false, true} {t.Run(fmt.Sprintf("mapped-first-%t", reverse), func(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)entered, finish := make(chan struct{}), make(chan struct{})var commands, configs atomic.Int32_, ipv4, mapped := mappedEndpointServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -165,7 +165,7 @@ func TestGuestAgentIPv4MappedUncertainty(t *testing.T) {for _, reverse := range []bool{false, true} {for _, incomplete := range []bool{false, true} {t.Run(fmt.Sprintf("mapped-first-%t/incomplete-%t", reverse, incomplete), func(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)var commands, configs atomic.Int32_, ipv4, mapped := mappedEndpointServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch {diff --git a/pkg/proxmox/guest_agent_typed_response_test.go b/pkg/proxmox/guest_agent_typed_response_test.goindex 6b992a51d5..1f152a0e0b 100644--- a/pkg/proxmox/guest_agent_typed_response_test.go+++ b/pkg/proxmox/guest_agent_typed_response_test.go@@ -88,7 +88,7 @@ func TestGuestAgentTypedSuccessDefersEveryRead(t *testing.T) {}func TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes(t *testing.T) {- isolateGuestAgentCapacity(t)+ isolateGuestAgentRegistry(t)var commands atomic.Int32var healthy atomic.Boolhandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {