Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Stabilize private bootstrap and guest-agent HTTP fixtures

rcourtmand87c98cfabab
diff --git a/internal/api/configapi/install_command_test.go b/internal/api/configapi/install_command_test.go
index e619f7983c..849ec78e80 100644
--- a/internal/api/configapi/install_command_test.go
+++ b/internal/api/configapi/install_command_test.go
@@ -85,24 +85,28 @@ while time.monotonic() < deadline:
exit_sent = True
if b"\x1ePULSE_COMMAND_READY\x1f" in output:
command_ready = True
- # The interactive shell also echoes the copied command, which contains
- # this prompt literal. A PTY read may end there before the command has
- # even run. Never deliver a credential while terminal echo is enabled:
- # the real bootstrap disables it before printing the input prompt.
- silent = not (termios.tcgetattr(tty_watch)[3] & termios.ECHO)
- if output.endswith(b"(paste at this prompt, not in the command): ") and command_ready and silent and not sent:
- if p.get("signal_on_prompt"):
- # Keep the controlling terminal/session alive. The selected
- # tool records the privileged reader that owns EXIT cleanup;
- # signalling the entire session kills that observation owner.
- with open(os.path.join(env["FAKE_ROOT"], "reader-pid")) as f:
- reader = int(f.read())
+ # Readline also changes terminal modes while echoing a copied command.
+ # Wait for the real prompt and Bash's noncanonical read, then poll even
+ # without new output so that a running reader can become blocked.
+ mode = termios.tcgetattr(tty_watch)[3]
+ input_ready = not (mode & (termios.ECHO | termios.ICANON))
+ if output.endswith(b"(paste at this prompt, not in the command): ") and command_ready and input_ready and not sent:
+ if p.get("signal_on_prompt"):
+ # Keep the controlling terminal/session alive. The selected tool
+ # records the privileged reader that owns EXIT cleanup. A visible
+ # prompt precedes Bash completing its read setup, so on Linux also
+ # observe the reader sleeping before delivering the real signal.
+ with open(os.path.join(env["FAKE_ROOT"], "reader-pid")) as f:
+ reader = int(f.read())
+ state = reader_signal_state(reader)
+ if not sys.platform.startswith("linux") or state.get("State", "").startswith("S "):
signal_observation = {"signal": p["signal_on_prompt"], "sent": False,
- "before": reader_signal_state(reader)}
+ "before": state}
os.kill(reader, getattr(signal, "SIG" + p["signal_on_prompt"]))
signal_observation["sent"] = True
- else:
- os.write(master, (p["input"] + "\n").encode())
+ sent = True
+ else:
+ os.write(master, (p["input"] + "\n").encode())
sent = True
if child.poll() is not None and not ready: break
else:
diff --git a/internal/api/configapi/portable_bootstrap_test.go b/internal/api/configapi/portable_bootstrap_test.go
index beb6b52a9d..da128abaf9 100644
--- a/internal/api/configapi/portable_bootstrap_test.go
+++ b/internal/api/configapi/portable_bootstrap_test.go
@@ -7,6 +7,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "runtime"
"strings"
"testing"
)
@@ -242,6 +243,51 @@ func TestPrivateBootstrapRealBusyBoxInput(t *testing.T) {
}
}
+func TestPrivateBootstrapSignalWaitsForBlockedReader(t *testing.T) {
+ if runtime.GOOS != "linux" {
+ t.Skip("blocked-reader observation uses Linux process state")
+ }
+ root := t.TempDir()
+ python, err := exec.LookPath("python3")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Keep the prompt's owner running briefly after printing. Prompt visibility
+ // alone must not dispatch a signal before the reader is actually blocked.
+ program := `
+import os, pathlib, signal, sys, termios, time
+root = pathlib.Path(os.environ["FAKE_ROOT"])
+fd = os.open("/dev/tty", os.O_RDWR)
+original = termios.tcgetattr(fd)
+mode = termios.tcgetattr(fd)
+mode[3] &= ~(termios.ECHO | termios.ICANON)
+termios.tcsetattr(fd, termios.TCSANOW, mode)
+phase = "setting-up-reader"
+def terminate(signum, frame):
+ (root / "signal-phase").write_text(phase)
+ termios.tcsetattr(fd, termios.TCSANOW, original)
+ sys.exit(143)
+signal.signal(signal.SIGTERM, terminate)
+(root / "reader-pid").write_text(str(os.getpid()))
+os.write(fd, b"Pulse agent token (paste at this prompt, not in the command): ")
+until = time.monotonic() + 1
+while time.monotonic() < until:
+ pass
+phase = "blocked-reader"
+os.read(fd, 1)
+sys.exit("reader unexpectedly received input")
+`
+ command := posixShellQuote(python) + " -c " + posixShellQuote(program)
+ out, err := runPortableBootstrap(t, command, "", []string{"FAKE_ROOT=" + root}, "", "TERM")
+ if err == nil || strings.Contains(string(out), "PTY fixture deadline expired") || strings.Contains(string(out), "Bootstrap did not restore") {
+ t.Fatalf("controlled reader did not terminate with its terminal restored: %v\n%s", err, out)
+ }
+ phase, err := os.ReadFile(filepath.Join(root, "signal-phase"))
+ if err != nil || string(phase) != "blocked-reader" {
+ t.Fatalf("signal reached %q instead of a blocked reader: %v", phase, err)
+ }
+}
+
func TestPrivateBootstrapStandalonePreferredAndBusyBoxHistory(t *testing.T) {
for _, caller := range portableBootstrapCallers() {
for _, mode := range []string{"prefer-standalone", "busybox"} {
diff --git a/pkg/proxmox/cluster_client_endpoint_ownership_test.go b/pkg/proxmox/cluster_client_endpoint_ownership_test.go
index 5ad31ac18e..f8e447668e 100644
--- a/pkg/proxmox/cluster_client_endpoint_ownership_test.go
+++ b/pkg/proxmox/cluster_client_endpoint_ownership_test.go
@@ -68,7 +68,7 @@ func testClusterClientEndpointOwnershipRoutingAndTrust(t *testing.T) {
}
func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
entered, finish := make(chan struct{}), make(chan struct{})
var firstCommand atomic.Bool
var originalCommands, otherRequests atomic.Int32
diff --git a/pkg/proxmox/guest_agent_capacity_test.go b/pkg/proxmox/guest_agent_capacity_test.go
index fc09a1bdc7..e4a90f123f 100644
--- a/pkg/proxmox/guest_agent_capacity_test.go
+++ b/pkg/proxmox/guest_agent_capacity_test.go
@@ -12,23 +12,6 @@ import (
"time"
)
-// These tests are serial, as are the other guest-guard tests. Restore the
-// process-wide state before the package's parallel HTTP tests can start.
-func isolateGuestAgentCapacity(t *testing.T) {
- t.Helper()
- guestAgentGuards.Lock()
- aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup
- guestAgentGuards.aliases = make(map[string][]string)
- guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)
- guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)
- guestAgentGuards.Unlock()
- t.Cleanup(func() {
- guestAgentGuards.Lock()
- guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next
- guestAgentGuards.Unlock()
- })
-}
-
// Half the protected entries are busy; half retain uncertain completion.
func seedProtectedGuestAgentCapacity(count int) map[guestAgentGuardKey]guestAgentGuardEntry {
protected := make(map[guestAgentGuardKey]guestAgentGuardEntry, count)
@@ -61,7 +44,7 @@ func assertProtectedGuestAgentCapacity(t *testing.T, protected map[guestAgentGua
func TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {
for _, expiredAliases := range []int{1, 2} {
t.Run(fmt.Sprintf("expired-%d", expiredAliases), func(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
first, second := "https://first.capacity.invalid/api2/json", "https://second.capacity.invalid/api2/json"
registerGuestAgentEndpoints("https://first.capacity.invalid", []string{"https://second.capacity.invalid"})
protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 1 - expiredAliases)
@@ -96,7 +79,7 @@ func TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {
func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {
for name, read := range backupAgentReads() {
t.Run(name, func(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
var commands, configs atomic.Int32
var locked atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -148,7 +131,7 @@ func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {
func TestGuestAgentCapacityNeverEvictsProtectedWork(t *testing.T) {
for _, slotsLeft := range []int{0, 1} {
t.Run(fmt.Sprintf("slots-%d", slotsLeft), func(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
diff --git a/pkg/proxmox/guest_agent_fixture_test.go b/pkg/proxmox/guest_agent_fixture_test.go
new file mode 100644
index 0000000000..76e30cb8b1
--- /dev/null
+++ b/pkg/proxmox/guest_agent_fixture_test.go
@@ -0,0 +1,24 @@
+package proxmox
+
+import (
+ "testing"
+ "time"
+)
+
+// Serial guest-agent fixtures own one registry for their whole scenario.
+// Keep admission shared across clients and aliases within that fixture, then
+// restore prior state before the package's parallel HTTP tests can start.
+func isolateGuestAgentRegistry(t *testing.T) {
+ t.Helper()
+ guestAgentGuards.Lock()
+ aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup
+ guestAgentGuards.aliases = make(map[string][]string)
+ guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)
+ guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)
+ guestAgentGuards.Unlock()
+ t.Cleanup(func() {
+ guestAgentGuards.Lock()
+ guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next
+ guestAgentGuards.Unlock()
+ })
+}
diff --git a/pkg/proxmox/guest_agent_http_status_test.go b/pkg/proxmox/guest_agent_http_status_test.go
index eda4066a06..18c8961d66 100644
--- a/pkg/proxmox/guest_agent_http_status_test.go
+++ b/pkg/proxmox/guest_agent_http_status_test.go
@@ -39,6 +39,7 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
for name, first := range backupAgentReads() {
t.Run(name, func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
@@ -86,6 +87,7 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {
t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
for name, first := range backupAgentReads() {
t.Run(name, func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var commands, configs atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
@@ -129,9 +131,51 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {
}
}
+func TestGuestAgentHTTPFixtureDoesNotInheritPriorAdmission(t *testing.T) {
+ isolateGuestAgentRegistry(t)
+ var commands atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/config") {
+ fmt.Fprint(w, `{"data":{}}`)
+ return
+ }
+ commands.Add(1)
+ w.WriteHeader(http.StatusUpgradeRequired)
+ fmt.Fprint(w, "upstream unavailable")
+ }))
+ defer server.Close()
+ client := backupTestClient(t, server.URL)
+ // A closed fixture can leave this same origin in the process-wide guard
+ // until its one-minute uncertainty interval expires. Kernel port reuse must
+ // not turn the first request of a new HTTP case into an old cooldown.
+ release, err := acquireGuestAgent(client.baseURL, 105)
+ if err != nil {
+ t.Fatal(err)
+ }
+ release(true)
+ t.Run("new-http-fixture", func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
+ var response *apiResponseError
+ _, err := backupTestClient(t, server.URL).GetVMAgentVersion(context.Background(), "node", 105)
+ if GuestAgentDeferredReason(err) != "agent-completion-unverified" || !errors.As(err, &response) || response.statusCode != http.StatusUpgradeRequired {
+ t.Errorf("new fixture inherited an earlier origin's admission: %v", err)
+ }
+ if _, err := backupTestClient(t, server.URL).GetVMAgentVersion(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("clients inside one fixture no longer share uncertainty: %v", err)
+ }
+ if commands.Load() != 1 {
+ t.Errorf("new fixture dispatched %d commands, want one", commands.Load())
+ }
+ })
+ if _, err := acquireGuestAgent(client.baseURL, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("fixture isolation discarded the outer uncertainty fence: %v", err)
+ }
+}
+
func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {
for _, status := range []int{409, 425, 499} {
t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var commands atomic.Int32
var healthy atomic.Bool
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -206,6 +250,7 @@ func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {
}
func TestGuestAgentHTTPFailureDoesNotFailOver(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
@@ -263,6 +308,7 @@ func TestGuestAgentHTTPExplicitRefusalsRemainErrors(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
@@ -311,6 +357,7 @@ func TestGuestAgentHTTPAmbiguousFailuresDoNotClearAdmission(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
@@ -347,6 +394,7 @@ func TestGuestAgentHTTPTerminalFailuresRemainCommandBound(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
@@ -376,6 +424,7 @@ func TestGuestAgentHTTPTerminalFailuresRemainCommandBound(t *testing.T) {
}
func TestGuestAgentHTTPOrdinaryRequestPreservesStatusAndError(t *testing.T) {
+ isolateGuestAgentRegistry(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(502)
fmt.Fprint(w, "API error 403: permission denied")
@@ -393,6 +442,7 @@ func TestGuestAgentHTTPOrdinaryRequestPreservesStatusAndError(t *testing.T) {
}
func TestGuestAgentHTTPUncertaintyExpiresAndKeepsOtherVMsIndependent(t *testing.T) {
+ isolateGuestAgentRegistry(t)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/config") {
diff --git a/pkg/proxmox/guest_agent_ipv4mapped_test.go b/pkg/proxmox/guest_agent_ipv4mapped_test.go
index 400b05b685..d760fe2631 100644
--- a/pkg/proxmox/guest_agent_ipv4mapped_test.go
+++ b/pkg/proxmox/guest_agent_ipv4mapped_test.go
@@ -89,7 +89,7 @@ func mappedEndpointReadNames() []string {
func TestGuestAgentIPv4MappedInflight(t *testing.T) {
for _, reverse := range []bool{false, true} {
t.Run(fmt.Sprintf("mapped-first-%t", reverse), func(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
entered, finish := make(chan struct{}), make(chan struct{})
var commands, configs atomic.Int32
_, ipv4, mapped := mappedEndpointServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -165,7 +165,7 @@ func TestGuestAgentIPv4MappedUncertainty(t *testing.T) {
for _, reverse := range []bool{false, true} {
for _, incomplete := range []bool{false, true} {
t.Run(fmt.Sprintf("mapped-first-%t/incomplete-%t", reverse, incomplete), func(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
var commands, configs atomic.Int32
_, ipv4, mapped := mappedEndpointServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
diff --git a/pkg/proxmox/guest_agent_typed_response_test.go b/pkg/proxmox/guest_agent_typed_response_test.go
index 6b992a51d5..1f152a0e0b 100644
--- a/pkg/proxmox/guest_agent_typed_response_test.go
+++ b/pkg/proxmox/guest_agent_typed_response_test.go
@@ -88,7 +88,7 @@ func TestGuestAgentTypedSuccessDefersEveryRead(t *testing.T) {
}
func TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes(t *testing.T) {
- isolateGuestAgentCapacity(t)
+ isolateGuestAgentRegistry(t)
var commands atomic.Int32
var healthy atomic.Bool
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {