Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep guest filesystem failure guidance backup-safe

pulse-triage[bot]9daf5c6f7179
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 061a14d341..819e0ae7b5 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -231,6 +231,22 @@ required; all responses in this fixture are synthetic.
### Guest-agent coordination around backups — issue #2439
+Filesystem-query diagnostics preserve this safety boundary. Missing, empty,
+unsupported or refused readings do not prescribe installing, activating or
+restarting QGA, changing backup settings, re-running setup or broadening a
+shared Proxmox role. Operator logs identify the existing guest and fixed reason,
+lead to guest-local disk checks, and keep existing guest-agent/backup settings.
+An actual HTTP 401/403 leads only to checking the existing credential's access
+to that VM. A stopped-agent reason requires exact command-bound rejection
+evidence, not an HTTP 500 or a status/diagnosis quoted in provider text.
+Unsupported commands remain errors without claiming that QGA is stopped.
+Deferrals retain their fixed reasons and do not gain new diagnostic commands.
+`TestGuestFilesystemFailureGuidanceUsesObservedEvidence` crosses the real
+client/collector/log boundary and preserves unavailable disk sentinels and
+single-attempt uncertainty. The error-reason controls preserve original error
+text/causes and terminal-rejection admission. These synthetic diagnostics are
+not independent thaw, every-filesystem writes or native backup recovery.
+
All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfo
callers, share fail-fast per-guest admission within this Pulse process. Clients
for the same endpoint and configured cluster aliases coordinate across methods;
diff --git a/internal/monitoring/guest_disk_stability.go b/internal/monitoring/guest_disk_stability.go
index 53a99390d5..9335800814 100644
--- a/internal/monitoring/guest_disk_stability.go
+++ b/internal/monitoring/guest_disk_stability.go
@@ -1,6 +1,8 @@
package monitoring
import (
+ "context"
+ "errors"
"strings"
"time"
@@ -21,25 +23,17 @@ func classifyGuestAgentDiskStatusError(err error) string {
return ""
}
- if reason := proxmox.GuestAgentDeferredReason(err); reason != "" {
+ if reason := proxmox.GuestAgentErrorReason(err); reason != "" {
return reason
}
- errStr := err.Error()
- errStrLower := strings.ToLower(errStr)
-
+ // Non-Proxmox client implementations may return untyped local errors.
+ // Never infer HTTP status or a stopped agent from numbers/body substrings.
switch {
- case strings.Contains(errStr, "QEMU guest agent is not running"):
+ case err.Error() == "QEMU guest agent is not running":
return "agent-not-running"
- case strings.Contains(errStr, "timeout") || strings.Contains(errStr, "deadline exceeded"):
+ case errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), "timeout"):
return "agent-timeout"
- case strings.Contains(errStr, "500") && (strings.Contains(errStr, "not running") || strings.Contains(errStr, "not available")):
- return "agent-not-running"
- case (strings.Contains(errStr, "403") || strings.Contains(errStr, "401")) &&
- (strings.Contains(errStrLower, "permission") || strings.Contains(errStrLower, "forbidden") || strings.Contains(errStrLower, "not allowed")):
- return "permission-denied"
- case strings.Contains(errStr, "500"):
- return "agent-not-running"
default:
return "agent-error"
}
diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.go
index a8189bdafb..03b200e2ae 100644
--- a/internal/monitoring/monitor_pve_guest_builders.go
+++ b/internal/monitoring/monitor_pve_guest_builders.go
@@ -593,72 +593,40 @@ func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res pr
if reason := proxmox.GuestAgentDeferredReason(err); reason != "" {
return nil, reason, false
}
- // Log more helpful error messages based on the error type
- errMsg := err.Error()
- if strings.Contains(errMsg, "500") || strings.Contains(errMsg, "QEMU guest agent is not running") {
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Int("vmid", res.VMID).
- Msg("Guest agent enabled in VM config but not running inside guest OS. Install and start qemu-guest-agent in the VM")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("To verify: ssh into VM and run 'systemctl status qemu-guest-agent' or 'ps aux | grep qemu-ga'")
- } else if strings.Contains(errMsg, "timeout") {
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Int("vmid", res.VMID).
- Msg("Guest agent timeout - agent may be installed but not responding")
- } else if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "401") || strings.Contains(errMsg, "authentication error") {
- // Permission error - user/token lacks required permissions
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Int("vmid", res.VMID).
- Msg("VM disk monitoring permission denied. Check permissions:")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("• Proxmox 9: Ensure token/user has VM.GuestAgent.Audit privilege (Pulse setup adds this via PulseMonitor role)")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("• Proxmox 8: Ensure token/user has VM.Monitor privilege (Pulse setup adds this via PulseMonitor role)")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("• All versions: Sys.Audit is recommended for Ceph metrics and applied when available")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("• Re-run Pulse setup script if node was added before v4.7")
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Msg("• Verify guest agent is installed and running inside the VM")
- } else {
- log.Debug().
- Err(err).
- Str("instance", instanceName).
- Str("vm", res.Name).
- Int("vmid", res.VMID).
- Msg("Failed to get filesystem info from guest agent")
- }
+ reason := classifyGuestAgentDiskStatusError(err)
+ logVMFilesystemUnavailable(instanceName, res, reason)
return nil, classifyGuestAgentDiskStatusError(err), false
}
if len(fsInfo) == 0 {
- log.Info().
- Str("instance", instanceName).
- Str("vm", res.Name).
- Int("vmid", res.VMID).
- Msg("Guest agent returned no filesystem info - agent may need restart or VM may have no mounted filesystems")
+ logVMFilesystemUnavailable(instanceName, res, "no-filesystems")
return nil, "no-filesystems", false
}
return fsInfo, "", true
}
+// Missing optional readings do not establish a need to activate/restart QGA,
+// change backup policy, or reconfigure a shared Proxmox role. Keep this guidance
+// independent of provider error text and the guest's operating system.
+func logVMFilesystemUnavailable(instanceName string, res proxmox.ClusterResource, reason string) {
+ message := "Guest filesystem query failed; check disk usage inside the guest. Keep existing guest-agent and backup settings."
+ switch reason {
+ case "agent-not-running":
+ message = "Proxmox reports the guest agent is not running; check disk usage inside the guest. Keep existing guest-agent and backup settings."
+ case "permission-denied":
+ message = "Guest filesystem query was not authorised; check the existing credential's access to this VM. Do not broaden shared roles or change guest-agent and backup settings to diagnose missing readings."
+ case "agent-timeout":
+ message = "Guest filesystem query did not complete; check disk usage inside the guest. Keep existing guest-agent and backup settings."
+ case "no-filesystems":
+ message = "Guest agent returned no filesystem readings; check disk usage inside the guest. Keep existing guest-agent and backup settings."
+ }
+ log.Info().
+ Str("instance", instanceName).
+ Str("vm", res.Name).
+ Int("vmid", res.VMID).
+ Str("reason", reason).
+ Msg(message)
+}
+
func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterResource, fsInfo []proxmox.VMFileSystem) vmFSInfoSummary {
summary := vmFSInfoSummary{}
diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.go
index 93fd917bef..1d8beacbbf 100644
--- a/internal/monitoring/monitor_pve_guest_lxc_test.go
+++ b/internal/monitoring/monitor_pve_guest_lxc_test.go
@@ -1,13 +1,23 @@
package monitoring
import (
+ "bytes"
"context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
)
type stubPVEClientLXCStatus struct {
@@ -567,3 +577,129 @@ func TestIssue2148LXCRejectsAgentMemoryWithMismatchedTotal(t *testing.T) {
t.Fatalf("memory used = %d, want provider value %d", container.Memory.Used, resource.Mem)
}
}
+
+func TestGuestFilesystemStatusDoesNotDiagnoseFromErrorText(t *testing.T) {
+ cases := []struct {
+ err error
+ want string
+ }{
+ {nil, ""},
+ {errors.New("API error 500: internal server error"), "agent-error"},
+ {errors.New("API error 500: unsupported command: guest-get-fsinfo"), "agent-error"},
+ {errors.New("API error 500: QEMU guest agent is not running"), "agent-error"},
+ {errors.New("request for VM 500 failed"), "agent-error"},
+ {errors.New("API error 400: upstream API error 403: permission denied"), "agent-error"},
+ {errors.New("QEMU guest agent is not running"), "agent-not-running"},
+ {context.DeadlineExceeded, "agent-timeout"},
+ {fmt.Errorf("wrapped: %w", context.DeadlineExceeded), "agent-timeout"},
+ {errors.New("guest agent request timeout"), "agent-timeout"},
+ }
+ for _, tc := range cases {
+ if got := classifyGuestAgentDiskStatusError(tc.err); got != tc.want {
+ t.Errorf("%v: reason = %q, want %q", tc.err, got, tc.want)
+ }
+ }
+}
+
+// These synthetic responses exercise the production command guard, filesystem
+// collector, unavailable sentinel and emitted operator guidance together. They
+// perform no native QGA, backup, guest activation or recovery action.
+func TestGuestFilesystemFailureGuidanceUsesObservedEvidence(t *testing.T) {
+ cases := []struct {
+ name, body, reason, message string
+ status int
+ logged bool
+ }{
+ {"stopped", `{"message":"QEMU guest agent is not running"}`, "agent-not-running", "Proxmox reports the guest agent is not running", 500, true},
+ {"unsupported", "unsupported command: guest-get-fsinfo", "agent-error", "Guest filesystem query failed", 500, true},
+ {"forbidden", "provider-private-detail", "permission-denied", "Guest filesystem query was not authorised", 403, true},
+ {"unauthorised-quotes-stopped", "API error 500: QEMU guest agent is not running", "permission-denied", "Guest filesystem query was not authorised", 401, true},
+ {"bad-request-quotes-permission", "API error 403: permission denied", "agent-error", "Guest filesystem query failed", 400, true},
+ {"empty", `{"data":{"result":[]}}`, "no-filesystems", "Guest agent returned no filesystem readings", 200, true},
+ {"malformed", `{"data":`, "agent-error", "Guest filesystem query failed", 200, true},
+ {"server-uncertain", "provider-private-detail", "agent-completion-unverified", "", 500, false},
+ {"gateway-quotes-stopped", "QEMU guest agent is not running", "agent-completion-unverified", "", 502, false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/status/current") {
+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":0,"diskwrite":null,"netin":42}}`)
+ return
+ }
+ if strings.HasSuffix(r.URL.Path, "/config") {
+ fmt.Fprint(w, `{"data":{}}`)
+ return
+ }
+ calls.Add(1)
+ w.WriteHeader(tc.status)
+ fmt.Fprint(w, tc.body)
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var output bytes.Buffer
+ previousLogger := log.Logger
+ log.Logger = zerolog.New(&output)
+ defer func() { log.Logger = previousLogger }()
+ m := &Monitor{guestAgentFSInfoTimeout: time.Second, guestAgentRetries: 2}
+ res := proxmox.ClusterResource{Node: "node", VMID: 105, Name: "fixture-vm", Type: "qemu", Status: "running", MaxDisk: 1000}
+ total, used, free, usage, disks, fromAgent, reason := m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "fixture-instance", res, client, 1000, 0, 0)
+ if reason != tc.reason || fromAgent || usage != -1 || total != 1000 || used != 0 || free != 1000 || disks != nil {
+ t.Errorf("unavailable reading = %d/%d/%d/%v/%v/%t/%q", total, used, free, usage, disks, fromAgent, reason)
+ }
+ beforeStatus := time.Now()
+ status, err := client.GetVMStatus(context.Background(), res.Node, res.VMID)
+ afterStatus := time.Now()
+ if err != nil {
+ t.Fatal(err)
+ }
+ presence := status.IOCounters.Effective()
+ if status.CPU != 0.25 || status.DiskRead != 0 || status.NetIn != 42 || !presence.DiskRead || !presence.NetworkIn || presence.DiskWrite || presence.NetworkOut || status.ObservedAt.Before(beforeStatus) || status.ObservedAt.After(afterStatus) {
+ t.Errorf("filesystem failure changed independent live counters/presence/receipt: %+v", status)
+ }
+ if calls.Load() != 1 {
+ t.Errorf("wire guest commands = %d, want one even with retries configured", calls.Load())
+ }
+ guidance := 0
+ for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) {
+ if len(line) == 0 {
+ continue
+ }
+ var event map[string]interface{}
+ if json.Unmarshal(line, &event) != nil {
+ t.Fatalf("invalid structured log: %s", line)
+ }
+ if event["vmid"] != float64(res.VMID) {
+ continue // Request-layer status logs are separate, not guidance.
+ }
+ guidance++
+ message, _ := event["message"].(string)
+ if !tc.logged || event["level"] != "info" || event["instance"] != "fixture-instance" || event["vm"] != "fixture-vm" || event["reason"] != tc.reason || !strings.HasPrefix(message, tc.message) {
+ t.Errorf("observed-evidence guidance lost: %v", event)
+ }
+ if !strings.Contains(message, "guest-agent and backup settings") {
+ t.Errorf("backup-settings precaution missing: %s", message)
+ }
+ if tc.reason == "permission-denied" && (!strings.Contains(message, "this VM") || !strings.Contains(message, "Do not broaden shared roles")) {
+ t.Errorf("credential advice is not scoped: %s", message)
+ }
+ for _, unsafe := range []string{"Install and start", "restart", "systemctl", "ps aux", "Re-run", "PulseMonitor", "VM.Monitor", "Sys.Audit", "provider-private-detail"} {
+ if strings.Contains(message, unsafe) {
+ t.Errorf("unsafe/unobserved advice %q: %s", unsafe, message)
+ }
+ }
+ }
+ wantGuidance := 0
+ if tc.logged {
+ wantGuidance = 1
+ }
+ if guidance != wantGuidance {
+ t.Errorf("guidance records = %d, want %d", guidance, wantGuidance)
+ }
+ })
+ }
+}
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index 2c6d24d098..dc4ffe9422 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -639,9 +639,11 @@ func (c *Client) requestWithRetryUsingClient(ctx context.Context, method, path s
apiErr = fmt.Errorf("authentication error: %w", apiErr)
}
+ guestFailureReason := guestAgentTerminalFailureReason(path, body)
return nil, &apiResponseError{
statusCode: resp.StatusCode,
- guestCommandRejected: resp.StatusCode == http.StatusInternalServerError && guestAgentTerminalFailure(path, body),
+ guestCommandRejected: resp.StatusCode == http.StatusInternalServerError && guestFailureReason != "",
+ guestFailureReason: guestFailureReason,
cause: apiErr,
}
}
diff --git a/pkg/proxmox/guest_agent_error_reason_test.go b/pkg/proxmox/guest_agent_error_reason_test.go
new file mode 100644
index 0000000000..902fc25de2
--- /dev/null
+++ b/pkg/proxmox/guest_agent_error_reason_test.go
@@ -0,0 +1,58 @@
+package proxmox
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+ "testing"
+)
+
+func TestGuestAgentErrorReasonKeepsWireAndCommandEvidence(t *testing.T) {
+ const path = "/nodes/node/qemu/105/agent/get-fsinfo"
+ cases := []struct {
+ name, body string
+ status int
+ want string
+ rejected bool
+ }{
+ {"stopped", `{"message":"QEMU guest agent is not running"}`, 500, "agent-not-running", true},
+ {"stopped-same-target", "VM 105 qmp command 'guest-get-fsinfo' failed - QEMU guest agent is not running", 500, "agent-not-running", true},
+ {"unsupported", "unsupported command: guest-get-fsinfo", 500, "agent-error", true},
+ {"missing-command", "The command guest-get-fsinfo has not been found", 500, "agent-error", true},
+ {"unexplained", "provider-private-detail", 500, "agent-error", false},
+ {"other-guest", "VM 106 qmp command 'guest-get-fsinfo' failed - QEMU guest agent is not running", 500, "agent-error", false},
+ {"other-command", "unsupported command: guest-get-osinfo", 500, "agent-error", false},
+ {"conflicting", `{"message":"QEMU guest agent is not running","errors":{"message":"upstream unavailable"}}`, 500, "agent-error", false},
+ {"proxy-quotes-stopped", "QEMU guest agent is not running", 502, "agent-error", false},
+ {"bad-request-quotes-status", "API error 403: permission denied", 400, "agent-error", false},
+ {"unauthorised-quotes-stopped", "API error 500: QEMU guest agent is not running", 401, "permission-denied", false},
+ {"forbidden", "provider-private-detail", 403, "permission-denied", false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ reason := guestAgentTerminalFailureReason(path, []byte(tc.body))
+ err := &apiResponseError{
+ statusCode: tc.status, guestCommandRejected: tc.status == http.StatusInternalServerError && reason != "",
+ guestFailureReason: reason, cause: errors.New(tc.body),
+ }
+ if err.guestCommandRejected != tc.rejected {
+ t.Fatalf("terminal rejection = %t, want %t", err.guestCommandRejected, tc.rejected)
+ }
+ if got := GuestAgentErrorReason(fmt.Errorf("wrapped: %w", err)); got != tc.want {
+ t.Errorf("fixed reason = %q, want %q", got, tc.want)
+ }
+ if err.Error() != tc.body || !errors.Is(err, err.cause) {
+ t.Fatal("observed error text/cause was changed")
+ }
+ })
+ }
+ for _, err := range []error{nil, errors.New("API error 500: QEMU guest agent is not running")} {
+ if got := GuestAgentErrorReason(err); got != "" {
+ t.Errorf("untyped error acquired wire evidence: %q", got)
+ }
+ }
+ deferred := &guestAgentDeferredError{reason: "agent-cooldown"}
+ if got := GuestAgentErrorReason(fmt.Errorf("wrapped: %w", deferred)); got != "agent-cooldown" {
+ t.Errorf("uncertainty was reclassified: %q", got)
+ }
+}
diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.go
index 29aea7c3fb..9ca881dd12 100644
--- a/pkg/proxmox/guest_agent_response.go
+++ b/pkg/proxmox/guest_agent_response.go
@@ -3,8 +3,10 @@ package proxmox
import (
"bytes"
"encoding/json"
+ "errors"
"fmt"
"io"
+ "net/http"
"strings"
)
@@ -13,32 +15,56 @@ import (
type apiResponseError struct {
statusCode int
guestCommandRejected bool
+ guestFailureReason string
cause error
}
func (e *apiResponseError) Error() string { return e.cause.Error() }
func (e *apiResponseError) Unwrap() error { return e.cause }
+// GuestAgentErrorReason preserves observed HTTP/command evidence for callers
+// without treating numbers or instructions quoted in an error body as status.
+// It returns only existing fixed disk-status reasons, never provider text.
+func GuestAgentErrorReason(err error) string {
+ if reason := GuestAgentDeferredReason(err); reason != "" {
+ return reason
+ }
+ var response *apiResponseError
+ if !errors.As(err, &response) {
+ return ""
+ }
+ switch response.statusCode {
+ case http.StatusUnauthorized, http.StatusForbidden:
+ return "permission-denied"
+ case http.StatusInternalServerError:
+ if response.guestCommandRejected {
+ return response.guestFailureReason
+ }
+ }
+ return "agent-error"
+}
+
// Only specific, complete terminal failures from the command endpoint allow
// another command after an HTTP 500. An unexplained 5xx, even with a complete
// body, does not establish that a serial QGA command has finished.
-func guestAgentTerminalFailure(path string, body []byte) bool {
+func guestAgentTerminalFailureReason(path string, body []byte) string {
_, vmid, ok := guestAgentPath(path)
if !ok {
- return false
+ return ""
}
message, ok := guestAgentErrorMessage(body)
if !ok {
- return false
+ return ""
}
commandPath := strings.SplitN(path, "?", 2)[0]
command := "guest-" + commandPath[strings.LastIndex(commandPath, "/")+1:]
message = strings.TrimPrefix(message, fmt.Sprintf("VM %d qmp command '%s' failed - ", vmid, command))
switch message {
- case "QEMU guest agent is not running",
- "unsupported command: " + command,
+ case "QEMU guest agent is not running":
+ return "agent-not-running"
+ case "unsupported command: " + command,
"The command " + command + " has not been found":
- return true
+ return "agent-error"
}
// Preserve the existing OpenBSD OS-info skip rather than repeatedly probing
// its missing os-release file. Do not apply it to another guest command.
@@ -46,11 +72,11 @@ func guestAgentTerminalFailure(path string, body []byte) bool {
message = strings.TrimPrefix(message, "guest agent command failed: ")
for _, filename := range []string{"/etc/os-release", "/usr/lib/os-release"} {
if message == "Failed to open file '"+filename+"': No such file or directory" {
- return true
+ return "agent-error"
}
}
}
- return false
+ return ""
}
func guestAgentErrorMessage(body []byte) (string, bool) {