Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep guest filesystem failure guidance backup-safe
pulse-triage[bot]9daf5c6f7179
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 061a14d341..819e0ae7b5 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -231,6 +231,22 @@ required; all responses in this fixture are synthetic.### Guest-agent coordination around backups — issue #2439+Filesystem-query diagnostics preserve this safety boundary. Missing, empty,+unsupported or refused readings do not prescribe installing, activating or+restarting QGA, changing backup settings, re-running setup or broadening a+shared Proxmox role. Operator logs identify the existing guest and fixed reason,+lead to guest-local disk checks, and keep existing guest-agent/backup settings.+An actual HTTP 401/403 leads only to checking the existing credential's access+to that VM. A stopped-agent reason requires exact command-bound rejection+evidence, not an HTTP 500 or a status/diagnosis quoted in provider text.+Unsupported commands remain errors without claiming that QGA is stopped.+Deferrals retain their fixed reasons and do not gain new diagnostic commands.+`TestGuestFilesystemFailureGuidanceUsesObservedEvidence` crosses the real+client/collector/log boundary and preserves unavailable disk sentinels and+single-attempt uncertainty. The error-reason controls preserve original error+text/causes and terminal-rejection admission. These synthetic diagnostics are+not independent thaw, every-filesystem writes or native backup recovery.+All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfocallers, share fail-fast per-guest admission within this Pulse process. Clientsfor the same endpoint and configured cluster aliases coordinate across methods;diff --git a/internal/monitoring/guest_disk_stability.go b/internal/monitoring/guest_disk_stability.goindex 53a99390d5..9335800814 100644--- a/internal/monitoring/guest_disk_stability.go+++ b/internal/monitoring/guest_disk_stability.go@@ -1,6 +1,8 @@package monitoringimport (+ "context"+ "errors""strings""time"@@ -21,25 +23,17 @@ func classifyGuestAgentDiskStatusError(err error) string {return ""}- if reason := proxmox.GuestAgentDeferredReason(err); reason != "" {+ if reason := proxmox.GuestAgentErrorReason(err); reason != "" {return reason}- errStr := err.Error()- errStrLower := strings.ToLower(errStr)-+ // Non-Proxmox client implementations may return untyped local errors.+ // Never infer HTTP status or a stopped agent from numbers/body substrings.switch {- case strings.Contains(errStr, "QEMU guest agent is not running"):+ case err.Error() == "QEMU guest agent is not running":return "agent-not-running"- case strings.Contains(errStr, "timeout") || strings.Contains(errStr, "deadline exceeded"):+ case errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), "timeout"):return "agent-timeout"- case strings.Contains(errStr, "500") && (strings.Contains(errStr, "not running") || strings.Contains(errStr, "not available")):- return "agent-not-running"- case (strings.Contains(errStr, "403") || strings.Contains(errStr, "401")) &&- (strings.Contains(errStrLower, "permission") || strings.Contains(errStrLower, "forbidden") || strings.Contains(errStrLower, "not allowed")):- return "permission-denied"- case strings.Contains(errStr, "500"):- return "agent-not-running"default:return "agent-error"}diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex a8189bdafb..03b200e2ae 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -593,72 +593,40 @@ func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res prif reason := proxmox.GuestAgentDeferredReason(err); reason != "" {return nil, reason, false}- // Log more helpful error messages based on the error type- errMsg := err.Error()- if strings.Contains(errMsg, "500") || strings.Contains(errMsg, "QEMU guest agent is not running") {- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Int("vmid", res.VMID).- Msg("Guest agent enabled in VM config but not running inside guest OS. Install and start qemu-guest-agent in the VM")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("To verify: ssh into VM and run 'systemctl status qemu-guest-agent' or 'ps aux | grep qemu-ga'")- } else if strings.Contains(errMsg, "timeout") {- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Int("vmid", res.VMID).- Msg("Guest agent timeout - agent may be installed but not responding")- } else if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "401") || strings.Contains(errMsg, "authentication error") {- // Permission error - user/token lacks required permissions- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Int("vmid", res.VMID).- Msg("VM disk monitoring permission denied. Check permissions:")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("• Proxmox 9: Ensure token/user has VM.GuestAgent.Audit privilege (Pulse setup adds this via PulseMonitor role)")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("• Proxmox 8: Ensure token/user has VM.Monitor privilege (Pulse setup adds this via PulseMonitor role)")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("• All versions: Sys.Audit is recommended for Ceph metrics and applied when available")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("• Re-run Pulse setup script if node was added before v4.7")- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Msg("• Verify guest agent is installed and running inside the VM")- } else {- log.Debug().- Err(err).- Str("instance", instanceName).- Str("vm", res.Name).- Int("vmid", res.VMID).- Msg("Failed to get filesystem info from guest agent")- }+ reason := classifyGuestAgentDiskStatusError(err)+ logVMFilesystemUnavailable(instanceName, res, reason)return nil, classifyGuestAgentDiskStatusError(err), false}if len(fsInfo) == 0 {- log.Info().- Str("instance", instanceName).- Str("vm", res.Name).- Int("vmid", res.VMID).- Msg("Guest agent returned no filesystem info - agent may need restart or VM may have no mounted filesystems")+ logVMFilesystemUnavailable(instanceName, res, "no-filesystems")return nil, "no-filesystems", false}return fsInfo, "", true}+// Missing optional readings do not establish a need to activate/restart QGA,+// change backup policy, or reconfigure a shared Proxmox role. Keep this guidance+// independent of provider error text and the guest's operating system.+func logVMFilesystemUnavailable(instanceName string, res proxmox.ClusterResource, reason string) {+ message := "Guest filesystem query failed; check disk usage inside the guest. Keep existing guest-agent and backup settings."+ switch reason {+ case "agent-not-running":+ message = "Proxmox reports the guest agent is not running; check disk usage inside the guest. Keep existing guest-agent and backup settings."+ case "permission-denied":+ message = "Guest filesystem query was not authorised; check the existing credential's access to this VM. Do not broaden shared roles or change guest-agent and backup settings to diagnose missing readings."+ case "agent-timeout":+ message = "Guest filesystem query did not complete; check disk usage inside the guest. Keep existing guest-agent and backup settings."+ case "no-filesystems":+ message = "Guest agent returned no filesystem readings; check disk usage inside the guest. Keep existing guest-agent and backup settings."+ }+ log.Info().+ Str("instance", instanceName).+ Str("vm", res.Name).+ Int("vmid", res.VMID).+ Str("reason", reason).+ Msg(message)+}+func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterResource, fsInfo []proxmox.VMFileSystem) vmFSInfoSummary {summary := vmFSInfoSummary{}diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.goindex 93fd917bef..1d8beacbbf 100644--- a/internal/monitoring/monitor_pve_guest_lxc_test.go+++ b/internal/monitoring/monitor_pve_guest_lxc_test.go@@ -1,13 +1,23 @@package monitoringimport (+ "bytes""context"+ "encoding/json"+ "errors"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic""testing""time""github.com/rcourtman/pulse-go-rewrite/internal/models"agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/rs/zerolog"+ "github.com/rs/zerolog/log")type stubPVEClientLXCStatus struct {@@ -567,3 +577,129 @@ func TestIssue2148LXCRejectsAgentMemoryWithMismatchedTotal(t *testing.T) {t.Fatalf("memory used = %d, want provider value %d", container.Memory.Used, resource.Mem)}}++func TestGuestFilesystemStatusDoesNotDiagnoseFromErrorText(t *testing.T) {+ cases := []struct {+ err error+ want string+ }{+ {nil, ""},+ {errors.New("API error 500: internal server error"), "agent-error"},+ {errors.New("API error 500: unsupported command: guest-get-fsinfo"), "agent-error"},+ {errors.New("API error 500: QEMU guest agent is not running"), "agent-error"},+ {errors.New("request for VM 500 failed"), "agent-error"},+ {errors.New("API error 400: upstream API error 403: permission denied"), "agent-error"},+ {errors.New("QEMU guest agent is not running"), "agent-not-running"},+ {context.DeadlineExceeded, "agent-timeout"},+ {fmt.Errorf("wrapped: %w", context.DeadlineExceeded), "agent-timeout"},+ {errors.New("guest agent request timeout"), "agent-timeout"},+ }+ for _, tc := range cases {+ if got := classifyGuestAgentDiskStatusError(tc.err); got != tc.want {+ t.Errorf("%v: reason = %q, want %q", tc.err, got, tc.want)+ }+ }+}++// These synthetic responses exercise the production command guard, filesystem+// collector, unavailable sentinel and emitted operator guidance together. They+// perform no native QGA, backup, guest activation or recovery action.+func TestGuestFilesystemFailureGuidanceUsesObservedEvidence(t *testing.T) {+ cases := []struct {+ name, body, reason, message string+ status int+ logged bool+ }{+ {"stopped", `{"message":"QEMU guest agent is not running"}`, "agent-not-running", "Proxmox reports the guest agent is not running", 500, true},+ {"unsupported", "unsupported command: guest-get-fsinfo", "agent-error", "Guest filesystem query failed", 500, true},+ {"forbidden", "provider-private-detail", "permission-denied", "Guest filesystem query was not authorised", 403, true},+ {"unauthorised-quotes-stopped", "API error 500: QEMU guest agent is not running", "permission-denied", "Guest filesystem query was not authorised", 401, true},+ {"bad-request-quotes-permission", "API error 403: permission denied", "agent-error", "Guest filesystem query failed", 400, true},+ {"empty", `{"data":{"result":[]}}`, "no-filesystems", "Guest agent returned no filesystem readings", 200, true},+ {"malformed", `{"data":`, "agent-error", "Guest filesystem query failed", 200, true},+ {"server-uncertain", "provider-private-detail", "agent-completion-unverified", "", 500, false},+ {"gateway-quotes-stopped", "QEMU guest agent is not running", "agent-completion-unverified", "", 502, false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/status/current") {+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":0,"diskwrite":null,"netin":42}}`)+ return+ }+ if strings.HasSuffix(r.URL.Path, "/config") {+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ calls.Add(1)+ w.WriteHeader(tc.status)+ fmt.Fprint(w, tc.body)+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ var output bytes.Buffer+ previousLogger := log.Logger+ log.Logger = zerolog.New(&output)+ defer func() { log.Logger = previousLogger }()+ m := &Monitor{guestAgentFSInfoTimeout: time.Second, guestAgentRetries: 2}+ res := proxmox.ClusterResource{Node: "node", VMID: 105, Name: "fixture-vm", Type: "qemu", Status: "running", MaxDisk: 1000}+ total, used, free, usage, disks, fromAgent, reason := m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "fixture-instance", res, client, 1000, 0, 0)+ if reason != tc.reason || fromAgent || usage != -1 || total != 1000 || used != 0 || free != 1000 || disks != nil {+ t.Errorf("unavailable reading = %d/%d/%d/%v/%v/%t/%q", total, used, free, usage, disks, fromAgent, reason)+ }+ beforeStatus := time.Now()+ status, err := client.GetVMStatus(context.Background(), res.Node, res.VMID)+ afterStatus := time.Now()+ if err != nil {+ t.Fatal(err)+ }+ presence := status.IOCounters.Effective()+ if status.CPU != 0.25 || status.DiskRead != 0 || status.NetIn != 42 || !presence.DiskRead || !presence.NetworkIn || presence.DiskWrite || presence.NetworkOut || status.ObservedAt.Before(beforeStatus) || status.ObservedAt.After(afterStatus) {+ t.Errorf("filesystem failure changed independent live counters/presence/receipt: %+v", status)+ }+ if calls.Load() != 1 {+ t.Errorf("wire guest commands = %d, want one even with retries configured", calls.Load())+ }+ guidance := 0+ for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) {+ if len(line) == 0 {+ continue+ }+ var event map[string]interface{}+ if json.Unmarshal(line, &event) != nil {+ t.Fatalf("invalid structured log: %s", line)+ }+ if event["vmid"] != float64(res.VMID) {+ continue // Request-layer status logs are separate, not guidance.+ }+ guidance+++ message, _ := event["message"].(string)+ if !tc.logged || event["level"] != "info" || event["instance"] != "fixture-instance" || event["vm"] != "fixture-vm" || event["reason"] != tc.reason || !strings.HasPrefix(message, tc.message) {+ t.Errorf("observed-evidence guidance lost: %v", event)+ }+ if !strings.Contains(message, "guest-agent and backup settings") {+ t.Errorf("backup-settings precaution missing: %s", message)+ }+ if tc.reason == "permission-denied" && (!strings.Contains(message, "this VM") || !strings.Contains(message, "Do not broaden shared roles")) {+ t.Errorf("credential advice is not scoped: %s", message)+ }+ for _, unsafe := range []string{"Install and start", "restart", "systemctl", "ps aux", "Re-run", "PulseMonitor", "VM.Monitor", "Sys.Audit", "provider-private-detail"} {+ if strings.Contains(message, unsafe) {+ t.Errorf("unsafe/unobserved advice %q: %s", unsafe, message)+ }+ }+ }+ wantGuidance := 0+ if tc.logged {+ wantGuidance = 1+ }+ if guidance != wantGuidance {+ t.Errorf("guidance records = %d, want %d", guidance, wantGuidance)+ }+ })+ }+}diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex 2c6d24d098..dc4ffe9422 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -639,9 +639,11 @@ func (c *Client) requestWithRetryUsingClient(ctx context.Context, method, path sapiErr = fmt.Errorf("authentication error: %w", apiErr)}+ guestFailureReason := guestAgentTerminalFailureReason(path, body)return nil, &apiResponseError{statusCode: resp.StatusCode,- guestCommandRejected: resp.StatusCode == http.StatusInternalServerError && guestAgentTerminalFailure(path, body),+ guestCommandRejected: resp.StatusCode == http.StatusInternalServerError && guestFailureReason != "",+ guestFailureReason: guestFailureReason,cause: apiErr,}}diff --git a/pkg/proxmox/guest_agent_error_reason_test.go b/pkg/proxmox/guest_agent_error_reason_test.gonew file mode 100644index 0000000000..902fc25de2--- /dev/null+++ b/pkg/proxmox/guest_agent_error_reason_test.go@@ -0,0 +1,58 @@+package proxmox++import (+ "errors"+ "fmt"+ "net/http"+ "testing"+)++func TestGuestAgentErrorReasonKeepsWireAndCommandEvidence(t *testing.T) {+ const path = "/nodes/node/qemu/105/agent/get-fsinfo"+ cases := []struct {+ name, body string+ status int+ want string+ rejected bool+ }{+ {"stopped", `{"message":"QEMU guest agent is not running"}`, 500, "agent-not-running", true},+ {"stopped-same-target", "VM 105 qmp command 'guest-get-fsinfo' failed - QEMU guest agent is not running", 500, "agent-not-running", true},+ {"unsupported", "unsupported command: guest-get-fsinfo", 500, "agent-error", true},+ {"missing-command", "The command guest-get-fsinfo has not been found", 500, "agent-error", true},+ {"unexplained", "provider-private-detail", 500, "agent-error", false},+ {"other-guest", "VM 106 qmp command 'guest-get-fsinfo' failed - QEMU guest agent is not running", 500, "agent-error", false},+ {"other-command", "unsupported command: guest-get-osinfo", 500, "agent-error", false},+ {"conflicting", `{"message":"QEMU guest agent is not running","errors":{"message":"upstream unavailable"}}`, 500, "agent-error", false},+ {"proxy-quotes-stopped", "QEMU guest agent is not running", 502, "agent-error", false},+ {"bad-request-quotes-status", "API error 403: permission denied", 400, "agent-error", false},+ {"unauthorised-quotes-stopped", "API error 500: QEMU guest agent is not running", 401, "permission-denied", false},+ {"forbidden", "provider-private-detail", 403, "permission-denied", false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ reason := guestAgentTerminalFailureReason(path, []byte(tc.body))+ err := &apiResponseError{+ statusCode: tc.status, guestCommandRejected: tc.status == http.StatusInternalServerError && reason != "",+ guestFailureReason: reason, cause: errors.New(tc.body),+ }+ if err.guestCommandRejected != tc.rejected {+ t.Fatalf("terminal rejection = %t, want %t", err.guestCommandRejected, tc.rejected)+ }+ if got := GuestAgentErrorReason(fmt.Errorf("wrapped: %w", err)); got != tc.want {+ t.Errorf("fixed reason = %q, want %q", got, tc.want)+ }+ if err.Error() != tc.body || !errors.Is(err, err.cause) {+ t.Fatal("observed error text/cause was changed")+ }+ })+ }+ for _, err := range []error{nil, errors.New("API error 500: QEMU guest agent is not running")} {+ if got := GuestAgentErrorReason(err); got != "" {+ t.Errorf("untyped error acquired wire evidence: %q", got)+ }+ }+ deferred := &guestAgentDeferredError{reason: "agent-cooldown"}+ if got := GuestAgentErrorReason(fmt.Errorf("wrapped: %w", deferred)); got != "agent-cooldown" {+ t.Errorf("uncertainty was reclassified: %q", got)+ }+}diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.goindex 29aea7c3fb..9ca881dd12 100644--- a/pkg/proxmox/guest_agent_response.go+++ b/pkg/proxmox/guest_agent_response.go@@ -3,8 +3,10 @@ package proxmoximport ("bytes""encoding/json"+ "errors""fmt""io"+ "net/http""strings")@@ -13,32 +15,56 @@ import (type apiResponseError struct {statusCode intguestCommandRejected bool+ guestFailureReason stringcause error}func (e *apiResponseError) Error() string { return e.cause.Error() }func (e *apiResponseError) Unwrap() error { return e.cause }+// GuestAgentErrorReason preserves observed HTTP/command evidence for callers+// without treating numbers or instructions quoted in an error body as status.+// It returns only existing fixed disk-status reasons, never provider text.+func GuestAgentErrorReason(err error) string {+ if reason := GuestAgentDeferredReason(err); reason != "" {+ return reason+ }+ var response *apiResponseError+ if !errors.As(err, &response) {+ return ""+ }+ switch response.statusCode {+ case http.StatusUnauthorized, http.StatusForbidden:+ return "permission-denied"+ case http.StatusInternalServerError:+ if response.guestCommandRejected {+ return response.guestFailureReason+ }+ }+ return "agent-error"+}+// Only specific, complete terminal failures from the command endpoint allow// another command after an HTTP 500. An unexplained 5xx, even with a complete// body, does not establish that a serial QGA command has finished.-func guestAgentTerminalFailure(path string, body []byte) bool {+func guestAgentTerminalFailureReason(path string, body []byte) string {_, vmid, ok := guestAgentPath(path)if !ok {- return false+ return ""}message, ok := guestAgentErrorMessage(body)if !ok {- return false+ return ""}commandPath := strings.SplitN(path, "?", 2)[0]command := "guest-" + commandPath[strings.LastIndex(commandPath, "/")+1:]message = strings.TrimPrefix(message, fmt.Sprintf("VM %d qmp command '%s' failed - ", vmid, command))switch message {- case "QEMU guest agent is not running",- "unsupported command: " + command,+ case "QEMU guest agent is not running":+ return "agent-not-running"+ case "unsupported command: " + command,"The command " + command + " has not been found":- return true+ return "agent-error"}// Preserve the existing OpenBSD OS-info skip rather than repeatedly probing// its missing os-release file. Do not apply it to another guest command.@@ -46,11 +72,11 @@ func guestAgentTerminalFailure(path string, body []byte) bool {message = strings.TrimPrefix(message, "guest agent command failed: ")for _, filename := range []string{"/etc/os-release", "/usr/lib/os-release"} {if message == "Failed to open file '"+filename+"': No such file or directory" {- return true+ return "agent-error"}}}- return false+ return ""}func guestAgentErrorMessage(body []byte) (string, bool) {