Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reproduce false VM command and Discovery success
pulse-triage[bot]4188c2a9bad3
diff --git a/internal/hostagent/command_client_test.go b/internal/hostagent/command_client_test.goindex 7c9f5ee1fe..5c5d8f04d2 100644--- a/internal/hostagent/command_client_test.go+++ b/internal/hostagent/command_client_test.go@@ -526,3 +526,7 @@ func TestGuestExecRealServerAgentDiscoveryAdmission(t *testing.T) {func TestCommandClientGuestCompletionContract(t *testing.T) {TestGuestExecZeroCLIExitDoesNotProveGuestCompletion(t)}++func TestCommandClientGuestOutcomeContract(t *testing.T) {+ testGuestExecProjectsGuestOutcome(t)+}diff --git a/internal/hostagent/guest_exec_outcome_test.go b/internal/hostagent/guest_exec_outcome_test.gonew file mode 100644index 0000000000..1ca29dec94--- /dev/null+++ b/internal/hostagent/guest_exec_outcome_test.go@@ -0,0 +1,78 @@+package hostagent++import (+ "context"+ "os/exec"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/agentexec"+)++// The provider is a local fixture, never QGA. Exercise the production command+// boundary on the unchanged parent as well as on the corrected source.+func testGuestExecProjectsGuestOutcome(t *testing.T) {+ for _, tc := range []struct {+ name, reply, stdout, stderr string+ exitCode int+ success, deferred bool+ }{+ {"successful guest", `{"exited":1,"exitcode":0,"out-data":"guest\nobservation café","err-data":"guest warning"}`, "guest\nobservation café", "guest warning", 0, true, false},+ {"empty success", `{"exited":true,"exitcode":0}`, "", "", 0, true, false},+ {"guest failure", `{"exited":1,"exitcode":7,"out-data":"misleading service identity","err-data":"failed probe"}`, "misleading service identity", "failed probe", 7, false, false},+ {"guest signal", `{"exited":1,"signal":15,"out-data":"partial"}`, "partial", "", 143, false, false},+ {"truncated stdout", `{"exited":1,"exitcode":0,"out-data":"partial","out-truncated":true}`, "partial", "", 0, false, false},+ {"truncated stderr", `{"exited":1,"exitcode":0,"err-data":"partial","err-truncated":1}`, "", "partial", 0, false, false},+ {"missing outcome", `{"exited":1}`, "", "", 0, false, true},+ {"null outcome", `{"exited":1,"exitcode":null}`, "", "", 0, false, true},+ {"string outcome", `{"exited":1,"exitcode":"0"}`, "", "", 0, false, true},+ {"fractional outcome", `{"exited":1,"exitcode":0.5}`, "", "", 0, false, true},+ {"negative outcome", `{"exited":1,"exitcode":-1}`, "", "", 0, false, true},+ {"out of range outcome", `{"exited":1,"exitcode":256}`, "", "", 0, false, true},+ {"competing outcomes", `{"exited":1,"exitcode":0,"signal":15}`, "", "", 0, false, true},+ {"duplicate outcome", `{"exited":1,"exitcode":7,"exitcode":0}`, "", "", 0, false, true},+ {"case conflicting outcome", `{"exited":1,"exitcode":0,"EXITCODE":7}`, "", "", 0, false, true},+ {"wrong output type", `{"exited":1,"exitcode":0,"out-data":{}}`, "", "", 0, false, true},+ {"ambiguous truncation", `{"exited":1,"exitcode":0,"out-truncated":"false"}`, "", "", 0, false, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ var calls, reads atomic.Int32+ guard := newGuestExecGuard(func(context.Context, string) ([]byte, error) {+ reads.Add(1)+ return []byte("name: vm\n"), nil+ })+ c := &CommandClient{guestExecAdmission: guard}+ oldExec := execCommandContext+ execCommandContext = func(ctx context.Context, _ string, _ ...string) *exec.Cmd {+ calls.Add(1)+ return exec.CommandContext(ctx, "sh", "-c", "printf '%s' "+shellQuote(tc.reply))+ }+ t.Cleanup(func() { execCommandContext = oldExec })+ payload := testApprovedCommandPayload(t, c, executeCommandPayload{Command: "echo observed", TargetType: "vm", TargetID: "105", Trusted: true})+ got := c.executeCommand(context.Background(), payload)+ if tc.deferred {+ if got.Success || !agentexec.IsGuestExecDeferred(got.Error) || got.Stdout != tc.reply || got.ExitCode != 0 {+ t.Fatalf("ambiguous result became guest evidence: %#v", got)+ }+ next := c.executeCommand(context.Background(), payload)+ if next.Success || calls.Load() != 1 || reads.Load() != 1 || !agentexec.IsGuestExecDeferred(next.Error) {+ t.Fatalf("ambiguous result released admission: %#v calls=%d reads=%d", next, calls.Load(), reads.Load())+ }+ return+ }+ if got.Success != tc.success || got.ExitCode != tc.exitCode || got.Stdout != tc.stdout || got.Stderr != tc.stderr || agentexec.IsGuestExecDeferred(got.Error) || (got.Error == "") != tc.success {+ t.Fatalf("CLI success replaced actual guest outcome: %#v", got)+ }+ if strings.Contains(got.Error, "misleading") || strings.Contains(got.Error, "failed probe") {+ t.Fatal("guest output entered the authored error")+ }+ // A known failed/truncated terminal result is not an unknown QGA+ // handoff. A distinct explicit request still checks the VM lock.+ _ = c.executeCommand(context.Background(), payload)+ if calls.Load() != 2 || reads.Load() != 4 {+ t.Fatalf("terminal failure acquired uncertainty or skipped lock checks: calls=%d reads=%d", calls.Load(), reads.Load())+ }+ })+ }+}diff --git a/internal/servicediscovery/command_evidence_test.go b/internal/servicediscovery/command_evidence_test.gonew file mode 100644index 0000000000..18063f323a--- /dev/null+++ b/internal/servicediscovery/command_evidence_test.go@@ -0,0 +1,105 @@+package servicediscovery++import (+ "context"+ "encoding/json"+ "strings"+ "testing"+ "time"+)++type commandEvidenceExecutor struct {+ *stubExecutor+ mode string+}++func (e *commandEvidenceExecutor) ExecuteCommand(_ context.Context, _ string, p ExecuteCommandPayload) (*CommandResultPayload, error) {+ if e.mode == "nil" {+ return nil, nil+ }+ if e.mode == "healthy" || (e.mode == "partial" && p.Command == "hostname") {+ return &CommandResultPayload{RequestID: p.RequestID, Success: true, ExitCode: 0, Stdout: "verified observation", Stderr: "warning is not evidence"}, nil+ }+ return &CommandResultPayload{RequestID: p.RequestID, Success: e.mode == "contradictory", ExitCode: 7, Stdout: "misleading service identity", Stderr: "private failure text"}, nil+}++func testDiscoveryRejectsFailedCommandEvidence(t *testing.T) {+ for _, rt := range []ResourceType{ResourceTypeVM, ResourceTypeDockerVM, ResourceTypeSystemContainer, ResourceTypeDocker, ResourceTypeAgent} {+ for _, mode := range []string{"failure", "nil", "contradictory"} {+ t.Run(string(rt)+"/"+mode, func(t *testing.T) {+ id := "105"+ if rt == ResourceTypeDockerVM {+ id = "105:app"+ }+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: mode}+ scanner := NewDeepScanner(e)+ var terminal *DiscoveryProgress+ scanner.SetProgressCallback(func(p *DiscoveryProgress) { copy := *p; terminal = © })+ result, err := scanner.Scan(context.Background(), DiscoveryRequest{ResourceType: rt, ResourceID: id, TargetID: "node"})+ if result == nil || len(result.CommandOutputs) != 0 || err == nil || terminal == nil || terminal.Error == "" || terminal.PercentComplete == 100 {+ t.Fatalf("failed commands became fresh evidence: result=%+v err=%v progress=%+v", result, err, terminal)+ }+ if strings.Contains(err.Error(), "private") || strings.Contains(terminal.Error, "private") {+ t.Fatal("command output entered the scan-level error")+ }+ })+ }+ }+ for _, mode := range []string{"healthy", "partial"} {+ t.Run(mode, func(t *testing.T) {+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: mode}+ scanner := NewDeepScanner(e)+ result, err := scanner.ScanVM(context.Background(), "node", "node", "105")+ want := len(GetCommandsForResource(ResourceTypeVM))+ if mode == "partial" {+ want = 1 // Optional failures do not discard independent healthy evidence.+ }+ if err != nil || result == nil || len(result.CommandOutputs) != want {+ t.Fatalf("independent successful observations lost: %+v %v", result, err)+ }+ for _, output := range result.CommandOutputs {+ if output != "verified observation" {+ t.Fatalf("warning/failure became evidence: %q", output)+ }+ }+ })+ }+}++func testDiscoveryAllFailedCommandsKeepSavedEvidence(t *testing.T) {+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ store.crypto = nil+ saved := &ResourceDiscovery{ID: MakeResourceID(ResourceTypeVM, "node", "105"), ResourceType: ResourceTypeVM, ResourceID: "105", TargetID: "node", ServiceName: "saved observation", UserNotes: "keep notes", RawCommandOutput: map[string]string{"old": "retain"}, UpdatedAt: time.Now().Add(-time.Hour), DiscoveredAt: time.Now().Add(-time.Hour)}+ if err := store.Save(saved); err != nil {+ t.Fatal(err)+ }+ before, _ := store.Get(saved.ID)+ beforeJSON, _ := json.Marshal(before)+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: "failure"}+ scanner := NewDeepScanner(e)+ cfg := DefaultConfig()+ cfg.CommandScanning = true+ service := NewService(store, scanner, cfg)+ analyzer := &stubAnalyzer{response: `{"service_type":"nginx","confidence":0.9}`}+ service.SetAIAnalyzer(analyzer)+ req := DiscoveryRequest{ResourceType: ResourceTypeVM, ResourceID: "105", TargetID: "node", Force: true}+ if fresh, err := service.DiscoverResource(context.Background(), req); fresh != nil || err == nil {+ t.Fatalf("failed manual scan became replacement: %+v %v", fresh, err)+ }+ after, _ := store.Get(saved.ID)+ afterJSON, _ := json.Marshal(after)+ if string(beforeJSON) != string(afterJSON) || analyzer.calls != 0 {+ t.Fatal("failed scan changed saved evidence/notes or sent failures to the analyzer")+ }+ if got := service.enhanceWithDeepScan(context.Background(), saved, DockerHost{}); got != saved || analyzer.calls != 0 {+ t.Fatal("failed automatic scan fabricated new evidence")+ }+ // Recovery is an explicit new scan, not a retry of the failed command.+ e.mode = "healthy"+ if fresh, err := service.DiscoverResource(context.Background(), req); fresh == nil || err != nil || analyzer.calls != 1 || fresh.UserNotes != saved.UserNotes {+ t.Fatalf("fresh explicit scan failed to resume: %+v %v analyzer=%d", fresh, err, analyzer.calls)+ }+}diff --git a/internal/servicediscovery/service_test.go b/internal/servicediscovery/service_test.goindex 734f651557..a9828d1fae 100644--- a/internal/servicediscovery/service_test.go+++ b/internal/servicediscovery/service_test.go@@ -2765,6 +2765,11 @@ func TestDiscoveryGuestSafetyStopsAndPreservesSavedEvidence(t *testing.T) {testDiscoveryGuestSafetyStopsAndPreservesSavedEvidence(t)}+func TestDiscoveryFailedCommandEvidenceContract(t *testing.T) {+ testDiscoveryRejectsFailedCommandEvidence(t)+ testDiscoveryAllFailedCommandsKeepSavedEvidence(t)+}+func TestDiscoveryGuestCancellationContract(t *testing.T) {TestVMDiscoveryCancellationIsNotSuccessfulEvidence(t)}