Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reproduce false VM command and Discovery success

pulse-triage[bot]4188c2a9bad3
diff --git a/internal/hostagent/command_client_test.go b/internal/hostagent/command_client_test.go
index 7c9f5ee1fe..5c5d8f04d2 100644
--- a/internal/hostagent/command_client_test.go
+++ b/internal/hostagent/command_client_test.go
@@ -526,3 +526,7 @@ func TestGuestExecRealServerAgentDiscoveryAdmission(t *testing.T) {
func TestCommandClientGuestCompletionContract(t *testing.T) {
TestGuestExecZeroCLIExitDoesNotProveGuestCompletion(t)
}
+
+func TestCommandClientGuestOutcomeContract(t *testing.T) {
+ testGuestExecProjectsGuestOutcome(t)
+}
diff --git a/internal/hostagent/guest_exec_outcome_test.go b/internal/hostagent/guest_exec_outcome_test.go
new file mode 100644
index 0000000000..1ca29dec94
--- /dev/null
+++ b/internal/hostagent/guest_exec_outcome_test.go
@@ -0,0 +1,78 @@
+package hostagent
+
+import (
+ "context"
+ "os/exec"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
+)
+
+// The provider is a local fixture, never QGA. Exercise the production command
+// boundary on the unchanged parent as well as on the corrected source.
+func testGuestExecProjectsGuestOutcome(t *testing.T) {
+ for _, tc := range []struct {
+ name, reply, stdout, stderr string
+ exitCode int
+ success, deferred bool
+ }{
+ {"successful guest", `{"exited":1,"exitcode":0,"out-data":"guest\nobservation café","err-data":"guest warning"}`, "guest\nobservation café", "guest warning", 0, true, false},
+ {"empty success", `{"exited":true,"exitcode":0}`, "", "", 0, true, false},
+ {"guest failure", `{"exited":1,"exitcode":7,"out-data":"misleading service identity","err-data":"failed probe"}`, "misleading service identity", "failed probe", 7, false, false},
+ {"guest signal", `{"exited":1,"signal":15,"out-data":"partial"}`, "partial", "", 143, false, false},
+ {"truncated stdout", `{"exited":1,"exitcode":0,"out-data":"partial","out-truncated":true}`, "partial", "", 0, false, false},
+ {"truncated stderr", `{"exited":1,"exitcode":0,"err-data":"partial","err-truncated":1}`, "", "partial", 0, false, false},
+ {"missing outcome", `{"exited":1}`, "", "", 0, false, true},
+ {"null outcome", `{"exited":1,"exitcode":null}`, "", "", 0, false, true},
+ {"string outcome", `{"exited":1,"exitcode":"0"}`, "", "", 0, false, true},
+ {"fractional outcome", `{"exited":1,"exitcode":0.5}`, "", "", 0, false, true},
+ {"negative outcome", `{"exited":1,"exitcode":-1}`, "", "", 0, false, true},
+ {"out of range outcome", `{"exited":1,"exitcode":256}`, "", "", 0, false, true},
+ {"competing outcomes", `{"exited":1,"exitcode":0,"signal":15}`, "", "", 0, false, true},
+ {"duplicate outcome", `{"exited":1,"exitcode":7,"exitcode":0}`, "", "", 0, false, true},
+ {"case conflicting outcome", `{"exited":1,"exitcode":0,"EXITCODE":7}`, "", "", 0, false, true},
+ {"wrong output type", `{"exited":1,"exitcode":0,"out-data":{}}`, "", "", 0, false, true},
+ {"ambiguous truncation", `{"exited":1,"exitcode":0,"out-truncated":"false"}`, "", "", 0, false, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var calls, reads atomic.Int32
+ guard := newGuestExecGuard(func(context.Context, string) ([]byte, error) {
+ reads.Add(1)
+ return []byte("name: vm\n"), nil
+ })
+ c := &CommandClient{guestExecAdmission: guard}
+ oldExec := execCommandContext
+ execCommandContext = func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
+ calls.Add(1)
+ return exec.CommandContext(ctx, "sh", "-c", "printf '%s' "+shellQuote(tc.reply))
+ }
+ t.Cleanup(func() { execCommandContext = oldExec })
+ payload := testApprovedCommandPayload(t, c, executeCommandPayload{Command: "echo observed", TargetType: "vm", TargetID: "105", Trusted: true})
+ got := c.executeCommand(context.Background(), payload)
+ if tc.deferred {
+ if got.Success || !agentexec.IsGuestExecDeferred(got.Error) || got.Stdout != tc.reply || got.ExitCode != 0 {
+ t.Fatalf("ambiguous result became guest evidence: %#v", got)
+ }
+ next := c.executeCommand(context.Background(), payload)
+ if next.Success || calls.Load() != 1 || reads.Load() != 1 || !agentexec.IsGuestExecDeferred(next.Error) {
+ t.Fatalf("ambiguous result released admission: %#v calls=%d reads=%d", next, calls.Load(), reads.Load())
+ }
+ return
+ }
+ if got.Success != tc.success || got.ExitCode != tc.exitCode || got.Stdout != tc.stdout || got.Stderr != tc.stderr || agentexec.IsGuestExecDeferred(got.Error) || (got.Error == "") != tc.success {
+ t.Fatalf("CLI success replaced actual guest outcome: %#v", got)
+ }
+ if strings.Contains(got.Error, "misleading") || strings.Contains(got.Error, "failed probe") {
+ t.Fatal("guest output entered the authored error")
+ }
+ // A known failed/truncated terminal result is not an unknown QGA
+ // handoff. A distinct explicit request still checks the VM lock.
+ _ = c.executeCommand(context.Background(), payload)
+ if calls.Load() != 2 || reads.Load() != 4 {
+ t.Fatalf("terminal failure acquired uncertainty or skipped lock checks: calls=%d reads=%d", calls.Load(), reads.Load())
+ }
+ })
+ }
+}
diff --git a/internal/servicediscovery/command_evidence_test.go b/internal/servicediscovery/command_evidence_test.go
new file mode 100644
index 0000000000..18063f323a
--- /dev/null
+++ b/internal/servicediscovery/command_evidence_test.go
@@ -0,0 +1,105 @@
+package servicediscovery
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+)
+
+type commandEvidenceExecutor struct {
+ *stubExecutor
+ mode string
+}
+
+func (e *commandEvidenceExecutor) ExecuteCommand(_ context.Context, _ string, p ExecuteCommandPayload) (*CommandResultPayload, error) {
+ if e.mode == "nil" {
+ return nil, nil
+ }
+ if e.mode == "healthy" || (e.mode == "partial" && p.Command == "hostname") {
+ return &CommandResultPayload{RequestID: p.RequestID, Success: true, ExitCode: 0, Stdout: "verified observation", Stderr: "warning is not evidence"}, nil
+ }
+ return &CommandResultPayload{RequestID: p.RequestID, Success: e.mode == "contradictory", ExitCode: 7, Stdout: "misleading service identity", Stderr: "private failure text"}, nil
+}
+
+func testDiscoveryRejectsFailedCommandEvidence(t *testing.T) {
+ for _, rt := range []ResourceType{ResourceTypeVM, ResourceTypeDockerVM, ResourceTypeSystemContainer, ResourceTypeDocker, ResourceTypeAgent} {
+ for _, mode := range []string{"failure", "nil", "contradictory"} {
+ t.Run(string(rt)+"/"+mode, func(t *testing.T) {
+ id := "105"
+ if rt == ResourceTypeDockerVM {
+ id = "105:app"
+ }
+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: mode}
+ scanner := NewDeepScanner(e)
+ var terminal *DiscoveryProgress
+ scanner.SetProgressCallback(func(p *DiscoveryProgress) { copy := *p; terminal = &copy })
+ result, err := scanner.Scan(context.Background(), DiscoveryRequest{ResourceType: rt, ResourceID: id, TargetID: "node"})
+ if result == nil || len(result.CommandOutputs) != 0 || err == nil || terminal == nil || terminal.Error == "" || terminal.PercentComplete == 100 {
+ t.Fatalf("failed commands became fresh evidence: result=%+v err=%v progress=%+v", result, err, terminal)
+ }
+ if strings.Contains(err.Error(), "private") || strings.Contains(terminal.Error, "private") {
+ t.Fatal("command output entered the scan-level error")
+ }
+ })
+ }
+ }
+ for _, mode := range []string{"healthy", "partial"} {
+ t.Run(mode, func(t *testing.T) {
+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: mode}
+ scanner := NewDeepScanner(e)
+ result, err := scanner.ScanVM(context.Background(), "node", "node", "105")
+ want := len(GetCommandsForResource(ResourceTypeVM))
+ if mode == "partial" {
+ want = 1 // Optional failures do not discard independent healthy evidence.
+ }
+ if err != nil || result == nil || len(result.CommandOutputs) != want {
+ t.Fatalf("independent successful observations lost: %+v %v", result, err)
+ }
+ for _, output := range result.CommandOutputs {
+ if output != "verified observation" {
+ t.Fatalf("warning/failure became evidence: %q", output)
+ }
+ }
+ })
+ }
+}
+
+func testDiscoveryAllFailedCommandsKeepSavedEvidence(t *testing.T) {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ store.crypto = nil
+ saved := &ResourceDiscovery{ID: MakeResourceID(ResourceTypeVM, "node", "105"), ResourceType: ResourceTypeVM, ResourceID: "105", TargetID: "node", ServiceName: "saved observation", UserNotes: "keep notes", RawCommandOutput: map[string]string{"old": "retain"}, UpdatedAt: time.Now().Add(-time.Hour), DiscoveredAt: time.Now().Add(-time.Hour)}
+ if err := store.Save(saved); err != nil {
+ t.Fatal(err)
+ }
+ before, _ := store.Get(saved.ID)
+ beforeJSON, _ := json.Marshal(before)
+ e := &commandEvidenceExecutor{stubExecutor: &stubExecutor{agents: []ConnectedAgent{{AgentID: "node", Hostname: "node"}}}, mode: "failure"}
+ scanner := NewDeepScanner(e)
+ cfg := DefaultConfig()
+ cfg.CommandScanning = true
+ service := NewService(store, scanner, cfg)
+ analyzer := &stubAnalyzer{response: `{"service_type":"nginx","confidence":0.9}`}
+ service.SetAIAnalyzer(analyzer)
+ req := DiscoveryRequest{ResourceType: ResourceTypeVM, ResourceID: "105", TargetID: "node", Force: true}
+ if fresh, err := service.DiscoverResource(context.Background(), req); fresh != nil || err == nil {
+ t.Fatalf("failed manual scan became replacement: %+v %v", fresh, err)
+ }
+ after, _ := store.Get(saved.ID)
+ afterJSON, _ := json.Marshal(after)
+ if string(beforeJSON) != string(afterJSON) || analyzer.calls != 0 {
+ t.Fatal("failed scan changed saved evidence/notes or sent failures to the analyzer")
+ }
+ if got := service.enhanceWithDeepScan(context.Background(), saved, DockerHost{}); got != saved || analyzer.calls != 0 {
+ t.Fatal("failed automatic scan fabricated new evidence")
+ }
+ // Recovery is an explicit new scan, not a retry of the failed command.
+ e.mode = "healthy"
+ if fresh, err := service.DiscoverResource(context.Background(), req); fresh == nil || err != nil || analyzer.calls != 1 || fresh.UserNotes != saved.UserNotes {
+ t.Fatalf("fresh explicit scan failed to resume: %+v %v analyzer=%d", fresh, err, analyzer.calls)
+ }
+}
diff --git a/internal/servicediscovery/service_test.go b/internal/servicediscovery/service_test.go
index 734f651557..a9828d1fae 100644
--- a/internal/servicediscovery/service_test.go
+++ b/internal/servicediscovery/service_test.go
@@ -2765,6 +2765,11 @@ func TestDiscoveryGuestSafetyStopsAndPreservesSavedEvidence(t *testing.T) {
testDiscoveryGuestSafetyStopsAndPreservesSavedEvidence(t)
}
+func TestDiscoveryFailedCommandEvidenceContract(t *testing.T) {
+ testDiscoveryRejectsFailedCommandEvidence(t)
+ testDiscoveryAllFailedCommandsKeepSavedEvidence(t)
+}
+
func TestDiscoveryGuestCancellationContract(t *testing.T) {
TestVMDiscoveryCancellationIsNotSuccessfulEvidence(t)
}