Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package resourceapi
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// Report-merge on a resource an operator link folded together splits it on
// every surface, as unlink does. It used to record exclusions only against
// candidate IDs derived from the merged resource's type, which name neither
// side of the link, so the link row survived and the pair stayed merged while
// the API answered "Merge reported" and the drawer toasted "Resource split
// applied". REST seeds from the monitor's already-linked listing, as in
// production, so the folded row is never in its registry.
func TestResourceReportMergeSplitsOperatorLinkedPair(t *testing.T) {
now := time.Now().UTC()
vm := models.VM{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}
agent := models.Host{ID: "host-box", Hostname: "box-agent", MachineID: "fedcba9876543210", Status: "online", LastSeen: now}
node := models.Node{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://10.0.0.5:8006", Status: "online", LastSeen: now}
// The agent and the Docker host take canonical IDs derived from their
// machine IDs, which no source-specific candidate reproduces, so a
// same-type link between them fails the same way a cross-type one does.
docker := models.DockerHost{ID: "docker-box", Hostname: "dock-box", MachineID: "0011223344556677", Status: "online", LastSeen: now}
// Two linked agents share their only source, which the API used to refuse
// as "Resource is not merged".
spare := models.Host{ID: "host-spare", Hostname: "spare-agent", MachineID: "8899aabbccddeeff", Status: "online", LastSeen: now}
for _, tc := range []struct {
name string
snapshot models.StateSnapshot
// linkFromAgent links from the agent's side, as the agent's drawer
// would; otherwise from the other resource's side.
linkFromAgent bool
}{
{"vm-agent", models.StateSnapshot{VMs: []models.VM{vm}, Hosts: []models.Host{agent}}, false},
{"vm-agent-from-agent", models.StateSnapshot{VMs: []models.VM{vm}, Hosts: []models.Host{agent}}, true},
{"node-agent", models.StateSnapshot{Nodes: []models.Node{node}, Hosts: []models.Host{agent}}, false},
{"node-agent-from-agent", models.StateSnapshot{Nodes: []models.Node{node}, Hosts: []models.Host{agent}}, true},
{"docker-agent", models.StateSnapshot{DockerHosts: []models.DockerHost{docker}, Hosts: []models.Host{agent}}, false},
{"docker-agent-from-agent", models.StateSnapshot{DockerHosts: []models.DockerHost{docker}, Hosts: []models.Host{agent}}, true},
{"agent-agent", models.StateSnapshot{Hosts: []models.Host{spare, agent}}, false},
} {
t.Run(tc.name, func(t *testing.T) {
tc.snapshot.LastUpdate = now
h, store, rebuild := newReportMergeHarness(t, tc.snapshot)
resources := rebuild()
if len(resources) != 2 {
t.Fatalf("fixture produced %d resources, want 2 apart: %+v", len(resources), resources)
}
var agentID, otherID string
for _, resource := range resources {
if resource.Name == agent.Hostname {
agentID = resource.ID
} else {
otherID = resource.ID
}
}
if agentID == "" || otherID == "" {
t.Fatalf("fixture lacks a standalone agent: %+v", resources)
}
fromID, targetID := otherID, agentID
if tc.linkFromAgent {
fromID, targetID = agentID, otherID
}
postResourceAction(t, h, fromID, "link", map[string]any{"targetId": targetID})
linked := rebuild()
if len(linked) != 1 {
t.Fatalf("link left %d monitor resources, want one: %+v", len(linked), linked)
}
assertRESTResourceCount(t, h, 1)
// Report as the drawer's Split dialog does: every merged source.
sources := make([]string, 0, len(linked[0].Sources))
for _, source := range linked[0].Sources {
sources = append(sources, string(source))
}
postResourceAction(t, h, linked[0].ID, "report-merge", map[string]any{"sources": sources})
if split := rebuild(); len(split) != 2 {
t.Fatalf("report-merge left %d monitor resources, want the pair apart: %+v", len(split), split)
}
assertRESTResourceCount(t, h, 2)
if links, err := store.GetLinks(); err != nil || len(links) != 0 {
t.Fatalf("report-merge left links %+v (err %v)", links, err)
}
exclusions, err := store.GetExclusions()
if err != nil {
t.Fatal(err)
}
excludedPair := false
for _, exclusion := range exclusions {
if (exclusion.ResourceA == agentID && exclusion.ResourceB == otherID) ||
(exclusion.ResourceA == otherID && exclusion.ResourceB == agentID) {
excludedPair = true
}
}
if !excludedPair {
t.Fatalf("report-merge did not exclude the linked pair %s/%s: %+v", agentID, otherID, exclusions)
}
})
}
}
// A source filter undoes only the links whose folded side brought a named
// source in.
func TestResourceReportMergeSourceFilterSelectsLinks(t *testing.T) {
now := time.Now().UTC()
snapshot := models.StateSnapshot{
LastUpdate: now,
VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}},
Hosts: []models.Host{{ID: "host-box", Hostname: "box-agent", MachineID: "fedcba9876543210", Status: "online", LastSeen: now}},
}
h, store, rebuild := newReportMergeHarness(t, snapshot)
var vmID, agentID string
for _, resource := range rebuild() {
switch resource.Type {
case unified.ResourceTypeVM:
vmID = resource.ID
case unified.ResourceTypeAgent:
agentID = resource.ID
}
}
postResourceAction(t, h, vmID, "link", map[string]any{"targetId": agentID})
rebuild()
// The VM's own source was not folded in by the link.
postResourceAction(t, h, vmID, "report-merge", map[string]any{"sources": []string{"proxmox"}})
if links, err := store.GetLinks(); err != nil || len(links) != 1 {
t.Fatalf("a filter naming only the VM's own source changed the link: %+v (err %v)", links, err)
}
postResourceAction(t, h, vmID, "report-merge", map[string]any{"sources": []string{"agent"}})
if links, err := store.GetLinks(); err != nil || len(links) != 0 {
t.Fatalf("a filter naming the agent left the link: %+v (err %v)", links, err)
}
if split := rebuild(); len(split) != 2 {
t.Fatalf("report-merge left %d monitor resources, want two: %+v", len(split), split)
}
}
// A source filter detaches the members of a chain that carry the source. A
// Docker host linked into an agent that is linked into a VM leaves two folds,
// and the agent's fold lists the Docker source as well, because its subtree
// holds the Docker host. Naming Docker used to undo the VM-agent link too, and
// the pair was apart on every surface; now only the link that holds the Docker
// host goes. Naming the agent detaches the agent from the VM and keeps the
// Docker host in the agent, which the report does not name.
func TestResourceReportMergeSourceFilterDetachesChainMembers(t *testing.T) {
now := time.Now().UTC()
snapshot := models.StateSnapshot{
LastUpdate: now,
VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}},
Hosts: []models.Host{{ID: "host-box", Hostname: "box-agent", MachineID: "fedcba9876543210", Status: "online", LastSeen: now}},
DockerHosts: []models.DockerHost{{ID: "docker-box", Hostname: "dock-box", MachineID: "0011223344556677", Status: "online", LastSeen: now}},
}
for _, tc := range []struct {
name string
sources []string
// want lists the sources of each resource left standing, and links
// how many of the chain's two links the report leaves in the store.
want []string
links int
}{
{"leaf member", []string{"docker"}, []string{"agent+proxmox", "docker"}, 1},
{"member that took another in", []string{"agent"}, []string{"agent+docker", "proxmox"}, 1},
{"both members", []string{"agent", "docker"}, []string{"agent", "docker", "proxmox"}, 0},
{"every source", []string{"proxmox", "agent", "docker"}, []string{"agent", "docker", "proxmox"}, 0},
{"no filter", nil, []string{"agent", "docker", "proxmox"}, 0},
} {
t.Run(tc.name, func(t *testing.T) {
h, store, rebuild := newReportMergeHarness(t, snapshot)
var vmID, agentID, dockerID string
for _, resource := range rebuild() {
switch {
case resource.Type == unified.ResourceTypeVM:
vmID = resource.ID
case resource.Agent != nil:
agentID = resource.ID
case resource.Docker != nil:
dockerID = resource.ID
}
}
if vmID == "" || agentID == "" || dockerID == "" {
t.Fatalf("fixture lacks a standalone VM, agent and Docker host: %+v", rebuild())
}
// Links apply in store order: the agent takes in the Docker host,
// then the VM takes in the agent.
postResourceAction(t, h, agentID, "link", map[string]any{"targetId": dockerID})
postResourceAction(t, h, vmID, "link", map[string]any{"targetId": agentID})
chained := rebuild()
if got := resourceSourceGroups(chained); !slices.Equal(got, []string{"agent+docker+proxmox"}) {
t.Fatalf("links left %v, want the chain folded into one resource", got)
}
assertRESTSourceGroups(t, h, []string{"agent+docker+proxmox"})
body := map[string]any{}
if tc.sources != nil {
body["sources"] = tc.sources
}
postResourceAction(t, h, chained[0].ID, "report-merge", body)
if got := resourceSourceGroups(rebuild()); !slices.Equal(got, tc.want) {
t.Fatalf("monitor lists %v after reporting %v, want %v", got, tc.sources, tc.want)
}
assertRESTSourceGroups(t, h, tc.want)
if links, err := store.GetLinks(); err != nil || len(links) != tc.links {
t.Fatalf("report-merge left %d links (err %v), want %d: %+v", len(links), err, tc.links, links)
}
})
}
}
// resourceSourceGroups lists the sources of each resource, sorted, as
// "a+b" groups in sorted order.
func resourceSourceGroups(resources []unified.Resource) []string {
groups := make([]string, 0, len(resources))
for _, resource := range resources {
sources := make([]string, 0, len(resource.Sources))
for _, source := range resource.Sources {
sources = append(sources, string(source))
}
slices.Sort(sources)
groups = append(groups, strings.Join(sources, "+"))
}
slices.Sort(groups)
return groups
}
func assertRESTSourceGroups(t *testing.T, h *QueryService, want []string) {
t.Helper()
rec := httptest.NewRecorder()
h.HandleListResources(rec, httptest.NewRequest(http.MethodGet, "/api/resources", nil))
if rec.Code != http.StatusOK {
t.Fatalf("list status = %d body=%s", rec.Code, rec.Body.String())
}
var resp ResourcesResponse
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("decode list: %v", err)
}
if got := resourceSourceGroups(resp.Data); !slices.Equal(got, want) {
t.Fatalf("REST lists %v, want %v", got, want)
}
}
// Report-merge on a pair identity matching merged keeps it apart through the
// monitor's rebuilds, whose identity pins persist on one generation and steer
// the next.
func TestResourceReportMergeSplitsIdentityMergedPair(t *testing.T) {
now := time.Now().UTC()
interfaces := []models.HostNetworkInterface{{Name: "eth0", MAC: "aa:bb:cc:dd:ee:ff", Addresses: []string{"10.0.0.5"}}}
snapshot := models.StateSnapshot{
LastUpdate: now,
Hosts: []models.Host{{ID: "host-1", Hostname: "alpha", MachineID: "0123456789abcdef", Status: "online", LastSeen: now, NetworkInterfaces: interfaces}},
DockerHosts: []models.DockerHost{{ID: "docker-1", Hostname: "alpha", MachineID: "0123456789abcdef", Status: "online", CPUs: 4, LastSeen: now, NetworkInterfaces: interfaces}},
}
h, _, rebuild := newReportMergeHarness(t, snapshot)
merged := rebuild()
if len(merged) != 1 {
t.Fatalf("fixture produced %d resources, want one merged: %+v", len(merged), merged)
}
sources := make([]string, 0, len(merged[0].Sources))
for _, source := range merged[0].Sources {
sources = append(sources, string(source))
}
postResourceAction(t, h, merged[0].ID, "report-merge", map[string]any{"sources": sources})
if split := rebuild(); len(split) != 2 {
t.Fatalf("report-merge left %d monitor resources, want two: %+v", len(split), split)
}
assertRESTResourceCount(t, h, 2)
}
// newReportMergeHarness serves the resources API from a monitor adapter on the
// same SQLite store, rebuilt three times per step: identity pins persist on one
// rebuild and steer the next.
func newReportMergeHarness(t *testing.T, snapshot models.StateSnapshot) (*QueryService, unified.ResourceStore, func() []unified.Resource) {
t.Helper()
h := NewQueryService(&config.Config{DataPath: t.TempDir()})
store, err := h.getStore("default")
if err != nil {
t.Fatalf("getStore: %v", err)
}
adapter := unified.NewMonitorAdapter(unified.NewRegistry(store))
h.SetStateProvider(&monitorAdapterSeedProvider{adapter: adapter, snapshot: snapshot})
rebuild := func() []unified.Resource {
t.Helper()
for i := 0; i < 3; i++ {
adapter.PopulateFromSnapshot(snapshot)
}
return adapter.GetAll()
}
return h, store, rebuild
}
func postResourceAction(t *testing.T, h *QueryService, resourceID, action string, payload map[string]any) {
t.Helper()
body, _ := json.Marshal(payload)
rec := httptest.NewRecorder()
h.HandleResourceRoutes(rec, httptest.NewRequest(http.MethodPost, "/api/resources/"+resourceID+"/"+action, bytes.NewReader(body)))
if rec.Code != http.StatusOK {
t.Fatalf("%s %s status = %d body=%s", action, resourceID, rec.Code, rec.Body.String())
}
}
func assertRESTResourceCount(t *testing.T, h *QueryService, want int) {
t.Helper()
rec := httptest.NewRecorder()
h.HandleListResources(rec, httptest.NewRequest(http.MethodGet, "/api/resources", nil))
if rec.Code != http.StatusOK {
t.Fatalf("list status = %d body=%s", rec.Code, rec.Body.String())
}
var resp ResourcesResponse
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("decode list: %v", err)
}
if len(resp.Data) != want {
ids := make([]string, 0, len(resp.Data))
for _, resource := range resp.Data {
ids = append(ids, resource.ID+"("+string(resource.Type)+")")
}
t.Fatalf("REST lists %d resources %v, want %d", len(resp.Data), ids, want)
}
}
// Canonical-ID succession re-keys link rows with UPDATE OR IGNORE but moves
// primary_id unconditionally, so a row whose re-key collided with the
// successor's own row keeps the retired endpoint and gains the successor as
// its primary. That row used to merge the successor through a pair it does
// not name, so reporting (or unlinking) the merged pair left it behind and
// the next rebuild merged the pair again.
func TestResourceReportMergeSplitsPairBehindASuccessionShadowedRow(t *testing.T) {
now := time.Now().UTC()
snapshot := models.StateSnapshot{
LastUpdate: now,
VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}},
Hosts: []models.Host{{ID: "host-box", Hostname: "box-agent", MachineID: "fedcba9876543210", Status: "online", LastSeen: now}},
}
h, store, rebuild := newReportMergeHarness(t, snapshot)
var vmID, agentID string
for _, resource := range rebuild() {
switch resource.Type {
case unified.ResourceTypeVM:
vmID = resource.ID
case unified.ResourceTypeAgent:
agentID = resource.ID
}
}
if err := store.AddLink(unified.ResourceLink{ResourceA: "agent-retired0000000", ResourceB: vmID, PrimaryID: agentID, CreatedAt: now.Add(-time.Hour)}); err != nil {
t.Fatal(err)
}
postResourceAction(t, h, vmID, "link", map[string]any{"targetId": agentID})
linked := rebuild()
if len(linked) != 1 {
t.Fatalf("link left %d monitor resources, want one: %+v", len(linked), linked)
}
postResourceAction(t, h, linked[0].ID, "report-merge", map[string]any{})
if split := rebuild(); len(split) != 2 {
t.Fatalf("report-merge left %d monitor resources, want the pair apart: %+v", len(split), split)
}
assertRESTResourceCount(t, h, 2)
}