Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package dockeragent
import (
"compress/gzip"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
systemtypes "github.com/moby/moby/api/types/system"
"github.com/rcourtman/pulse-go-rewrite/internal/agenthelper"
"github.com/rcourtman/pulse-go-rewrite/internal/hostmetrics"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rs/zerolog"
)
type helperInventoryStub struct {
result agenthelper.ContainerInventoryResult
err error
calls int
}
type helperOperationStatusEvent struct {
operation string
err error
}
type helperOperationStatusRecorderStub struct {
events []helperOperationStatusEvent
}
func (s *helperOperationStatusRecorderStub) Record(operation string, err error) {
s.events = append(s.events, helperOperationStatusEvent{operation: operation, err: err})
}
func (s *helperOperationStatusRecorderStub) ModuleStatus() agentshost.ModuleStatus {
status := agentshost.ModuleStatus{
Name: agentshost.ModuleNameTypedPrivilegeHelper, Enabled: true, State: "running",
UpdatedAt: time.Now().UTC(),
}
if len(s.events) > 0 && s.events[len(s.events)-1].err != nil {
status.State = "degraded"
status.LastError = "container.inventory: helper operation failed"
}
return status
}
func TestCollectHelperFailureDeliversIncompleteDegradationStatus(t *testing.T) {
reports := make(chan agentsdocker.Report, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reader, err := gzip.NewReader(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer reader.Close()
var report agentsdocker.Report
if err := json.NewDecoder(reader).Decode(&report); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
reports <- report
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
status := &helperOperationStatusRecorderStub{}
helper := &helperInventoryStub{err: &agenthelper.RemoteError{
Code: agenthelper.ErrorProviderUnavailable, Message: "token=must-not-leak", RequestID: "secret-id",
}}
agent := &Agent{
cfg: Config{
AgentID: "docker-only-agent", AgentType: "unified", Interval: 30 * time.Second,
HelperOperationStatus: status,
},
helperInventory: helper,
runtimePref: RuntimeDocker,
runtime: RuntimeDocker,
agentVersion: "6.4.2",
hostName: "docker-only",
machineID: "machine-docker-only",
targets: []TargetConfig{{
Name: "primary", URL: server.URL, Token: "token", Authoritative: true,
}},
httpClients: map[bool]*http.Client{false: server.Client()},
logger: zerolog.Nop(),
reportStreamID: "docker-status-test-stream",
}
returned, err := agent.collectOnceWithReport(context.Background())
if err == nil {
t.Fatal("helper inventory failure was hidden")
}
if returned.InventoryComplete == nil || *returned.InventoryComplete {
t.Fatalf("returned failure report inventoryComplete = %v", returned.InventoryComplete)
}
select {
case report := <-reports:
if report.InventoryComplete == nil || *report.InventoryComplete {
t.Fatalf("delivered failure report inventoryComplete = %v", report.InventoryComplete)
}
if len(report.Containers) != 0 || len(report.Agent.Modules) != 1 || report.Agent.Modules[0].State != "degraded" {
t.Fatalf("delivered helper failure report = %+v", report)
}
if stream, sequence, ok := agentshost.ParseReportSequenceID(report.SequenceID); !ok || stream != "docker-status-test-stream" || sequence != 1 {
t.Fatalf("delivered helper status sequence = %q (%q, %d, %t)", report.SequenceID, stream, sequence, ok)
}
serialized, marshalErr := json.Marshal(report.Agent.Modules)
if marshalErr != nil {
t.Fatalf("marshal delivered helper modules: %v", marshalErr)
}
for _, secret := range []string{"must-not-leak", "secret-id"} {
if strings.Contains(string(serialized), secret) {
t.Fatalf("raw helper detail %q reached Docker status report: %s", secret, serialized)
}
}
case <-time.After(time.Second):
t.Fatal("helper degradation status report was not delivered")
}
}
func TestIncompleteHelperStatusIsNeverBufferedForStaleReplay(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "offline", http.StatusServiceUnavailable)
}))
defer server.Close()
incomplete := false
agent := &Agent{
targets: []TargetConfig{{
Name: "primary", URL: server.URL, Token: "token", Authoritative: true,
}},
httpClients: map[bool]*http.Client{false: server.Client()},
logger: zerolog.Nop(),
}
if err := agent.deliverReport(context.Background(), agentsdocker.Report{InventoryComplete: &incomplete}); err != nil {
t.Fatalf("deliver incomplete helper status: %v", err)
}
if got := agent.reportBuffers["primary"].Len(); got != 0 {
t.Fatalf("incomplete helper status buffered for stale replay: %d reports", got)
}
}
func (s *helperInventoryStub) Inventory(context.Context) (agenthelper.ContainerInventoryResult, error) {
s.calls++
return s.result, s.err
}
func TestSelectHelperRuntimeHonorsPreferenceAndAvailability(t *testing.T) {
result := agenthelper.ContainerInventoryResult{Runtimes: []agenthelper.ContainerRuntimeSnapshot{
{Runtime: "docker", Available: true},
{Runtime: "podman", Available: true},
}}
selected, err := selectHelperRuntime(result, RuntimeAuto)
if err != nil || selected.Runtime != "docker" {
t.Fatalf("auto selection = %+v, %v; want docker", selected, err)
}
selected, err = selectHelperRuntime(result, RuntimePodman)
if err != nil || selected.Runtime != "podman" {
t.Fatalf("podman selection = %+v, %v", selected, err)
}
if _, err := selectHelperRuntime(agenthelper.ContainerInventoryResult{}, RuntimeDocker); err == nil {
t.Fatal("unavailable requested runtime was accepted")
}
}
func TestNewFallsBackToTypedHelperWithoutActionAuthority(t *testing.T) {
originalConnect := connectCollectorRuntimeFn
t.Cleanup(func() { connectCollectorRuntimeFn = originalConnect })
connectCollectorRuntimeFn = func(RuntimeKind, *zerolog.Logger) (dockerClient, systemtypes.Info, RuntimeKind, error) {
return nil, systemtypes.Info{}, RuntimeAuto, errors.New("permission denied opening daemon socket")
}
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{Runtime: "docker", Available: true}},
}}
logger := zerolog.Nop()
agent, err := New(Config{
PulseURL: "http://127.0.0.1:7655", APIToken: "token", Runtime: "auto",
HelperInventory: helper, Logger: &logger,
})
if err != nil {
t.Fatalf("New helper fallback: %v", err)
}
defer agent.Close()
if agent.helperInventory != helper || agent.docker != nil {
t.Fatalf("helper fallback wiring = helper:%T docker:%T", agent.helperInventory, agent.docker)
}
if agent.ContainerActionsAvailable() {
t.Fatal("summary-only helper fallback exposed container action authority")
}
if helper.calls != 1 {
t.Fatalf("helper probe calls = %d, want 1", helper.calls)
}
}
func TestNewRecordsTypedHelperProbeFailure(t *testing.T) {
originalConnect := connectCollectorRuntimeFn
t.Cleanup(func() { connectCollectorRuntimeFn = originalConnect })
connectCollectorRuntimeFn = func(RuntimeKind, *zerolog.Logger) (dockerClient, systemtypes.Info, RuntimeKind, error) {
return nil, systemtypes.Info{}, RuntimeAuto, errors.New("permission denied opening daemon socket")
}
helperErr := &agenthelper.RemoteError{
Code: agenthelper.ErrorProviderUnavailable, Message: "token=must-not-persist", RequestID: "secret-id",
}
helper := &helperInventoryStub{err: helperErr}
status := &helperOperationStatusRecorderStub{}
logger := zerolog.Nop()
agent, err := New(Config{
PulseURL: "http://127.0.0.1:7655", APIToken: "token", Runtime: "auto",
HelperInventory: helper, HelperOperationStatus: status, Logger: &logger,
})
if agent != nil || err == nil {
t.Fatalf("failed helper probe = agent:%T err:%v", agent, err)
}
if len(status.events) != 1 || status.events[0].operation != agenthelper.OperationContainerInventory || !errors.Is(status.events[0].err, helperErr) {
t.Fatalf("failed helper probe status = %+v", status.events)
}
}
func TestNewRejectsDirectRootfulSocketWhenTypedHelperIsConfigured(t *testing.T) {
originalConnect := connectCollectorRuntimeFn
t.Cleanup(func() { connectCollectorRuntimeFn = originalConnect })
closed := false
connectCollectorRuntimeFn = func(RuntimeKind, *zerolog.Logger) (dockerClient, systemtypes.Info, RuntimeKind, error) {
return &fakeDockerClient{
daemonHost: "unix:///var/run/docker.sock",
closeFn: func() error { closed = true; return nil },
}, systemtypes.Info{ID: "rootful-daemon"}, RuntimeDocker, nil
}
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{Runtime: "docker", Available: true}},
}}
status := &helperOperationStatusRecorderStub{}
logger := zerolog.Nop()
agent, err := New(Config{
PulseURL: "http://127.0.0.1:7655", APIToken: "token", Runtime: "auto",
HelperInventory: helper, HelperOperationStatus: status, Logger: &logger,
})
if err != nil {
t.Fatalf("New helper fallback: %v", err)
}
defer agent.Close()
if !closed {
t.Fatal("rejected rootful direct client was not closed")
}
if agent.helperInventory != helper || agent.ContainerActionsAvailable() {
t.Fatalf("rootful endpoint bypassed helper boundary: helper=%T actions=%t", agent.helperInventory, agent.ContainerActionsAvailable())
}
if len(status.events) != 1 || status.events[0].operation != agenthelper.OperationContainerInventory || status.events[0].err != nil {
t.Fatalf("rootful rejection/helper recovery status = %+v", status.events)
}
}
func TestHelperInventoryStatusTracksFailureAndCompleteRecovery(t *testing.T) {
originalMetrics := hostmetricsCollectWithDiskFilters
t.Cleanup(func() { hostmetricsCollectWithDiskFilters = originalMetrics })
hostmetricsCollectWithDiskFilters = func(context.Context, []string, []string) (hostmetrics.Snapshot, error) {
return hostmetrics.Snapshot{}, nil
}
status := &helperOperationStatusRecorderStub{}
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{Runtime: "docker", Available: true}},
}}
agent := &Agent{
cfg: Config{HelperOperationStatus: status},
helperInventory: helper,
runtimePref: RuntimeDocker,
runtime: RuntimeDocker,
logger: zerolog.Nop(),
}
if _, err := agent.buildHelperInventoryReport(context.Background()); err != nil {
t.Fatalf("complete helper inventory: %v", err)
}
helper.err = errors.New("helper transport token=must-not-persist")
if _, err := agent.buildHelperInventoryReport(context.Background()); err == nil {
t.Fatal("helper transport failure was accepted")
}
helper.err = nil
helper.result = agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{Runtime: "docker", Available: false}},
}
if _, err := agent.buildHelperInventoryReport(context.Background()); err == nil {
t.Fatal("unavailable helper runtime was accepted")
}
helper.result.Runtimes[0].Available = true
if _, err := agent.buildHelperInventoryReport(context.Background()); err != nil {
t.Fatalf("recovered helper inventory: %v", err)
}
if len(status.events) != 4 {
t.Fatalf("status events = %+v, want success/failure/failure/success", status.events)
}
for _, event := range status.events {
if event.operation != agenthelper.OperationContainerInventory {
t.Fatalf("status operation = %q", event.operation)
}
}
if status.events[0].err != nil || status.events[1].err == nil || status.events[2].err == nil || status.events[3].err != nil {
t.Fatalf("status outcomes = %+v", status.events)
}
}
func TestDirectRootlessLossFallsBackToCompleteHelperInventory(t *testing.T) {
originalMetrics := hostmetricsCollectWithDiskFilters
t.Cleanup(func() { hostmetricsCollectWithDiskFilters = originalMetrics })
hostmetricsCollectWithDiskFilters = func(context.Context, []string, []string) (hostmetrics.Snapshot, error) {
return hostmetrics.Snapshot{}, nil
}
closed := false
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{
Runtime: "docker", Available: true,
Containers: []agenthelper.ContainerSummary{{ID: "container-1", Names: []string{"app"}, State: "running"}},
}},
}}
agent := &Agent{
cfg: Config{
HelperInventory: helper, IncludeServices: true, IncludeTasks: true,
},
docker: newSwappableDockerClient(&fakeDockerClient{closeFn: func() error { closed = true; return nil }}),
runtime: RuntimeDocker,
runtimePref: RuntimeDocker,
runtimeGoneStreak: runtimeReconnectFailureThreshold,
registryChecker: newRegistryCheckerWithConfig(zerolog.Nop(), true),
allowedStates: map[string]struct{}{"running": {}},
logger: zerolog.Nop(),
}
if !agent.maybeFallbackToHelperInventory(context.Background()) {
t.Fatal("complete helper inventory did not become the recovery source")
}
if !closed || agent.docker != nil || agent.helperInventory != helper {
t.Fatalf("fallback state = closed:%t docker:%T helper:%T", closed, agent.docker, agent.helperInventory)
}
if agent.cfg.IncludeServices || agent.cfg.IncludeTasks || !agent.cfg.DisableUpdateChecks || agent.registryChecker.Enabled() {
t.Fatalf("helper fallback retained direct-only configuration: %+v", agent.cfg)
}
report, err := agent.buildHelperInventoryReport(context.Background())
if err != nil {
t.Fatalf("build recovered helper report: %v", err)
}
if report.InventoryComplete == nil || !*report.InventoryComplete || report.Host.CollectionMode != agentsdocker.CollectionModeTypedHelperSummary || len(report.Containers) != 1 {
t.Fatalf("recovered helper report = %+v", report)
}
}
func TestDirectRootlessLossDefersHelperFallbackWhileBackgroundTaskRuns(t *testing.T) {
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{Runtime: "docker", Available: true}},
}}
agent := &Agent{
cfg: Config{HelperInventory: helper},
docker: &fakeDockerClient{},
runtimeGoneStreak: runtimeReconnectFailureThreshold,
cleanupTaskRunning: true,
logger: zerolog.Nop(),
}
if agent.maybeFallbackToHelperInventory(context.Background()) {
t.Fatal("helper fallback raced an active direct-runtime cleanup")
}
if helper.calls != 0 || agent.helperInventory != nil || agent.docker == nil {
t.Fatalf("busy fallback mutated state: calls=%d helper=%T docker=%T", helper.calls, agent.helperInventory, agent.docker)
}
}
func TestMonitoringOnlyCollectorNeverSchedulesLegacyMutationTasks(t *testing.T) {
helper := &helperInventoryStub{}
agent := &Agent{
cfg: Config{HelperInventory: helper},
docker: &fakeDockerClient{},
logger: zerolog.Nop(),
}
agent.scheduleDirectCleanup()
agent.scheduleDirectUpdateCheck()
if agent.cleanupTaskRunning || agent.updateCheckRunning {
t.Fatalf("monitoring-only collector scheduled legacy tasks: cleanup=%t update=%t", agent.cleanupTaskRunning, agent.updateCheckRunning)
}
agent.cleanupOrphanedBackups(context.Background())
agent.checkForUpdates(context.Background())
if agent.cleanupTaskRunning || agent.updateCheckRunning {
t.Fatalf("direct legacy task entrypoints bypassed profile gate: cleanup=%t update=%t", agent.cleanupTaskRunning, agent.updateCheckRunning)
}
}
func TestBuildHelperInventoryReportIsExplicitlySummaryOnly(t *testing.T) {
originalMetrics := hostmetricsCollectWithDiskFilters
t.Cleanup(func() { hostmetricsCollectWithDiskFilters = originalMetrics })
hostmetricsCollectWithDiskFilters = func(context.Context, []string, []string) (hostmetrics.Snapshot, error) {
return hostmetrics.Snapshot{
CPUUsagePercent: 12.5,
CPUCount: 4,
Memory: agentshost.MemoryMetric{TotalBytes: 4096, UsedBytes: 1024},
}, nil
}
helper := &helperInventoryStub{result: agenthelper.ContainerInventoryResult{
Runtimes: []agenthelper.ContainerRuntimeSnapshot{{
Runtime: "docker", Available: true,
Containers: []agenthelper.ContainerSummary{
{ID: "1234567890abcdef", Names: []string{"/app"}, Image: "repo/app:1", State: "running", Status: "Up", Created: 1_700_000_000},
{ID: "backup", Names: []string{"app_pulse_backup_20260830_120000"}, State: "exited"},
{ID: "stopped", Names: []string{"stopped"}, State: "exited"},
},
}},
}}
agent := &Agent{
cfg: Config{
AgentID: "agent-1", AgentType: "unified", Interval: 15 * time.Second,
},
helperInventory: helper,
runtimePref: RuntimeAuto,
runtime: RuntimeDocker,
agentVersion: "6.0.0",
hostName: "node-1",
machineID: "machine-1",
allowedStates: map[string]struct{}{"running": {}},
logger: zerolog.Nop(),
}
report, err := agent.buildHelperInventoryReport(context.Background())
if err != nil {
t.Fatalf("buildHelperInventoryReport: %v", err)
}
if report.Host.CollectionMode != "typed-helper-summary" {
t.Fatalf("collection mode = %q", report.Host.CollectionMode)
}
if len(report.Containers) != 1 {
t.Fatalf("containers = %+v, want only running non-backup summary", report.Containers)
}
container := report.Containers[0]
if container.ID != "1234567890abcdef" || container.Name != "app" || container.Image != "repo/app:1" {
t.Fatalf("container summary = %+v", container)
}
if container.CPUPercent != 0 || container.MemoryUsageBytes != 0 || len(container.Env) != 0 || len(container.Mounts) != 0 {
t.Fatalf("summary-only report fabricated privileged detail: %+v", container)
}
if report.Host.TotalMemoryBytes != 4096 || report.Host.TotalCPU != 4 || report.Agent.IntervalSeconds != 15 {
t.Fatalf("host/agent summary = %+v / %+v", report.Host, report.Agent)
}
}