Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package proxmox
import (
"context"
"fmt"
"testing"
"time"
)
func TestGetHealthyClientSingleEndpointFallbackKeepsObservedFailure(t *testing.T) {
endpoint := "http://example.invalid"
cc := &ClusterClient{
name: "single",
endpoints: []string{endpoint},
clients: map[string]*Client{endpoint: {}},
nodeHealth: map[string]bool{endpoint: false},
lastError: map[string]string{endpoint: "previous failed observation"},
lastHealthCheck: map[string]time.Time{endpoint: time.Now()},
rateLimitUntil: make(map[string]time.Time),
}
client, err := cc.getHealthyClient(context.Background())
if err != nil {
t.Fatalf("getHealthyClient failed: %v", err)
}
if client == nil {
t.Fatal("expected client, got nil")
}
if cc.nodeHealth[endpoint] || cc.lastError[endpoint] != "previous failed observation" {
t.Fatal("cached fallback selection must not erase an observed failure")
}
if err := cc.executeWithFailover(context.Background(), func(*Client) error {
return fmt.Errorf("API error 500: fixture unavailable")
}); err == nil || cc.nodeHealth[endpoint] {
t.Fatal("failed last-resort request must not establish recovery")
}
if err := cc.executeWithFailover(context.Background(), func(*Client) error { return nil }); err != nil {
t.Fatal(err)
}
if !cc.nodeHealth[endpoint] || cc.lastError[endpoint] != "" {
t.Fatal("accepted ordinary request must clear its own recorded failure")
}
}
func TestExecuteWithFailoverMovesToAnotherEndpoint(t *testing.T) {
endpoint1 := "node1"
endpoint2 := "node2"
now := time.Now()
cc := &ClusterClient{
name: "test",
endpoints: []string{endpoint1, endpoint2},
clients: map[string]*Client{
endpoint1: {},
endpoint2: {},
},
nodeHealth: map[string]bool{
endpoint1: true,
endpoint2: true,
},
lastError: make(map[string]string),
lastHealthCheck: map[string]time.Time{
endpoint1: now,
endpoint2: now,
},
rateLimitUntil: make(map[string]time.Time),
}
usedEndpoints := make([]string, 0, 2)
attempts := 0
err := cc.executeWithFailover(context.Background(), func(c *Client) error {
endpoint := ""
cc.mu.RLock()
for ep, client := range cc.clients {
if client == c {
endpoint = ep
break
}
}
cc.mu.RUnlock()
if endpoint == "" {
return fmt.Errorf("failed to resolve endpoint for client")
}
usedEndpoints = append(usedEndpoints, endpoint)
attempts++
if attempts == 1 {
// Must be a connectivity error to trigger failover to next endpoint.
// Application-level errors (HTTP 4xx/5xx) no longer mark endpoints unhealthy.
return fmt.Errorf("dial tcp node1:8006: connect: connection refused")
}
return nil
})
if err != nil {
t.Fatalf("executeWithFailover failed: %v", err)
}
if len(usedEndpoints) < 2 {
t.Fatalf("expected failover to second endpoint, used: %v", usedEndpoints)
}
if usedEndpoints[0] == usedEndpoints[1] {
t.Fatalf("expected failover to a different endpoint, used: %v", usedEndpoints)
}
health := cc.GetHealthStatus()
if health[usedEndpoints[0]] {
t.Fatal("expected first endpoint to be unhealthy after failure")
}
if !health[usedEndpoints[1]] {
t.Fatal("expected second endpoint to be healthy after success")
}
}