Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Try reachable alternatives in the cached infrastructure DNS answer
pulse-triage[bot]4295476940d8
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 494c94fc30..28f003914e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -6599,6 +6599,19 @@ the temperature SSH key on disk) and from every system-settings savesettings API), so repairing the key — or saving settings after repairing SSHaccess any other way — is retried on the next cycle rather than after a windowthat has compounded to fifteen minutes.+The shared socket dialler also tries the usable addresses in that one cached+answer rather than discarding all but the first. It preserves answer order,+filters explicit IPv4/IPv6 networks and deduplicates equivalent literals.+Serial socket attempts share a ten-second ceiling or the earlier caller+deadline; each attempt reserves a share for the remaining addresses.+Cancellation stops later attempts and closes a connection that arrives after+the caller ends. This is TCP/socket establishment, not HTTP replay, a change+to discovery admission, new DNS resolution or a relaxation of TLS/authority.+`pkg/tlsutil/dns_fallback_test.go` covers actual second-address sockets, retained+cache reuse, caller budgets, cancellation, family selection and pinned HTTPS.+The existing macOS private-address subprocess route is preserved; Linux socket+controls do not establish native macOS/VPN acceptance.+`internal/monitoring/issue1638_dns_cache_test.go` is the registered proof thatrepeat polls stay on the DNS cache, that the link-local blocklist still rejectshostname endpoints resolving into it, and that the SSH backoffs suppress,diff --git a/pkg/tlsutil/dns_fallback_test.go b/pkg/tlsutil/dns_fallback_test.gonew file mode 100644index 0000000000..9f850e2bad--- /dev/null+++ b/pkg/tlsutil/dns_fallback_test.go@@ -0,0 +1,321 @@+package tlsutil++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++type fallbackDNSResolver struct {+ answers map[string][]string+ queries atomic.Int32+}++func (r *fallbackDNSResolver) LookupHost(_ context.Context, host string) ([]string, error) {+ r.queries.Add(1)+ if answer, ok := r.answers[host]; ok {+ return answer, nil+ }+ return nil, &net.DNSError{Err: "no fixture answer", Name: host, IsNotFound: true}+}++func (r *fallbackDNSResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {+ return nil, &net.DNSError{Err: "no fixture reverse answer", Name: addr, IsNotFound: true}+}++func useFallbackDNS(t *testing.T, answers map[string][]string) *fallbackDNSResolver {+ t.Helper()+ resolver := GetDNSResolver()+ previous := resolver.Resolver+ fixture := &fallbackDNSResolver{answers: answers}+ resolver.Resolver = fixture+ t.Cleanup(func() { resolver.Resolver = previous })+ return fixture+}++func TestDialContextWithCacheFallback(t *testing.T) {+ for _, tc := range []struct {+ name, network string+ answers []string+ }{+ {"refused-ipv4", "tcp", []string{"127.0.0.2", "127.0.0.1"}},+ {"refused-ipv6", "tcp", []string{"::1", "127.0.0.1"}},+ {"tcp4-family", "tcp4", []string{"::1", "127.0.0.1"}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ listener, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ defer listener.Close()+ accepted := make(chan error, 1)+ go func() {+ conn, err := listener.Accept()+ if err == nil {+ err = conn.Close()+ }+ accepted <- err+ }()+ host := "cached-fallback-" + tc.name + ".invalid"+ dns := useFallbackDNS(t, map[string][]string{host: tc.answers})+ _, port, _ := net.SplitHostPort(listener.Addr().String())+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ for read := 0; read < 2; read++ {+ conn, err := DialContextWithCache(ctx, tc.network, net.JoinHostPort(host, port))+ if err != nil {+ listener.Close()+ <-accepted+ t.Fatalf("reachable cached alternative was not tried: %v", err)+ }+ if !strings.HasPrefix(conn.RemoteAddr().String(), "127.0.0.1:") {+ t.Errorf("dial reached unexpected address: %s", conn.RemoteAddr())+ }+ conn.Close()+ if read == 0 {+ if err := <-accepted; err != nil {+ t.Fatal(err)+ }+ go func() {+ conn, err := listener.Accept()+ if err == nil {+ err = conn.Close()+ }+ accepted <- err+ }()+ }+ }+ if err := <-accepted; err != nil {+ t.Fatal(err)+ }+ if dns.queries.Load() != 1 {+ t.Fatalf("two dials repeated DNS: %d queries", dns.queries.Load())+ }+ t.Log("two connections reached the second cached answer, one resolver query")+ })+ }+}++func TestCachedFallbackKeepsTLSTrust(t *testing.T) {+ for _, pinMatches := range []bool{true, false} {+ name := "rejected-pin"+ if pinMatches {+ name = "accepted-pin"+ }+ t.Run(name, func(t *testing.T) {+ host := "cached-tls-" + name + ".invalid"+ var authenticated atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.TLS.ServerName != host {+ t.Errorf("SNI changed from hostname to dialled IP: %q", r.TLS.ServerName)+ }+ if r.Header.Get("Authorization") != "fixture-token" {+ t.Error("ordinary fixture header lost")+ }+ authenticated.Add(1)+ io.WriteString(w, `{"data":[{"node":"cached-node","status":"online"}]}`)+ }))+ defer server.Close()+ _, port, _ := net.SplitHostPort(server.Listener.Addr().String())+ dns := useFallbackDNS(t, map[string][]string{host: {"127.0.0.2", "127.0.0.1"}})+ digest := sha256.Sum256(server.Certificate().Raw)+ pin := hex.EncodeToString(digest[:])+ if !pinMatches {+ pin = strings.Repeat("0", 64)+ }+ client := CreateHTTPClientWithTimeout(true, pin, 2*time.Second)+ // The fixture hostname is private evidence, not a proxy request.+ client.Transport.(*http.Transport).Proxy = nil+ defer client.CloseIdleConnections()+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+net.JoinHostPort(host, port)+"/api2/json/nodes", nil)+ req.Header.Set("Authorization", "fixture-token")+ resp, err := client.Do(req)+ if !pinMatches {+ if resp != nil {+ resp.Body.Close()+ }+ var mismatch *FingerprintMismatchError+ if !errors.As(err, &mismatch) || authenticated.Load() != 0 {+ t.Fatalf("alternative bypassed pin or never checked it: requests=%d err=%v", authenticated.Load(), err)+ }+ } else {+ if err != nil {+ t.Fatalf("ordinary pinned read could not reach healthy alternative: %v", err)+ }+ body, err := io.ReadAll(resp.Body)+ resp.Body.Close()+ if err != nil || !strings.Contains(string(body), `"node":"cached-node"`) || authenticated.Load() != 1 {+ t.Fatalf("ordinary fixture read changed: requests=%d err=%v body=%s", authenticated.Load(), err, body)+ }+ }+ if dns.queries.Load() != 1 {+ t.Errorf("fallback re-resolved hostname: queries=%d", dns.queries.Load())+ }+ t.Logf("matching pin=%t; authenticated requests=%d; one cached hostname resolution", pinMatches, authenticated.Load())+ })+ }+}++func TestDialCachedAddressesBudgetAndStop(t *testing.T) {+ t.Run("stalled-first-reserves-caller-budget", func(t *testing.T) {+ listener, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ defer listener.Close()+ accepted := make(chan error, 1)+ go func() {+ conn, err := listener.Accept()+ if err == nil {+ err = conn.Close()+ }+ accepted <- err+ }()+ _, port, _ := net.SplitHostPort(listener.Addr().String())+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ calls := 0+ dial := func(ctx context.Context, network, address string, timeout time.Duration) (net.Conn, error) {+ calls+++ if calls == 1 {+ if timeout <= 0 || timeout > time.Second {+ t.Errorf("first address consumed caller budget: %s", timeout)+ }+ timer := time.NewTimer(timeout)+ defer timer.Stop()+ select {+ case <-timer.C:+ return nil, context.DeadlineExceeded+ case <-ctx.Done():+ return nil, ctx.Err()+ }+ }+ return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, address)+ }+ conn, err := dialCachedAddresses(ctx, "tcp4", "budget.invalid", port, []string{"127.0.0.2", "127.0.0.1"}, dial)+ if err != nil {+ listener.Close()+ <-accepted+ t.Fatal(err)+ }+ conn.Close()+ if err := <-accepted; err != nil {+ t.Fatal(err)+ }+ if calls != 2 || ctx.Err() != nil {+ t.Fatalf("later address lost its share: calls=%d err=%v", calls, ctx.Err())+ }+ t.Log("timed-out first attempt left enough of the original two-second budget for an actual second socket")+ })+ t.Run("ten-second-cap-and-first-error", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)+ defer cancel()+ firstErr := errors.New("first socket failed")+ calls := 0+ conn, err := dialCachedAddresses(ctx, "tcp", "cap.invalid", "443", []string{"127.0.0.2", "127.0.0.1"}, func(_ context.Context, _, _ string, timeout time.Duration) (net.Conn, error) {+ calls+++ limit := 10 * time.Second+ if calls == 1 {+ limit = 5 * time.Second+ }+ if timeout <= 0 || timeout > limit {+ t.Errorf("attempt %d timeout %s exceeds %s", calls, timeout, limit)+ }+ if calls == 1 {+ return nil, firstErr+ }+ return nil, errors.New("second socket failed")+ })+ if conn != nil || !errors.Is(err, firstErr) || calls != 2 {+ t.Fatalf("all-failed result: calls=%d conn=%v err=%v", calls, conn, err)+ }+ })+ t.Run("cancelled-first-stops-alternatives", func(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ calls := 0+ conn, err := dialCachedAddresses(ctx, "tcp", "cancel.invalid", "443", []string{"127.0.0.2", "127.0.0.1"}, func(_ context.Context, _, _ string, _ time.Duration) (net.Conn, error) {+ calls+++ cancel()+ return nil, errors.New("socket stopped")+ })+ if conn != nil || !errors.Is(err, context.Canceled) || calls != 1 {+ t.Fatalf("cancelled caller started another socket: calls=%d conn=%v err=%v", calls, conn, err)+ }+ })+ t.Run("late-success-after-cancellation-is-closed", func(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ local, peer := net.Pipe()+ defer peer.Close()+ conn, err := dialCachedAddresses(ctx, "tcp", "late.invalid", "443", []string{"127.0.0.1"}, func(_ context.Context, _, _ string, _ time.Duration) (net.Conn, error) {+ cancel()+ return local, nil+ })+ if conn != nil || !errors.Is(err, context.Canceled) {+ t.Fatalf("cancelled caller received a connection: %v, %v", conn, err)+ }+ peer.SetReadDeadline(time.Now().Add(time.Second))+ if _, err := peer.Read(make([]byte, 1)); !errors.Is(err, io.EOF) {+ t.Fatalf("cancelled late-success socket stayed open: %v", err)+ }+ })+}++func TestDialCachedAddressesFamilyAndValidation(t *testing.T) {+ for _, tc := range []struct {+ name, network string+ ips, want []string+ }{+ {"tcp4", "tcp4", []string{"bad.invalid", "::1", "::ffff:127.0.0.1", "127.0.0.1", "127.0.0.2"}, []string{"127.0.0.1:8006", "127.0.0.2:8006"}},+ {"tcp6", "tcp6", []string{"127.0.0.1", "::ffff:127.0.0.1", "::1", "0:0:0:0:0:0:0:1", "fe80::1%FixtureZone"}, []string{"[::1]:8006", "[fe80::1%FixtureZone]:8006"}},+ {"udp4", "udp4", []string{"::1", "127.0.0.1"}, []string{"127.0.0.1:8006"}},+ {"no-usable-address", "tcp6", []string{"bad.invalid", "127.0.0.1"}, nil},+ {"empty-answer", "tcp", nil, nil},+ } {+ t.Run(tc.name, func(t *testing.T) {+ var attempted []string+ firstErr := errors.New("fixture socket failure")+ conn, err := dialCachedAddresses(context.Background(), tc.network, "family.invalid", "8006", tc.ips, func(_ context.Context, network, address string, _ time.Duration) (net.Conn, error) {+ if network != tc.network {+ t.Errorf("network changed: %s", network)+ }+ attempted = append(attempted, address)+ return nil, firstErr+ })+ if conn != nil || strings.Join(attempted, ",") != strings.Join(tc.want, ",") {+ t.Fatalf("addresses tried=%v, want=%v", attempted, tc.want)+ }+ if len(tc.want) == 0 {+ var dnsErr *net.DNSError+ if !errors.As(err, &dnsErr) {+ t.Fatalf("no suitable answer was not a DNS failure: %v", err)+ }+ } else if !errors.Is(err, firstErr) {+ t.Fatalf("first socket error lost: %v", err)+ }+ })+ }+}++func TestDialContextWithCacheCancelledBeforeDNS(t *testing.T) {+ const host = "cached-pre-cancelled.invalid"+ dns := useFallbackDNS(t, map[string][]string{host: {"127.0.0.1"}})+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ conn, err := DialContextWithCache(ctx, "tcp", host+":443")+ if conn != nil || !errors.Is(err, context.Canceled) || dns.queries.Load() != 0 {+ t.Fatalf("cancelled caller queried DNS: conn=%v err=%v queries=%d", conn, err, dns.queries.Load())+ }+}diff --git a/pkg/tlsutil/dnscache.go b/pkg/tlsutil/dnscache.goindex e17ed65567..09de6de234 100644--- a/pkg/tlsutil/dnscache.go+++ b/pkg/tlsutil/dnscache.go@@ -3,6 +3,7 @@ package tlsutilimport ("context""net"+ "net/netip""runtime""sync""time"@@ -104,29 +105,85 @@ func LookupHostCached(ctx context.Context, host string) ([]net.IP, error) {// On macOS, connections to RFC 1918 addresses are routed through a subprocess// (nc) to bypass VPN/NECP routing captures that affect the host process.func DialContextWithCache(ctx context.Context, network, address string) (net.Conn, error) {- resolver := GetDNSResolver()-+ if err := ctx.Err(); err != nil {+ return nil, err+ }host, port, err := net.SplitHostPort(address)if err != nil {return nil, err}// Look up the IP address using the cached resolver- ips, err := resolver.LookupHost(ctx, host)+ ips, err := GetDNSResolver().LookupHost(ctx, host)if err != nil {return nil, err}- // Use the first IP address- if len(ips) == 0 {- return nil, &net.DNSError{- Err: "no IP addresses found",- Name: host,+ return dialCachedAddresses(ctx, network, host, port, ips, dialResolvedAddress)+}++// Retry only socket establishment, using this one cached DNS answer. A failed+// first address must not conceal a reachable alternative or start another DNS+// lookup. Keep the resolver's order and divide the remaining dial budget so a+// stalled address cannot consume the time reserved for later candidates.+func dialCachedAddresses(ctx context.Context, network, host, port string, ips []string, dial func(context.Context, string, string, time.Duration) (net.Conn, error)) (net.Conn, error) {+ deadline := time.Now().Add(10 * time.Second)+ if callerDeadline, ok := ctx.Deadline(); ok && callerDeadline.Before(deadline) {+ deadline = callerDeadline+ }+ addresses := make([]string, 0, len(ips))+ seen := make(map[netip.Addr]bool, len(ips))+ for _, raw := range ips {+ ip, err := netip.ParseAddr(raw)+ if err != nil {+ continue+ }+ ip = ip.Unmap()+ if ((network == "tcp4" || network == "udp4") && !ip.Is4()) ||+ ((network == "tcp6" || network == "udp6") && !ip.Is6()) || seen[ip] {+ continue+ }+ seen[ip] = true+ addresses = append(addresses, net.JoinHostPort(ip.String(), port))+ }+ if len(addresses) == 0 {+ return nil, &net.DNSError{Err: "no suitable IP addresses found", Name: host}+ }++ var firstErr error+ for i, address := range addresses {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ remaining := time.Until(deadline)+ if remaining <= 0 {+ return nil, &net.OpError{Op: "dial", Net: network, Err: context.DeadlineExceeded}+ }+ timeout := max(time.Nanosecond, remaining/time.Duration(len(addresses)-i))+ conn, err := dial(ctx, network, address, timeout)+ if err == nil {+ if contextErr := ctx.Err(); contextErr != nil {+ conn.Close()+ return nil, contextErr+ }+ return conn, nil+ }+ if firstErr == nil {+ firstErr = err}}+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ return nil, firstErr+}- resolvedIP := net.ParseIP(ips[0])- resolvedAddr := net.JoinHostPort(ips[0], port)+func dialResolvedAddress(ctx context.Context, network, resolvedAddr string, timeout time.Duration) (net.Conn, error) {+ host, _, err := net.SplitHostPort(resolvedAddr)+ if err != nil {+ return nil, err+ }+ resolvedIP := net.ParseIP(host)// On macOS, route local-network connections through a subprocess to bypass// Tailscale NECP routing captures that prevent the host process from@@ -137,7 +194,7 @@ func DialContextWithCache(ctx context.Context, network, address string) (net.Con// Create a dialer with the resolved IPdialer := &net.Dialer{- Timeout: 10 * time.Second,+ Timeout: timeout,KeepAlive: 30 * time.Second,}