Skip to content

Agent Mesh

router: count and name refused inbound connections, restate identity hourly

Antonio Ojeaab83d104707d
diff --git a/internal/router/metrics.go b/internal/router/metrics.go
index 10cca0b0..21dc5498 100644
--- a/internal/router/metrics.go
+++ b/internal/router/metrics.go
@@ -124,7 +124,7 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
- c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,
+ c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,
} {
ch <- d
}
@@ -132,6 +132,7 @@ func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
r := c.r
+ ch <- prometheus.MustNewConstMetric(connsPerSourceIPDesc, prometheus.GaugeValue, float64(effectiveConnsPerSourceIP(r.config.ConnsPerSourceIP)))
// isReady is stored after Host and DHT are assigned, so observing it
// true is what makes reading them from this goroutine safe.
if !r.isReady.Load() {
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
index ce63eb25..446b8115 100644
--- a/internal/router/metrics_test.go
+++ b/internal/router/metrics_test.go
@@ -99,6 +99,8 @@ func TestRouterMetricsListener(t *testing.T) {
for _, want := range []string{
"sam_router_ready 1",
`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,
+ "sam_router_conns_per_source_ip_limit 8",
+ "sam_router_inbound_connections_refused_total",
"sam_router_authenticated_peers 2",
"sam_router_banned_peers 1",
"sam_router_connected_peers 0",
diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.go
new file mode 100644
index 00000000..5cc3e030
--- /dev/null
+++ b/internal/router/resource_observer.go
@@ -0,0 +1,178 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "errors"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/connmgr"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ manet "github.com/multiformats/go-multiaddr/net"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// Why the resource manager refused an inbound connection. A refusal happens
+// before any byte of TLS: the dialer sees the TCP connection close at once,
+// and libp2p's own metrics count only the scope-limit case.
+const (
+ refusedPerIP = "per_ip_limit"
+ refusedRateLimit = "rate_limit"
+ refusedScope = "scope_limit"
+ refusedOther = "other"
+)
+
+var inboundRefusedTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_router_inbound_connections_refused_total",
+ Help: "Inbound connections the resource manager refused before the handshake, by reason",
+ },
+ []string{"reason"},
+)
+
+// refusalLogInterval bounds how often one source IP's refusals are logged;
+// the counter carries the rate.
+const refusalLogInterval = time.Minute
+
+func init() {
+ for _, v := range []string{refusedPerIP, refusedRateLimit, refusedScope, refusedOther} {
+ inboundRefusedTotal.WithLabelValues(v)
+ }
+}
+
+// observedResourceManager counts and logs what the wrapped manager refuses.
+// It keeps the wrapped manager's connection limit visible to libp2p's
+// connmgr sanity check, and leaves every scope it returns untouched.
+type observedResourceManager struct {
+ network.ResourceManager
+ mu sync.Mutex
+ lastLogAt map[string]time.Time
+ now func() time.Time
+}
+
+func observeResourceManager(inner network.ResourceManager) *observedResourceManager {
+ return &observedResourceManager{ResourceManager: inner, lastLogAt: map[string]time.Time{}, now: time.Now}
+}
+
+func (m *observedResourceManager) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) {
+ scope, err := m.ResourceManager.OpenConnection(dir, usefd, endpoint)
+ if err != nil && dir == network.DirInbound {
+ m.refused(endpoint, err)
+ }
+ return scope, err
+}
+
+// GetConnLimit is what libp2p compares the connection manager's high water
+// mark against at host construction.
+func (m *observedResourceManager) GetConnLimit() int {
+ if l, ok := m.ResourceManager.(connmgr.GetConnLimiter); ok {
+ return l.GetConnLimit()
+ }
+ return 0
+}
+
+func (m *observedResourceManager) refused(endpoint multiaddr.Multiaddr, err error) {
+ reason := refusalReason(err)
+ inboundRefusedTotal.WithLabelValues(reason).Inc()
+
+ source := "?"
+ if ip, ipErr := manet.ToIP(endpoint); ipErr == nil {
+ source = ip.String()
+ }
+ m.mu.Lock()
+ last, seen := m.lastLogAt[source]
+ now := m.now()
+ shouldLog := !seen || now.Sub(last) >= refusalLogInterval
+ if shouldLog {
+ m.lastLogAt[source] = now
+ for k, t := range m.lastLogAt {
+ if now.Sub(t) >= refusalLogInterval {
+ delete(m.lastLogAt, k)
+ }
+ }
+ }
+ m.mu.Unlock()
+ if shouldLog {
+ logger.Warnf("[Conn] Refused inbound connection from %s (%s): %v; the dialer sees the connection close before TLS", source, reason, err)
+ }
+}
+
+// refusalReason maps the resource manager's errors to the counter's labels.
+// The per-IP and rate-limit paths return plain strings, not sentinel errors.
+func refusalReason(err error) string {
+ if errors.Is(err, network.ErrResourceLimitExceeded) {
+ return refusedScope
+ }
+ msg := err.Error()
+ switch {
+ case strings.Contains(msg, "connections per ip limit"):
+ return refusedPerIP
+ case strings.Contains(msg, "rate limit exceeded"):
+ return refusedRateLimit
+ }
+ return refusedOther
+}
+
+// connsPerSourceIPDesc exports the cap in force, so a dashboard can read the
+// refusal counter against it.
+var connsPerSourceIPDesc = prometheus.NewDesc(
+ "sam_router_conns_per_source_ip_limit",
+ "Inbound connections allowed from one source IP",
+ nil, nil)
+
+// defaultConnsPerSourceIP is libp2p's cap when --conns-per-source-ip is unset.
+const defaultConnsPerSourceIP = 8
+
+var _ connmgr.GetConnLimiter = (*observedResourceManager)(nil)
+
+// effectiveConnsPerSourceIP is the cap a dashboard should compare against.
+func effectiveConnsPerSourceIP(configured int) int {
+ if configured > 0 {
+ return configured
+ }
+ return defaultConnsPerSourceIP
+}
+
+// perPeerLogLimiter admits one log line per peer per interval. Entries older
+// than the interval are dropped on the way, so it holds only recent peers.
+type perPeerLogLimiter struct {
+ mu sync.Mutex
+ lastAt map[peer.ID]time.Time
+}
+
+const perPeerLogInterval = time.Minute
+
+func (l *perPeerLogLimiter) allow(p peer.ID, now time.Time) bool {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.lastAt == nil {
+ l.lastAt = map[peer.ID]time.Time{}
+ }
+ if last, ok := l.lastAt[p]; ok && now.Sub(last) < perPeerLogInterval {
+ return false
+ }
+ for k, t := range l.lastAt {
+ if now.Sub(t) >= perPeerLogInterval {
+ delete(l.lastAt, k)
+ }
+ }
+ l.lastAt[p] = now
+ return true
+}
diff --git a/internal/router/resource_observer_test.go b/internal/router/resource_observer_test.go
new file mode 100644
index 00000000..866e77b9
--- /dev/null
+++ b/internal/router/resource_observer_test.go
@@ -0,0 +1,148 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "errors"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func counterValue(t *testing.T, c prometheus.Counter) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := c.Write(&m); err != nil {
+ t.Fatal(err)
+ }
+ return m.GetCounter().GetValue()
+}
+
+// refusingResourceManager answers every OpenConnection with one error.
+type refusingResourceManager struct {
+ network.NullResourceManager
+ err error
+ limit int
+}
+
+func (m *refusingResourceManager) OpenConnection(network.Direction, bool, multiaddr.Multiaddr) (network.ConnManagementScope, error) {
+ if m.err != nil {
+ return nil, m.err
+ }
+ return &network.NullScope{}, nil
+}
+
+func (m *refusingResourceManager) GetConnLimit() int { return m.limit }
+
+func TestObservedResourceManagerCountsAndClassifiesRefusals(t *testing.T) {
+ endpoint := multiaddr.StringCast("/ip4/104.197.44.137/tcp/51234")
+ for _, tc := range []struct {
+ err error
+ reason string
+ }{
+ {fmt.Errorf("connections per ip limit exceeded for %s", endpoint), refusedPerIP},
+ {errors.New("rate limit exceeded"), refusedRateLimit},
+ {fmt.Errorf("cannot reserve inbound connection: %w", network.ErrResourceLimitExceeded), refusedScope},
+ {errors.New("something else"), refusedOther},
+ } {
+ before := counterValue(t, inboundRefusedTotal.WithLabelValues(tc.reason))
+ m := observeResourceManager(&refusingResourceManager{err: tc.err})
+ if _, err := m.OpenConnection(network.DirInbound, true, endpoint); err == nil {
+ t.Fatalf("%s: refusal not returned", tc.reason)
+ }
+ if got := counterValue(t, inboundRefusedTotal.WithLabelValues(tc.reason)); got != before+1 {
+ t.Errorf("%s: counter %v, want %v", tc.reason, got, before+1)
+ }
+ }
+
+ // An outbound failure is not an inbound refusal.
+ before := counterValue(t, inboundRefusedTotal.WithLabelValues(refusedOther))
+ m := observeResourceManager(&refusingResourceManager{err: errors.New("dial side")})
+ _, _ = m.OpenConnection(network.DirOutbound, true, endpoint)
+ if got := counterValue(t, inboundRefusedTotal.WithLabelValues(refusedOther)); got != before {
+ t.Errorf("outbound failure counted as an inbound refusal")
+ }
+
+ // A granted connection is passed through and not counted.
+ ok := observeResourceManager(&refusingResourceManager{limit: 42})
+ if _, err := ok.OpenConnection(network.DirInbound, true, endpoint); err != nil {
+ t.Fatalf("granted connection refused: %v", err)
+ }
+ if ok.GetConnLimit() != 42 {
+ t.Errorf("GetConnLimit = %d, want the wrapped manager's 42", ok.GetConnLimit())
+ }
+}
+
+func TestObservedResourceManagerLogsOneSourceOncePerInterval(t *testing.T) {
+ now := time.Unix(1_700_000_000, 0)
+ m := observeResourceManager(&refusingResourceManager{err: errors.New("rate limit exceeded")})
+ m.now = func() time.Time { return now }
+ a := multiaddr.StringCast("/ip4/10.0.0.1/tcp/1")
+ b := multiaddr.StringCast("/ip4/10.0.0.2/tcp/1")
+
+ for i := 0; i < 3; i++ {
+ _, _ = m.OpenConnection(network.DirInbound, true, a)
+ }
+ _, _ = m.OpenConnection(network.DirInbound, true, b)
+ if len(m.lastLogAt) != 2 {
+ t.Fatalf("sources remembered = %d, want 2", len(m.lastLogAt))
+ }
+ if !m.lastLogAt["10.0.0.1"].Equal(now) {
+ t.Errorf("first source logged at %v, want the first refusal's time", m.lastLogAt["10.0.0.1"])
+ }
+
+ // Past the interval the source is logged again and stale ones are dropped.
+ now = now.Add(refusalLogInterval)
+ _, _ = m.OpenConnection(network.DirInbound, true, a)
+ if !m.lastLogAt["10.0.0.1"].Equal(now) {
+ t.Errorf("source not logged again after the interval")
+ }
+ if _, kept := m.lastLogAt["10.0.0.2"]; kept {
+ t.Errorf("a source silent for the whole interval is still remembered")
+ }
+}
+
+func TestPerPeerLogLimiter(t *testing.T) {
+ var l perPeerLogLimiter
+ now := time.Unix(1_700_000_000, 0)
+ p, q := peer.ID("peer-a"), peer.ID("peer-b")
+ if !l.allow(p, now) || l.allow(p, now.Add(perPeerLogInterval-time.Second)) {
+ t.Fatal("first line allowed once, then held for the interval")
+ }
+ if !l.allow(q, now) {
+ t.Fatal("another peer is not held by the first")
+ }
+ if !l.allow(p, now.Add(perPeerLogInterval)) {
+ t.Fatal("the peer is logged again after the interval")
+ }
+ if _, kept := l.lastAt[q]; kept {
+ t.Fatal("a peer silent for the whole interval is still remembered")
+ }
+}
+
+func TestEffectiveConnsPerSourceIP(t *testing.T) {
+ if got := effectiveConnsPerSourceIP(0); got != defaultConnsPerSourceIP {
+ t.Errorf("unset = %d, want libp2p's default %d", got, defaultConnsPerSourceIP)
+ }
+ if got := effectiveConnsPerSourceIP(64); got != 64 {
+ t.Errorf("configured = %d, want 64", got)
+ }
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index d2232849..029879ec 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -202,6 +202,10 @@ type Router struct {
metricsServer *http.Server
metricsAddr net.Addr
+
+ // identityLoggedAt is when the "Router Online" line was last written.
+ identityLoggedAt atomic.Pointer[time.Time]
+ authFailureLog perPeerLogLimiter
}
// NewRouter initializes the router.
@@ -301,6 +305,14 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
)
}
+// defaultResourceManager is what libp2p installs when none is given, built
+// here so it can be wrapped.
+func defaultResourceManager() (network.ResourceManager, error) {
+ limits := rcmgr.DefaultLimits
+ libp2p.SetDefaultServiceLimits(&limits)
+ return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits.AutoScale()))
+}
+
// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.
func (r *Router) Start() error {
// The operator listener comes up first so /healthz answers while
@@ -376,13 +388,19 @@ func (r *Router) Start() error {
}),
}
+ // Every refusal the resource manager makes is counted and logged with
+ // its source, whether the per-IP cap is the configured one or libp2p's
+ // default; a refused dialer only sees its connection close before TLS.
+ var mgr network.ResourceManager
if r.config.ConnsPerSourceIP > 0 {
- mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)
- if err != nil {
- return fmt.Errorf("failed to create resource manager: %w", err)
- }
- p2pOpts = append(p2pOpts, libp2p.ResourceManager(mgr))
+ mgr, err = perIPConnResourceManager(r.config.ConnsPerSourceIP)
+ } else {
+ mgr, err = defaultResourceManager()
}
+ if err != nil {
+ return fmt.Errorf("failed to create resource manager: %w", err)
+ }
+ p2pOpts = append(p2pOpts, libp2p.ResourceManager(observeResourceManager(mgr)))
hostNode, err := libp2p.New(p2pOpts...)
if err != nil {
@@ -1023,11 +1041,26 @@ func (r *Router) renewLease() {
} else {
logger.Debugf("Lease renewed successfully. Expires at: %s", leaseResp.GetExpireTime().AsTime().Format(time.RFC3339))
leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()
+ r.logIdentityPeriodically()
}
return
}
}
+// identityLogInterval is how often the router restates who it is. Container
+// logs rotate, and the startup line is the first to go; an operator reading
+// the log should still find the peer ID within the last hour of it.
+const identityLogInterval = time.Hour
+
+func (r *Router) logIdentityPeriodically() {
+ now := time.Now()
+ if last := r.identityLoggedAt.Load(); last != nil && now.Sub(*last) < identityLogInterval {
+ return
+ }
+ r.identityLoggedAt.Store(&now)
+ logger.Infof("Router Online. PeerID: %s, ListenAddrs: %v, ConnectedPeers: %d", r.Host.ID(), r.Host.Addrs(), len(r.Host.Network().Peers()))
+}
+
// reconcileBannedPeers replaces the local blocklist with the control plane's
// ban set. It is a replacement rather than a merge: the control plane can
// unban a peer, and there is no event for that, so a peer that has dropped off
@@ -1233,7 +1266,12 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
// Verify Biscuit
_, err = identity.VerifyBiscuit(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)
if err != nil {
- logger.Warnf("[AuthN] Authorization failed for peer %s: %v", remotePeer, err)
+ // A peer stuck on a bad credential retries every few seconds for
+ // days; one line a minute per peer keeps the log readable and the
+ // counter carries the rate.
+ if r.authFailureLog.allow(remotePeer, time.Now()) {
+ logger.Warnf("[AuthN] Authorization failed for peer %s: %v (further failures from this peer are logged once per minute)", remotePeer, err)
+ }
authHandshakesTotal.WithLabelValues(handshakeUnauthorized).Inc()
_ = s.Reset()
return