Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: count and name refused inbound connections, restate identity hourly
Antonio Ojeaab83d104707d
diff --git a/internal/router/metrics.go b/internal/router/metrics.goindex 10cca0b0..21dc5498 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -124,7 +124,7 @@ func newRouterStateCollector(r *Router) *routerStateCollector {func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{- c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,+ c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,} {ch <- d}@@ -132,6 +132,7 @@ func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {r := c.r+ ch <- prometheus.MustNewConstMetric(connsPerSourceIPDesc, prometheus.GaugeValue, float64(effectiveConnsPerSourceIP(r.config.ConnsPerSourceIP)))// isReady is stored after Host and DHT are assigned, so observing it// true is what makes reading them from this goroutine safe.if !r.isReady.Load() {diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex ce63eb25..446b8115 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -99,6 +99,8 @@ func TestRouterMetricsListener(t *testing.T) {for _, want := range []string{"sam_router_ready 1",`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,+ "sam_router_conns_per_source_ip_limit 8",+ "sam_router_inbound_connections_refused_total","sam_router_authenticated_peers 2","sam_router_banned_peers 1","sam_router_connected_peers 0",diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.gonew file mode 100644index 00000000..5cc3e030--- /dev/null+++ b/internal/router/resource_observer.go@@ -0,0 +1,178 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "errors"+ "strings"+ "sync"+ "time"++ "github.com/libp2p/go-libp2p/core/connmgr"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr"+ manet "github.com/multiformats/go-multiaddr/net"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++// Why the resource manager refused an inbound connection. A refusal happens+// before any byte of TLS: the dialer sees the TCP connection close at once,+// and libp2p's own metrics count only the scope-limit case.+const (+ refusedPerIP = "per_ip_limit"+ refusedRateLimit = "rate_limit"+ refusedScope = "scope_limit"+ refusedOther = "other"+)++var inboundRefusedTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_router_inbound_connections_refused_total",+ Help: "Inbound connections the resource manager refused before the handshake, by reason",+ },+ []string{"reason"},+)++// refusalLogInterval bounds how often one source IP's refusals are logged;+// the counter carries the rate.+const refusalLogInterval = time.Minute++func init() {+ for _, v := range []string{refusedPerIP, refusedRateLimit, refusedScope, refusedOther} {+ inboundRefusedTotal.WithLabelValues(v)+ }+}++// observedResourceManager counts and logs what the wrapped manager refuses.+// It keeps the wrapped manager's connection limit visible to libp2p's+// connmgr sanity check, and leaves every scope it returns untouched.+type observedResourceManager struct {+ network.ResourceManager+ mu sync.Mutex+ lastLogAt map[string]time.Time+ now func() time.Time+}++func observeResourceManager(inner network.ResourceManager) *observedResourceManager {+ return &observedResourceManager{ResourceManager: inner, lastLogAt: map[string]time.Time{}, now: time.Now}+}++func (m *observedResourceManager) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) {+ scope, err := m.ResourceManager.OpenConnection(dir, usefd, endpoint)+ if err != nil && dir == network.DirInbound {+ m.refused(endpoint, err)+ }+ return scope, err+}++// GetConnLimit is what libp2p compares the connection manager's high water+// mark against at host construction.+func (m *observedResourceManager) GetConnLimit() int {+ if l, ok := m.ResourceManager.(connmgr.GetConnLimiter); ok {+ return l.GetConnLimit()+ }+ return 0+}++func (m *observedResourceManager) refused(endpoint multiaddr.Multiaddr, err error) {+ reason := refusalReason(err)+ inboundRefusedTotal.WithLabelValues(reason).Inc()++ source := "?"+ if ip, ipErr := manet.ToIP(endpoint); ipErr == nil {+ source = ip.String()+ }+ m.mu.Lock()+ last, seen := m.lastLogAt[source]+ now := m.now()+ shouldLog := !seen || now.Sub(last) >= refusalLogInterval+ if shouldLog {+ m.lastLogAt[source] = now+ for k, t := range m.lastLogAt {+ if now.Sub(t) >= refusalLogInterval {+ delete(m.lastLogAt, k)+ }+ }+ }+ m.mu.Unlock()+ if shouldLog {+ logger.Warnf("[Conn] Refused inbound connection from %s (%s): %v; the dialer sees the connection close before TLS", source, reason, err)+ }+}++// refusalReason maps the resource manager's errors to the counter's labels.+// The per-IP and rate-limit paths return plain strings, not sentinel errors.+func refusalReason(err error) string {+ if errors.Is(err, network.ErrResourceLimitExceeded) {+ return refusedScope+ }+ msg := err.Error()+ switch {+ case strings.Contains(msg, "connections per ip limit"):+ return refusedPerIP+ case strings.Contains(msg, "rate limit exceeded"):+ return refusedRateLimit+ }+ return refusedOther+}++// connsPerSourceIPDesc exports the cap in force, so a dashboard can read the+// refusal counter against it.+var connsPerSourceIPDesc = prometheus.NewDesc(+ "sam_router_conns_per_source_ip_limit",+ "Inbound connections allowed from one source IP",+ nil, nil)++// defaultConnsPerSourceIP is libp2p's cap when --conns-per-source-ip is unset.+const defaultConnsPerSourceIP = 8++var _ connmgr.GetConnLimiter = (*observedResourceManager)(nil)++// effectiveConnsPerSourceIP is the cap a dashboard should compare against.+func effectiveConnsPerSourceIP(configured int) int {+ if configured > 0 {+ return configured+ }+ return defaultConnsPerSourceIP+}++// perPeerLogLimiter admits one log line per peer per interval. Entries older+// than the interval are dropped on the way, so it holds only recent peers.+type perPeerLogLimiter struct {+ mu sync.Mutex+ lastAt map[peer.ID]time.Time+}++const perPeerLogInterval = time.Minute++func (l *perPeerLogLimiter) allow(p peer.ID, now time.Time) bool {+ l.mu.Lock()+ defer l.mu.Unlock()+ if l.lastAt == nil {+ l.lastAt = map[peer.ID]time.Time{}+ }+ if last, ok := l.lastAt[p]; ok && now.Sub(last) < perPeerLogInterval {+ return false+ }+ for k, t := range l.lastAt {+ if now.Sub(t) >= perPeerLogInterval {+ delete(l.lastAt, k)+ }+ }+ l.lastAt[p] = now+ return true+}diff --git a/internal/router/resource_observer_test.go b/internal/router/resource_observer_test.gonew file mode 100644index 00000000..866e77b9--- /dev/null+++ b/internal/router/resource_observer_test.go@@ -0,0 +1,148 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "errors"+ "fmt"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go"+)++func counterValue(t *testing.T, c prometheus.Counter) float64 {+ t.Helper()+ var m dto.Metric+ if err := c.Write(&m); err != nil {+ t.Fatal(err)+ }+ return m.GetCounter().GetValue()+}++// refusingResourceManager answers every OpenConnection with one error.+type refusingResourceManager struct {+ network.NullResourceManager+ err error+ limit int+}++func (m *refusingResourceManager) OpenConnection(network.Direction, bool, multiaddr.Multiaddr) (network.ConnManagementScope, error) {+ if m.err != nil {+ return nil, m.err+ }+ return &network.NullScope{}, nil+}++func (m *refusingResourceManager) GetConnLimit() int { return m.limit }++func TestObservedResourceManagerCountsAndClassifiesRefusals(t *testing.T) {+ endpoint := multiaddr.StringCast("/ip4/104.197.44.137/tcp/51234")+ for _, tc := range []struct {+ err error+ reason string+ }{+ {fmt.Errorf("connections per ip limit exceeded for %s", endpoint), refusedPerIP},+ {errors.New("rate limit exceeded"), refusedRateLimit},+ {fmt.Errorf("cannot reserve inbound connection: %w", network.ErrResourceLimitExceeded), refusedScope},+ {errors.New("something else"), refusedOther},+ } {+ before := counterValue(t, inboundRefusedTotal.WithLabelValues(tc.reason))+ m := observeResourceManager(&refusingResourceManager{err: tc.err})+ if _, err := m.OpenConnection(network.DirInbound, true, endpoint); err == nil {+ t.Fatalf("%s: refusal not returned", tc.reason)+ }+ if got := counterValue(t, inboundRefusedTotal.WithLabelValues(tc.reason)); got != before+1 {+ t.Errorf("%s: counter %v, want %v", tc.reason, got, before+1)+ }+ }++ // An outbound failure is not an inbound refusal.+ before := counterValue(t, inboundRefusedTotal.WithLabelValues(refusedOther))+ m := observeResourceManager(&refusingResourceManager{err: errors.New("dial side")})+ _, _ = m.OpenConnection(network.DirOutbound, true, endpoint)+ if got := counterValue(t, inboundRefusedTotal.WithLabelValues(refusedOther)); got != before {+ t.Errorf("outbound failure counted as an inbound refusal")+ }++ // A granted connection is passed through and not counted.+ ok := observeResourceManager(&refusingResourceManager{limit: 42})+ if _, err := ok.OpenConnection(network.DirInbound, true, endpoint); err != nil {+ t.Fatalf("granted connection refused: %v", err)+ }+ if ok.GetConnLimit() != 42 {+ t.Errorf("GetConnLimit = %d, want the wrapped manager's 42", ok.GetConnLimit())+ }+}++func TestObservedResourceManagerLogsOneSourceOncePerInterval(t *testing.T) {+ now := time.Unix(1_700_000_000, 0)+ m := observeResourceManager(&refusingResourceManager{err: errors.New("rate limit exceeded")})+ m.now = func() time.Time { return now }+ a := multiaddr.StringCast("/ip4/10.0.0.1/tcp/1")+ b := multiaddr.StringCast("/ip4/10.0.0.2/tcp/1")++ for i := 0; i < 3; i++ {+ _, _ = m.OpenConnection(network.DirInbound, true, a)+ }+ _, _ = m.OpenConnection(network.DirInbound, true, b)+ if len(m.lastLogAt) != 2 {+ t.Fatalf("sources remembered = %d, want 2", len(m.lastLogAt))+ }+ if !m.lastLogAt["10.0.0.1"].Equal(now) {+ t.Errorf("first source logged at %v, want the first refusal's time", m.lastLogAt["10.0.0.1"])+ }++ // Past the interval the source is logged again and stale ones are dropped.+ now = now.Add(refusalLogInterval)+ _, _ = m.OpenConnection(network.DirInbound, true, a)+ if !m.lastLogAt["10.0.0.1"].Equal(now) {+ t.Errorf("source not logged again after the interval")+ }+ if _, kept := m.lastLogAt["10.0.0.2"]; kept {+ t.Errorf("a source silent for the whole interval is still remembered")+ }+}++func TestPerPeerLogLimiter(t *testing.T) {+ var l perPeerLogLimiter+ now := time.Unix(1_700_000_000, 0)+ p, q := peer.ID("peer-a"), peer.ID("peer-b")+ if !l.allow(p, now) || l.allow(p, now.Add(perPeerLogInterval-time.Second)) {+ t.Fatal("first line allowed once, then held for the interval")+ }+ if !l.allow(q, now) {+ t.Fatal("another peer is not held by the first")+ }+ if !l.allow(p, now.Add(perPeerLogInterval)) {+ t.Fatal("the peer is logged again after the interval")+ }+ if _, kept := l.lastAt[q]; kept {+ t.Fatal("a peer silent for the whole interval is still remembered")+ }+}++func TestEffectiveConnsPerSourceIP(t *testing.T) {+ if got := effectiveConnsPerSourceIP(0); got != defaultConnsPerSourceIP {+ t.Errorf("unset = %d, want libp2p's default %d", got, defaultConnsPerSourceIP)+ }+ if got := effectiveConnsPerSourceIP(64); got != 64 {+ t.Errorf("configured = %d, want 64", got)+ }+}diff --git a/internal/router/router.go b/internal/router/router.goindex d2232849..029879ec 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -202,6 +202,10 @@ type Router struct {metricsServer *http.ServermetricsAddr net.Addr++ // identityLoggedAt is when the "Router Online" line was last written.+ identityLoggedAt atomic.Pointer[time.Time]+ authFailureLog perPeerLogLimiter}// NewRouter initializes the router.@@ -301,6 +305,14 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {)}+// defaultResourceManager is what libp2p installs when none is given, built+// here so it can be wrapped.+func defaultResourceManager() (network.ResourceManager, error) {+ limits := rcmgr.DefaultLimits+ libp2p.SetDefaultServiceLimits(&limits)+ return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits.AutoScale()))+}+// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.func (r *Router) Start() error {// The operator listener comes up first so /healthz answers while@@ -376,13 +388,19 @@ func (r *Router) Start() error {}),}+ // Every refusal the resource manager makes is counted and logged with+ // its source, whether the per-IP cap is the configured one or libp2p's+ // default; a refused dialer only sees its connection close before TLS.+ var mgr network.ResourceManagerif r.config.ConnsPerSourceIP > 0 {- mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)- if err != nil {- return fmt.Errorf("failed to create resource manager: %w", err)- }- p2pOpts = append(p2pOpts, libp2p.ResourceManager(mgr))+ mgr, err = perIPConnResourceManager(r.config.ConnsPerSourceIP)+ } else {+ mgr, err = defaultResourceManager()}+ if err != nil {+ return fmt.Errorf("failed to create resource manager: %w", err)+ }+ p2pOpts = append(p2pOpts, libp2p.ResourceManager(observeResourceManager(mgr)))hostNode, err := libp2p.New(p2pOpts...)if err != nil {@@ -1023,11 +1041,26 @@ func (r *Router) renewLease() {} else {logger.Debugf("Lease renewed successfully. Expires at: %s", leaseResp.GetExpireTime().AsTime().Format(time.RFC3339))leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()+ r.logIdentityPeriodically()}return}}+// identityLogInterval is how often the router restates who it is. Container+// logs rotate, and the startup line is the first to go; an operator reading+// the log should still find the peer ID within the last hour of it.+const identityLogInterval = time.Hour++func (r *Router) logIdentityPeriodically() {+ now := time.Now()+ if last := r.identityLoggedAt.Load(); last != nil && now.Sub(*last) < identityLogInterval {+ return+ }+ r.identityLoggedAt.Store(&now)+ logger.Infof("Router Online. PeerID: %s, ListenAddrs: %v, ConnectedPeers: %d", r.Host.ID(), r.Host.Addrs(), len(r.Host.Network().Peers()))+}+// reconcileBannedPeers replaces the local blocklist with the control plane's// ban set. It is a replacement rather than a merge: the control plane can// unban a peer, and there is no event for that, so a peer that has dropped off@@ -1233,7 +1266,12 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {// Verify Biscuit_, err = identity.VerifyBiscuit(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)if err != nil {- logger.Warnf("[AuthN] Authorization failed for peer %s: %v", remotePeer, err)+ // A peer stuck on a bad credential retries every few seconds for+ // days; one line a minute per peer keeps the log readable and the+ // counter carries the rate.+ if r.authFailureLog.allow(remotePeer, time.Now()) {+ logger.Warnf("[AuthN] Authorization failed for peer %s: %v (further failures from this peer are logged once per minute)", remotePeer, err)+ }authHandshakesTotal.WithLabelValues(handshakeUnauthorized).Inc()_ = s.Reset()return