Skip to content

Agent Mesh

refactor and fix bug

Antonio Ojeaab3ee8a0b10b
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 19b376f7..32d0945f 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -24,6 +24,7 @@ import (
"net/http"
"os"
"strings"
+ "sync/atomic"
"time"
"github.com/biscuit-auth/biscuit-go/v2"
@@ -42,7 +43,7 @@ import (
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
"github.com/libp2p/go-msgio"
- "github.com/prometheus/client_golang/prometheus/promhttp"
+
"golang.org/x/time/rate"
"github.com/libp2p/go-libp2p/p2p/transport/tcp"
@@ -56,8 +57,15 @@ const (
// Rate limiting defaults
EnrollRateLimit = 10
EnrollBurst = 20
+
+ // ConnManager limits
+ LowWaterMark = 100
+ HighWaterMark = 400
+ ConnGracePeriod = 1 * time.Minute
)
+var isHubReady atomic.Bool
+
var (
oidcIssuer string
clientID string
@@ -70,7 +78,7 @@ var (
keyRotationInterval time.Duration
keyGracePeriod time.Duration
keysDBPath string
- metricsAddr string
+ bindAddress string
adminToken string
tlsCertFile string
tlsKeyFile string
@@ -98,7 +106,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig) (*Hub, error) {
gater := newHubConnGate()
// Connection Manager for DoS protection
- cm, err := connmgr.NewConnManager(100, 400, connmgr.WithGracePeriod(time.Minute))
+ cm, err := connmgr.NewConnManager(LowWaterMark, HighWaterMark, connmgr.WithGracePeriod(ConnGracePeriod))
if err != nil {
return nil, fmt.Errorf("failed to create connection manager: %w", err)
}
@@ -629,59 +637,10 @@ func main() {
// Watchdog: Expel peers that connect but never finish authentication
h.startWatchdog(ctx)
- go func() {
- mux := http.NewServeMux()
- mux.Handle("/metrics", promhttp.Handler())
- mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte("ok"))
- })
-
- mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {
- if adminToken != "" {
- authHeader := r.Header.Get("Authorization")
- if authHeader != "Bearer "+adminToken {
- http.Error(w, "Unauthorized", http.StatusUnauthorized)
- return
- }
- }
- handleBan(h)(w, r)
- })
-
- server := &http.Server{
- Addr: metricsAddr,
- Handler: mux,
- }
-
- // Configure TLS if requested
- if tlsCertFile != "" && tlsKeyFile != "" {
- tlsConfig := &tls.Config{}
- if tlsCAFile != "" {
- caCert, err := os.ReadFile(tlsCAFile)
- if err != nil {
- logger.Fatalf("Failed to read CA cert: %v", err)
- }
- caCertPool := x509.NewCertPool()
- caCertPool.AppendCertsFromPEM(caCert)
- tlsConfig.ClientCAs = caCertPool
- tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
- logger.Info("mTLS enabled for admin service")
- }
- server.TLSConfig = tlsConfig
-
- logger.Infof("Starting HTTPS server on %s", metricsAddr)
- if err := server.ListenAndServeTLS(tlsCertFile, tlsKeyFile); err != nil {
- logger.Errorf("HTTPS server failed: %v", err)
- }
- } else {
- logger.Infof("Starting HTTP server on %s", metricsAddr)
- if err := server.ListenAndServe(); err != nil {
- logger.Errorf("HTTP server failed: %v", err)
- }
- }
- }()
+ startHTTPServer(h, bindAddress, adminToken, tlsCertFile, tlsKeyFile, tlsCAFile, &isHubReady)
logger.Infof("SAM Hub Online (QUIC + TCP)")
+ isHubReady.Store(true)
logger.Infof("MeshID: %s", h.MeshID)
logger.Infof("PeerID: %s", h.Host.ID())
@@ -705,7 +664,7 @@ func main() {
rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 0, "Key rotation interval (e.g. 12h). 0 disables rotation.")
rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h).")
rootCmd.Flags().StringVar(&keysDBPath, "keys-db", "keys.db", "Path to BoltDB file for keys")
- rootCmd.PersistentFlags().StringVar(&metricsAddr, "metrics-addr", ":9090", "Address to listen on for Prometheus metrics")
+ rootCmd.PersistentFlags().StringVar(&bindAddress, "bind-address", ":9090", "Address to listen on for HTTP/HTTPS service")
rootCmd.PersistentFlags().StringVar(&adminToken, "admin-token", "", "Secret token for authorizing admin requests")
rootCmd.PersistentFlags().StringVar(&tlsCertFile, "tls-cert-file", "", "Path to TLS certificate for the server")
rootCmd.PersistentFlags().StringVar(&tlsKeyFile, "tls-key-file", "", "Path to TLS private key for the server")
@@ -728,7 +687,7 @@ func main() {
logger.Fatal("Missing --peer flag")
}
- targetAddr := metricsAddr
+ targetAddr := bindAddress
if strings.HasPrefix(targetAddr, ":") {
targetAddr = "localhost" + targetAddr
}
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
new file mode 100644
index 00000000..9a32c25e
--- /dev/null
+++ b/cmd/sam-hub/server.go
@@ -0,0 +1,87 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "net/http"
+ "os"
+ "sync/atomic"
+
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// startHTTPServer starts the HTTP/HTTPS server for metrics, healthz, and admin commands.
+func startHTTPServer(h *Hub, bindAddr string, adminToken string, tlsCertFile string, tlsKeyFile string, tlsCAFile string, isHubReady *atomic.Bool) {
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.Handler())
+ mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+ if isHubReady.Load() {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ } else {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ _, _ = w.Write([]byte("not ready"))
+ }
+ })
+
+ mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {
+ if adminToken != "" {
+ authHeader := r.Header.Get("Authorization")
+ if authHeader != "Bearer "+adminToken {
+ http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ return
+ }
+ }
+ handleBan(h)(w, r)
+ })
+
+ server := &http.Server{
+ Addr: bindAddr,
+ Handler: mux,
+ }
+
+ // Configure TLS if requested
+ if tlsCertFile != "" && tlsKeyFile != "" {
+ tlsConfig := &tls.Config{}
+ if tlsCAFile != "" {
+ caCert, err := os.ReadFile(tlsCAFile)
+ if err != nil {
+ logger.Fatalf("Failed to read CA cert: %v", err)
+ }
+ caCertPool := x509.NewCertPool()
+ caCertPool.AppendCertsFromPEM(caCert)
+ tlsConfig.ClientCAs = caCertPool
+ tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+ logger.Info("mTLS enabled for admin service")
+ }
+ server.TLSConfig = tlsConfig
+
+ logger.Infof("Starting HTTPS server on %s", bindAddr)
+ go func() {
+ if err := server.ListenAndServeTLS(tlsCertFile, tlsKeyFile); err != nil && err != http.ErrServerClosed {
+ logger.Errorf("HTTPS server failed: %v", err)
+ }
+ }()
+ } else {
+ logger.Infof("Starting HTTP server on %s", bindAddr)
+ go func() {
+ if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ logger.Errorf("HTTP server failed: %v", err)
+ }
+ }()
+ }
+}
diff --git a/cmd/sam-node/gate.go b/cmd/sam-node/gate.go
index 9c3411ed..cd5c8755 100644
--- a/cmd/sam-node/gate.go
+++ b/cmd/sam-node/gate.go
@@ -40,6 +40,9 @@ type nodeConnGate struct {
// InterceptPeerDial controls who we are allowed to call (Outbound)
func (g *nodeConnGate) InterceptPeerDial(p peer.ID) (allow bool) {
+ if g.node.revokedPeers.Contains(p.String()) {
+ return false
+ }
return !g.node.Store.IsBanned(p)
}
@@ -55,6 +58,10 @@ func (g *nodeConnGate) InterceptAccept(n network.ConnMultiaddrs) (allow bool) {
// InterceptSecured is called after TLS handshake. This is our Layer 2 Check.
func (g *nodeConnGate) InterceptSecured(dir network.Direction, p peer.ID, n network.ConnMultiaddrs) (allow bool) {
+ if g.node.revokedPeers.Contains(p.String()) {
+ fmt.Printf("[Layer 2] Dropping connection: Peer %s is in revoked cache\n", p)
+ return false
+ }
if g.node.Store.IsBanned(p) {
fmt.Printf("[Layer 2] Dropping connection: Peer %s is explicitly BANNED\n", p)
return false
diff --git a/cmd/sam-node/gate_test.go b/cmd/sam-node/gate_test.go
new file mode 100644
index 00000000..195c819d
--- /dev/null
+++ b/cmd/sam-node/gate_test.go
@@ -0,0 +1,96 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "testing"
+ "time"
+
+ lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "go.etcd.io/bbolt"
+)
+
+func TestConnectionGater(t *testing.T) {
+ dir := t.TempDir()
+ store, err := NewStore(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.Close()
+
+ cache, err := lru.New[string, int64](100)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node := &SamNode{
+ Store: store,
+ revokedPeers: cache,
+ }
+ gater := &nodeConnGate{node: node}
+
+ // Generate test peer IDs
+ priv1, _, _ := crypto.GenerateEd25519Key(nil)
+ peer1, _ := peer.IDFromPrivateKey(priv1)
+
+ priv2, _, _ := crypto.GenerateEd25519Key(nil)
+ peer2, _ := peer.IDFromPrivateKey(priv2)
+
+ priv3, _, _ := crypto.GenerateEd25519Key(nil)
+ peer3, _ := peer.IDFromPrivateKey(priv3)
+
+ // Case 1: Peer is not banned
+ if !gater.InterceptPeerDial(peer1) {
+ t.Errorf("expected InterceptPeerDial to allow peer1")
+ }
+ if !gater.InterceptSecured(network.DirInbound, peer1, nil) {
+ t.Errorf("expected InterceptSecured to allow peer1")
+ }
+
+ // Case 2: Peer is in revoked cache
+ node.revokedPeers.Add(peer2.String(), time.Now().Unix())
+ if gater.InterceptPeerDial(peer2) {
+ t.Errorf("expected InterceptPeerDial to deny peer2 (in revoked cache)")
+ }
+ if !gater.InterceptSecured(network.DirInbound, peer2, nil) {
+ // Wait, InterceptSecured should ALSO deny it!
+ // Let's check my implementation of InterceptSecured in task 21!
+ // I added check for revoked cache there too!
+ // So it should return false!
+ }
+ // Let's fix the assertion to expect false!
+ if gater.InterceptSecured(network.DirInbound, peer2, nil) {
+ t.Errorf("expected InterceptSecured to deny peer2 (in revoked cache)")
+ }
+
+ // Case 3: Peer is in persistent store (banned)
+ err = store.db.Update(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte(bucketBannedPeers))
+ return b.Put([]byte(peer3.String()), []byte("true"))
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if gater.InterceptPeerDial(peer3) {
+ t.Errorf("expected InterceptPeerDial to deny peer3 (in store)")
+ }
+ if gater.InterceptSecured(network.DirInbound, peer3, nil) {
+ t.Errorf("expected InterceptSecured to deny peer3 (in store)")
+ }
+}
diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash
index c566dda6..91be8d97 100644
--- a/tests/e2e/lib/container_mesh.bash
+++ b/tests/e2e/lib/container_mesh.bash
@@ -184,6 +184,26 @@ sys.exit(1)
return 1
}
+ mesh_wait_for_peer_disconnection() {
+ local idx="$1"
+ local target_peer="$2"
+ local timeout_s="${3:-20}"
+ local i
+ for ((i=0; i<timeout_s; i++)); do
+ local output
+ output="$(timeout 15s docker run --rm -v "${MESH_SOCKET_DIR}:/sockets" -v "$(pwd)/bin/mcp-client:/mcp-client" python:3.12 /mcp-client -socket "/sockets/node-${idx}.sock" 2>/dev/null)"
+ echo "[$(date +%T)] Node ${idx} get_mesh_info raw output: ${output}"
+ local connected
+ connected="$(echo "${output}" | jq -r --arg peer "$target_peer" '.connected_peers | index($peer) != null')"
+ echo "[$(date +%T)] Node ${idx} connection to ${target_peer}: ${connected}"
+ if [[ "${connected}" == "false" ]]; then
+ return 0
+ fi
+ sleep 1
+ done
+ return 1
+ }
+
mesh_start_mock_oidc() {
local name="${MESH_PREFIX}-oidc"
docker run -d \
diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.bats
index 818f86c6..89dda035 100644
--- a/tests/e2e/revocation_test.bats
+++ b/tests/e2e/revocation_test.bats
@@ -110,6 +110,14 @@ teardown() {
[[ "$status" -eq 0 ]]
# Verify Node 1 disconnects from Node 2
- # In a full test we would check active connections, but checking the log is a good proxy
- # since node.go calls ClosePeer.
+ echo "[$(date +%T)] Waiting for disconnection"
+ run mesh_wait_for_peer_disconnection 1 "${node2_peer_id}" 20
+ [[ "$status" -eq 0 ]]
+
+ # Verify Node 1 cannot reconnect to Node 2
+ echo "[$(date +%T)] Attempting to reconnect (should fail)"
+ local node2_addr="/dns4/sam-node-2/tcp/5002/p2p/${node2_peer_id}"
+ run docker run --rm -v "${MESH_SOCKET_DIR}:/sockets" -v "$(pwd)/bin/mcp-client:/mcp-client" python:3.12 /mcp-client -socket "/sockets/node-1.sock" -tool "connect_peer" -args "{\"peer_addr\":\"${node2_addr}\"}"
+ echo "Reconnect output: $output"
+ [[ "$output" == *"gater disallows connection"* ]]
}