Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
Refactor: simplify Hub network architecture to use single connection websocket
Antonio Ojea868f58fcf5c0
diff --git a/.github/k8s/sam-hub-template.yaml b/.github/k8s/sam-hub-template.yamlindex 38aced54..b332b62f 100644--- a/.github/k8s/sam-hub-template.yaml+++ b/.github/k8s/sam-hub-template.yaml@@ -11,14 +11,6 @@ spec:sessionAffinity: ClientIPexternalTrafficPolicy: Localports:- - port: 4001- targetPort: 4001- protocol: TCP- name: libp2p-tcp- - port: 4001- targetPort: 4001- protocol: UDP- name: libp2p-udp- port: 443targetPort: 9090protocol: TCP@@ -68,12 +60,6 @@ spec:key: sam-hub-keyoptional: true # Make it optional so it doesn't block local kind testing if not createdports:- - containerPort: 4001- protocol: TCP- name: libp2p-tcp- - containerPort: 4001- protocol: UDP- name: libp2p-udp- containerPort: 9090protocol: TCPname: metrics@@ -92,7 +78,6 @@ spec:initialDelaySeconds: 15periodSeconds: 20args:- - "--listen=/ip4/0.0.0.0/tcp/4001,/ip4/0.0.0.0/udp/4001/quic-v1"- "--external-multiaddr=${EXTERNAL_MULTIADDR}"# 1. Trust Dex for humans/Google/GitHub. 2. Trust GKE for internal Canaries.- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.goindex de7ff2a5..c8d61aa7 100644--- a/cmd/sam-hub/main.go+++ b/cmd/sam-hub/main.go@@ -16,6 +16,8 @@ package mainimport ("github.com/libp2p/go-libp2p/core/network"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promhttp""sync"@@ -48,12 +50,11 @@ import ("github.com/libp2p/go-libp2p/p2p/net/connmgr""github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"- libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"+ "github.com/libp2p/go-libp2p/p2p/transport/websocket""github.com/multiformats/go-multiaddr""golang.org/x/time/rate"- "github.com/libp2p/go-libp2p/p2p/transport/tcp""github.com/spf13/cobra""google.golang.org/protobuf/proto")@@ -139,7 +140,7 @@ type Hub struct {}// NewHub starts a host supporting both QUIC and TCP (with TLS 1.3)-func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (*Hub, error) {+func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool, mux *http.ServeMux) (*Hub, error) {// Connection Manager for DoS protectioncm, err := connmgr.NewConnManager(LowWaterMark, HighWaterMark, connmgr.WithGracePeriod(ConnGracePeriod))@@ -147,9 +148,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (return nil, fmt.Errorf("failed to create connection manager: %w", err)}- h, err := libp2p.New(- libp2p.Transport(libp2pquic.NewTransport),- libp2p.Transport(tcp.NewTCPTransport),+ opts := []libp2p.Option{libp2p.ListenAddrStrings(listenAddrs...),// FIPS compliant Securitylibp2p.Security(libp2ptls.ID, libp2ptls.New),@@ -168,7 +167,12 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (}return filtered}),- )+ }+ if mux != nil {+ opts = append(opts, libp2p.Transport(websocket.New, websocket.WithHTTPHandler(mux)))+ }++ h, err := libp2p.New(opts...)if err != nil {return nil, err}@@ -606,17 +610,64 @@ func main() {logger.Fatal(err)}- h, err := NewHub(ctx, policyConfig, allowLoopbackFlag)+ // Add websocket listen address based on bindAddress+ host, port, err := net.SplitHostPort(bindAddress)+ if err != nil {+ // if bindAddress is just ":9090"+ if strings.HasPrefix(bindAddress, ":") {+ host = "0.0.0.0"+ port = strings.TrimPrefix(bindAddress, ":")+ } else {+ logger.Fatalf("Invalid bind address: %v", err)+ }+ }+ if host == "" {+ host = "0.0.0.0"+ }+ wsAddr := fmt.Sprintf("/ip4/%s/tcp/%s/ws", host, port)+ listenAddrs = append(listenAddrs, wsAddr)++ mux := http.NewServeMux()+ mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))+ mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {+ if isHubReady.Load() {+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("ok"))+ } else {+ w.WriteHeader(http.StatusServiceUnavailable)+ _, _ = w.Write([]byte("not ready"))+ }+ })++ h, err := NewHub(ctx, policyConfig, allowLoopbackFlag, mux)if err != nil {logger.Fatal(err)}+ // Register Hub-specific HTTP routes now that Hub is created+ mux.HandleFunc("/register", handleRegisterHTTP(h))+ mux.HandleFunc("/info", handleInfoHTTP(h))+ mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {+ if adminToken != "" {+ authHeader := r.Header.Get("Authorization")+ if authHeader != "Bearer "+adminToken {+ http.Error(w, "Unauthorized", http.StatusUnauthorized)+ return+ }+ }+ handleBan(h)(w, r)+ })+// Start key rotation if enabledh.startRotation(ctx)- startHTTPServer(h, bindAddress, adminToken, tlsCertFile, tlsKeyFile, tlsCAFile, &isHubReady)+ // We don't start a separate HTTP server unless TLS is specifically required for admin separately,+ // but WithHTTPHandler already shares the port on the websocket listener.+ if tlsCertFile != "" && tlsKeyFile != "" {+ logger.Warn("Separate TLS server not supported via CLI when using WithHTTPHandler. Ignoring tlsCertFile.")+ }- logger.Infof("SAM Hub Online (QUIC + TCP)")+ logger.Infof("SAM Hub Online (QUIC + TCP + WS on %s)", bindAddress)isHubReady.Store(true)logger.Infof("MeshID: %s", h.MeshID)logger.Infof("PeerID: %s", h.Host.ID())@@ -633,7 +684,6 @@ func main() {rootCmd.Flags().StringVar(&oidcIssuer, "issuer", defIssuer, "OIDC Issuer URL")rootCmd.Flags().StringVar(&clientID, "client-id", os.Getenv("SAM_OIDC_ID"), "OIDC Client ID")rootCmd.Flags().StringVar(&biscuitHex, "key", os.Getenv("SAM_HUB_KEY"), "Hub Private Key (32-byte Hex)")- rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{"/ip4/0.0.0.0/udp/8080/quic-v1", "/ip4/0.0.0.0/tcp/8080"}, "libp2p Listen Addrs")rootCmd.Flags().StringSliceVar(&externalMultiaddrs, "external-multiaddr", []string{}, "External multiaddrs to announce")rootCmd.Flags().StringVar(&meshName, "mesh", DefaultMeshName, "Mesh federation name")rootCmd.Flags().StringVar(&allowedAudiencesFlag, "allowed-audiences", api.DefaultAudience, "Comma-separated list of allowed OIDC audiences")diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.goindex 62cd7037..2efb89db 100644--- a/cmd/sam-hub/server.go+++ b/cmd/sam-hub/server.go@@ -16,86 +16,16 @@ package mainimport ("context"- "crypto/tls"- "crypto/x509""io""net/http"- "os""sort""strings"- "sync/atomic""github.com/google/sam/api""github.com/libp2p/go-libp2p/core/peer"- "github.com/prometheus/client_golang/prometheus"- "github.com/prometheus/client_golang/prometheus/promhttp""google.golang.org/protobuf/proto")-// startHTTPServer starts the HTTP/HTTPS server for metrics, healthz, and admin commands.-func startHTTPServer(h *Hub, bindAddr string, adminToken string, tlsCertFile string, tlsKeyFile string, tlsCAFile string, isHubReady *atomic.Bool) {- mux := http.NewServeMux()- mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))- mux.HandleFunc("/register", handleRegisterHTTP(h))- mux.HandleFunc("/info", handleInfoHTTP(h))- mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {- if isHubReady.Load() {- w.WriteHeader(http.StatusOK)- _, _ = w.Write([]byte("ok"))- } else {- w.WriteHeader(http.StatusServiceUnavailable)- _, _ = w.Write([]byte("not ready"))- }- })-- mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {- if adminToken != "" {- authHeader := r.Header.Get("Authorization")- if authHeader != "Bearer "+adminToken {- http.Error(w, "Unauthorized", http.StatusUnauthorized)- return- }- }- handleBan(h)(w, r)- })-- server := &http.Server{- Addr: bindAddr,- Handler: mux,- }-- // Configure TLS if requested- if tlsCertFile != "" && tlsKeyFile != "" {- tlsConfig := &tls.Config{}- if tlsCAFile != "" {- caCert, err := os.ReadFile(tlsCAFile)- if err != nil {- logger.Fatalf("Failed to read CA cert: %v", err)- }- caCertPool := x509.NewCertPool()- caCertPool.AppendCertsFromPEM(caCert)- tlsConfig.ClientCAs = caCertPool- tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert- logger.Info("mTLS enabled for admin service")- }- server.TLSConfig = tlsConfig-- logger.Infof("Starting HTTPS server on %s", bindAddr)- go func() {- if err := server.ListenAndServeTLS(tlsCertFile, tlsKeyFile); err != nil && err != http.ErrServerClosed {- logger.Errorf("HTTPS server failed: %v", err)- }- }()- } else {- logger.Infof("Starting HTTP server on %s", bindAddr)- go func() {- if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {- logger.Errorf("HTTP server failed: %v", err)- }- }()- }-}-func handleRegisterHTTP(h *Hub) http.HandlerFunc {return func(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodPost {diff --git a/go.mod b/go.modindex 3d3c7f71..761f569d 100644--- a/go.mod+++ b/go.mod@@ -9,7 +9,7 @@ require (github.com/hashicorp/golang-lru/v2 v2.0.7github.com/ipfs/go-cid v0.6.1github.com/ipfs/go-log/v2 v2.9.2- github.com/libp2p/go-libp2p v0.48.0+ github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286github.com/libp2p/go-libp2p-gostream v0.6.0github.com/libp2p/go-libp2p-http v0.5.0github.com/libp2p/go-libp2p-kad-dht v0.39.2@@ -18,6 +18,7 @@ require (github.com/modelcontextprotocol/go-sdk v1.6.0github.com/multiformats/go-multiaddr v0.16.1github.com/multiformats/go-multihash v0.2.3+ github.com/openai/openai-go v1.12.0github.com/prometheus/client_golang v1.23.2github.com/prometheus/client_model v0.6.2github.com/spf13/cobra v1.10.2@@ -83,7 +84,6 @@ require (github.com/multiformats/go-multistream v0.6.1 // indirectgithub.com/multiformats/go-varint v0.1.0 // indirectgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect- github.com/openai/openai-go v1.12.0 // indirectgithub.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirectgithub.com/pion/datachannel v1.6.0 // indirectgithub.com/pion/dtls/v3 v3.1.2 // indirectdiff --git a/go.sum b/go.sumindex 1f1b278f..7aee74cc 100644--- a/go.sum+++ b/go.sum@@ -282,8 +282,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38ygithub.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784=github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo=-github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo=-github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk=+github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286 h1:Y1LIBzKrtU/NbptugPNuem74f355oyWdLRIm0edgozo=+github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286/go.mod h1:+zGTonNiePk+PlraDn51k+8grAbHh9df7IIAVOMwqZo=github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=diff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.batsindex 5599ea26..7586443c 100644--- a/tests/e2e/key_rotation_test.bats+++ b/tests/e2e/key_rotation_test.bats@@ -36,9 +36,7 @@ teardown() {--issuer "http://mock-oidc:18080" \--client-id "sam-e2e" \--key "${key}" \- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \- --listen "/ip4/0.0.0.0/tcp/4002" \- --external-multiaddr "/dns4/sam-hub/tcp/4002" \+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \--mesh "e2e-mesh" \--key-rotation-interval 5s >/dev/nulldiff --git a/tests/e2e/kind_wi.bats b/tests/e2e/kind_wi.batsindex adef8741..6337a4e3 100644--- a/tests/e2e/kind_wi.bats+++ b/tests/e2e/kind_wi.bats@@ -113,8 +113,6 @@ spec:- "${issuer}"- "--key"- "0000000000000000000000000000000000000000000000000000000000000000"- - "--listen"- - "/ip4/0.0.0.0/tcp/4002"- "--insecure-skip-tls-verify"envFrom:- configMapRef:@@ -127,9 +125,6 @@ metadata:spec:type: ClusterIPports:- - name: p2p- port: 4002- targetPort: 4002- name: httpport: 9090targetPort: 9090diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bashindex f871c905..b796a2c9 100644--- a/tests/e2e/lib/container_mesh.bash+++ b/tests/e2e/lib/container_mesh.bash@@ -211,9 +211,7 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then--client-id "sam-e2e" \--allowed-audiences "sam-e2e,sam-mesh-audience" \--key "${key}" \- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \- --listen "/ip4/0.0.0.0/tcp/4002" \- --external-multiaddr "/dns4/sam-hub/tcp/4002" \+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \--mesh "e2e-mesh" >/dev/nullMESH_CONTAINERS+=("${name}")diff --git a/tests/e2e/policy.bats b/tests/e2e/policy.batsindex a8211f22..e79ad309 100644--- a/tests/e2e/policy.bats+++ b/tests/e2e/policy.bats@@ -193,8 +193,7 @@ EOF"--client-id "sam-e2e" \--allowed-audiences "sam-e2e" \--key "${key}" \- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \- --listen "/ip4/0.0.0.0/tcp/4002" \+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \--mesh "e2e-mesh" \--policy-file "/etc/sam/policies.yaml" >/dev/nulldiff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.batsindex 5c17afea..76bd1ee6 100644--- a/tests/e2e/revocation_test.bats+++ b/tests/e2e/revocation_test.bats@@ -35,8 +35,6 @@ teardown() {--issuer "http://mock-oidc:18080" \--client-id "sam-e2e" \--key "${hub_key}" \- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \- --listen "/ip4/0.0.0.0/tcp/4002" \--mesh "e2e-mesh" \--admin-token "e2e-token" \--log-level debug >/dev/null@@ -91,7 +89,7 @@ teardown() {# Publish ban event for Node 2echo "[$(date +%T)] Publishing ban event"- run docker exec "${hub_name}" /sam-hub admin ban --peer "${node2_peer_id}" --connect "/dns4/sam-hub/tcp/4002/p2p/${hub_peer_id}" --admin-token "e2e-token"+ run docker exec "${hub_name}" /sam-hub admin ban --peer "${node2_peer_id}" --connect "/dns4/sam-hub/tcp/9090/ws/p2p/${hub_peer_id}" --admin-token "e2e-token"echo "admin ban output: $output"[[ "$status" -eq 0 ]]diff --git a/tests/integration/multimaster_test.go b/tests/integration/multimaster_test.goindex 36a4710c..52bba897 100644--- a/tests/integration/multimaster_test.go+++ b/tests/integration/multimaster_test.go@@ -27,6 +27,7 @@ import ("github.com/libp2p/go-libp2p""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/p2p/transport/websocket""github.com/multiformats/go-multiaddr")@@ -60,8 +61,7 @@ roles: {}// Start Replica AcmdA := exec.Command(hubBin,- "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", portA),- "--bind-address", "127.0.0.1:0",+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", portA),"--policy-file", policyFile,"--keys-db", filepath.Join(tmpDir, "keysA.db"),"--allow-loopback",@@ -77,8 +77,7 @@ roles: {}// Start Replica BcmdB := exec.Command(hubBin,- "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", portB),- "--bind-address", "127.0.0.1:0",+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", portB),"--policy-file", policyFile,"--keys-db", filepath.Join(tmpDir, "keysB.db"),"--allow-loopback",@@ -134,14 +133,17 @@ roles: {}t.Logf("Replicas successfully started. Replica A: %s, Replica B: %s", peerIDA, peerIDB)// Create a separate libp2p client host- clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ clientHost, err := libp2p.New(+ libp2p.NoListenAddrs,+ libp2p.Transport(websocket.New),+ )if err != nil {t.Fatal(err)}defer func() { _ = clientHost.Close() }()// Validate connecting to Replica A- addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", portA, peerIDA))+ addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws/p2p/%s", portA, peerIDA))if err != nil {t.Fatal(err)}@@ -159,7 +161,7 @@ roles: {}t.Log("Successfully connected and authenticated with Hub Replica A!")// Validate connecting to Replica B- addrB, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", portB, peerIDB))+ addrB, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws/p2p/%s", portB, peerIDB))if err != nil {t.Fatal(err)}