Skip to content

Agent Mesh

Refactor: simplify Hub network architecture to use single connection websocket

Antonio Ojea868f58fcf5c0
diff --git a/.github/k8s/sam-hub-template.yaml b/.github/k8s/sam-hub-template.yaml
index 38aced54..b332b62f 100644
--- a/.github/k8s/sam-hub-template.yaml
+++ b/.github/k8s/sam-hub-template.yaml
@@ -11,14 +11,6 @@ spec:
sessionAffinity: ClientIP
externalTrafficPolicy: Local
ports:
- - port: 4001
- targetPort: 4001
- protocol: TCP
- name: libp2p-tcp
- - port: 4001
- targetPort: 4001
- protocol: UDP
- name: libp2p-udp
- port: 443
targetPort: 9090
protocol: TCP
@@ -68,12 +60,6 @@ spec:
key: sam-hub-key
optional: true # Make it optional so it doesn't block local kind testing if not created
ports:
- - containerPort: 4001
- protocol: TCP
- name: libp2p-tcp
- - containerPort: 4001
- protocol: UDP
- name: libp2p-udp
- containerPort: 9090
protocol: TCP
name: metrics
@@ -92,7 +78,6 @@ spec:
initialDelaySeconds: 15
periodSeconds: 20
args:
- - "--listen=/ip4/0.0.0.0/tcp/4001,/ip4/0.0.0.0/udp/4001/quic-v1"
- "--external-multiaddr=${EXTERNAL_MULTIADDR}"
# 1. Trust Dex for humans/Google/GitHub. 2. Trust GKE for internal Canaries.
- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index de7ff2a5..c8d61aa7 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -16,6 +16,8 @@ package main
import (
"github.com/libp2p/go-libp2p/core/network"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"sync"
@@ -48,12 +50,11 @@ import (
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
- libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
+ "github.com/libp2p/go-libp2p/p2p/transport/websocket"
"github.com/multiformats/go-multiaddr"
"golang.org/x/time/rate"
- "github.com/libp2p/go-libp2p/p2p/transport/tcp"
"github.com/spf13/cobra"
"google.golang.org/protobuf/proto"
)
@@ -139,7 +140,7 @@ type Hub struct {
}
// NewHub starts a host supporting both QUIC and TCP (with TLS 1.3)
-func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (*Hub, error) {
+func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool, mux *http.ServeMux) (*Hub, error) {
// Connection Manager for DoS protection
cm, err := connmgr.NewConnManager(LowWaterMark, HighWaterMark, connmgr.WithGracePeriod(ConnGracePeriod))
@@ -147,9 +148,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (
return nil, fmt.Errorf("failed to create connection manager: %w", err)
}
- h, err := libp2p.New(
- libp2p.Transport(libp2pquic.NewTransport),
- libp2p.Transport(tcp.NewTCPTransport),
+ opts := []libp2p.Option{
libp2p.ListenAddrStrings(listenAddrs...),
// FIPS compliant Security
libp2p.Security(libp2ptls.ID, libp2ptls.New),
@@ -168,7 +167,12 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (
}
return filtered
}),
- )
+ }
+ if mux != nil {
+ opts = append(opts, libp2p.Transport(websocket.New, websocket.WithHTTPHandler(mux)))
+ }
+
+ h, err := libp2p.New(opts...)
if err != nil {
return nil, err
}
@@ -606,17 +610,64 @@ func main() {
logger.Fatal(err)
}
- h, err := NewHub(ctx, policyConfig, allowLoopbackFlag)
+ // Add websocket listen address based on bindAddress
+ host, port, err := net.SplitHostPort(bindAddress)
+ if err != nil {
+ // if bindAddress is just ":9090"
+ if strings.HasPrefix(bindAddress, ":") {
+ host = "0.0.0.0"
+ port = strings.TrimPrefix(bindAddress, ":")
+ } else {
+ logger.Fatalf("Invalid bind address: %v", err)
+ }
+ }
+ if host == "" {
+ host = "0.0.0.0"
+ }
+ wsAddr := fmt.Sprintf("/ip4/%s/tcp/%s/ws", host, port)
+ listenAddrs = append(listenAddrs, wsAddr)
+
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
+ mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+ if isHubReady.Load() {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ } else {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ _, _ = w.Write([]byte("not ready"))
+ }
+ })
+
+ h, err := NewHub(ctx, policyConfig, allowLoopbackFlag, mux)
if err != nil {
logger.Fatal(err)
}
+ // Register Hub-specific HTTP routes now that Hub is created
+ mux.HandleFunc("/register", handleRegisterHTTP(h))
+ mux.HandleFunc("/info", handleInfoHTTP(h))
+ mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {
+ if adminToken != "" {
+ authHeader := r.Header.Get("Authorization")
+ if authHeader != "Bearer "+adminToken {
+ http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ return
+ }
+ }
+ handleBan(h)(w, r)
+ })
+
// Start key rotation if enabled
h.startRotation(ctx)
- startHTTPServer(h, bindAddress, adminToken, tlsCertFile, tlsKeyFile, tlsCAFile, &isHubReady)
+ // We don't start a separate HTTP server unless TLS is specifically required for admin separately,
+ // but WithHTTPHandler already shares the port on the websocket listener.
+ if tlsCertFile != "" && tlsKeyFile != "" {
+ logger.Warn("Separate TLS server not supported via CLI when using WithHTTPHandler. Ignoring tlsCertFile.")
+ }
- logger.Infof("SAM Hub Online (QUIC + TCP)")
+ logger.Infof("SAM Hub Online (QUIC + TCP + WS on %s)", bindAddress)
isHubReady.Store(true)
logger.Infof("MeshID: %s", h.MeshID)
logger.Infof("PeerID: %s", h.Host.ID())
@@ -633,7 +684,6 @@ func main() {
rootCmd.Flags().StringVar(&oidcIssuer, "issuer", defIssuer, "OIDC Issuer URL")
rootCmd.Flags().StringVar(&clientID, "client-id", os.Getenv("SAM_OIDC_ID"), "OIDC Client ID")
rootCmd.Flags().StringVar(&biscuitHex, "key", os.Getenv("SAM_HUB_KEY"), "Hub Private Key (32-byte Hex)")
- rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{"/ip4/0.0.0.0/udp/8080/quic-v1", "/ip4/0.0.0.0/tcp/8080"}, "libp2p Listen Addrs")
rootCmd.Flags().StringSliceVar(&externalMultiaddrs, "external-multiaddr", []string{}, "External multiaddrs to announce")
rootCmd.Flags().StringVar(&meshName, "mesh", DefaultMeshName, "Mesh federation name")
rootCmd.Flags().StringVar(&allowedAudiencesFlag, "allowed-audiences", api.DefaultAudience, "Comma-separated list of allowed OIDC audiences")
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
index 62cd7037..2efb89db 100644
--- a/cmd/sam-hub/server.go
+++ b/cmd/sam-hub/server.go
@@ -16,86 +16,16 @@ package main
import (
"context"
- "crypto/tls"
- "crypto/x509"
"io"
"net/http"
- "os"
"sort"
"strings"
- "sync/atomic"
"github.com/google/sam/api"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/protobuf/proto"
)
-// startHTTPServer starts the HTTP/HTTPS server for metrics, healthz, and admin commands.
-func startHTTPServer(h *Hub, bindAddr string, adminToken string, tlsCertFile string, tlsKeyFile string, tlsCAFile string, isHubReady *atomic.Bool) {
- mux := http.NewServeMux()
- mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
- mux.HandleFunc("/register", handleRegisterHTTP(h))
- mux.HandleFunc("/info", handleInfoHTTP(h))
- mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
- if isHubReady.Load() {
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte("ok"))
- } else {
- w.WriteHeader(http.StatusServiceUnavailable)
- _, _ = w.Write([]byte("not ready"))
- }
- })
-
- mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {
- if adminToken != "" {
- authHeader := r.Header.Get("Authorization")
- if authHeader != "Bearer "+adminToken {
- http.Error(w, "Unauthorized", http.StatusUnauthorized)
- return
- }
- }
- handleBan(h)(w, r)
- })
-
- server := &http.Server{
- Addr: bindAddr,
- Handler: mux,
- }
-
- // Configure TLS if requested
- if tlsCertFile != "" && tlsKeyFile != "" {
- tlsConfig := &tls.Config{}
- if tlsCAFile != "" {
- caCert, err := os.ReadFile(tlsCAFile)
- if err != nil {
- logger.Fatalf("Failed to read CA cert: %v", err)
- }
- caCertPool := x509.NewCertPool()
- caCertPool.AppendCertsFromPEM(caCert)
- tlsConfig.ClientCAs = caCertPool
- tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
- logger.Info("mTLS enabled for admin service")
- }
- server.TLSConfig = tlsConfig
-
- logger.Infof("Starting HTTPS server on %s", bindAddr)
- go func() {
- if err := server.ListenAndServeTLS(tlsCertFile, tlsKeyFile); err != nil && err != http.ErrServerClosed {
- logger.Errorf("HTTPS server failed: %v", err)
- }
- }()
- } else {
- logger.Infof("Starting HTTP server on %s", bindAddr)
- go func() {
- if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
- logger.Errorf("HTTP server failed: %v", err)
- }
- }()
- }
-}
-
func handleRegisterHTTP(h *Hub) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
diff --git a/go.mod b/go.mod
index 3d3c7f71..761f569d 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/ipfs/go-cid v0.6.1
github.com/ipfs/go-log/v2 v2.9.2
- github.com/libp2p/go-libp2p v0.48.0
+ github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286
github.com/libp2p/go-libp2p-gostream v0.6.0
github.com/libp2p/go-libp2p-http v0.5.0
github.com/libp2p/go-libp2p-kad-dht v0.39.2
@@ -18,6 +18,7 @@ require (
github.com/modelcontextprotocol/go-sdk v1.6.0
github.com/multiformats/go-multiaddr v0.16.1
github.com/multiformats/go-multihash v0.2.3
+ github.com/openai/openai-go v1.12.0
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2
github.com/spf13/cobra v1.10.2
@@ -83,7 +84,6 @@ require (
github.com/multiformats/go-multistream v0.6.1 // indirect
github.com/multiformats/go-varint v0.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
- github.com/openai/openai-go v1.12.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pion/datachannel v1.6.0 // indirect
github.com/pion/dtls/v3 v3.1.2 // indirect
diff --git a/go.sum b/go.sum
index 1f1b278f..7aee74cc 100644
--- a/go.sum
+++ b/go.sum
@@ -282,8 +282,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y
github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784=
github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo=
-github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo=
-github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk=
+github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286 h1:Y1LIBzKrtU/NbptugPNuem74f355oyWdLRIm0edgozo=
+github.com/libp2p/go-libp2p v0.48.1-0.20260603104142-e6309a296286/go.mod h1:+zGTonNiePk+PlraDn51k+8grAbHh9df7IIAVOMwqZo=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=
diff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.bats
index 5599ea26..7586443c 100644
--- a/tests/e2e/key_rotation_test.bats
+++ b/tests/e2e/key_rotation_test.bats
@@ -36,9 +36,7 @@ teardown() {
--issuer "http://mock-oidc:18080" \
--client-id "sam-e2e" \
--key "${key}" \
- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
- --listen "/ip4/0.0.0.0/tcp/4002" \
- --external-multiaddr "/dns4/sam-hub/tcp/4002" \
+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \
--mesh "e2e-mesh" \
--key-rotation-interval 5s >/dev/null
diff --git a/tests/e2e/kind_wi.bats b/tests/e2e/kind_wi.bats
index adef8741..6337a4e3 100644
--- a/tests/e2e/kind_wi.bats
+++ b/tests/e2e/kind_wi.bats
@@ -113,8 +113,6 @@ spec:
- "${issuer}"
- "--key"
- "0000000000000000000000000000000000000000000000000000000000000000"
- - "--listen"
- - "/ip4/0.0.0.0/tcp/4002"
- "--insecure-skip-tls-verify"
envFrom:
- configMapRef:
@@ -127,9 +125,6 @@ metadata:
spec:
type: ClusterIP
ports:
- - name: p2p
- port: 4002
- targetPort: 4002
- name: http
port: 9090
targetPort: 9090
diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash
index f871c905..b796a2c9 100644
--- a/tests/e2e/lib/container_mesh.bash
+++ b/tests/e2e/lib/container_mesh.bash
@@ -211,9 +211,7 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
--client-id "sam-e2e" \
--allowed-audiences "sam-e2e,sam-mesh-audience" \
--key "${key}" \
- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
- --listen "/ip4/0.0.0.0/tcp/4002" \
- --external-multiaddr "/dns4/sam-hub/tcp/4002" \
+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \
--mesh "e2e-mesh" >/dev/null
MESH_CONTAINERS+=("${name}")
diff --git a/tests/e2e/policy.bats b/tests/e2e/policy.bats
index a8211f22..e79ad309 100644
--- a/tests/e2e/policy.bats
+++ b/tests/e2e/policy.bats
@@ -193,8 +193,7 @@ EOF"
--client-id "sam-e2e" \
--allowed-audiences "sam-e2e" \
--key "${key}" \
- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
- --listen "/ip4/0.0.0.0/tcp/4002" \
+ --external-multiaddr "/dns4/sam-hub/tcp/9090/ws" \
--mesh "e2e-mesh" \
--policy-file "/etc/sam/policies.yaml" >/dev/null
diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.bats
index 5c17afea..76bd1ee6 100644
--- a/tests/e2e/revocation_test.bats
+++ b/tests/e2e/revocation_test.bats
@@ -35,8 +35,6 @@ teardown() {
--issuer "http://mock-oidc:18080" \
--client-id "sam-e2e" \
--key "${hub_key}" \
- --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
- --listen "/ip4/0.0.0.0/tcp/4002" \
--mesh "e2e-mesh" \
--admin-token "e2e-token" \
--log-level debug >/dev/null
@@ -91,7 +89,7 @@ teardown() {
# Publish ban event for Node 2
echo "[$(date +%T)] Publishing ban event"
- run docker exec "${hub_name}" /sam-hub admin ban --peer "${node2_peer_id}" --connect "/dns4/sam-hub/tcp/4002/p2p/${hub_peer_id}" --admin-token "e2e-token"
+ run docker exec "${hub_name}" /sam-hub admin ban --peer "${node2_peer_id}" --connect "/dns4/sam-hub/tcp/9090/ws/p2p/${hub_peer_id}" --admin-token "e2e-token"
echo "admin ban output: $output"
[[ "$status" -eq 0 ]]
diff --git a/tests/integration/multimaster_test.go b/tests/integration/multimaster_test.go
index 36a4710c..52bba897 100644
--- a/tests/integration/multimaster_test.go
+++ b/tests/integration/multimaster_test.go
@@ -27,6 +27,7 @@ import (
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/transport/websocket"
"github.com/multiformats/go-multiaddr"
)
@@ -60,8 +61,7 @@ roles: {}
// Start Replica A
cmdA := exec.Command(hubBin,
- "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", portA),
- "--bind-address", "127.0.0.1:0",
+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", portA),
"--policy-file", policyFile,
"--keys-db", filepath.Join(tmpDir, "keysA.db"),
"--allow-loopback",
@@ -77,8 +77,7 @@ roles: {}
// Start Replica B
cmdB := exec.Command(hubBin,
- "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", portB),
- "--bind-address", "127.0.0.1:0",
+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", portB),
"--policy-file", policyFile,
"--keys-db", filepath.Join(tmpDir, "keysB.db"),
"--allow-loopback",
@@ -134,14 +133,17 @@ roles: {}
t.Logf("Replicas successfully started. Replica A: %s, Replica B: %s", peerIDA, peerIDB)
// Create a separate libp2p client host
- clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ clientHost, err := libp2p.New(
+ libp2p.NoListenAddrs,
+ libp2p.Transport(websocket.New),
+ )
if err != nil {
t.Fatal(err)
}
defer func() { _ = clientHost.Close() }()
// Validate connecting to Replica A
- addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", portA, peerIDA))
+ addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws/p2p/%s", portA, peerIDA))
if err != nil {
t.Fatal(err)
}
@@ -159,7 +161,7 @@ roles: {}
t.Log("Successfully connected and authenticated with Hub Replica A!")
// Validate connecting to Replica B
- addrB, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", portB, peerIDB))
+ addrB, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws/p2p/%s", portB, peerIDB))
if err != nil {
t.Fatal(err)
}