Skip to content

Agent Mesh

refactor sam hhub serve and enable all protocols

Antonio Ojea06fd22eccb9a
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index edb88b51..9c942141 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -50,7 +50,6 @@ import (
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
- "github.com/libp2p/go-libp2p/p2p/transport/websocket"
"github.com/multiformats/go-multiaddr"
"golang.org/x/time/rate"
@@ -150,8 +149,7 @@ type Hub struct {
}
// NewHub starts a host supporting both QUIC and TCP (with TLS 1.3)
-func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool, mux *http.ServeMux, serverTlsConfig *tls.Config) (*Hub, error) {
-
+func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (*Hub, error) {
// Connection Manager for DoS protection
cm, err := connmgr.NewConnManager(LowWaterMark, HighWaterMark, connmgr.WithGracePeriod(ConnGracePeriod))
if err != nil {
@@ -159,6 +157,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool, m
}
opts := []libp2p.Option{
+ libp2p.DefaultTransports,
libp2p.ListenAddrStrings(listenAddrs...),
// FIPS compliant Security
libp2p.Security(libp2ptls.ID, libp2ptls.New),
@@ -178,13 +177,6 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool, m
return filtered
}),
}
- if mux != nil {
- if serverTlsConfig != nil {
- opts = append(opts, libp2p.Transport(websocket.New, websocket.WithHTTPHandler(mux), websocket.WithTLSConfig(serverTlsConfig)))
- } else {
- opts = append(opts, libp2p.Transport(websocket.New, websocket.WithHTTPHandler(mux)))
- }
- }
h, err := libp2p.New(opts...)
if err != nil {
@@ -624,26 +616,7 @@ func main() {
logger.Fatal(err)
}
- // Add websocket listen address based on bindAddress
- host, port, err := net.SplitHostPort(bindAddress)
- if err != nil {
- // if bindAddress is just ":9090"
- if strings.HasPrefix(bindAddress, ":") {
- host = "0.0.0.0"
- port = strings.TrimPrefix(bindAddress, ":")
- } else {
- logger.Fatalf("Invalid bind address: %v", err)
- }
- }
- if host == "" {
- host = "0.0.0.0"
- }
- ipVersion := "ip4"
- if strings.Contains(host, ":") {
- ipVersion = "ip6"
- }
var serverTlsConfig *tls.Config
- wsScheme := "ws"
if tlsCertFile != "" && tlsKeyFile != "" {
cert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile)
if err != nil {
@@ -652,12 +625,15 @@ func main() {
serverTlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
}
- wsScheme = "wss"
}
- wsAddr := fmt.Sprintf("/%s/%s/tcp/%s/%s", ipVersion, host, port, wsScheme)
- listenAddrs = append(listenAddrs, wsAddr)
- var hRef atomic.Pointer[Hub]
+ h, err := NewHub(ctx, policyConfig, allowLoopbackFlag)
+ if err != nil {
+ logger.Fatal(err)
+ }
+
+ // Start key rotation if enabled
+ h.startRotation(ctx)
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
@@ -671,21 +647,8 @@ func main() {
}
})
- // Register Hub-specific HTTP routes using an atomic pointer to avoid data races
- mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
- if h := hRef.Load(); h != nil {
- handleRegisterHTTP(h)(w, r)
- } else {
- http.Error(w, "Hub not ready", http.StatusServiceUnavailable)
- }
- })
- mux.HandleFunc("/info", func(w http.ResponseWriter, r *http.Request) {
- if h := hRef.Load(); h != nil {
- handleInfoHTTP(h)(w, r)
- } else {
- http.Error(w, "Hub not ready", http.StatusServiceUnavailable)
- }
- })
+ mux.HandleFunc("/register", h.handleRegisterHTTP)
+ mux.HandleFunc("/info", h.handleInfoHTTP)
mux.HandleFunc("/admin/ban", func(w http.ResponseWriter, r *http.Request) {
if adminToken != "" {
authHeader := r.Header.Get("Authorization")
@@ -694,27 +657,38 @@ func main() {
return
}
}
- if h := hRef.Load(); h != nil {
- handleBan(h)(w, r)
- } else {
- http.Error(w, "Hub not ready", http.StatusServiceUnavailable)
- }
+ handleBan(h)(w, r)
})
- h, err := NewHub(ctx, policyConfig, allowLoopbackFlag, mux, serverTlsConfig)
- if err != nil {
- logger.Fatal(err)
+
+
+ server := &http.Server{
+ Addr: bindAddress,
+ Handler: mux,
}
- hRef.Store(h)
- // Start key rotation if enabled
- h.startRotation(ctx)
+ go func() {
+ if serverTlsConfig != nil {
+ server.TLSConfig = serverTlsConfig
+ if err := server.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
+ logger.Fatalf("HTTP server failed: %v", err)
+ }
+ } else {
+ if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ logger.Fatalf("HTTP server failed: %v", err)
+ }
+ }
+ }()
logger.Infof("SAM Hub Online (HTTP on %s, P2P on %v)", bindAddress, listenAddrs)
isHubReady.Store(true)
logger.Infof("MeshID: %s", h.MeshID)
logger.Infof("PeerID: %s", h.Host.ID())
<-ctx.Done()
+
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ _ = server.Shutdown(shutdownCtx)
},
}
diff --git a/cmd/sam-hub/main_test.go b/cmd/sam-hub/main_test.go
index 41b87e94..47df075c 100644
--- a/cmd/sam-hub/main_test.go
+++ b/cmd/sam-hub/main_test.go
@@ -238,12 +238,10 @@ func TestHandleInfoHTTP(t *testing.T) {
Host: h,
}
- handler := handleInfoHTTP(hub)
-
req := httptest.NewRequest("GET", "/info", nil)
w := httptest.NewRecorder()
- handler(w, req)
+ hub.handleInfoHTTP(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusOK {
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
index 066d9f44..79f4eb7e 100644
--- a/cmd/sam-hub/server.go
+++ b/cmd/sam-hub/server.go
@@ -26,145 +26,141 @@ import (
"google.golang.org/protobuf/proto"
)
-func handleRegisterHTTP(h *Hub) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodPost {
- http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
- return
- }
- if r.URL.Path != "/register" {
- http.NotFound(w, r)
- return
- }
+func (h *Hub) handleRegisterHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if r.URL.Path != "/register" {
+ http.NotFound(w, r)
+ return
+ }
- body, err := io.ReadAll(r.Body)
- if err != nil {
- http.Error(w, "Failed to read body", http.StatusBadRequest)
- return
- }
- defer func() {
- if err := r.Body.Close(); err != nil {
- logger.Errorf("failed to close request body: %v", err)
- }
- }()
-
- var req api.EnrollRequest
- if err := proto.Unmarshal(body, &req); err != nil {
- http.Error(w, "Invalid request format", http.StatusBadRequest)
- return
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ http.Error(w, "Failed to read body", http.StatusBadRequest)
+ return
+ }
+ defer func() {
+ if err := r.Body.Close(); err != nil {
+ logger.Errorf("failed to close request body: %v", err)
}
+ }()
- if !h.limiter.Allow() {
- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
- return
- }
+ var req api.EnrollRequest
+ if err := proto.Unmarshal(body, &req); err != nil {
+ http.Error(w, "Invalid request format", http.StatusBadRequest)
+ return
+ }
- logger.Infow("New HTTP enrollment request", "peer_id", req.PeerId)
+ if !h.limiter.Allow() {
+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+ return
+ }
- ctx, cancel := context.WithTimeout(context.Background(), JWTVerificationTimeout)
- defer cancel()
+ logger.Infow("New HTTP enrollment request", "peer_id", req.PeerId)
- claims, token, err := h.parseAndVerifyJWT(ctx, req.Jwt, h.AllowedAudiences)
- if err != nil {
- logger.Errorw("JWT verification failed", "peer_id", req.PeerId, "error", err)
- http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)
- return
- }
+ ctx, cancel := context.WithTimeout(context.Background(), JWTVerificationTimeout)
+ defer cancel()
- pID, err := peer.Decode(req.PeerId)
- if err != nil {
- logger.Errorw("Invalid Peer ID", "peer_id", req.PeerId, "error", err)
- http.Error(w, "Invalid Peer ID", http.StatusBadRequest)
- return
- }
-
- biscuitData, err := h.mintBiscuitToken(claims, token, pID)
- if err != nil {
- logger.Errorw("Biscuit minting failed", "peer_id", req.PeerId, "error", err)
- http.Error(w, "Failed to mint biscuit", http.StatusInternalServerError)
- return
- }
- allHubAddrs := getMyHubAddrs(h)
+ claims, token, err := h.parseAndVerifyJWT(ctx, req.Jwt, h.AllowedAudiences)
+ if err != nil {
+ logger.Errorw("JWT verification failed", "peer_id", req.PeerId, "error", err)
+ http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)
+ return
+ }
- resp := &api.EnrollResponse{
- BiscuitToken: biscuitData,
- HubPublicKey: h.KeyRing.GetCurrentPublicKey(),
- HubAddresses: allHubAddrs,
- Expiration: token.Expiry.Unix(),
- }
+ pID, err := peer.Decode(req.PeerId)
+ if err != nil {
+ logger.Errorw("Invalid Peer ID", "peer_id", req.PeerId, "error", err)
+ http.Error(w, "Invalid Peer ID", http.StatusBadRequest)
+ return
+ }
- respData, err := proto.Marshal(resp)
- if err != nil {
- logger.Errorf("[Enroll] Failed to marshal response: %v", err)
- http.Error(w, "Failed to serialize response", http.StatusInternalServerError)
- return
- }
+ biscuitData, err := h.mintBiscuitToken(claims, token, pID)
+ if err != nil {
+ logger.Errorw("Biscuit minting failed", "peer_id", req.PeerId, "error", err)
+ http.Error(w, "Failed to mint biscuit", http.StatusInternalServerError)
+ return
+ }
+ allHubAddrs := getMyHubAddrs(h)
- w.Header().Set("Content-Type", "application/x-protobuf")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write(respData)
+ resp := &api.EnrollResponse{
+ BiscuitToken: biscuitData,
+ HubPublicKey: h.KeyRing.GetCurrentPublicKey(),
+ HubAddresses: allHubAddrs,
+ Expiration: token.Expiry.Unix(),
+ }
- logger.Infow("Successfully enrolled peer via HTTP", "peer_id", req.PeerId)
- samHubEnrollmentTotal.WithLabelValues("success").Inc()
+ respData, err := proto.Marshal(resp)
+ if err != nil {
+ logger.Errorf("[Enroll] Failed to marshal response: %v", err)
+ http.Error(w, "Failed to serialize response", http.StatusInternalServerError)
+ return
}
-}
-func handleInfoHTTP(h *Hub) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodGet {
- http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
- return
- }
- if r.URL.Path != "/info" {
- http.NotFound(w, r)
- return
- }
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(respData)
- // Find the OIDC issuer deterministically by sorting the map keys
- issuer := ""
- if len(h.Providers) > 0 {
- issuers := make([]string, 0, len(h.Providers))
- for iss := range h.Providers {
- issuers = append(issuers, iss)
- }
- sort.Strings(issuers)
- issuer = issuers[0]
- }
+ logger.Infow("Successfully enrolled peer via HTTP", "peer_id", req.PeerId)
+ samHubEnrollmentTotal.WithLabelValues("success").Inc()
+}
- // Fallback if Providers isn't fully populated (e.g. in unit tests)
- if issuer == "" {
- issuer = oidcIssuer
- }
- // If multiple issuers in fallback, pick the first one deterministically
- if strings.Contains(issuer, ",") {
- parts := strings.Split(issuer, ",")
- issuer = strings.TrimSpace(parts[0])
- }
+func (h *Hub) handleInfoHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if r.URL.Path != "/info" {
+ http.NotFound(w, r)
+ return
+ }
- // Get primary client ID / audience
- aud := api.DefaultAudience
- if len(h.AllowedAudiences) > 0 {
- aud = h.AllowedAudiences[0]
+ // Find the OIDC issuer deterministically by sorting the map keys
+ issuer := ""
+ if len(h.Providers) > 0 {
+ issuers := make([]string, 0, len(h.Providers))
+ for iss := range h.Providers {
+ issuers = append(issuers, iss)
}
+ sort.Strings(issuers)
+ issuer = issuers[0]
+ }
- resp := &api.HubInfoResponse{
- OidcIssuer: issuer,
- ClientId: aud,
- Audience: aud,
- HubAddresses: getMyHubAddrs(h),
- }
+ // Fallback if Providers isn't fully populated (e.g. in unit tests)
+ if issuer == "" {
+ issuer = oidcIssuer
+ }
+ // If multiple issuers in fallback, pick the first one deterministically
+ if strings.Contains(issuer, ",") {
+ parts := strings.Split(issuer, ",")
+ issuer = strings.TrimSpace(parts[0])
+ }
- respData, err := proto.Marshal(resp)
- if err != nil {
- logger.Errorf("[Info] Failed to marshal info response: %v", err)
- http.Error(w, "Failed to serialize response", http.StatusInternalServerError)
- return
- }
+ // Get primary client ID / audience
+ aud := api.DefaultAudience
+ if len(h.AllowedAudiences) > 0 {
+ aud = h.AllowedAudiences[0]
+ }
+
+ resp := &api.HubInfoResponse{
+ OidcIssuer: issuer,
+ ClientId: aud,
+ Audience: aud,
+ HubAddresses: getMyHubAddrs(h),
+ }
- w.Header().Set("Content-Type", "application/x-protobuf")
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write(respData)
+ respData, err := proto.Marshal(resp)
+ if err != nil {
+ logger.Errorf("[Info] Failed to marshal info response: %v", err)
+ http.Error(w, "Failed to serialize response", http.StatusInternalServerError)
+ return
}
+
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(respData)
}
func getMyHubAddrs(h *Hub) []string {
diff --git a/tests/integration/multimaster_test.go b/tests/integration/multimaster_test.go
index 52bba897..4501509f 100644
--- a/tests/integration/multimaster_test.go
+++ b/tests/integration/multimaster_test.go
@@ -27,7 +27,6 @@ import (
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/libp2p/go-libp2p/p2p/transport/websocket"
"github.com/multiformats/go-multiaddr"
)
@@ -58,10 +57,13 @@ roles: {}
portA := getFreePort(t)
portB := getFreePort(t)
+ p2pPortA := getFreePort(t)
+ p2pPortB := getFreePort(t)
// Start Replica A
cmdA := exec.Command(hubBin,
"--bind-address", fmt.Sprintf("127.0.0.1:%d", portA),
+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortA),
"--policy-file", policyFile,
"--keys-db", filepath.Join(tmpDir, "keysA.db"),
"--allow-loopback",
@@ -78,6 +80,7 @@ roles: {}
// Start Replica B
cmdB := exec.Command(hubBin,
"--bind-address", fmt.Sprintf("127.0.0.1:%d", portB),
+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortB),
"--policy-file", policyFile,
"--keys-db", filepath.Join(tmpDir, "keysB.db"),
"--allow-loopback",
@@ -135,7 +138,6 @@ roles: {}
// Create a separate libp2p client host
clientHost, err := libp2p.New(
libp2p.NoListenAddrs,
- libp2p.Transport(websocket.New),
)
if err != nil {
t.Fatal(err)
@@ -143,7 +145,7 @@ roles: {}
defer func() { _ = clientHost.Close() }()
// Validate connecting to Replica A
- addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws/p2p/%s", portA, peerIDA))
+ addrA, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", p2pPortA, peerIDA))
if err != nil {
t.Fatal(err)
}