Skip to content

Agent Mesh

remove odl code

Antonio Ojea92aae7076fda
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.go
index 99ca2f60..70693bb9 100644
--- a/cmd/sam-hub/hooks.go
+++ b/cmd/sam-hub/hooks.go
@@ -34,13 +34,11 @@ var _ connmgr.ConnectionGater = (*hubConnGate)(nil)
type hubConnGate struct {
mu sync.RWMutex
authenticated map[peer.ID]bool
- pending map[peer.ID]time.Time
}
func newHubConnGate() *hubConnGate {
return &hubConnGate{
authenticated: make(map[peer.ID]bool),
- pending: make(map[peer.ID]time.Time),
}
}
@@ -50,11 +48,6 @@ func (g *hubConnGate) InterceptAddrDial(p peer.ID, m multiaddr.Multiaddr) bool {
func (g *hubConnGate) InterceptAccept(c network.ConnMultiaddrs) bool { return true }
func (g *hubConnGate) InterceptSecured(dir network.Direction, p peer.ID, mas network.ConnMultiaddrs) bool {
- g.mu.Lock()
- defer g.mu.Unlock()
- if !g.authenticated[p] {
- g.pending[p] = time.Now()
- }
return true
}
@@ -78,7 +71,6 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
n.hub.gater.mu.Lock()
wasAuth := n.hub.gater.authenticated[p]
delete(n.hub.gater.authenticated, p)
- delete(n.hub.gater.pending, p)
n.hub.gater.mu.Unlock()
logger.Infow("Peer disconnected", "peer_id", p, "was_authenticated", wasAuth)
diff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.go
index a2677753..c6a4a3b6 100644
--- a/cmd/sam-hub/keyring.go
+++ b/cmd/sam-hub/keyring.go
@@ -192,6 +192,18 @@ func (kr *KeyRing) GetAllValidKeys() []ed25519.PrivateKey {
return keys
}
+// GetAllValidPublicKeys returns all public keys that are still valid.
+func (kr *KeyRing) GetAllValidPublicKeys() []ed25519.PublicKey {
+ kr.mu.RLock()
+ defer kr.mu.RUnlock()
+
+ keys := []ed25519.PublicKey{ed25519.PublicKey(kr.Current.Public)}
+ for _, kp := range kr.Previous {
+ keys = append(keys, ed25519.PublicKey(kp.Public))
+ }
+ return keys
+}
+
// Close closes the BoltDB database.
func (kr *KeyRing) Close() error {
return kr.db.Close()
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index a9aecebc..8b88eda7 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -230,90 +230,54 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig) (*Hub, error) {
return hub, nil
}
-func (h *Hub) handleEnroll(s network.Stream) {
+func (h *Hub) handleAuthHandshake(s network.Stream) {
defer func() {
- _ = s.Close()
+ if err := s.Close(); err != nil {
+ logger.Errorf("[AuthN] Failed to close auth stream: %v", err)
+ }
}()
remotePeer := s.Conn().RemotePeer()
- if !h.limiter.Allow() {
- logger.Warnw("Rate limit exceeded during enrollment", "peer_id", remotePeer)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
- _ = s.Reset()
- return
- }
-
- logger.Infow("New enrollment request", "peer_id", remotePeer)
-
reader := msgio.NewVarintReaderSize(s, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
- logger.Errorw("Failed to read enrollment message", "peer_id", remotePeer, "error", err)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
+ logger.Errorf("[AuthN] Failed to read handshake from %s: %v", remotePeer, err)
return
}
defer reader.ReleaseMsg(msg)
- var req api.EnrollRequest
- if err := proto.Unmarshal(msg, &req); err != nil {
- logger.Errorw("Failed to unmarshal enrollment request", "peer_id", remotePeer, "error", err)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
- return
- }
-
- if req.PeerId != remotePeer.String() {
- logger.Warnw("Peer ID mismatch in enrollment", "peer_id", remotePeer, "requested_peer_id", req.PeerId)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
- h.sendEnrollResponse(s, nil, "Peer ID mismatch", 0, nil, nil)
+ var exchange api.AuthFrame
+ if err := proto.Unmarshal(msg, &exchange); err != nil {
+ logger.Warnf("[AuthN] Invalid protobuf from %s", remotePeer)
return
}
- ctx, cancel := context.WithTimeout(context.Background(), JWTVerificationTimeout)
- defer cancel()
- claims, token, err := h.parseAndVerifyJWT(ctx, req.Jwt, h.AllowedAudiences)
+ b, err := h.verifyBiscuit(exchange.Biscuit, remotePeer)
if err != nil {
- logger.Errorw("JWT verification failed", "peer_id", remotePeer, "error", err)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
- h.sendEnrollResponse(s, nil, err.Error(), 0, nil, nil)
+ logger.Warnf("[AuthN] Authorization failed for %s: %v", remotePeer, err)
return
}
- biscuitData, err := h.mintBiscuitToken(claims, token, remotePeer)
- if err != nil {
- logger.Errorw("Biscuit minting failed", "peer_id", remotePeer, "error", err)
- samHubEnrollmentTotal.WithLabelValues("failed").Inc()
- h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)
+ // Enforce hardware binding: token must include node(<remotePeerID>)
+ boundFact := biscuit.Fact{Predicate: biscuit.Predicate{
+ Name: "node",
+ IDs: []biscuit.Term{biscuit.String(remotePeer.String())},
+ }}
+ if _, err := b.GetBlockID(boundFact); err != nil {
+ logger.Warnf("[AuthN] Token is not bound to peer %s", remotePeer)
return
}
+ logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
+
+ // Update active nodes gauge and gater state
h.gater.mu.Lock()
- // Update active nodes gauge if this is a new node
if !h.gater.authenticated[remotePeer] {
samHubActiveNodes.Inc()
}
h.gater.authenticated[remotePeer] = true
- delete(h.gater.pending, remotePeer)
-
- // Collect authenticated peers
- var authPeers []peer.ID
- for p := range h.gater.authenticated {
- authPeers = append(authPeers, p)
- }
h.gater.mu.Unlock()
- var knownPeers []string
- for _, p := range authPeers {
- knownPeers = append(knownPeers, p.String())
- }
-
- policies := []string{
- `allow if operation($op)`,
- }
-
- h.sendEnrollResponse(s, biscuitData, "", token.Expiry.Unix(), knownPeers, policies)
- logger.Infow("Successfully enrolled peer", "peer_id", remotePeer)
- samHubEnrollmentTotal.WithLabelValues("success").Inc()
-
// Publish JOIN event
event := &api.MeshEvent{
Type: api.MeshEvent_JOIN,
@@ -335,43 +299,46 @@ func (h *Hub) handleEnroll(s network.Stream) {
}
}
}
+
+ // Send ACK back to client
+ writer := msgio.NewVarintWriter(s)
+ resp := &api.AuthResponse{Success: true}
+ respBytes, _ := proto.Marshal(resp)
+ if err := writer.WriteMsg(respBytes); err != nil {
+ logger.Errorf("[AuthN] Failed to write ACK to %s: %v", remotePeer, err)
+ }
}
-func (h *Hub) sendEnrollResponse(s network.Stream, biscuitToken []byte, errMsg string, expiration int64, knownPeers []string, policies []string) {
- var hubAddrs []string
- if len(h.ExternalAddrs) > 0 {
- for _, addr := range h.ExternalAddrs {
- fullAddr := addr
- if !strings.Contains(addr, "/p2p/") {
- fullAddr = addr + "/p2p/" + h.Host.ID().String()
- }
- hubAddrs = append(hubAddrs, fullAddr)
- }
- } else {
- for _, addr := range h.Host.Addrs() {
- hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())
- }
+func (h *Hub) verifyBiscuit(biscuitData []byte, remotePeer peer.ID) (*biscuit.Biscuit, error) {
+ b, err := biscuit.Unmarshal(biscuitData)
+ if err != nil {
+ return nil, fmt.Errorf("malformed biscuit: %w", err)
}
- pubKey := h.KeyRing.GetCurrentPublicKey()
+ keys := h.KeyRing.GetAllValidPublicKeys()
+ var lastErr error
+ for _, pubKey := range keys {
+ authorizer, err := b.Authorizer(pubKey)
+ if err != nil {
+ lastErr = err
+ continue
+ }
- resp := &api.EnrollResponse{
- BiscuitToken: biscuitToken,
- ErrorMessage: errMsg,
- HubPublicKey: pubKey,
- HubAddresses: hubAddrs,
- Expiration: expiration,
- KnownPeers: knownPeers,
- }
- data, err := proto.Marshal(resp)
- if err != nil {
- logger.Errorf("[Enroll] Failed to marshal response: %v", err)
- return
- }
- writer := msgio.NewVarintWriter(s)
- if err := writer.WriteMsg(data); err != nil {
- logger.Errorf("[Enroll] Failed to write response: %v", err)
+ rule, err := parser.FromStringPolicy("allow if true")
+ if err != nil {
+ lastErr = err
+ continue
+ }
+ authorizer.AddPolicy(rule)
+
+ if err := authorizer.Authorize(); err == nil {
+ return b, nil
+ } else {
+ lastErr = err
+ }
}
+
+ return nil, fmt.Errorf("no valid key found for verification: %v", lastErr)
}
func (h *Hub) parseAndVerifyJWT(ctx context.Context, jwtStr string, allowedAudiences []string) (jwt.MapClaims, *oidc.IDToken, error) {
@@ -539,31 +506,7 @@ func (h *Hub) mintBiscuitToken(claims jwt.MapClaims, token *oidc.IDToken, remote
// startWatchdog periodically checks for peers that have connected but not completed OIDC
// authentication within the grace period, and evicts them from the network.
-func (h *Hub) startWatchdog(ctx context.Context) {
- go func() {
- ticker := time.NewTicker(10 * time.Second)
- defer ticker.Stop()
- for range ticker.C {
- select {
- case <-ctx.Done():
- return
- default:
- }
- h.gater.mu.Lock()
- now := time.Now()
- for pID, connectedAt := range h.gater.pending {
- if now.Sub(connectedAt) > GracePeriod {
- logger.Warnf("[Security] Evicting unauthenticated peer: %s", pID)
- if err := h.Host.Network().ClosePeer(pID); err != nil {
- logger.Errorf("[Security] Closing unauthenticated peer %s: %v", pID, err)
- }
- delete(h.gater.pending, pID)
- }
- }
- h.gater.mu.Unlock()
- }
- }()
-}
+
func (h *Hub) startRotation(ctx context.Context) {
if keyRotationInterval <= 0 {
@@ -673,13 +616,12 @@ func main() {
if err != nil {
logger.Fatal(err)
}
- h.Host.SetStreamHandler(api.EnrollProtocolID, h.handleEnroll)
+ h.Host.SetStreamHandler(api.AuthProtocolID, h.handleAuthHandshake)
// Start key rotation if enabled
h.startRotation(ctx)
- // Watchdog: Expel peers that connect but never finish authentication
- h.startWatchdog(ctx)
+
startHTTPServer(h, bindAddress, adminToken, tlsCertFile, tlsKeyFile, tlsCAFile, &isHubReady)
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
index e6c17272..e34fba9a 100644
--- a/cmd/sam-hub/server.go
+++ b/cmd/sam-hub/server.go
@@ -149,13 +149,6 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {
return
}
- h.gater.mu.Lock()
- if !h.gater.authenticated[pID] {
- samHubActiveNodes.Inc()
- }
- h.gater.authenticated[pID] = true
- h.gater.mu.Unlock()
-
var authPeers []peer.ID
h.gater.mu.Lock()
for p := range h.gater.authenticated {
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 2a505384..d9770701 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -32,7 +32,6 @@ import (
"github.com/google/sam/api"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
- "github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
"github.com/spf13/cobra"
"google.golang.org/protobuf/proto"
@@ -477,23 +476,16 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {
}
n.mu.Unlock()
- // Connect to hub after enrollment to join the mesh
+ // Connect and Auth to hub after enrollment to join the mesh
for _, addrStr := range enrollResp.HubAddresses {
addr, err := multiaddr.NewMultiaddr(addrStr)
if err != nil {
logger.Warnf("Failed to parse hub address from response: %v", err)
continue
}
- addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
- if err != nil {
- logger.Warnf("Failed to get AddrInfo from hub address: %v", err)
- continue
- }
- if err := n.Host.Connect(ctx, *addrInfo); err != nil {
- logger.Warnf("Failed to connect to hub after enrollment: %v", err)
+ if err := n.ConnectAndAuthWithHub(ctx, addr); err != nil {
+ logger.Warnf("Failed to connect and auth with hub after enrollment: %v", err)
} else {
- logger.Infof("Successfully connected to hub via libp2p after enrollment")
- n.HubPeerID = addrInfo.ID
break
}
}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 8738f71e..2b84e207 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -191,32 +191,11 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
return nil, fmt.Errorf("failed to bootstrap DHT: %w", err)
}
- // Bootstrap: Connect to the Hub
+ // Bootstrap: Connect and Auth to the Hub
for _, addr := range hubAddrs {
- addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
- if err != nil || addrInfo == nil {
- // Try to connect without Peer ID in address
- addrInfo = &peer.AddrInfo{
- Addrs: []multiaddr.Multiaddr{addr},
- }
- logger.Warn("[AuthN] Connecting to hub without Peer ID verification in address.")
- }
- if err := h.Connect(ctx, *addrInfo); err != nil {
- logger.Warnf("[AuthN] Failed to bootstrap to hub %s: %v", addr, err)
+ if err := node.ConnectAndAuthWithHub(ctx, addr); err != nil {
+ logger.Warnf("[AuthN] Failed to bootstrap and auth with hub %s: %v", addr, err)
} else {
- if addrInfo.ID == "" {
- // Discover Peer ID from connection
- for _, c := range h.Network().Conns() {
- if c.RemoteMultiaddr().Equal(addr) {
- node.HubPeerID = c.RemotePeer()
- logger.Infof("[AuthN] Connected to hub (discovered PeerID): %s", node.HubPeerID)
- break
- }
- }
- } else {
- node.HubPeerID = addrInfo.ID
- logger.Infof("[AuthN] Connected to hub: %s", addrInfo.ID)
- }
break
}
}
@@ -301,6 +280,69 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
return node, nil
}
+func (n *SamNode) ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error {
+ addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
+ if err != nil {
+ return fmt.Errorf("failed to get AddrInfo from multiaddr: %w", err)
+ }
+
+ if err := n.Host.Connect(ctx, *addrInfo); err != nil {
+ return fmt.Errorf("failed to connect to hub %s: %w", addr, err)
+ }
+
+ n.HubPeerID = addrInfo.ID
+ logger.Infof("[AuthN] Connected to hub: %s", addrInfo.ID)
+
+ // Load biscuit from store
+ biscuitBytes, err := n.Store.LoadIdentity()
+ if err != nil {
+ return fmt.Errorf("failed to load identity from store: %w", err)
+ }
+ if len(biscuitBytes) == 0 {
+ return fmt.Errorf("no identity biscuit found in store")
+ }
+
+ // Open auth stream
+ s, err := n.Host.NewStream(ctx, addrInfo.ID, api.AuthProtocolID)
+ if err != nil {
+ return fmt.Errorf("failed to open auth stream to hub: %w", err)
+ }
+ defer func() {
+ if err := s.Close(); err != nil {
+ logger.Errorf("failed to close stream: %v", err)
+ }
+ }()
+
+ writer := msgio.NewVarintWriter(s)
+ authFrame := &api.AuthFrame{Biscuit: biscuitBytes}
+ data, err := proto.Marshal(authFrame)
+ if err != nil {
+ return fmt.Errorf("failed to marshal auth frame: %w", err)
+ }
+ if err := writer.WriteMsg(data); err != nil {
+ return fmt.Errorf("failed to write auth frame: %w", err)
+ }
+
+ reader := msgio.NewVarintReaderSize(s, 1024*64)
+ respMsg, err := reader.ReadMsg()
+ if err != nil {
+ return fmt.Errorf("failed to read auth response: %w", err)
+ }
+ defer reader.ReleaseMsg(respMsg)
+
+ var resp api.AuthResponse
+ if err := proto.Unmarshal(respMsg, &resp); err != nil {
+ return fmt.Errorf("failed to unmarshal auth response: %w", err)
+ }
+
+ if !resp.Success {
+ return fmt.Errorf("auth failed: %s", resp.Error)
+ }
+
+ logger.Infof("[AuthN] Successfully authenticated with hub via libp2p")
+ return nil
+}
+
func (n *SamNode) StartRenewalLoop(ctx context.Context, issuerURL, clientID, clientSecret, jwtPath string) {
go func() {
for {