Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
solve the multimaster problem
Antonio Ojea846ed89b8e0d
diff --git a/api/constants.go b/api/constants.goindex cfc47860..84ca5b8e 100644--- a/api/constants.go+++ b/api/constants.go@@ -19,6 +19,7 @@ import "github.com/libp2p/go-libp2p/core/protocol"const EnrollProtocolID protocol.ID = "/sam/enroll/1.0.0"const MCPProtocolID protocol.ID = "/sam/mcp/1.0.0"const GossipEvents = "/sam/mesh/events/v1"+const GossipHubSync = "/sam/hub/sync/v1"const AuthProtocolID protocol.ID = "/sam/auth/1.0.0"const DefaultAudience = "sam-mesh-audience"diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.goindex 70693bb9..b7e76444 100644--- a/cmd/sam-hub/hooks.go+++ b/cmd/sam-hub/hooks.go@@ -72,12 +72,19 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {wasAuth := n.hub.gater.authenticated[p]delete(n.hub.gater.authenticated, p)n.hub.gater.mu.Unlock()-+logger.Infow("Peer disconnected", "peer_id", p, "was_authenticated", wasAuth)if wasAuth {samHubActiveNodes.Dec() // Decrement active nodes+ // Notify other hubs to clean up this peer+ n.hub.publishSyncMessage(context.Background(), HubSyncMessage{+ Action: "REMOVE",+ PeerID: p.String(),+ Timestamp: time.Now().Unix(),+ })+event := &api.MeshEvent{Type: api.MeshEvent_EXIT,PeerId: p.String(),diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.goindex 871ad7a0..cc70ce05 100644--- a/cmd/sam-hub/main.go+++ b/cmd/sam-hub/main.go@@ -101,8 +101,6 @@ var (allowLoopbackFlag bool)--var logger = golog.Logger("sam-hub")// Hub handles identity bridging and network discovery@@ -114,6 +112,8 @@ type Hub struct {MeshID stringPubSub *pubsub.PubSubEventTopic *pubsub.Topic+ SyncTopic *pubsub.Topic+ otherHubAddrs map[peer.ID][]stringgater *hubConnGatePolicy *api.PolicyConfiglimiter *rate.Limiter@@ -235,6 +235,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (ExternalAddrs: externalMultiaddrs,AllowedAudiences: auds,AllowLoopback: allowLoopback,+ otherHubAddrs: make(map[peer.ID][]string),}h.Network().Notify(¬ifier{hub: hub})@@ -246,8 +247,17 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (if err != nil {return nil, err}++ syncTopic, err := ps.Join(api.GossipHubSync)+ if err != nil {+ return nil, err+ }+hub.PubSub = pshub.EventTopic = topic+ hub.SyncTopic = syncTopic+ hub.startSyncListener(ctx)+return hub, nil}@@ -299,6 +309,13 @@ func (h *Hub) handleAuthHandshake(s network.Stream) {h.gater.authenticated[remotePeer] = trueh.gater.mu.Unlock()+ // NEW: Notify other hubs of this peer+ h.publishSyncMessage(context.Background(), HubSyncMessage{+ Action: "ADD",+ PeerID: remotePeer.String(),+ Timestamp: time.Now().Unix(),+ })+// Publish JOIN eventevent := &api.MeshEvent{Type: api.MeshEvent_JOIN,diff --git a/cmd/sam-hub/main_test.go b/cmd/sam-hub/main_test.goindex 94472ad0..1ca3611c 100644--- a/cmd/sam-hub/main_test.go+++ b/cmd/sam-hub/main_test.go@@ -15,6 +15,7 @@package mainimport (+ "context""io""net/http""net/http/httptest"@@ -261,4 +262,129 @@ func TestHandleInfoHTTP(t *testing.T) {}}+func TestHubReplicaSynchronization(t *testing.T) {+ // 1. Save and restore original global variables to avoid side effects+ origOidcIssuer := oidcIssuer+ origListenAddrs := listenAddrs+ origKeysDBPath := keysDBPath+ origBiscuitHex := biscuitHex+ origKeyGracePeriod := keyGracePeriod+ origAllowLoopbackFlag := allowLoopbackFlag++ defer func() {+ oidcIssuer = origOidcIssuer+ listenAddrs = origListenAddrs+ keysDBPath = origKeysDBPath+ biscuitHex = origBiscuitHex+ keyGracePeriod = origKeyGracePeriod+ allowLoopbackFlag = origAllowLoopbackFlag+ }()++ // Configure parameters for test+ oidcIssuer = "https://accounts.google.com"+ listenAddrs = []string{"/ip4/127.0.0.1/tcp/0"}+ keyGracePeriod = 24 * time.Hour+ allowLoopbackFlag = true+ // Use same hex key so they can decrypt each other's messages!+ biscuitHex = "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ tmpDir := t.TempDir()++ // 2. Spin up Hub Replica A+ keysDBPath = filepath.Join(tmpDir, "keysA.db")+ hubA, err := NewHub(ctx, nil, true)+ if err != nil {+ t.Fatalf("failed to create hub A: %v", err)+ }+ defer func() { _ = hubA.Host.Close() }()++ // 3. Spin up Hub Replica B+ keysDBPath = filepath.Join(tmpDir, "keysB.db")+ hubB, err := NewHub(ctx, nil, true)+ if err != nil {+ t.Fatalf("failed to create hub B: %v", err)+ }+ defer func() { _ = hubB.Host.Close() }()++ // 4. Connect Hub A and Hub B directly+ err = hubA.Host.Connect(ctx, peer.AddrInfo{+ ID: hubB.Host.ID(),+ Addrs: hubB.Host.Addrs(),+ })+ if err != nil {+ t.Fatalf("failed to connect Hub A to Hub B: %v", err)+ }++ // Wait a moment for GossipSub sub and mesh connection+ time.Sleep(200 * time.Millisecond)++ // 5. Create a dummy peer to authenticate+ priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ peerC, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }++ // 6. Authenticate peerC on Hub A+ hubA.gater.mu.Lock()+ hubA.gater.authenticated[peerC] = true+ hubA.gater.mu.Unlock()++ // 7. Broadcast ADD event from Hub A+ hubA.publishSyncMessage(ctx, HubSyncMessage{+ Action: "ADD",+ PeerID: peerC.String(),+ Timestamp: time.Now().Unix(),+ })++ // 8. Verify Hub B receives the ADD and authenticates peerC+ authedOnB := false+ for i := 0; i < 100; i++ {+ hubB.gater.mu.RLock()+ authedOnB = hubB.gater.authenticated[peerC]+ hubB.gater.mu.RUnlock()+ if authedOnB {+ break+ }+ time.Sleep(50 * time.Millisecond)+ }+ if !authedOnB {+ t.Fatal("Expected Hub B to authenticate peerC via sync ADD event, but it did not")+ }++ // 9. Remove peerC from Hub A and broadcast REMOVE+ hubA.gater.mu.Lock()+ delete(hubA.gater.authenticated, peerC)+ hubA.gater.mu.Unlock()++ hubA.publishSyncMessage(ctx, HubSyncMessage{+ Action: "REMOVE",+ PeerID: peerC.String(),+ Timestamp: time.Now().Unix(),+ })++ // 10. Verify Hub B receives the REMOVE and removes peerC+ removedOnB := false+ for i := 0; i < 100; i++ {+ hubB.gater.mu.RLock()+ authed := hubB.gater.authenticated[peerC]+ hubB.gater.mu.RUnlock()+ if !authed {+ removedOnB = true+ break+ }+ time.Sleep(50 * time.Millisecond)+ }+ if !removedOnB {+ t.Fatal("Expected Hub B to de-authenticate peerC via sync REMOVE event, but it remained authenticated")+ }+}++diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.goindex b55cfd1d..d6a3d00e 100644--- a/cmd/sam-hub/server.go+++ b/cmd/sam-hub/server.go@@ -24,6 +24,7 @@ import ("sort""strings""sync/atomic"+ "time""github.com/google/sam/api""github.com/libp2p/go-libp2p/core/peer"@@ -158,11 +159,25 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {h.gater.authenticated[pID] = trueh.gater.mu.Unlock()+ // NEW: Propagate addition to other hubs+ h.publishSyncMessage(context.Background(), HubSyncMessage{+ Action: "ADD",+ PeerID: pID.String(),+ Timestamp: time.Now().Unix(),+ })++ // NEW: Compile addresses of ALL Hub Replicas + Known Peersvar authPeers []peer.IDh.gater.mu.Lock()for p := range h.gater.authenticated {authPeers = append(authPeers, p)}++ var allHubAddrs []string+ allHubAddrs = append(allHubAddrs, h.getMyHubAddrs()...)+ for _, addrs := range h.otherHubAddrs {+ allHubAddrs = append(allHubAddrs, addrs...)+ }h.gater.mu.Unlock()var knownPeers []string@@ -170,28 +185,10 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {knownPeers = append(knownPeers, p.String())}- var hubAddrs []string- if len(h.ExternalAddrs) > 0 {- for _, addr := range h.ExternalAddrs {- fullAddr := addr- if !strings.Contains(addr, "/p2p/") {- fullAddr = addr + "/p2p/" + h.Host.ID().String()- }- hubAddrs = append(hubAddrs, fullAddr)- }- } else {- for _, addr := range h.Host.Addrs() {- if !h.AllowLoopback && isLoopbackOrLinkLocal(addr) {- continue- }- hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())- }- }-resp := &api.EnrollResponse{BiscuitToken: biscuitData,HubPublicKey: h.KeyRing.GetCurrentPublicKey(),- HubAddresses: hubAddrs,+ HubAddresses: allHubAddrs,Expiration: token.Expiry.Unix(),KnownPeers: knownPeers,}diff --git a/cmd/sam-hub/sync.go b/cmd/sam-hub/sync.gonew file mode 100644index 00000000..4faba250--- /dev/null+++ b/cmd/sam-hub/sync.go@@ -0,0 +1,207 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "context"+ "crypto/aes"+ "crypto/cipher"+ "crypto/rand"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "io"+ "strings"+ "time"++ "github.com/libp2p/go-libp2p/core/peer"+)++type HubSyncMessage struct {+ Action string `json:"action"` // ADD, REMOVE, FULL_SYNC+ PeerID string `json:"peer_id,omitempty"`+ Peers []string `json:"peers,omitempty"`+ HubAddrs []string `json:"hub_addrs,omitempty"`+ Timestamp int64 `json:"timestamp"`+}++func (h *Hub) encryptMsg(plaintext []byte) ([]byte, error) {+ key := sha256.Sum256(h.KeyRing.GetCurrentKey())+ block, err := aes.NewCipher(key[:])+ if err != nil {+ return nil, err+ }+ gcm, err := cipher.NewGCM(block)+ if err != nil {+ return nil, err+ }+ nonce := make([]byte, gcm.NonceSize())+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {+ return nil, err+ }+ return gcm.Seal(nonce, nonce, plaintext, nil), nil+}++func (h *Hub) decryptMsg(ciphertext []byte) ([]byte, error) {+ // Try all valid keys in case a rotation just happened+ for _, priv := range h.KeyRing.GetAllValidKeys() {+ key := sha256.Sum256(priv)+ block, err := aes.NewCipher(key[:])+ if err != nil {+ continue+ }+ gcm, err := cipher.NewGCM(block)+ if err != nil {+ continue+ }+ nonceSize := gcm.NonceSize()+ if len(ciphertext) < nonceSize {+ continue+ }+ nonce, actualCiphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]+ plaintext, err := gcm.Open(nil, nonce, actualCiphertext, nil)+ if err == nil {+ return plaintext, nil+ }+ }+ return nil, fmt.Errorf("failed to decrypt sync message")+}++func (h *Hub) startSyncListener(ctx context.Context) {+ sub, err := h.SyncTopic.Subscribe()+ if err != nil {+ logger.Errorf("Failed to subscribe to sync topic: %v", err)+ return+ }++ // Message processor+ go func() {+ for {+ msg, err := sub.Next(ctx)+ if err != nil {+ return+ }+ if msg.ReceivedFrom == h.Host.ID() {+ continue // Skip our own broadcast+ }++ plaintext, err := h.decryptMsg(msg.Data)+ if err != nil {+ continue // Invalid encryption/key, ignore+ }+ var syncMsg HubSyncMessage+ if err := json.Unmarshal(plaintext, &syncMsg); err != nil {+ continue+ }++ h.gater.mu.Lock()+ switch syncMsg.Action {+ case "ADD":+ p, err := peer.Decode(syncMsg.PeerID)+ if err == nil && !h.gater.authenticated[p] {+ samHubActiveNodes.Inc()+ h.gater.authenticated[p] = true+ }+ case "REMOVE":+ p, err := peer.Decode(syncMsg.PeerID)+ if err == nil && h.gater.authenticated[p] {+ samHubActiveNodes.Dec()+ delete(h.gater.authenticated, p)+ }+ case "FULL_SYNC":+ for _, pStr := range syncMsg.Peers {+ p, err := peer.Decode(pStr)+ if err == nil && !h.gater.authenticated[p] {+ samHubActiveNodes.Inc()+ h.gater.authenticated[p] = true+ }+ }+ if syncMsg.PeerID != "" && len(syncMsg.HubAddrs) > 0 {+ p, err := peer.Decode(syncMsg.PeerID)+ if err == nil {+ h.otherHubAddrs[p] = syncMsg.HubAddrs+ }+ }+ }+ h.gater.mu.Unlock()+ }+ }()++ // Periodic synchronizer+ go func() {+ ticker := time.NewTicker(15 * time.Second)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ h.publishFullSync(ctx)+ }+ }+ }()++ // Immediate sync broadcast upon starting+ h.publishFullSync(ctx)+}++func (h *Hub) publishFullSync(ctx context.Context) {+ h.gater.mu.RLock()+ var peers []string+ for p := range h.gater.authenticated {+ peers = append(peers, p.String())+ }+ h.gater.mu.RUnlock()++ msg := HubSyncMessage{+ Action: "FULL_SYNC",+ PeerID: h.Host.ID().String(),+ Peers: peers,+ HubAddrs: h.getMyHubAddrs(),+ Timestamp: time.Now().Unix(),+ }+ h.publishSyncMessage(ctx, msg)+}++func (h *Hub) publishSyncMessage(ctx context.Context, msg HubSyncMessage) {+ data, _ := json.Marshal(msg)+ ciphertext, err := h.encryptMsg(data)+ if err != nil {+ logger.Errorf("Failed to encrypt sync msg: %v", err)+ return+ }+ _ = h.SyncTopic.Publish(ctx, ciphertext)+}++func (h *Hub) getMyHubAddrs() []string {+ var hubAddrs []string+ if len(h.ExternalAddrs) > 0 {+ for _, addr := range h.ExternalAddrs {+ fullAddr := addr+ if !strings.Contains(addr, "/p2p/") {+ fullAddr = addr + "/p2p/" + h.Host.ID().String()+ }+ hubAddrs = append(hubAddrs, fullAddr)+ }+ } else {+ for _, addr := range h.Host.Addrs() {+ if !h.AllowLoopback && isLoopbackOrLinkLocal(addr) {+ continue+ }+ hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())+ }+ }+ return hubAddrs+}