Skip to content

Agent Mesh

solve the multimaster problem

Antonio Ojea846ed89b8e0d
diff --git a/api/constants.go b/api/constants.go
index cfc47860..84ca5b8e 100644
--- a/api/constants.go
+++ b/api/constants.go
@@ -19,6 +19,7 @@ import "github.com/libp2p/go-libp2p/core/protocol"
const EnrollProtocolID protocol.ID = "/sam/enroll/1.0.0"
const MCPProtocolID protocol.ID = "/sam/mcp/1.0.0"
const GossipEvents = "/sam/mesh/events/v1"
+const GossipHubSync = "/sam/hub/sync/v1"
const AuthProtocolID protocol.ID = "/sam/auth/1.0.0"
const DefaultAudience = "sam-mesh-audience"
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.go
index 70693bb9..b7e76444 100644
--- a/cmd/sam-hub/hooks.go
+++ b/cmd/sam-hub/hooks.go
@@ -72,12 +72,19 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
wasAuth := n.hub.gater.authenticated[p]
delete(n.hub.gater.authenticated, p)
n.hub.gater.mu.Unlock()
-
+
logger.Infow("Peer disconnected", "peer_id", p, "was_authenticated", wasAuth)
if wasAuth {
samHubActiveNodes.Dec() // Decrement active nodes
+ // Notify other hubs to clean up this peer
+ n.hub.publishSyncMessage(context.Background(), HubSyncMessage{
+ Action: "REMOVE",
+ PeerID: p.String(),
+ Timestamp: time.Now().Unix(),
+ })
+
event := &api.MeshEvent{
Type: api.MeshEvent_EXIT,
PeerId: p.String(),
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 871ad7a0..cc70ce05 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -101,8 +101,6 @@ var (
allowLoopbackFlag bool
)
-
-
var logger = golog.Logger("sam-hub")
// Hub handles identity bridging and network discovery
@@ -114,6 +112,8 @@ type Hub struct {
MeshID string
PubSub *pubsub.PubSub
EventTopic *pubsub.Topic
+ SyncTopic *pubsub.Topic
+ otherHubAddrs map[peer.ID][]string
gater *hubConnGate
Policy *api.PolicyConfig
limiter *rate.Limiter
@@ -235,6 +235,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (
ExternalAddrs: externalMultiaddrs,
AllowedAudiences: auds,
AllowLoopback: allowLoopback,
+ otherHubAddrs: make(map[peer.ID][]string),
}
h.Network().Notify(&notifier{hub: hub})
@@ -246,8 +247,17 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (
if err != nil {
return nil, err
}
+
+ syncTopic, err := ps.Join(api.GossipHubSync)
+ if err != nil {
+ return nil, err
+ }
+
hub.PubSub = ps
hub.EventTopic = topic
+ hub.SyncTopic = syncTopic
+ hub.startSyncListener(ctx)
+
return hub, nil
}
@@ -299,6 +309,13 @@ func (h *Hub) handleAuthHandshake(s network.Stream) {
h.gater.authenticated[remotePeer] = true
h.gater.mu.Unlock()
+ // NEW: Notify other hubs of this peer
+ h.publishSyncMessage(context.Background(), HubSyncMessage{
+ Action: "ADD",
+ PeerID: remotePeer.String(),
+ Timestamp: time.Now().Unix(),
+ })
+
// Publish JOIN event
event := &api.MeshEvent{
Type: api.MeshEvent_JOIN,
diff --git a/cmd/sam-hub/main_test.go b/cmd/sam-hub/main_test.go
index 94472ad0..1ca3611c 100644
--- a/cmd/sam-hub/main_test.go
+++ b/cmd/sam-hub/main_test.go
@@ -15,6 +15,7 @@
package main
import (
+ "context"
"io"
"net/http"
"net/http/httptest"
@@ -261,4 +262,129 @@ func TestHandleInfoHTTP(t *testing.T) {
}
}
+func TestHubReplicaSynchronization(t *testing.T) {
+ // 1. Save and restore original global variables to avoid side effects
+ origOidcIssuer := oidcIssuer
+ origListenAddrs := listenAddrs
+ origKeysDBPath := keysDBPath
+ origBiscuitHex := biscuitHex
+ origKeyGracePeriod := keyGracePeriod
+ origAllowLoopbackFlag := allowLoopbackFlag
+
+ defer func() {
+ oidcIssuer = origOidcIssuer
+ listenAddrs = origListenAddrs
+ keysDBPath = origKeysDBPath
+ biscuitHex = origBiscuitHex
+ keyGracePeriod = origKeyGracePeriod
+ allowLoopbackFlag = origAllowLoopbackFlag
+ }()
+
+ // Configure parameters for test
+ oidcIssuer = "https://accounts.google.com"
+ listenAddrs = []string{"/ip4/127.0.0.1/tcp/0"}
+ keyGracePeriod = 24 * time.Hour
+ allowLoopbackFlag = true
+ // Use same hex key so they can decrypt each other's messages!
+ biscuitHex = "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ tmpDir := t.TempDir()
+
+ // 2. Spin up Hub Replica A
+ keysDBPath = filepath.Join(tmpDir, "keysA.db")
+ hubA, err := NewHub(ctx, nil, true)
+ if err != nil {
+ t.Fatalf("failed to create hub A: %v", err)
+ }
+ defer func() { _ = hubA.Host.Close() }()
+
+ // 3. Spin up Hub Replica B
+ keysDBPath = filepath.Join(tmpDir, "keysB.db")
+ hubB, err := NewHub(ctx, nil, true)
+ if err != nil {
+ t.Fatalf("failed to create hub B: %v", err)
+ }
+ defer func() { _ = hubB.Host.Close() }()
+
+ // 4. Connect Hub A and Hub B directly
+ err = hubA.Host.Connect(ctx, peer.AddrInfo{
+ ID: hubB.Host.ID(),
+ Addrs: hubB.Host.Addrs(),
+ })
+ if err != nil {
+ t.Fatalf("failed to connect Hub A to Hub B: %v", err)
+ }
+
+ // Wait a moment for GossipSub sub and mesh connection
+ time.Sleep(200 * time.Millisecond)
+
+ // 5. Create a dummy peer to authenticate
+ priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ peerC, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // 6. Authenticate peerC on Hub A
+ hubA.gater.mu.Lock()
+ hubA.gater.authenticated[peerC] = true
+ hubA.gater.mu.Unlock()
+
+ // 7. Broadcast ADD event from Hub A
+ hubA.publishSyncMessage(ctx, HubSyncMessage{
+ Action: "ADD",
+ PeerID: peerC.String(),
+ Timestamp: time.Now().Unix(),
+ })
+
+ // 8. Verify Hub B receives the ADD and authenticates peerC
+ authedOnB := false
+ for i := 0; i < 100; i++ {
+ hubB.gater.mu.RLock()
+ authedOnB = hubB.gater.authenticated[peerC]
+ hubB.gater.mu.RUnlock()
+ if authedOnB {
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ if !authedOnB {
+ t.Fatal("Expected Hub B to authenticate peerC via sync ADD event, but it did not")
+ }
+
+ // 9. Remove peerC from Hub A and broadcast REMOVE
+ hubA.gater.mu.Lock()
+ delete(hubA.gater.authenticated, peerC)
+ hubA.gater.mu.Unlock()
+
+ hubA.publishSyncMessage(ctx, HubSyncMessage{
+ Action: "REMOVE",
+ PeerID: peerC.String(),
+ Timestamp: time.Now().Unix(),
+ })
+
+ // 10. Verify Hub B receives the REMOVE and removes peerC
+ removedOnB := false
+ for i := 0; i < 100; i++ {
+ hubB.gater.mu.RLock()
+ authed := hubB.gater.authenticated[peerC]
+ hubB.gater.mu.RUnlock()
+ if !authed {
+ removedOnB = true
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ if !removedOnB {
+ t.Fatal("Expected Hub B to de-authenticate peerC via sync REMOVE event, but it remained authenticated")
+ }
+}
+
+
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
index b55cfd1d..d6a3d00e 100644
--- a/cmd/sam-hub/server.go
+++ b/cmd/sam-hub/server.go
@@ -24,6 +24,7 @@ import (
"sort"
"strings"
"sync/atomic"
+ "time"
"github.com/google/sam/api"
"github.com/libp2p/go-libp2p/core/peer"
@@ -158,11 +159,25 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {
h.gater.authenticated[pID] = true
h.gater.mu.Unlock()
+ // NEW: Propagate addition to other hubs
+ h.publishSyncMessage(context.Background(), HubSyncMessage{
+ Action: "ADD",
+ PeerID: pID.String(),
+ Timestamp: time.Now().Unix(),
+ })
+
+ // NEW: Compile addresses of ALL Hub Replicas + Known Peers
var authPeers []peer.ID
h.gater.mu.Lock()
for p := range h.gater.authenticated {
authPeers = append(authPeers, p)
}
+
+ var allHubAddrs []string
+ allHubAddrs = append(allHubAddrs, h.getMyHubAddrs()...)
+ for _, addrs := range h.otherHubAddrs {
+ allHubAddrs = append(allHubAddrs, addrs...)
+ }
h.gater.mu.Unlock()
var knownPeers []string
@@ -170,28 +185,10 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {
knownPeers = append(knownPeers, p.String())
}
- var hubAddrs []string
- if len(h.ExternalAddrs) > 0 {
- for _, addr := range h.ExternalAddrs {
- fullAddr := addr
- if !strings.Contains(addr, "/p2p/") {
- fullAddr = addr + "/p2p/" + h.Host.ID().String()
- }
- hubAddrs = append(hubAddrs, fullAddr)
- }
- } else {
- for _, addr := range h.Host.Addrs() {
- if !h.AllowLoopback && isLoopbackOrLinkLocal(addr) {
- continue
- }
- hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())
- }
- }
-
resp := &api.EnrollResponse{
BiscuitToken: biscuitData,
HubPublicKey: h.KeyRing.GetCurrentPublicKey(),
- HubAddresses: hubAddrs,
+ HubAddresses: allHubAddrs,
Expiration: token.Expiry.Unix(),
KnownPeers: knownPeers,
}
diff --git a/cmd/sam-hub/sync.go b/cmd/sam-hub/sync.go
new file mode 100644
index 00000000..4faba250
--- /dev/null
+++ b/cmd/sam-hub/sync.go
@@ -0,0 +1,207 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+type HubSyncMessage struct {
+ Action string `json:"action"` // ADD, REMOVE, FULL_SYNC
+ PeerID string `json:"peer_id,omitempty"`
+ Peers []string `json:"peers,omitempty"`
+ HubAddrs []string `json:"hub_addrs,omitempty"`
+ Timestamp int64 `json:"timestamp"`
+}
+
+func (h *Hub) encryptMsg(plaintext []byte) ([]byte, error) {
+ key := sha256.Sum256(h.KeyRing.GetCurrentKey())
+ block, err := aes.NewCipher(key[:])
+ if err != nil {
+ return nil, err
+ }
+ gcm, err := cipher.NewGCM(block)
+ if err != nil {
+ return nil, err
+ }
+ nonce := make([]byte, gcm.NonceSize())
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return nil, err
+ }
+ return gcm.Seal(nonce, nonce, plaintext, nil), nil
+}
+
+func (h *Hub) decryptMsg(ciphertext []byte) ([]byte, error) {
+ // Try all valid keys in case a rotation just happened
+ for _, priv := range h.KeyRing.GetAllValidKeys() {
+ key := sha256.Sum256(priv)
+ block, err := aes.NewCipher(key[:])
+ if err != nil {
+ continue
+ }
+ gcm, err := cipher.NewGCM(block)
+ if err != nil {
+ continue
+ }
+ nonceSize := gcm.NonceSize()
+ if len(ciphertext) < nonceSize {
+ continue
+ }
+ nonce, actualCiphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
+ plaintext, err := gcm.Open(nil, nonce, actualCiphertext, nil)
+ if err == nil {
+ return plaintext, nil
+ }
+ }
+ return nil, fmt.Errorf("failed to decrypt sync message")
+}
+
+func (h *Hub) startSyncListener(ctx context.Context) {
+ sub, err := h.SyncTopic.Subscribe()
+ if err != nil {
+ logger.Errorf("Failed to subscribe to sync topic: %v", err)
+ return
+ }
+
+ // Message processor
+ go func() {
+ for {
+ msg, err := sub.Next(ctx)
+ if err != nil {
+ return
+ }
+ if msg.ReceivedFrom == h.Host.ID() {
+ continue // Skip our own broadcast
+ }
+
+ plaintext, err := h.decryptMsg(msg.Data)
+ if err != nil {
+ continue // Invalid encryption/key, ignore
+ }
+ var syncMsg HubSyncMessage
+ if err := json.Unmarshal(plaintext, &syncMsg); err != nil {
+ continue
+ }
+
+ h.gater.mu.Lock()
+ switch syncMsg.Action {
+ case "ADD":
+ p, err := peer.Decode(syncMsg.PeerID)
+ if err == nil && !h.gater.authenticated[p] {
+ samHubActiveNodes.Inc()
+ h.gater.authenticated[p] = true
+ }
+ case "REMOVE":
+ p, err := peer.Decode(syncMsg.PeerID)
+ if err == nil && h.gater.authenticated[p] {
+ samHubActiveNodes.Dec()
+ delete(h.gater.authenticated, p)
+ }
+ case "FULL_SYNC":
+ for _, pStr := range syncMsg.Peers {
+ p, err := peer.Decode(pStr)
+ if err == nil && !h.gater.authenticated[p] {
+ samHubActiveNodes.Inc()
+ h.gater.authenticated[p] = true
+ }
+ }
+ if syncMsg.PeerID != "" && len(syncMsg.HubAddrs) > 0 {
+ p, err := peer.Decode(syncMsg.PeerID)
+ if err == nil {
+ h.otherHubAddrs[p] = syncMsg.HubAddrs
+ }
+ }
+ }
+ h.gater.mu.Unlock()
+ }
+ }()
+
+ // Periodic synchronizer
+ go func() {
+ ticker := time.NewTicker(15 * time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ h.publishFullSync(ctx)
+ }
+ }
+ }()
+
+ // Immediate sync broadcast upon starting
+ h.publishFullSync(ctx)
+}
+
+func (h *Hub) publishFullSync(ctx context.Context) {
+ h.gater.mu.RLock()
+ var peers []string
+ for p := range h.gater.authenticated {
+ peers = append(peers, p.String())
+ }
+ h.gater.mu.RUnlock()
+
+ msg := HubSyncMessage{
+ Action: "FULL_SYNC",
+ PeerID: h.Host.ID().String(),
+ Peers: peers,
+ HubAddrs: h.getMyHubAddrs(),
+ Timestamp: time.Now().Unix(),
+ }
+ h.publishSyncMessage(ctx, msg)
+}
+
+func (h *Hub) publishSyncMessage(ctx context.Context, msg HubSyncMessage) {
+ data, _ := json.Marshal(msg)
+ ciphertext, err := h.encryptMsg(data)
+ if err != nil {
+ logger.Errorf("Failed to encrypt sync msg: %v", err)
+ return
+ }
+ _ = h.SyncTopic.Publish(ctx, ciphertext)
+}
+
+func (h *Hub) getMyHubAddrs() []string {
+ var hubAddrs []string
+ if len(h.ExternalAddrs) > 0 {
+ for _, addr := range h.ExternalAddrs {
+ fullAddr := addr
+ if !strings.Contains(addr, "/p2p/") {
+ fullAddr = addr + "/p2p/" + h.Host.ID().String()
+ }
+ hubAddrs = append(hubAddrs, fullAddr)
+ }
+ } else {
+ for _, addr := range h.Host.Addrs() {
+ if !h.AllowLoopback && isLoopbackOrLinkLocal(addr) {
+ continue
+ }
+ hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())
+ }
+ }
+ return hubAddrs
+}