Skip to content

Agent Mesh

fix tests and race

Antonio Ojea028d5c998099
diff --git a/cmd/sam-hub/admin.go b/cmd/sam-hub/admin.go
index cff783e4..87d6de36 100644
--- a/cmd/sam-hub/admin.go
+++ b/cmd/sam-hub/admin.go
@@ -47,7 +47,7 @@ func handleBan(admin HubAdmin) http.HandlerFunc {
event := &api.MeshEvent{
Type: api.MeshEvent_BANNED,
PeerId: peerID,
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
}
logger.Infof("[Admin API] Publishing ban event for peer: %s", peerID)
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.go
index 8d0acbe1..71831727 100644
--- a/cmd/sam-hub/hooks.go
+++ b/cmd/sam-hub/hooks.go
@@ -92,7 +92,7 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
event := &api.MeshEvent{
Type: api.MeshEvent_EXIT,
PeerId: p.String(),
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
}
// Sign event
if err := n.hub.signEvent(event); err != nil {
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 49562f7c..b5ac9eaa 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -114,6 +114,7 @@ type Hub struct {
EventTopic *pubsub.Topic
SyncTopic *pubsub.Topic
otherHubAddrs map[peer.ID][]string
+ otherHubLastSeen map[peer.ID]time.Time
gater *hubConnGate
Policy *api.PolicyConfig
limiter *rate.Limiter
@@ -236,6 +237,7 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig, allowLoopback bool) (
AllowedAudiences: auds,
AllowLoopback: allowLoopback,
otherHubAddrs: make(map[peer.ID][]string),
+ otherHubLastSeen: make(map[peer.ID]time.Time),
}
h.Network().Notify(&notifier{hub: hub})
@@ -322,7 +324,7 @@ func (h *Hub) handleAuthHandshake(s network.Stream) {
event := &api.MeshEvent{
Type: api.MeshEvent_JOIN,
PeerId: remotePeer.String(),
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
}
if err := h.signEvent(event); err != nil {
logger.Errorw("Failed to sign mesh event", "peer_id", remotePeer, "error", err)
@@ -631,7 +633,7 @@ func (h *Hub) startRotation(ctx context.Context) {
// Broadcast event
event := &api.MeshEvent{
Type: api.MeshEvent_KEY_ROTATION,
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
NewPublicKey: newPub,
}
diff --git a/cmd/sam-hub/sync.go b/cmd/sam-hub/sync.go
index e6242e9a..c4b015b5 100644
--- a/cmd/sam-hub/sync.go
+++ b/cmd/sam-hub/sync.go
@@ -126,6 +126,11 @@ func (h *Hub) startSyncListener(ctx context.Context) {
}
}
case api.HubSyncMessage_FULL_SYNC:
+ activePeers := make(map[string]bool)
+ for _, activeStr := range syncMsg.Peers {
+ activePeers[activeStr] = true
+ }
+
for pStr, ts := range syncMsg.PeerTimestamps {
p, err := peer.Decode(pStr)
if err != nil {
@@ -135,15 +140,7 @@ func (h *Hub) startSyncListener(ctx context.Context) {
h.gater.lastUpdated[p] = ts
// Determine if the peer is active in the incoming state
- isActive := false
- for _, activeStr := range syncMsg.Peers {
- if activeStr == pStr {
- isActive = true
- break
- }
- }
-
- if isActive {
+ if activePeers[pStr] {
if !h.gater.authenticated[p] {
samHubActiveNodes.Inc()
h.gater.authenticated[p] = true
@@ -160,6 +157,7 @@ func (h *Hub) startSyncListener(ctx context.Context) {
p, err := peer.Decode(syncMsg.PeerId)
if err == nil {
h.otherHubAddrs[p] = syncMsg.HubAddrs
+ h.otherHubLastSeen[p] = time.Now()
}
}
}
@@ -177,6 +175,7 @@ func (h *Hub) startSyncListener(ctx context.Context) {
return
case <-ticker.C:
h.publishFullSync(ctx)
+ h.pruneStaleData()
}
}
}()
@@ -244,3 +243,26 @@ func (h *Hub) getMyHubAddrs() []string {
}
return hubAddrs
}
+
+func (h *Hub) pruneStaleData() {
+ h.gater.mu.Lock()
+ defer h.gater.mu.Unlock()
+ now := time.Now()
+ for p, lastSeen := range h.otherHubLastSeen {
+ if now.Sub(lastSeen) > 45*time.Second {
+ delete(h.otherHubAddrs, p)
+ delete(h.otherHubLastSeen, p)
+ logger.Infof("Pruned stale hub %s (last seen %s ago)", p.String(), now.Sub(lastSeen))
+ }
+ }
+
+ nowMilli := now.UnixMilli()
+ for p, ts := range h.gater.lastUpdated {
+ if !h.gater.authenticated[p] {
+ if nowMilli-ts > 3600*1000 { // 1 hour
+ delete(h.gater.lastUpdated, p)
+ logger.Debugf("Pruned transient peer %s from lastUpdated", p.String())
+ }
+ }
+ }
+}
diff --git a/cmd/sam-node/middleware_test.go b/cmd/sam-node/middleware_test.go
index d0310227..32f94a66 100644
--- a/cmd/sam-node/middleware_test.go
+++ b/cmd/sam-node/middleware_test.go
@@ -416,7 +416,7 @@ func TestVerifyEvent(t *testing.T) {
event := &api.MeshEvent{
Type: api.MeshEvent_BANNED,
PeerId: "attacker-peer",
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
}
// Sign it
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 217fcd09..7f3db553 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -82,6 +82,7 @@ type SamNode struct {
Store *Store
HubPeerID peer.ID
knownPeers map[string]bool
+ peerLastEventTime map[string]int64
receivedMsgs map[string][]string
topics map[string]*pubsub.Topic
mu sync.Mutex
@@ -106,8 +107,9 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
node := &SamNode{
Store: store,
trustedKeys: trustedKeys,
- knownPeers: make(map[string]bool),
- receivedMsgs: make(map[string][]string),
+ knownPeers: make(map[string]bool),
+ peerLastEventTime: make(map[string]int64),
+ receivedMsgs: make(map[string][]string),
topics: make(map[string]*pubsub.Topic),
LocalPolicy: nodeConfig,
AllowLoopback: allowLoopback,
@@ -469,7 +471,7 @@ func (n *SamNode) listenForHubEvents(ctx context.Context) {
}
// Freshness check: reject events older than the threshold to prevent replay attacks
- if time.Since(time.Unix(event.Timestamp, 0)) > FreshnessThreshold {
+ if time.Since(time.UnixMilli(event.Timestamp)) > FreshnessThreshold {
logger.Warnf("[Mesh Event] Dropping stale event from %s (timestamp: %d)", msg.ReceivedFrom, event.Timestamp)
continue
}
@@ -490,6 +492,15 @@ func (n *SamNode) listenForHubEvents(ctx context.Context) {
func (n *SamNode) handleJoinEvent(event *api.MeshEvent) {
n.mu.Lock()
defer n.mu.Unlock()
+ if n.peerLastEventTime == nil {
+ n.peerLastEventTime = make(map[string]int64)
+ }
+ if event.Timestamp < n.peerLastEventTime[event.PeerId] {
+ logger.Warnf("[Mesh Event] Dropping out-of-order JOIN event for peer %s (event timestamp: %d, last processed: %d)", event.PeerId, event.Timestamp, n.peerLastEventTime[event.PeerId])
+ return
+ }
+ n.peerLastEventTime[event.PeerId] = event.Timestamp
+
if n.Host == nil || event.PeerId != n.Host.ID().String() {
n.knownPeers[event.PeerId] = true
}
@@ -499,12 +510,31 @@ func (n *SamNode) handleJoinEvent(event *api.MeshEvent) {
func (n *SamNode) handleExitEvent(event *api.MeshEvent) {
n.mu.Lock()
defer n.mu.Unlock()
+ if n.peerLastEventTime == nil {
+ n.peerLastEventTime = make(map[string]int64)
+ }
+ if event.Timestamp < n.peerLastEventTime[event.PeerId] {
+ logger.Warnf("[Mesh Event] Dropping out-of-order EXIT event for peer %s (event timestamp: %d, last processed: %d)", event.PeerId, event.Timestamp, n.peerLastEventTime[event.PeerId])
+ return
+ }
+ n.peerLastEventTime[event.PeerId] = event.Timestamp
+
delete(n.knownPeers, event.PeerId)
logger.Infof("[Mesh Event] Peer left: %s", event.PeerId)
}
func (n *SamNode) handleBannedEvent(event *api.MeshEvent) {
n.mu.Lock()
+ if n.peerLastEventTime == nil {
+ n.peerLastEventTime = make(map[string]int64)
+ }
+ if event.Timestamp < n.peerLastEventTime[event.PeerId] {
+ logger.Warnf("[Mesh Event] Dropping out-of-order BANNED event for peer %s (event timestamp: %d, last processed: %d)", event.PeerId, event.Timestamp, n.peerLastEventTime[event.PeerId])
+ n.mu.Unlock()
+ return
+ }
+ n.peerLastEventTime[event.PeerId] = event.Timestamp
+
delete(n.knownPeers, event.PeerId)
n.mu.Unlock()
diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.go
index b18b36ae..74c24dcc 100644
--- a/cmd/sam-node/node_test.go
+++ b/cmd/sam-node/node_test.go
@@ -29,8 +29,9 @@ func TestHandleJoinEvent(t *testing.T) {
}
event := &api.MeshEvent{
- Type: api.MeshEvent_JOIN,
- PeerId: "12D3KooWAFv4iJst5G6MjwXhZ66K5zS1tP7A9vSg4vK8f1T7X8t9",
+ Type: api.MeshEvent_JOIN,
+ PeerId: "12D3KooWAFv4iJst5G6MjwXhZ66K5zS1tP7A9vSg4vK8f1T7X8t9",
+ Timestamp: time.Now().UnixMilli(),
}
node.handleJoinEvent(event)
@@ -48,8 +49,9 @@ func TestHandleExitEvent(t *testing.T) {
}
event := &api.MeshEvent{
- Type: api.MeshEvent_EXIT,
- PeerId: "12D3KooWAFv4iJst5G6MjwXhZ66K5zS1tP7A9vSg4vK8f1T7X8t9",
+ Type: api.MeshEvent_EXIT,
+ PeerId: "12D3KooWAFv4iJst5G6MjwXhZ66K5zS1tP7A9vSg4vK8f1T7X8t9",
+ Timestamp: time.Now().UnixMilli(),
}
node.handleExitEvent(event)
@@ -71,7 +73,7 @@ func TestHandleBannedEvent(t *testing.T) {
event := &api.MeshEvent{
Type: api.MeshEvent_BANNED,
PeerId: "12D3KooWAFv4iJst5G6MjwXhZ66K5zS1tP7A9vSg4vK8f1T7X8t9",
- Timestamp: time.Now().Unix(),
+ Timestamp: time.Now().UnixMilli(),
}
node.handleBannedEvent(event)
@@ -93,6 +95,7 @@ func TestHandleKeyRotationEvent(t *testing.T) {
event := &api.MeshEvent{
Type: api.MeshEvent_KEY_ROTATION,
NewPublicKey: pub,
+ Timestamp: time.Now().UnixMilli(),
}
node.handleKeyRotationEvent(event)