Skip to content

Agent Mesh

address comments

Antonio Ojea7f52aad94723
diff --git a/api/sam.pb.go b/api/sam.pb.go
index 15545a48..319ac929 100644
--- a/api/sam.pb.go
+++ b/api/sam.pb.go
@@ -830,14 +830,15 @@ func (x *HubInfoResponse) GetAudience() string {
}
type HubSyncMessage struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- Action HubSyncMessage_Action `protobuf:"varint,1,opt,name=action,proto3,enum=sam.v1.HubSyncMessage_Action" json:"action,omitempty"`
- PeerId string `protobuf:"bytes,2,opt,name=peer_id,json=peerId,proto3" json:"peer_id,omitempty"`
- Peers []string `protobuf:"bytes,3,rep,name=peers,proto3" json:"peers,omitempty"`
- HubAddrs []string `protobuf:"bytes,4,rep,name=hub_addrs,json=hubAddrs,proto3" json:"hub_addrs,omitempty"`
- Timestamp int64 `protobuf:"varint,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ state protoimpl.MessageState `protogen:"open.v1"`
+ Action HubSyncMessage_Action `protobuf:"varint,1,opt,name=action,proto3,enum=sam.v1.HubSyncMessage_Action" json:"action,omitempty"`
+ PeerId string `protobuf:"bytes,2,opt,name=peer_id,json=peerId,proto3" json:"peer_id,omitempty"`
+ Peers []string `protobuf:"bytes,3,rep,name=peers,proto3" json:"peers,omitempty"`
+ HubAddrs []string `protobuf:"bytes,4,rep,name=hub_addrs,json=hubAddrs,proto3" json:"hub_addrs,omitempty"`
+ Timestamp int64 `protobuf:"varint,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+ PeerTimestamps map[string]int64 `protobuf:"bytes,6,rep,name=peer_timestamps,json=peerTimestamps,proto3" json:"peer_timestamps,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *HubSyncMessage) Reset() {
@@ -905,6 +906,13 @@ func (x *HubSyncMessage) GetTimestamp() int64 {
return 0
}
+func (x *HubSyncMessage) GetPeerTimestamps() map[string]int64 {
+ if x != nil {
+ return x.PeerTimestamps
+ }
+ return nil
+}
+
var File_api_sam_proto protoreflect.FileDescriptor
const file_api_sam_proto_rawDesc = "" +
@@ -965,13 +973,17 @@ const file_api_sam_proto_rawDesc = "" +
"\voidc_issuer\x18\x01 \x01(\tR\n" +
"oidcIssuer\x12\x1b\n" +
"\tclient_id\x18\x02 \x01(\tR\bclientId\x12\x1a\n" +
- "\baudience\x18\x03 \x01(\tR\baudience\"\xf7\x01\n" +
+ "\baudience\x18\x03 \x01(\tR\baudience\"\x8f\x03\n" +
"\x0eHubSyncMessage\x125\n" +
"\x06action\x18\x01 \x01(\x0e2\x1d.sam.v1.HubSyncMessage.ActionR\x06action\x12\x17\n" +
"\apeer_id\x18\x02 \x01(\tR\x06peerId\x12\x14\n" +
"\x05peers\x18\x03 \x03(\tR\x05peers\x12\x1b\n" +
"\thub_addrs\x18\x04 \x03(\tR\bhubAddrs\x12\x1c\n" +
- "\ttimestamp\x18\x05 \x01(\x03R\ttimestamp\"D\n" +
+ "\ttimestamp\x18\x05 \x01(\x03R\ttimestamp\x12S\n" +
+ "\x0fpeer_timestamps\x18\x06 \x03(\v2*.sam.v1.HubSyncMessage.PeerTimestampsEntryR\x0epeerTimestamps\x1aA\n" +
+ "\x13PeerTimestampsEntry\x12\x10\n" +
+ "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+ "\x05value\x18\x02 \x01(\x03R\x05value:\x028\x01\"D\n" +
"\x06Action\x12\x16\n" +
"\x12ACTION_UNSPECIFIED\x10\x00\x12\a\n" +
"\x03ADD\x10\x01\x12\n" +
@@ -997,7 +1009,7 @@ func file_api_sam_proto_rawDescGZIP() []byte {
}
var file_api_sam_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_api_sam_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
+var file_api_sam_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_api_sam_proto_goTypes = []any{
(ServiceType)(0), // 0: sam.v1.ServiceType
(MeshEvent_Type)(0), // 1: sam.v1.MeshEvent.Type
@@ -1014,6 +1026,7 @@ var file_api_sam_proto_goTypes = []any{
(*HubInfoResponse)(nil), // 12: sam.v1.HubInfoResponse
(*HubSyncMessage)(nil), // 13: sam.v1.HubSyncMessage
nil, // 14: sam.v1.CommandBackend.EnvEntry
+ nil, // 15: sam.v1.HubSyncMessage.PeerTimestampsEntry
}
var file_api_sam_proto_depIdxs = []int32{
1, // 0: sam.v1.MeshEvent.type:type_name -> sam.v1.MeshEvent.Type
@@ -1022,11 +1035,12 @@ var file_api_sam_proto_depIdxs = []int32{
8, // 3: sam.v1.RegisterServiceRequest.service:type_name -> sam.v1.ServiceInfo
9, // 4: sam.v1.RegisterServiceRequest.command:type_name -> sam.v1.CommandBackend
2, // 5: sam.v1.HubSyncMessage.action:type_name -> sam.v1.HubSyncMessage.Action
- 6, // [6:6] is the sub-list for method output_type
- 6, // [6:6] is the sub-list for method input_type
- 6, // [6:6] is the sub-list for extension type_name
- 6, // [6:6] is the sub-list for extension extendee
- 0, // [0:6] is the sub-list for field type_name
+ 15, // 6: sam.v1.HubSyncMessage.peer_timestamps:type_name -> sam.v1.HubSyncMessage.PeerTimestampsEntry
+ 7, // [7:7] is the sub-list for method output_type
+ 7, // [7:7] is the sub-list for method input_type
+ 7, // [7:7] is the sub-list for extension type_name
+ 7, // [7:7] is the sub-list for extension extendee
+ 0, // [0:7] is the sub-list for field type_name
}
func init() { file_api_sam_proto_init() }
@@ -1044,7 +1058,7 @@ func file_api_sam_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_sam_proto_rawDesc), len(file_api_sam_proto_rawDesc)),
NumEnums: 3,
- NumMessages: 12,
+ NumMessages: 13,
NumExtensions: 0,
NumServices: 0,
},
diff --git a/api/sam.proto b/api/sam.proto
index c963f9ce..9c206f81 100644
--- a/api/sam.proto
+++ b/api/sam.proto
@@ -108,4 +108,5 @@ message HubSyncMessage {
repeated string peers = 3;
repeated string hub_addrs = 4;
int64 timestamp = 5;
+ map<string, int64> peer_timestamps = 6;
}
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.go
index bc7a8a84..b13c806c 100644
--- a/cmd/sam-hub/hooks.go
+++ b/cmd/sam-hub/hooks.go
@@ -34,11 +34,13 @@ var _ connmgr.ConnectionGater = (*hubConnGate)(nil)
type hubConnGate struct {
mu sync.RWMutex
authenticated map[peer.ID]bool
+ lastUpdated map[peer.ID]int64
}
func newHubConnGate() *hubConnGate {
return &hubConnGate{
authenticated: make(map[peer.ID]bool),
+ lastUpdated: make(map[peer.ID]int64),
}
}
@@ -70,6 +72,8 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
p := c.RemotePeer()
n.hub.gater.mu.Lock()
wasAuth := n.hub.gater.authenticated[p]
+ now := time.Now().Unix()
+ n.hub.gater.lastUpdated[p] = now
delete(n.hub.gater.authenticated, p)
n.hub.gater.mu.Unlock()
@@ -82,7 +86,7 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
n.hub.publishSyncMessage(context.Background(), &api.HubSyncMessage{
Action: api.HubSyncMessage_REMOVE,
PeerId: p.String(),
- Timestamp: time.Now().Unix(),
+ Timestamp: now,
})
event := &api.MeshEvent{
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 45108e93..78da5964 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -303,6 +303,8 @@ func (h *Hub) handleAuthHandshake(s network.Stream) {
// Update active nodes gauge and gater state
h.gater.mu.Lock()
+ now := time.Now().Unix()
+ h.gater.lastUpdated[remotePeer] = now
if !h.gater.authenticated[remotePeer] {
samHubActiveNodes.Inc()
}
@@ -313,7 +315,7 @@ func (h *Hub) handleAuthHandshake(s network.Stream) {
h.publishSyncMessage(context.Background(), &api.HubSyncMessage{
Action: api.HubSyncMessage_ADD,
PeerId: remotePeer.String(),
- Timestamp: time.Now().Unix(),
+ Timestamp: now,
})
// Publish JOIN event
diff --git a/cmd/sam-hub/server.go b/cmd/sam-hub/server.go
index 70245dd6..49737d40 100644
--- a/cmd/sam-hub/server.go
+++ b/cmd/sam-hub/server.go
@@ -153,6 +153,8 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {
}
h.gater.mu.Lock()
+ now := time.Now().Unix()
+ h.gater.lastUpdated[pID] = now
if !h.gater.authenticated[pID] {
samHubActiveNodes.Inc()
}
@@ -163,7 +165,7 @@ func handleRegisterHTTP(h *Hub) http.HandlerFunc {
h.publishSyncMessage(context.Background(), &api.HubSyncMessage{
Action: api.HubSyncMessage_ADD,
PeerId: pID.String(),
- Timestamp: time.Now().Unix(),
+ Timestamp: now,
})
// NEW: Compile addresses of ALL Hub Replicas + Known Peers
diff --git a/cmd/sam-hub/sync.go b/cmd/sam-hub/sync.go
index d6325182..56dbcc97 100644
--- a/cmd/sam-hub/sync.go
+++ b/cmd/sam-hub/sync.go
@@ -81,12 +81,13 @@ func (h *Hub) startSyncListener(ctx context.Context) {
// Message processor
go func() {
+ defer sub.Cancel()
for {
msg, err := sub.Next(ctx)
if err != nil {
return
}
- if msg.ReceivedFrom == h.Host.ID() {
+ if msg.From == h.Host.ID() {
continue // Skip our own broadcast
}
@@ -103,22 +104,55 @@ func (h *Hub) startSyncListener(ctx context.Context) {
switch syncMsg.Action {
case api.HubSyncMessage_ADD:
p, err := peer.Decode(syncMsg.PeerId)
- if err == nil && !h.gater.authenticated[p] {
- samHubActiveNodes.Inc()
- h.gater.authenticated[p] = true
+ if err == nil {
+ if syncMsg.Timestamp > h.gater.lastUpdated[p] {
+ h.gater.lastUpdated[p] = syncMsg.Timestamp
+ if !h.gater.authenticated[p] {
+ samHubActiveNodes.Inc()
+ h.gater.authenticated[p] = true
+ }
+ }
}
case api.HubSyncMessage_REMOVE:
p, err := peer.Decode(syncMsg.PeerId)
- if err == nil && h.gater.authenticated[p] {
- samHubActiveNodes.Dec()
- delete(h.gater.authenticated, p)
+ if err == nil {
+ if syncMsg.Timestamp > h.gater.lastUpdated[p] {
+ h.gater.lastUpdated[p] = syncMsg.Timestamp
+ if h.gater.authenticated[p] {
+ samHubActiveNodes.Dec()
+ delete(h.gater.authenticated, p)
+ }
+ }
}
case api.HubSyncMessage_FULL_SYNC:
- for _, pStr := range syncMsg.Peers {
+ for pStr, ts := range syncMsg.PeerTimestamps {
p, err := peer.Decode(pStr)
- if err == nil && !h.gater.authenticated[p] {
- samHubActiveNodes.Inc()
- h.gater.authenticated[p] = true
+ if err != nil {
+ continue
+ }
+ if ts > h.gater.lastUpdated[p] {
+ h.gater.lastUpdated[p] = ts
+
+ // Determine if the peer is active in the incoming state
+ isActive := false
+ for _, activeStr := range syncMsg.Peers {
+ if activeStr == pStr {
+ isActive = true
+ break
+ }
+ }
+
+ if isActive {
+ if !h.gater.authenticated[p] {
+ samHubActiveNodes.Inc()
+ h.gater.authenticated[p] = true
+ }
+ } else {
+ if h.gater.authenticated[p] {
+ samHubActiveNodes.Dec()
+ delete(h.gater.authenticated, p)
+ }
+ }
}
}
if syncMsg.PeerId != "" && len(syncMsg.HubAddrs) > 0 {
@@ -156,14 +190,19 @@ func (h *Hub) publishFullSync(ctx context.Context) {
for p := range h.gater.authenticated {
peers = append(peers, p.String())
}
+ peerTimestamps := make(map[string]int64)
+ for p, ts := range h.gater.lastUpdated {
+ peerTimestamps[p.String()] = ts
+ }
h.gater.mu.RUnlock()
msg := api.HubSyncMessage{
- Action: api.HubSyncMessage_FULL_SYNC,
- PeerId: h.Host.ID().String(),
- Peers: peers,
- HubAddrs: h.getMyHubAddrs(),
- Timestamp: time.Now().Unix(),
+ Action: api.HubSyncMessage_FULL_SYNC,
+ PeerId: h.Host.ID().String(),
+ Peers: peers,
+ HubAddrs: h.getMyHubAddrs(),
+ Timestamp: time.Now().Unix(),
+ PeerTimestamps: peerTimestamps,
}
h.publishSyncMessage(ctx, &msg)
}
@@ -179,7 +218,9 @@ func (h *Hub) publishSyncMessage(ctx context.Context, msg *api.HubSyncMessage) {
logger.Errorf("Failed to encrypt sync msg: %v", err)
return
}
- _ = h.SyncTopic.Publish(ctx, ciphertext)
+ if err := h.SyncTopic.Publish(ctx, ciphertext); err != nil {
+ logger.Errorf("Failed to publish sync msg: %v", err)
+ }
}
func (h *Hub) getMyHubAddrs() []string {