Skip to content

Agent Mesh

add key rotation

Antonio Ojea805afd798bc7
diff --git a/api/sam.pb.go b/api/sam.pb.go
index 00ad0e59..2a58c8a5 100644
--- a/api/sam.pb.go
+++ b/api/sam.pb.go
@@ -38,9 +38,10 @@ const (
type MeshEvent_Type int32
const (
- MeshEvent_JOIN MeshEvent_Type = 0
- MeshEvent_EXIT MeshEvent_Type = 1
- MeshEvent_BANNED MeshEvent_Type = 2
+ MeshEvent_JOIN MeshEvent_Type = 0
+ MeshEvent_EXIT MeshEvent_Type = 1
+ MeshEvent_BANNED MeshEvent_Type = 2
+ MeshEvent_KEY_ROTATION MeshEvent_Type = 3
)
// Enum value maps for MeshEvent_Type.
@@ -49,11 +50,13 @@ var (
0: "JOIN",
1: "EXIT",
2: "BANNED",
+ 3: "KEY_ROTATION",
}
MeshEvent_Type_value = map[string]int32{
- "JOIN": 0,
- "EXIT": 1,
- "BANNED": 2,
+ "JOIN": 0,
+ "EXIT": 1,
+ "BANNED": 2,
+ "KEY_ROTATION": 3,
}
)
@@ -241,12 +244,14 @@ func (x *AuthResponse) GetError() string {
}
type HubConfig struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- PublicKeyHex string `protobuf:"bytes,1,opt,name=public_key_hex,json=publicKeyHex,proto3" json:"public_key_hex,omitempty"`
- MeshId string `protobuf:"bytes,2,opt,name=mesh_id,json=meshId,proto3" json:"mesh_id,omitempty"`
- BootstrapNodes []string `protobuf:"bytes,3,rep,name=bootstrap_nodes,json=bootstrapNodes,proto3" json:"bootstrap_nodes,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ state protoimpl.MessageState `protogen:"open.v1"`
+ PublicKeyHex string `protobuf:"bytes,1,opt,name=public_key_hex,json=publicKeyHex,proto3" json:"public_key_hex,omitempty"`
+ MeshId string `protobuf:"bytes,2,opt,name=mesh_id,json=meshId,proto3" json:"mesh_id,omitempty"`
+ BootstrapNodes []string `protobuf:"bytes,3,rep,name=bootstrap_nodes,json=bootstrapNodes,proto3" json:"bootstrap_nodes,omitempty"`
+ KeyRotationInterval int64 `protobuf:"varint,4,opt,name=key_rotation_interval,json=keyRotationInterval,proto3" json:"key_rotation_interval,omitempty"`
+ KeyGracePeriod int64 `protobuf:"varint,5,opt,name=key_grace_period,json=keyGracePeriod,proto3" json:"key_grace_period,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *HubConfig) Reset() {
@@ -300,6 +305,20 @@ func (x *HubConfig) GetBootstrapNodes() []string {
return nil
}
+func (x *HubConfig) GetKeyRotationInterval() int64 {
+ if x != nil {
+ return x.KeyRotationInterval
+ }
+ return 0
+}
+
+func (x *HubConfig) GetKeyGracePeriod() int64 {
+ if x != nil {
+ return x.KeyGracePeriod
+ }
+ return 0
+}
+
type PeerProfile struct {
state protoimpl.MessageState `protogen:"open.v1"`
Email string `protobuf:"bytes,1,opt,name=email,proto3" json:"email,omitempty"`
@@ -621,6 +640,8 @@ type MeshEvent struct {
Type MeshEvent_Type `protobuf:"varint,1,opt,name=type,proto3,enum=sam.v1.MeshEvent_Type" json:"type,omitempty"`
PeerId string `protobuf:"bytes,2,opt,name=peer_id,json=peerId,proto3" json:"peer_id,omitempty"`
Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+ NewPublicKey []byte `protobuf:"bytes,4,opt,name=new_public_key,json=newPublicKey,proto3" json:"new_public_key,omitempty"`
+ Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -676,6 +697,20 @@ func (x *MeshEvent) GetTimestamp() int64 {
return 0
}
+func (x *MeshEvent) GetNewPublicKey() []byte {
+ if x != nil {
+ return x.NewPublicKey
+ }
+ return nil
+}
+
+func (x *MeshEvent) GetSignature() []byte {
+ if x != nil {
+ return x.Signature
+ }
+ return nil
+}
+
type EnrollRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Jwt string `protobuf:"bytes,1,opt,name=jwt,proto3" json:"jwt,omitempty"`
@@ -825,11 +860,13 @@ const file_api_sam_proto_rawDesc = "" +
"\abiscuit\x18\x01 \x01(\fR\abiscuit\">\n" +
"\fAuthResponse\x12\x18\n" +
"\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" +
- "\x05error\x18\x02 \x01(\tR\x05error\"s\n" +
+ "\x05error\x18\x02 \x01(\tR\x05error\"\xd1\x01\n" +
"\tHubConfig\x12$\n" +
"\x0epublic_key_hex\x18\x01 \x01(\tR\fpublicKeyHex\x12\x17\n" +
"\amesh_id\x18\x02 \x01(\tR\x06meshId\x12'\n" +
- "\x0fbootstrap_nodes\x18\x03 \x03(\tR\x0ebootstrapNodes\";\n" +
+ "\x0fbootstrap_nodes\x18\x03 \x03(\tR\x0ebootstrapNodes\x122\n" +
+ "\x15key_rotation_interval\x18\x04 \x01(\x03R\x13keyRotationInterval\x12(\n" +
+ "\x10key_grace_period\x18\x05 \x01(\x03R\x0ekeyGracePeriod\";\n" +
"\vPeerProfile\x12\x14\n" +
"\x05email\x18\x01 \x01(\tR\x05email\x12\x16\n" +
"\x06groups\x18\x02 \x03(\tR\x06groups\"\x94\x01\n" +
@@ -858,16 +895,19 @@ const file_api_sam_proto_rawDesc = "" +
"\x0etarget_peer_id\x18\x01 \x01(\tR\ftargetPeerId\x12\x16\n" +
"\x06reason\x18\x02 \x01(\tR\x06reason\x12\x1c\n" +
"\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\x12\x1c\n" +
- "\tsignature\x18\x04 \x01(\fR\tsignature\"\x96\x01\n" +
+ "\tsignature\x18\x04 \x01(\fR\tsignature\"\xec\x01\n" +
"\tMeshEvent\x12*\n" +
"\x04type\x18\x01 \x01(\x0e2\x16.sam.v1.MeshEvent.TypeR\x04type\x12\x17\n" +
"\apeer_id\x18\x02 \x01(\tR\x06peerId\x12\x1c\n" +
- "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\"&\n" +
+ "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\x12$\n" +
+ "\x0enew_public_key\x18\x04 \x01(\fR\fnewPublicKey\x12\x1c\n" +
+ "\tsignature\x18\x05 \x01(\fR\tsignature\"8\n" +
"\x04Type\x12\b\n" +
"\x04JOIN\x10\x00\x12\b\n" +
"\x04EXIT\x10\x01\x12\n" +
"\n" +
- "\x06BANNED\x10\x02\":\n" +
+ "\x06BANNED\x10\x02\x12\x10\n" +
+ "\fKEY_ROTATION\x10\x03\":\n" +
"\rEnrollRequest\x12\x10\n" +
"\x03jwt\x18\x01 \x01(\tR\x03jwt\x12\x17\n" +
"\apeer_id\x18\x02 \x01(\tR\x06peerId\"\xe6\x01\n" +
diff --git a/api/sam.proto b/api/sam.proto
index 1125234f..575e9d0d 100644
--- a/api/sam.proto
+++ b/api/sam.proto
@@ -35,6 +35,8 @@ message HubConfig {
string public_key_hex = 1;
string mesh_id = 2;
repeated string bootstrap_nodes = 3;
+ int64 key_rotation_interval = 4;
+ int64 key_grace_period = 5;
}
message PeerProfile {
@@ -74,10 +76,13 @@ message MeshEvent {
JOIN = 0;
EXIT = 1;
BANNED = 2;
+ KEY_ROTATION = 3;
}
Type type = 1;
string peer_id = 2;
int64 timestamp = 3;
+ bytes new_public_key = 4;
+ bytes signature = 5;
}
message EnrollRequest {
diff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.go
new file mode 100644
index 00000000..969639c7
--- /dev/null
+++ b/cmd/sam-hub/keyring.go
@@ -0,0 +1,169 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "time"
+
+ "go.etcd.io/bbolt"
+)
+
+// KeyPair holds an ed25519 keypair and its expiration time if it is a previous key.
+type KeyPair struct {
+ Private []byte `json:"private"`
+ Public []byte `json:"public"`
+ Expiration time.Time `json:"expiration"`
+}
+
+// KeyRing manages the active and expired keys for the Hub.
+type KeyRing struct {
+ Current KeyPair `json:"current"`
+ Previous []KeyPair `json:"previous"`
+ mu sync.RWMutex
+ db *bbolt.DB
+}
+
+// NewKeyRing opens or creates a BoltDB file to store the keyring.
+func NewKeyRing(dbPath string, gracePeriod time.Duration) (*KeyRing, error) {
+ db, err := bbolt.Open(dbPath, 0600, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open db: %w", err)
+ }
+
+ kr := &KeyRing{db: db}
+
+ err = db.Update(func(tx *bbolt.Tx) error {
+ _, err := tx.CreateBucketIfNotExists([]byte("keyring"))
+ return err
+ })
+ if err != nil {
+ _ = db.Close()
+ return nil, fmt.Errorf("failed to create bucket: %w", err)
+ }
+
+ if err := kr.load(); err != nil {
+ _ = db.Close()
+ return nil, fmt.Errorf("failed to load keyring: %w", err)
+ }
+
+ // Generate initial key if empty
+ if len(kr.Current.Private) == 0 {
+ if err := kr.rotate(gracePeriod); err != nil {
+ _ = db.Close()
+ return nil, fmt.Errorf("failed to generate initial key: %w", err)
+ }
+ }
+
+ return kr, nil
+}
+
+func (kr *KeyRing) load() error {
+ return kr.db.View(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte("keyring"))
+ data := b.Get([]byte("data"))
+ if data == nil {
+ return nil
+ }
+ return json.Unmarshal(data, kr)
+ })
+}
+
+func (kr *KeyRing) save() error {
+ data, err := json.Marshal(kr)
+ if err != nil {
+ return err
+ }
+ return kr.db.Update(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte("keyring"))
+ return b.Put([]byte("data"), data)
+ })
+}
+
+// Rotate generates a new keypair and moves the current one to the previous list.
+func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, error) {
+ kr.mu.Lock()
+ defer kr.mu.Unlock()
+
+ err := kr.rotate(gracePeriod)
+ if err != nil {
+ return nil, err
+ }
+ return kr.Current.Public, nil
+}
+
+func (kr *KeyRing) rotate(gracePeriod time.Duration) error {
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+
+ if len(kr.Current.Private) > 0 {
+ kr.Current.Expiration = time.Now().Add(gracePeriod)
+ kr.Previous = append(kr.Previous, kr.Current)
+ }
+
+ kr.Current = KeyPair{
+ Private: priv,
+ Public: pub,
+ }
+
+ // Clean up expired keys
+ now := time.Now()
+ var activePrevious []KeyPair
+ for _, kp := range kr.Previous {
+ if kp.Expiration.After(now) {
+ activePrevious = append(activePrevious, kp)
+ }
+ }
+ kr.Previous = activePrevious
+
+ return kr.save()
+}
+
+// GetCurrentKey returns the current private key for signing.
+func (kr *KeyRing) GetCurrentKey() ed25519.PrivateKey {
+ kr.mu.RLock()
+ defer kr.mu.RUnlock()
+ return ed25519.PrivateKey(kr.Current.Private)
+}
+
+// GetCurrentPublicKey returns the current public key.
+func (kr *KeyRing) GetCurrentPublicKey() ed25519.PublicKey {
+ kr.mu.RLock()
+ defer kr.mu.RUnlock()
+ return ed25519.PublicKey(kr.Current.Public)
+}
+
+// GetAllValidKeys returns all private keys that are still valid (current + non-expired previous).
+func (kr *KeyRing) GetAllValidKeys() []ed25519.PrivateKey {
+ kr.mu.RLock()
+ defer kr.mu.RUnlock()
+
+ keys := []ed25519.PrivateKey{ed25519.PrivateKey(kr.Current.Private)}
+ for _, kp := range kr.Previous {
+ keys = append(keys, ed25519.PrivateKey(kp.Private))
+ }
+ return keys
+}
+
+// Close closes the BoltDB database.
+func (kr *KeyRing) Close() error {
+ return kr.db.Close()
+}
diff --git a/cmd/sam-hub/keyring_test.go b/cmd/sam-hub/keyring_test.go
new file mode 100644
index 00000000..0f459f73
--- /dev/null
+++ b/cmd/sam-hub/keyring_test.go
@@ -0,0 +1,133 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func TestNewKeyRing(t *testing.T) {
+ dir := t.TempDir()
+ dbPath := filepath.Join(dir, "test.db")
+
+ kr, err := NewKeyRing(dbPath, 24*time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer kr.Close()
+
+ if len(kr.Current.Private) == 0 {
+ t.Error("Expected current private key to be initialized")
+ }
+ if len(kr.Current.Public) == 0 {
+ t.Error("Expected current public key to be initialized")
+ }
+}
+
+func TestKeyRingRotate(t *testing.T) {
+ dir := t.TempDir()
+ dbPath := filepath.Join(dir, "test.db")
+
+ kr, err := NewKeyRing(dbPath, 24*time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer kr.Close()
+
+ oldPub := kr.Current.Public
+
+ newPub, err := kr.Rotate(24 * time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if bytes.Equal(oldPub, newPub) {
+ t.Error("Expected new public key to be different from old public key")
+ }
+
+ if len(kr.Previous) != 1 {
+ t.Errorf("Expected 1 previous key, got %d", len(kr.Previous))
+ }
+ if !bytes.Equal(kr.Previous[0].Public, oldPub) {
+ t.Error("Expected previous key to match old public key")
+ }
+}
+
+func TestKeyRingPersistence(t *testing.T) {
+ dir := t.TempDir()
+ dbPath := filepath.Join(dir, "test.db")
+
+ kr, err := NewKeyRing(dbPath, 24*time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ currentPub := kr.Current.Public
+ kr.Close()
+
+ // Reopen
+ kr2, err := NewKeyRing(dbPath, 24*time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer kr2.Close()
+
+ if !bytes.Equal(kr2.Current.Public, currentPub) {
+ t.Error("Expected persisted public key to match original")
+ }
+}
+
+func TestKeyRingCleanup(t *testing.T) {
+ dir := t.TempDir()
+ dbPath := filepath.Join(dir, "test.db")
+
+ kr, err := NewKeyRing(dbPath, 1*time.Millisecond) // Short grace period
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer kr.Close()
+
+ oldPub := kr.Current.Public
+
+ _, err = kr.Rotate(1 * time.Millisecond)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(kr.Previous) != 1 {
+ t.Errorf("Expected 1 previous key before expiration, got %d", len(kr.Previous))
+ }
+
+ time.Sleep(5 * time.Millisecond) // Wait for expiration
+
+ // Rotate again to trigger cleanup
+ _, err = kr.Rotate(1 * time.Millisecond)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // The first previous key should be cleaned up!
+ // So we should still have 1 previous key (the one from the second rotation),
+ // but its public key should NOT be oldPub!
+ if len(kr.Previous) != 1 {
+ t.Errorf("Expected 1 previous key after cleanup, got %d", len(kr.Previous))
+ }
+ if bytes.Equal(kr.Previous[0].Public, oldPub) {
+ t.Error("Expected expired key to be pruned")
+ }
+}
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index b152d212..31674c31 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -18,7 +18,6 @@ import (
"context"
"crypto/ed25519"
"crypto/tls"
- "encoding/hex"
"fmt"
"net/http"
"os"
@@ -59,6 +58,9 @@ var (
insecureSkipTLSVerify bool
logLevel string
policyFile string
+ keyRotationInterval time.Duration
+ keyGracePeriod time.Duration
+ keysDBPath string
)
var logger = golog.Logger("sam-hub")
@@ -68,7 +70,7 @@ type Hub struct {
Host host.Host
DHT *dht.IpfsDHT
Providers map[string]*oidc.Provider
- BiscuitKey ed25519.PrivateKey
+ KeyRing *KeyRing
MeshID string
PubSub *pubsub.PubSub
EventTopic *pubsub.Topic
@@ -130,21 +132,19 @@ func NewHub(ctx context.Context, policy *api.PolicyConfig) (*Hub, error) {
logger.Infof("[OIDC] Trusted issuer: %s", iss)
}
- // SAM_HUB_KEY stores an ed25519 seed as a 32-byte hex value.
- keyBytes, err := hex.DecodeString(biscuitHex)
- if err != nil || len(keyBytes) != 32 {
- return nil, fmt.Errorf("invalid SAM_HUB_KEY: must be 32-byte hex string")
+ kr, err := NewKeyRing(keysDBPath, keyGracePeriod)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create keyring: %w", err)
}
- bKey := ed25519.NewKeyFromSeed(keyBytes)
hub := &Hub{
- Host: h,
- DHT: kadDHT,
- gater: gater,
- Providers: providers,
- BiscuitKey: bKey,
- MeshID: meshName,
- Policy: policy,
+ Host: h,
+ DHT: kadDHT,
+ gater: gater,
+ Providers: providers,
+ KeyRing: kr,
+ MeshID: meshName,
+ Policy: policy,
}
h.Network().Notify(&notifier{hub: hub})
@@ -260,7 +260,7 @@ func (h *Hub) handleEnroll(s network.Stream) {
}
}
- builder := biscuit.NewBuilder(h.BiscuitKey)
+ builder := biscuit.NewBuilder(h.KeyRing.GetCurrentKey())
if err := builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{
Name: "expiration",
@@ -380,14 +380,18 @@ func (h *Hub) handleEnroll(s network.Stream) {
PeerId: remotePeer.String(),
Timestamp: time.Now().Unix(),
}
- eventData, err := proto.Marshal(event)
- if err != nil {
- logger.Errorf("[Enroll] Failed to marshal event: %v", err)
+ if err := h.signEvent(event); err != nil {
+ logger.Errorf("[Enroll] Failed to sign event: %v", err)
} else {
- if err := h.EventTopic.Publish(context.Background(), eventData); err != nil {
- logger.Errorf("[Enroll] Failed to publish event: %v", err)
+ eventData, err := proto.Marshal(event)
+ if err != nil {
+ logger.Errorf("[Enroll] Failed to marshal event: %v", err)
} else {
- logger.Infof("[Enroll] Published JOIN event for %s", remotePeer)
+ if err := h.EventTopic.Publish(context.Background(), eventData); err != nil {
+ logger.Errorf("[Enroll] Failed to publish event: %v", err)
+ } else {
+ logger.Infof("[Enroll] Published JOIN event for %s", remotePeer)
+ }
}
}
}
@@ -398,7 +402,7 @@ func (h *Hub) sendEnrollResponse(s network.Stream, biscuitToken []byte, errMsg s
hubAddrs = append(hubAddrs, addr.String()+"/p2p/"+h.Host.ID().String())
}
- pubKey := h.BiscuitKey.Public().(ed25519.PublicKey)
+ pubKey := h.KeyRing.GetCurrentPublicKey()
resp := &api.EnrollResponse{
BiscuitToken: biscuitToken,
@@ -421,10 +425,16 @@ func (h *Hub) sendEnrollResponse(s network.Stream, biscuitToken []byte, errMsg s
// startWatchdog periodically checks for peers that have connected but not completed OIDC
// authentication within the grace period, and evicts them from the network.
-func (h *Hub) startWatchdog() {
+func (h *Hub) startWatchdog(ctx context.Context) {
go func() {
ticker := time.NewTicker(10 * time.Second)
+ defer ticker.Stop()
for range ticker.C {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
h.gater.mu.Lock()
now := time.Now()
for pID, connectedAt := range h.gater.pending {
@@ -441,6 +451,60 @@ func (h *Hub) startWatchdog() {
}()
}
+func (h *Hub) startRotation(ctx context.Context) {
+ if keyRotationInterval <= 0 {
+ return
+ }
+ go func() {
+ ticker := time.NewTicker(keyRotationInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ logger.Info("[Rotation] Rotating keys...")
+ newPub, err := h.KeyRing.Rotate(keyGracePeriod)
+ if err != nil {
+ logger.Errorf("[Rotation] Failed to rotate keys: %v", err)
+ continue
+ }
+
+ // Broadcast event
+ event := &api.MeshEvent{
+ Type: api.MeshEvent_KEY_ROTATION,
+ Timestamp: time.Now().Unix(),
+ NewPublicKey: newPub,
+ }
+ if err := h.signEvent(event); err != nil {
+ logger.Errorf("[Rotation] Failed to sign event: %v", err)
+ continue
+ }
+ eventData, err := proto.Marshal(event)
+ if err != nil {
+ logger.Errorf("[Rotation] Failed to marshal event: %v", err)
+ continue
+ }
+ if err := h.EventTopic.Publish(ctx, eventData); err != nil {
+ logger.Errorf("[Rotation] Failed to publish event: %v", err)
+ } else {
+ logger.Infof("[Rotation] Broadcasted new public key: %x", newPub)
+ }
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+}
+
+func (h *Hub) signEvent(event *api.MeshEvent) error {
+ event.Signature = nil
+ data, err := proto.Marshal(event)
+ if err != nil {
+ return err
+ }
+ event.Signature = ed25519.Sign(h.KeyRing.GetCurrentKey(), data)
+ return nil
+}
+
func main() {
rootCmd := &cobra.Command{
Use: "sam-hub",
@@ -468,8 +532,11 @@ func main() {
}
h.Host.SetStreamHandler(api.EnrollProtocolID, h.handleEnroll)
+ // Start key rotation if enabled
+ h.startRotation(ctx)
+
// Watchdog: Expel peers that connect but never finish authentication
- h.startWatchdog()
+ h.startWatchdog(ctx)
logger.Infof("SAM Hub Online (QUIC + TCP)")
logger.Infof("MeshID: %s", h.MeshID)
@@ -492,6 +559,9 @@ func main() {
rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC issuers")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
rootCmd.Flags().StringVar(&policyFile, "policy-file", "policies.yaml", "Path to policies.yaml")
+ rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 0, "Key rotation interval (e.g. 12h). 0 disables rotation.")
+ rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h).")
+ rootCmd.Flags().StringVar(&keysDBPath, "keys-db", "keys.db", "Path to BoltDB file for keys")
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index a94ef5a3..ac3472dd 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -53,6 +53,7 @@ var (
enableRelayFlag bool
logLevelFlag string
localPolicyFile string
+ keyGracePeriodFlag time.Duration
)
var logger = golog.Logger("sam-node")
@@ -151,7 +152,7 @@ func main() {
logger.Fatal("Hub public key not found in store and not provided. Cannot verify peers.")
}
priv := getOrGenerateKey(store)
- node, err = NewSamNode(context.Background(), priv, hubPubKey, hubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy)
+ node, err = NewSamNode(context.Background(), priv, hubPubKey, hubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy, keyGracePeriodFlag)
if err != nil {
logger.Fatalf("Failed to start mesh node: %v", err)
}
@@ -182,7 +183,7 @@ func main() {
}
priv := getOrGenerateKey(store)
- node, err = NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy)
+ node, err = NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy, keyGracePeriodFlag)
if err != nil {
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
}
@@ -195,7 +196,9 @@ func main() {
storedPubKey, _, _ = store.LoadHubConfig()
hubPubKey = storedPubKey
- node.HubPublicKey = hubPubKey
+ node.keysMu.Lock()
+ node.trustedKeys = []TrustedKey{{Key: hubPubKey, ReceivedAt: time.Now()}}
+ node.keysMu.Unlock()
}
// Start renewal loop
@@ -349,7 +352,7 @@ func main() {
}
priv := getOrGenerateKey(store)
- node, err := NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy)
+ node, err := NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, localPolicy, keyGracePeriodFlag)
if err != nil {
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
}
@@ -375,6 +378,7 @@ func main() {
runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", "2s", "Polling interval for DHT discovery")
runCmd.Flags().BoolVar(&enableRelayFlag, "enable-relay", false, "Allow this node to serve as a relay for others")
runCmd.Flags().StringVar(&logLevelFlag, "log-level", "info", "Log level (debug, info, warn, error)")
+ runCmd.Flags().DurationVar(&keyGracePeriodFlag, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h)")
rootCmd.PersistentFlags().StringVar(&hubAddr, "hub", "http://localhost:8080", "Hub URL")
rootCmd.PersistentFlags().StringVar(&localPolicyFile, "local-policy", "local_policy.yaml", "Path to local_policy.yaml")
rootCmd.PersistentFlags().StringVar(&tokenURLFlag, "token-url", "", "OIDC Token URL")
@@ -477,6 +481,10 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {
return fmt.Errorf("failed to save hub config: %v", err)
}
+ n.keysMu.Lock()
+ n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: ed25519.PublicKey(resp.HubPublicKey), ReceivedAt: time.Now()})
+ n.keysMu.Unlock()
+
// Add known peers from response
diff --git a/cmd/sam-node/middleware.go b/cmd/sam-node/middleware.go
index 14f883b7..dd35acae 100644
--- a/cmd/sam-node/middleware.go
+++ b/cmd/sam-node/middleware.go
@@ -15,7 +15,13 @@
package main
import (
+ "context"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
+ "os"
+ "strings"
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/biscuit-auth/biscuit-go/v2/parser"
@@ -59,8 +65,6 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
logger.Warnf("[Auth] Invalid auth frame from %s", remotePeer)
return
}
-
- // Verify token
reqCtx := RequestContext{
PeerID: remotePeer,
User: "", // Not used in Authorize
@@ -69,10 +73,8 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
writer := msgio.NewVarintWriter(s)
- n.revocationMu.RLock()
- isRevoked := n.revokedPeers[remotePeer.String()]
- n.revocationMu.RUnlock()
- if isRevoked {
+ // Check revocation cache
+ if _, isRevoked := n.revokedPeers.Get(remotePeer.String()); isRevoked {
logger.Warnf("[Auth] Peer %s is revoked", remotePeer)
resp := &api.AuthResponse{Success: false, Error: "Peer is revoked"}
respBytes, _ := proto.Marshal(resp)
@@ -80,14 +82,93 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
return
}
- if err := n.Authorize(authFrame.Biscuit, reqCtx); err != nil {
- logger.Warnf("[Auth] AuthZ Denied %s: %v", remotePeer, err)
- resp := &api.AuthResponse{Success: false, Error: err.Error()}
+ // Check verification cache
+ tokenHash := sha256.Sum256(authFrame.Biscuit)
+ hashStr := hex.EncodeToString(tokenHash[:])
+
+ if valid, ok := n.verificationCache.Get(hashStr); ok && valid {
+ logger.Infof("[Auth] Token cache hit for %s", remotePeer)
+ // Valid
+ resp := &api.AuthResponse{Success: true}
+ respBytes, _ := proto.Marshal(resp)
+ if err := writer.WriteMsg(respBytes); err != nil {
+ logger.Errorf("[Auth] Failed to write ACK to %s: %v", remotePeer, err)
+ return
+ }
+ next(s)
+ return
+ }
+
+ n.keysMu.RLock()
+ keys := n.trustedKeys
+ n.keysMu.RUnlock()
+
+ var authorized bool
+ var lastErr error
+ for _, pubKey := range keys {
+ if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {
+ authorized = true
+ break
+ } else {
+ lastErr = err
+ }
+ }
+
+ if !authorized {
+ logger.Infof("[Auth] All keys failed, triggering re-enrollment fallback for %s", remotePeer)
+ var jwtStr string
+ var err error
+
+ if tokenURLFlag != "" {
+ jwtStr, err = n.FetchJWT(context.Background(), tokenURLFlag, clientIDFlag, clientSecretFlag)
+ if err != nil {
+ logger.Errorf("[Auth] Failed to fetch JWT for fallback: %v", err)
+ }
+ } else if jwtPathFlag != "" {
+ data, err := os.ReadFile(jwtPathFlag)
+ if err != nil {
+ logger.Errorf("[Auth] Failed to read JWT file for fallback: %v", err)
+ } else {
+ jwtStr = strings.TrimSpace(string(data))
+ }
+ }
+
+ if jwtStr != "" {
+ err = n.Enroll(context.Background(), jwtStr)
+ if err != nil {
+ logger.Errorf("[Auth] Fallback enrollment failed: %v", err)
+ } else {
+ // Retry authorization with new keys
+ n.keysMu.RLock()
+ keys = n.trustedKeys
+ n.keysMu.RUnlock()
+
+ for _, pubKey := range keys {
+ if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {
+ authorized = true
+ break
+ } else {
+ lastErr = err
+ }
+ }
+ }
+ }
+ }
+
+ if !authorized {
+ logger.Warnf("[Auth] AuthZ Denied %s: %v", remotePeer, lastErr)
+ resp := &api.AuthResponse{Success: false, Error: "Authorization failed"}
+ if lastErr != nil {
+ resp.Error = lastErr.Error()
+ }
respBytes, _ := proto.Marshal(resp)
_ = writer.WriteMsg(respBytes)
return
}
+ // Cache successful verification
+ n.verificationCache.Add(hashStr, true)
+
// Valid
resp := &api.AuthResponse{Success: true}
respBytes, _ := proto.Marshal(resp)
@@ -100,13 +181,13 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
}
}
-func (n *SamNode) Authorize(rawToken []byte, req RequestContext) error {
+func (n *SamNode) Authorize(rawToken []byte, req RequestContext, pubKey ed25519.PublicKey) error {
b, err := biscuit.Unmarshal(rawToken)
if err != nil {
return fmt.Errorf("invalid biscuit: %w", err)
}
- authorizer, err := b.Authorizer(n.HubPublicKey)
+ authorizer, err := b.Authorizer(pubKey)
if err != nil {
return err
}
diff --git a/cmd/sam-node/middleware_test.go b/cmd/sam-node/middleware_test.go
index 8732cfef..7233adb3 100644
--- a/cmd/sam-node/middleware_test.go
+++ b/cmd/sam-node/middleware_test.go
@@ -20,10 +20,12 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/biscuit-auth/biscuit-go/v2/parser"
"github.com/google/sam/api"
+ lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/protocol"
@@ -98,7 +100,7 @@ func TestAuthorize(t *testing.T) {
node := &SamNode{
Store: store,
- HubPublicKey: pub,
+ trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
}
req := RequestContext{
@@ -106,7 +108,7 @@ func TestAuthorize(t *testing.T) {
Protocol: "/test/proto",
}
- if err := node.Authorize(tokenBytes, req); err != nil {
+ if err := node.Authorize(tokenBytes, req, pub); err != nil {
t.Fatalf("Authorize failed: %v", err)
}
}
@@ -245,7 +247,7 @@ attenuation:
}
node := &SamNode{
- HubPublicKey: pub,
+ trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
LocalPolicy: localPolicy,
}
@@ -254,7 +256,7 @@ attenuation:
Protocol: protocol.ID(tt.operation),
}
- err = node.Authorize(tokenBytes, req)
+ err = node.Authorize(tokenBytes, req, pub)
if tt.expectSuccess {
if err != nil {
t.Errorf("expected success, got error: %v", err)
@@ -302,15 +304,14 @@ func TestRevocation(t *testing.T) {
t.Fatal(err)
}
+ cache, _ := lru.New[string, int64](10000)
node := &SamNode{
- HubPublicKey: pub,
- revokedPeers: make(map[string]bool),
+ trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
+ revokedPeers: cache,
}
// Mark as revoked
- node.revocationMu.Lock()
- node.revokedPeers[dummyPeer.String()] = true
- node.revocationMu.Unlock()
+ node.revokedPeers.Add(dummyPeer.String(), time.Now().Unix())
pr1, pw1 := io.Pipe()
pr2, pw2 := io.Pipe()
@@ -352,3 +353,45 @@ func TestRevocation(t *testing.T) {
t.Errorf("expected error 'Peer is revoked', got %q", resp.Error)
}
}
+
+func TestVerifyEvent(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node := &SamNode{
+ trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
+ }
+
+ event := &api.MeshEvent{
+ Type: api.MeshEvent_BANNED,
+ PeerId: "attacker-peer",
+ Timestamp: time.Now().Unix(),
+ }
+
+ // Sign it
+ event.Signature = nil
+ data, err := proto.Marshal(event)
+ if err != nil {
+ t.Fatal(err)
+ }
+ event.Signature = ed25519.Sign(priv, data)
+
+ // Case 1: Valid signature
+ if !node.verifyEvent(event) {
+ t.Error("Expected event to be verified, got false")
+ }
+
+ // Case 2: Invalid signature
+ event.Signature = []byte("invalid-sig")
+ if node.verifyEvent(event) {
+ t.Error("Expected event verification to fail, got true")
+ }
+
+ // Case 3: Missing signature
+ event.Signature = nil
+ if node.verifyEvent(event) {
+ t.Error("Expected event verification to fail for missing signature, got true")
+ }
+}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 3f217cf1..c5a32a20 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -17,12 +17,14 @@ package main
import (
"context"
"crypto/ed25519"
+ "fmt"
"sync"
"time"
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
+ lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p"
dht "github.com/libp2p/go-libp2p-kad-dht"
pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -43,32 +45,53 @@ import (
const AuthProtocolID = protocol.ID("/sam/auth/1.0.0")
+type TrustedKey struct {
+ Key ed25519.PublicKey
+ ReceivedAt time.Time
+}
+
type SamNode struct {
Host host.Host
DHT *dht.IpfsDHT
PubSub *pubsub.PubSub
Store *Store
- HubPublicKey ed25519.PublicKey
HubPeerID peer.ID
knownPeers map[string]bool
receivedMsgs map[string][]string
topics map[string]*pubsub.Topic
mu sync.Mutex
LocalPolicy *CompiledLocalPolicy
- revokedPeers map[string]bool
- revocationMu sync.RWMutex
+ revokedPeers *lru.Cache[string, int64]
+ verificationCache *lru.Cache[string, bool]
+ trustedKeys []TrustedKey
+ keysMu sync.RWMutex
}
// NewSamNode creates a new Agent instance secured with the 4-layer pipeline.
-func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.PublicKey, hubAddrs []multiaddr.Multiaddr, store *Store, meshID string, discoveryInterval string, listenAddrs []string, enableRelay bool, localPolicy *CompiledLocalPolicy) (*SamNode, error) {
+func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.PublicKey, hubAddrs []multiaddr.Multiaddr, store *Store, meshID string, discoveryInterval string, listenAddrs []string, enableRelay bool, localPolicy *CompiledLocalPolicy, keyGracePeriod time.Duration) (*SamNode, error) {
+ var trustedKeys []TrustedKey
+ if len(hubPubKey) > 0 {
+ trustedKeys = []TrustedKey{{Key: hubPubKey, ReceivedAt: time.Now()}}
+ }
+
node := &SamNode{
Store: store,
- HubPublicKey: hubPubKey,
+ trustedKeys: trustedKeys,
knownPeers: make(map[string]bool),
receivedMsgs: make(map[string][]string),
topics: make(map[string]*pubsub.Topic),
LocalPolicy: localPolicy,
- revokedPeers: make(map[string]bool),
+ }
+
+ var err error
+ node.revokedPeers, err = lru.New[string, int64](10000)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create revocation cache: %w", err)
+ }
+
+ node.verificationCache, err = lru.New[string, bool](1000)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create verification cache: %w", err)
}
// Layer 2: Attach the Bouncer (Gater)
@@ -175,6 +198,10 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
// Layer 3: Wire up MCP handler wrapped in middleware
node.Host.SetStreamHandler(api.MCPProtocolID, node.WithBiscuitAuth(node.HandleMCPStream))
+
+ // Start key pruning
+ node.startKeyPruning(ctx, keyGracePeriod)
+
return node, nil
}
@@ -205,6 +232,11 @@ func (n *SamNode) listenForHubEvents(ctx context.Context) {
continue
}
+ if !n.verifyEvent(&event) {
+ logger.Warnf("[Mesh Event] Potential spoofing attempt: invalid signature on event from %s", msg.ReceivedFrom)
+ continue
+ }
+
n.mu.Lock()
switch event.Type {
case api.MeshEvent_JOIN:
@@ -217,14 +249,70 @@ func (n *SamNode) listenForHubEvents(ctx context.Context) {
delete(n.knownPeers, event.PeerId)
logger.Infof("[Mesh Event] Peer banned: %s", event.PeerId)
- n.revocationMu.Lock()
- n.revokedPeers[event.PeerId] = true
- n.revocationMu.Unlock()
+ n.revokedPeers.Add(event.PeerId, event.Timestamp)
+ case api.MeshEvent_KEY_ROTATION:
+ logger.Infof("[Mesh Event] Key rotation received")
+ n.keysMu.Lock()
+ n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: ed25519.PublicKey(event.NewPublicKey), ReceivedAt: time.Now()})
+ n.keysMu.Unlock()
}
n.mu.Unlock()
}
}
+func (n *SamNode) startKeyPruning(ctx context.Context, gracePeriod time.Duration) {
+ if gracePeriod <= 0 {
+ return
+ }
+ go func() {
+ ticker := time.NewTicker(1 * time.Hour)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ logger.Info("[KeyPruning] Pruning expired keys...")
+ n.keysMu.Lock()
+ now := time.Now()
+ var activeKeys []TrustedKey
+ for _, tk := range n.trustedKeys {
+ if now.Sub(tk.ReceivedAt) <= gracePeriod {
+ activeKeys = append(activeKeys, tk)
+ }
+ }
+ n.trustedKeys = activeKeys
+ n.keysMu.Unlock()
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+}
+
+func (n *SamNode) verifyEvent(event *api.MeshEvent) bool {
+ sig := event.Signature
+ event.Signature = nil
+ data, err := proto.Marshal(event)
+ event.Signature = sig // Restore
+ if err != nil {
+ logger.Errorf("[Mesh Event] Failed to marshal event for verification: %v", err)
+ return false
+ }
+
+ n.keysMu.RLock()
+ keys := n.trustedKeys
+ n.keysMu.RUnlock()
+
+ for _, tk := range keys {
+ if len(tk.Key) != ed25519.PublicKeySize {
+ continue
+ }
+ if ed25519.Verify(tk.Key, data, sig) {
+ return true
+ }
+ }
+ return false
+}
+
func (n *SamNode) subscribeToTopic(ctx context.Context, topicName string) error {
n.mu.Lock()
topic, ok := n.topics[topicName]
@@ -328,14 +416,25 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {
return
}
- // 3. Verify signature chain against the trusted Hub key.
- authorizer, err := b.Authorizer(n.HubPublicKey)
- if err != nil {
- logger.Errorf("[AuthN] Signature verification setup failed for %s: %v", remotePeer, err)
- return
+ // 3. Verify signature chain against the trusted Hub keys.
+ n.keysMu.RLock()
+ keys := n.trustedKeys
+ n.keysMu.RUnlock()
+
+ var authorized bool
+ for _, tk := range keys {
+ authorizer, err := b.Authorizer(tk.Key)
+ if err != nil {
+ continue
+ }
+ if err := authorizer.Authorize(); err == nil {
+ authorized = true
+ break
+ }
}
- if err := authorizer.Authorize(); err != nil {
- logger.Warnf("[AuthN] Authorization failed for %s: %v", remotePeer, err)
+
+ if !authorized {
+ logger.Warnf("[AuthN] Authorization failed for %s: no valid key found", remotePeer)
return
}