Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: operator-owned signing keys from a PEM file
Antonio Ojeac79d69b26d71
diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.goindex fcec9af8..17293c4c 100644--- a/cmd/agentmesh-control-plane/main.go+++ b/cmd/agentmesh-control-plane/main.go@@ -47,6 +47,7 @@ var (allowedAudiencesFlag stringkeyRotationInterval time.DurationkeyGracePeriod time.Duration+ signingKeyPath stringleaseDuration time.DurationdhtReplication intdhtReprovideInterval time.Duration@@ -140,6 +141,7 @@ func main() {LeaseDuration: leaseDuration,KeyRotationInterval: keyRotationInterval,KeyGracePeriod: keyGracePeriod,+ SigningKeyPath: signingKeyPath,DHTReplication: dhtReplication,DHTReprovideInterval: dhtReprovideInterval,InsecureSkipTLSVerify: insecureSkipTLSVerify,@@ -205,6 +207,7 @@ func main() {rootCmd.Flags().StringVar(&allowedAudiencesFlag, "allowed-audiences", api.DefaultAudience, "Comma-separated list of allowed OIDC audiences")rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 24*time.Hour, "Key rotation interval (e.g. 24h). 0 disables rotation.")rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 1*time.Hour, "Key grace period for rotated keys.")+ rootCmd.Flags().StringVar(&signingKeyPath, "signing-key-path", "", "PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs; the first key of each kind signs, later ones only verify. Turns automatic key rotation off: rotate by publishing a new first key and keeping the previous one for the grace period")rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")rootCmd.Flags().IntVar(&dhtReplication, "dht-replication", api.DefaultDHTReplication, "How many routers hold each service a member publishes and answer each lookup: the routers closest to the key. Every member holds a session with each of them besides its --routers. More replicas survive more routers down at once; each one is a session more per key on every member. Published on /info for the whole mesh.")rootCmd.Flags().DurationVar(&dhtReprovideInterval, "dht-reprovide-interval", api.DefaultReprovideInterval, "How often a member re-announces each service it publishes. A provider that stopped without unpublishing stays listed for three intervals; the mesh pays providers / interval announcements a minute. Published on /info for the whole mesh.")diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.goindex a48728f6..a5d73349 100644--- a/cmd/agentmesh-one/main.go+++ b/cmd/agentmesh-one/main.go@@ -229,6 +229,7 @@ func main() {rootCmd.Flags().DurationVar(&cpTunables.DHTReprovideInterval, "dht-reprovide-interval", 0, "How often a member re-announces each service it publishes (0 keeps the component default of 5m); a stopped provider stays listed for three intervals; published on /info for the whole mesh")rootCmd.Flags().DurationVar(&cpTunables.KeyRotationInterval, "control-plane-key-rotation-interval", 0, "Biscuit signing key rotation interval (0 keeps the component default)")rootCmd.Flags().DurationVar(&cpTunables.KeyGracePeriod, "control-plane-key-grace-period", 0, "How long rotated-out keys stay valid for verification (0 keeps the component default)")+ rootCmd.Flags().StringVar(&cpTunables.SigningKeyPath, "signing-key-path", "", "PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs; the first key of each kind signs, later ones only verify. Turns automatic key rotation off: rotate by publishing a new first key and keeping the previous one for the grace period")rootCmd.Flags().DurationVar(&cpTunables.BiscuitTTL, "control-plane-biscuit-ttl", 0, "Lifespan minted into issued biscuits (0 keeps the component default)")rootCmd.Flags().DurationVar(&cpTunables.WorkloadSessionTTL, "control-plane-workload-session-ttl", 0, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh (0 keeps the component default)")rootCmd.Flags().DurationVar(&cpTunables.NodeRetention, "control-plane-node-retention", 0, "How long a node record is kept after its session expired, or after its last refresh for a bootstrap enrollment, before it is deleted (0 keeps the component default)")diff --git a/internal/controlplane/config.go b/internal/controlplane/config.goindex 269ff62f..d50af498 100644--- a/internal/controlplane/config.go+++ b/internal/controlplane/config.go@@ -38,6 +38,13 @@ type Options struct {LeaseDuration time.DurationKeyRotationInterval time.DurationKeyGracePeriod time.Duration+ // SigningKeyPath names a PEM file with the control plane's private+ // signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the+ // ES256 JWTs. For each kind the first key signs and later keys only+ // verify. When set, the operator owns rotation: the control plane+ // installs these keys through its normal rotation path at start-up and+ // never rotates on its own.+ SigningKeyPath string// DHTReplication and DHTReprovideInterval are the mesh's directory// parameters, published on /info for every member and router// (api.DirectoryParameters); 0 is the api default of each.diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.goindex 6f7fb2b0..222baa40 100644--- a/internal/controlplane/metrics.go+++ b/internal/controlplane/metrics.go@@ -57,6 +57,17 @@ var (},[]string{"issuer"},)++ // operatorSigningKeysGauge counts the signing keys the operator supplied+ // through --signing-key-path, per algorithm; absent when the control+ // plane generates and rotates its own.+ operatorSigningKeysGauge = promauto.NewGaugeVec(+ prometheus.GaugeOpts{+ Name: "agentmesh_control_plane_operator_signing_keys",+ Help: "Signing keys supplied by the operator (--signing-key-path) by algorithm; automatic rotation is off while any are present",+ },+ []string{"algorithm"},+ ))// observeRoute counts and times requests to one registered mux pattern. Thediff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 7cbd245e..dd4a17f8 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -141,7 +141,10 @@ type Server struct {// resolving holds the workload issuers a background resolver is working// on, so that a second pass over the configuration (Init is exported for// embedders) starts no second resolver for the same issuer.- resolving map[string]bool+ resolving map[string]bool+ // operatorKeys are the signing keys the operator supplied, nil when the+ // control plane generates and rotates its own.+ operatorKeys *operatorSigningKeysproviders map[string]*oidc.ProviderworkloadIssuers map[string]boolworkloadEmailSuffixes map[string][]string@@ -191,6 +194,18 @@ func NewServer(config Options, store storage.Store) (*Server, error) {return nil, err}+ var operatorKeys *operatorSigningKeys+ if config.SigningKeyPath != "" {+ var err error+ operatorKeys, err = loadOperatorSigningKeys(config.SigningKeyPath)+ if err != nil {+ return nil, err+ }+ if err := reconcileES256Keyring(context.Background(), store, operatorKeys.es256, config.KeyGracePeriod); err != nil {+ return nil, fmt.Errorf("operator signing keys: %w", err)+ }+ }+signer := config.OIDCSignerif signer == nil {var err error@@ -224,6 +239,7 @@ func NewServer(config Options, store storage.Store) (*Server, error) {oauthCodes: make(map[string]*oauthAuthCode),providers: make(map[string]*oidc.Provider),workloadIssuers: workloadIssuers,+ operatorKeys: operatorKeys,workloadEmailSuffixes: workloadEmailSuffixes,catalog: make(map[string]nodeCatalogEntry),metricsRegistry: reg,@@ -292,6 +308,18 @@ func (s *Server) Start() error {func (s *Server) Init() error {// Initialize Keyringctx := context.Background()+ if s.operatorKeys != nil {+ if err := reconcileEd25519Keyring(ctx, s.store, s.operatorKeys.ed25519, s.config.KeyGracePeriod); err != nil {+ return fmt.Errorf("operator signing keys: %w", err)+ }+ operatorSigningKeysGauge.WithLabelValues("ed25519").Set(float64(len(s.operatorKeys.ed25519)))+ operatorSigningKeysGauge.WithLabelValues("es256").Set(float64(len(s.operatorKeys.es256)))+ logger.Infow("Signing keys are operator-managed; automatic key rotation is off",+ "path", s.config.SigningKeyPath,+ "ed25519_keys", len(s.operatorKeys.ed25519),+ "es256_keys", len(s.operatorKeys.es256),+ "rotation", "put the new key first in the file and keep the previous one for the grace period")+ }_, _, err := s.store.GetCurrentKey(ctx)if err == storage.ErrNotFound {logger.Info("Generating initial control plane signing keys...")@@ -307,8 +335,9 @@ func (s *Server) Init() error {}// The rotation deadline lives in the store, shared by the replicas and// kept across restarts; a store that never had one gets its first an- // interval from now.- if interval := s.config.KeyRotationInterval; interval > 0 {+ // interval from now. Operator-managed keys rotate when the operator+ // publishes a new file, never here.+ if interval := s.config.KeyRotationInterval; interval > 0 && s.operatorKeys == nil {first := time.Now().Add(interval)scheduled, err := s.store.ScheduleKeyRotationIfUnset(ctx, first)if err != nil {@@ -327,8 +356,10 @@ func (s *Server) Init() error {}// Start key rotation routine- s.wg.Add(1)- go s.runKeyRotationLoop()+ if s.operatorKeys == nil {+ s.wg.Add(1)+ go s.runKeyRotationLoop()+ }s.wg.Add(1)go s.runNodeGCLoop()diff --git a/internal/controlplane/signingkeys.go b/internal/controlplane/signingkeys.gonew file mode 100644index 00000000..29346312--- /dev/null+++ b/internal/controlplane/signingkeys.go@@ -0,0 +1,211 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "crypto/ecdsa"+ "crypto/ed25519"+ "crypto/elliptic"+ "crypto/sha256"+ "crypto/x509"+ "encoding/hex"+ "encoding/pem"+ "errors"+ "fmt"+ "os"+ "time"++ "github.com/google/agentmesh/internal/storage"+)++// operatorSigningKeys are the control plane's private signing keys when the+// operator supplies them (Options.SigningKeyPath) instead of letting the+// control plane generate and rotate its own. The operator then owns the root+// of trust: the keys live where the operator keeps them (a secret manager, a+// KMS-wrapped file) and rotate on the operator's schedule.+//+// The file is PEM. Each "PRIVATE KEY" (PKCS#8) or "EC PRIVATE KEY" (SEC 1)+// block is one key; Ed25519 keys sign Biscuits, P-256 keys sign the ES256+// JWTs behind /sts/token and /jwks. For each kind, the first key in the file+// signs and every later one only verifies, which is how a rotation is+// expressed: put the new key first, keep the previous one until every+// credential it signed has expired, then drop it.+type operatorSigningKeys struct {+ ed25519 []ed25519.PrivateKey+ es256 []*ecdsa.PrivateKey+}++// loadOperatorSigningKeys reads and validates the PEM file at path.+func loadOperatorSigningKeys(path string) (*operatorSigningKeys, error) {+ data, err := os.ReadFile(path)+ if err != nil {+ return nil, fmt.Errorf("read signing keys: %w", err)+ }+ keys := &operatorSigningKeys{}+ for block, rest := pem.Decode(data); block != nil; block, rest = pem.Decode(rest) {+ var parsed any+ switch block.Type {+ case "PRIVATE KEY":+ parsed, err = x509.ParsePKCS8PrivateKey(block.Bytes)+ case "EC PRIVATE KEY":+ parsed, err = x509.ParseECPrivateKey(block.Bytes)+ default:+ return nil, fmt.Errorf("signing keys %s: unsupported PEM block %q (want PRIVATE KEY or EC PRIVATE KEY)", path, block.Type)+ }+ if err != nil {+ return nil, fmt.Errorf("signing keys %s: %w", path, err)+ }+ switch k := parsed.(type) {+ case ed25519.PrivateKey:+ keys.ed25519 = append(keys.ed25519, k)+ case *ecdsa.PrivateKey:+ if k.Curve != elliptic.P256() {+ return nil, fmt.Errorf("signing keys %s: EC key on %s, want P-256", path, k.Curve.Params().Name)+ }+ keys.es256 = append(keys.es256, k)+ default:+ return nil, fmt.Errorf("signing keys %s: unsupported key type %T (want Ed25519 or P-256)", path, parsed)+ }+ }+ if len(keys.ed25519) == 0 && len(keys.es256) == 0 {+ return nil, fmt.Errorf("signing keys %s: no private key found", path)+ }+ return keys, nil+}++// keyFingerprint identifies a public key in logs without revealing anything+// a verifier does not already learn from /keys or /jwks.+func keyFingerprint(pub []byte) string {+ sum := sha256.Sum256(pub)+ return hex.EncodeToString(sum[:8])+}++// reconcileEd25519Keyring makes the store's Biscuit keyring reflect keys:+// keys[0] current, the rest valid for verification until gracePeriod from+// now. It goes through the same SaveInitialKey / RotateKeys path as+// automatic rotation, so a store that already matches is left alone, a store+// that holds a key the operator does not know (one the control plane+// generated before the operator took over) keeps it valid for the grace+// period, and every verifier learns the new set from /keys the usual way.+func reconcileEd25519Keyring(ctx context.Context, store storage.Store, keys []ed25519.PrivateKey, gracePeriod time.Duration) error {+ if len(keys) == 0 {+ return nil+ }+ valid, err := store.GetAllValidKeys(ctx)+ if err != nil && !errors.Is(err, storage.ErrNotFound) {+ return fmt.Errorf("read keyring: %w", err)+ }+ known := func(pub ed25519.PublicKey) bool {+ for _, k := range valid {+ if k.Public.Equal(pub) {+ return true+ }+ }+ return false+ }+ // Oldest first, so that the first key in the file ends up current.+ for i := len(keys) - 1; i >= 0; i-- {+ priv := keys[i]+ pub := priv.Public().(ed25519.PublicKey)+ if known(pub) {+ continue+ }+ if len(valid) == 0 {+ if err := store.SaveInitialKey(ctx, priv, pub); err != nil {+ return fmt.Errorf("save signing key %s: %w", keyFingerprint(pub), err)+ }+ } else if err := store.RotateKeys(ctx, priv, pub, gracePeriod); err != nil {+ return fmt.Errorf("install signing key %s: %w", keyFingerprint(pub), err)+ }+ valid = append(valid, storage.KeyPair{Private: priv, Public: pub})+ logger.Infow("Installed operator signing key", "algorithm", "ed25519", "fingerprint", keyFingerprint(pub))+ }+ _, currentPub, err := store.GetCurrentKey(ctx)+ if err != nil {+ return fmt.Errorf("read current signing key: %w", err)+ }+ if wantPub := keys[0].Public().(ed25519.PublicKey); !currentPub.Equal(wantPub) {+ // The first key in the file is known but not current, which happens+ // when the operator moves a previous key back to the front; the+ // store's current key becomes a verification-only key.+ if err := store.RotateKeys(ctx, keys[0], wantPub, gracePeriod); err != nil {+ return fmt.Errorf("make signing key %s current: %w", keyFingerprint(wantPub), err)+ }+ logger.Infow("Operator signing key made current", "algorithm", "ed25519", "fingerprint", keyFingerprint(wantPub))+ }+ return nil+}++// reconcileES256Keyring is reconcileEd25519Keyring for the ES256 keys behind+// /sts/token and /jwks. The key id is the one generateES256KeyEntry derives,+// so a verifier that pinned a kid from /jwks keeps matching.+func reconcileES256Keyring(ctx context.Context, store storage.Store, keys []*ecdsa.PrivateKey, gracePeriod time.Duration) error {+ if len(keys) == 0 {+ return nil+ }+ valid, err := store.GetAllValidOIDCKeys(ctx)+ if err != nil && !errors.Is(err, storage.ErrNotFound) {+ return fmt.Errorf("read ES256 keyring: %w", err)+ }+ known := make(map[string]bool, len(valid))+ for _, k := range valid {+ known[k.Kid] = true+ }+ encode := func(priv *ecdsa.PrivateKey) (kid string, privBytes, pubBytes []byte, err error) {+ pubBytes, err = priv.PublicKey.Bytes()+ if err != nil || len(pubBytes) != 65 {+ return "", nil, nil, fmt.Errorf("encode P-256 public key: %w", err)+ }+ privBytes, err = x509.MarshalECPrivateKey(priv)+ if err != nil {+ return "", nil, nil, fmt.Errorf("encode P-256 private key: %w", err)+ }+ return es256Kid(pubBytes), privBytes, pubBytes, nil+ }+ for i := len(keys) - 1; i >= 0; i-- {+ kid, privBytes, pubBytes, err := encode(keys[i])+ if err != nil {+ return err+ }+ if known[kid] {+ continue+ }+ if len(known) == 0 {+ if err := store.SaveInitialOIDCKey(ctx, kid, privBytes, pubBytes); err != nil {+ return fmt.Errorf("save ES256 key %s: %w", kid, err)+ }+ } else if err := store.RotateOIDCKeys(ctx, kid, privBytes, pubBytes, gracePeriod); err != nil {+ return fmt.Errorf("install ES256 key %s: %w", kid, err)+ }+ known[kid] = true+ logger.Infow("Installed operator signing key", "algorithm", "es256", "kid", kid)+ }+ wantKid, privBytes, pubBytes, err := encode(keys[0])+ if err != nil {+ return err+ }+ current, err := store.GetCurrentOIDCKey(ctx)+ if err != nil {+ return fmt.Errorf("read current ES256 key: %w", err)+ }+ if current == nil || current.Kid != wantKid {+ if err := store.RotateOIDCKeys(ctx, wantKid, privBytes, pubBytes, gracePeriod); err != nil {+ return fmt.Errorf("make ES256 key %s current: %w", wantKid, err)+ }+ logger.Infow("Operator signing key made current", "algorithm", "es256", "kid", wantKid)+ }+ return nil+}diff --git a/internal/controlplane/signingkeys_test.go b/internal/controlplane/signingkeys_test.gonew file mode 100644index 00000000..3cc4c867--- /dev/null+++ b/internal/controlplane/signingkeys_test.go@@ -0,0 +1,380 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "crypto/ecdsa"+ "crypto/ed25519"+ "crypto/elliptic"+ "crypto/rand"+ "crypto/rsa"+ "crypto/x509"+ "encoding/json"+ "encoding/pem"+ "io"+ "net/http"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/storage"+ "google.golang.org/protobuf/proto"+)++// pemPKCS8 encodes a private key as one "PRIVATE KEY" block.+func pemPKCS8(t *testing.T, key any) []byte {+ t.Helper()+ der, err := x509.MarshalPKCS8PrivateKey(key)+ if err != nil {+ t.Fatalf("MarshalPKCS8PrivateKey: %v", err)+ }+ return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})+}++func newEd25519(t *testing.T) ed25519.PrivateKey {+ t.Helper()+ _, priv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatalf("GenerateKey: %v", err)+ }+ return priv+}++func newP256(t *testing.T) *ecdsa.PrivateKey {+ t.Helper()+ priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)+ if err != nil {+ t.Fatalf("ecdsa.GenerateKey: %v", err)+ }+ return priv+}++func writeKeyFile(t *testing.T, blocks ...[]byte) string {+ t.Helper()+ path := filepath.Join(t.TempDir(), "signing-keys.pem")+ var data []byte+ for _, b := range blocks {+ data = append(data, b...)+ }+ if err := os.WriteFile(path, data, 0o600); err != nil {+ t.Fatalf("WriteFile: %v", err)+ }+ return path+}++func TestLoadOperatorSigningKeys(t *testing.T) {+ edA, edB := newEd25519(t), newEd25519(t)+ ecA := newP256(t)++ t.Run("pkcs8 ed25519 and p256 in file order", func(t *testing.T) {+ path := writeKeyFile(t, pemPKCS8(t, edA), pemPKCS8(t, ecA), pemPKCS8(t, edB))+ keys, err := loadOperatorSigningKeys(path)+ if err != nil {+ t.Fatalf("loadOperatorSigningKeys: %v", err)+ }+ if len(keys.ed25519) != 2 || !keys.ed25519[0].Equal(edA) || !keys.ed25519[1].Equal(edB) {+ t.Fatalf("ed25519 keys = %d in wrong order, want [A B]", len(keys.ed25519))+ }+ if len(keys.es256) != 1 || !keys.es256[0].Equal(ecA) {+ t.Fatalf("es256 keys = %d, want [A]", len(keys.es256))+ }+ })++ t.Run("sec1 ec block", func(t *testing.T) {+ der, err := x509.MarshalECPrivateKey(ecA)+ if err != nil {+ t.Fatalf("MarshalECPrivateKey: %v", err)+ }+ path := writeKeyFile(t, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}))+ keys, err := loadOperatorSigningKeys(path)+ if err != nil {+ t.Fatalf("loadOperatorSigningKeys: %v", err)+ }+ if len(keys.es256) != 1 || !keys.es256[0].Equal(ecA) {+ t.Fatalf("es256 keys = %d, want the SEC 1 key", len(keys.es256))+ }+ })++ t.Run("rsa is refused", func(t *testing.T) {+ rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)+ if err != nil {+ t.Fatalf("rsa.GenerateKey: %v", err)+ }+ _, err = loadOperatorSigningKeys(writeKeyFile(t, pemPKCS8(t, rsaKey)))+ if err == nil || !strings.Contains(err.Error(), "unsupported key type") {+ t.Fatalf("err = %v, want unsupported key type", err)+ }+ })++ t.Run("other curve is refused", func(t *testing.T) {+ p384, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)+ if err != nil {+ t.Fatalf("ecdsa.GenerateKey: %v", err)+ }+ _, err = loadOperatorSigningKeys(writeKeyFile(t, pemPKCS8(t, p384)))+ if err == nil || !strings.Contains(err.Error(), "want P-256") {+ t.Fatalf("err = %v, want P-256 complaint", err)+ }+ })++ t.Run("certificate block is refused", func(t *testing.T) {+ _, err := loadOperatorSigningKeys(writeKeyFile(t, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte{1}})))+ if err == nil || !strings.Contains(err.Error(), "unsupported PEM block") {+ t.Fatalf("err = %v, want unsupported PEM block", err)+ }+ })++ t.Run("empty file is refused", func(t *testing.T) {+ _, err := loadOperatorSigningKeys(writeKeyFile(t, []byte("not pem at all\n")))+ if err == nil || !strings.Contains(err.Error(), "no private key found") {+ t.Fatalf("err = %v, want no private key found", err)+ }+ })++ t.Run("missing file is refused", func(t *testing.T) {+ if _, err := loadOperatorSigningKeys(filepath.Join(t.TempDir(), "absent.pem")); err == nil {+ t.Fatal("loadOperatorSigningKeys on a missing file returned nil")+ }+ })+}++func TestReconcileEd25519Keyring(t *testing.T) {+ ctx := context.Background()+ newStore := func(t *testing.T) storage.Store {+ t.Helper()+ store, err := storage.NewSQLStore("sqlite", ":memory:")+ if err != nil {+ t.Fatalf("NewSQLStore: %v", err)+ }+ t.Cleanup(func() { _ = store.Close() })+ return store+ }+ pubOf := func(k ed25519.PrivateKey) ed25519.PublicKey { return k.Public().(ed25519.PublicKey) }+ validPubs := func(t *testing.T, store storage.Store) map[string]bool {+ t.Helper()+ valid, err := store.GetAllValidKeys(ctx)+ if err != nil {+ t.Fatalf("GetAllValidKeys: %v", err)+ }+ out := map[string]bool{}+ for _, k := range valid {+ out[string(k.Public)] = true+ }+ return out+ }+ current, previous := newEd25519(t), newEd25519(t)++ t.Run("empty store: first key current, rest valid", func(t *testing.T) {+ store := newStore(t)+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current, previous}, time.Hour); err != nil {+ t.Fatalf("reconcile: %v", err)+ }+ _, gotPub, err := store.GetCurrentKey(ctx)+ if err != nil {+ t.Fatalf("GetCurrentKey: %v", err)+ }+ if !gotPub.Equal(pubOf(current)) {+ t.Fatalf("current key = %s, want %s (first in file)", keyFingerprint(gotPub), keyFingerprint(pubOf(current)))+ }+ valid := validPubs(t, store)+ if len(valid) != 2 || !valid[string(pubOf(current))] || !valid[string(pubOf(previous))] {+ t.Fatalf("valid keys = %d, want both file keys", len(valid))+ }+ })++ t.Run("matching store is left alone", func(t *testing.T) {+ store := newStore(t)+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {+ t.Fatalf("first reconcile: %v", err)+ }+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {+ t.Fatalf("second reconcile: %v", err)+ }+ if valid := validPubs(t, store); len(valid) != 1 {+ t.Fatalf("valid keys = %d after a no-op reconcile, want 1", len(valid))+ }+ })++ t.Run("generated key stays valid for the grace period", func(t *testing.T) {+ store := newStore(t)+ generated := newEd25519(t)+ if err := store.SaveInitialKey(ctx, generated, pubOf(generated)); err != nil {+ t.Fatalf("SaveInitialKey: %v", err)+ }+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {+ t.Fatalf("reconcile: %v", err)+ }+ _, gotPub, err := store.GetCurrentKey(ctx)+ if err != nil {+ t.Fatalf("GetCurrentKey: %v", err)+ }+ if !gotPub.Equal(pubOf(current)) {+ t.Fatalf("current key = %s, want the operator key", keyFingerprint(gotPub))+ }+ valid := validPubs(t, store)+ if !valid[string(pubOf(generated))] {+ t.Fatal("the key the control plane generated before the operator took over is no longer valid; credentials it signed would break at once")+ }+ })++ t.Run("operator rotation: new key first, previous kept", func(t *testing.T) {+ store := newStore(t)+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{previous}, time.Hour); err != nil {+ t.Fatalf("reconcile v1: %v", err)+ }+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current, previous}, time.Hour); err != nil {+ t.Fatalf("reconcile v2: %v", err)+ }+ _, gotPub, err := store.GetCurrentKey(ctx)+ if err != nil {+ t.Fatalf("GetCurrentKey: %v", err)+ }+ if !gotPub.Equal(pubOf(current)) {+ t.Fatalf("current key = %s after rotation, want the new first key", keyFingerprint(gotPub))+ }+ if valid := validPubs(t, store); len(valid) != 2 || !valid[string(pubOf(previous))] {+ t.Fatalf("valid keys = %d, want the previous key still valid", len(valid))+ }+ })+}++func TestReconcileES256Keyring(t *testing.T) {+ ctx := context.Background()+ store, err := storage.NewSQLStore("sqlite", ":memory:")+ if err != nil {+ t.Fatalf("NewSQLStore: %v", err)+ }+ defer func() { _ = store.Close() }()+ kidOf := func(t *testing.T, k *ecdsa.PrivateKey) string {+ t.Helper()+ pub, err := k.PublicKey.Bytes()+ if err != nil {+ t.Fatalf("PublicKey.Bytes: %v", err)+ }+ return es256Kid(pub)+ }+ v1, v2 := newP256(t), newP256(t)++ if err := reconcileES256Keyring(ctx, store, []*ecdsa.PrivateKey{v1}, time.Hour); err != nil {+ t.Fatalf("reconcile v1: %v", err)+ }+ cur, err := store.GetCurrentOIDCKey(ctx)+ if err != nil || cur == nil {+ t.Fatalf("GetCurrentOIDCKey: %v (%v)", err, cur)+ }+ if cur.Kid != kidOf(t, v1) {+ t.Fatalf("current kid = %s, want %s", cur.Kid, kidOf(t, v1))+ }+ if err := reconcileES256Keyring(ctx, store, []*ecdsa.PrivateKey{v2, v1}, time.Hour); err != nil {+ t.Fatalf("reconcile v2: %v", err)+ }+ cur, err = store.GetCurrentOIDCKey(ctx)+ if err != nil || cur == nil {+ t.Fatalf("GetCurrentOIDCKey: %v (%v)", err, cur)+ }+ if cur.Kid != kidOf(t, v2) {+ t.Fatalf("current kid after rotation = %s, want %s", cur.Kid, kidOf(t, v2))+ }+ all, err := store.GetAllValidOIDCKeys(ctx)+ if err != nil {+ t.Fatalf("GetAllValidOIDCKeys: %v", err)+ }+ if len(all) != 2 {+ t.Fatalf("valid ES256 keys = %d after rotation, want 2 (previous kept for the grace period)", len(all))+ }+}++// TestServerOperatorSigningKeys starts a control plane on operator keys and+// checks what verifiers see: /keys carries exactly the file's Ed25519 keys,+// /jwks the file's P-256 key, and no automatic rotation is scheduled.+func TestServerOperatorSigningKeys(t *testing.T) {+ issuer, _ := startCustomMockOIDC(t)+ edCurrent, edPrevious := newEd25519(t), newEd25519(t)+ ec := newP256(t)+ path := writeKeyFile(t, pemPKCS8(t, edCurrent), pemPKCS8(t, edPrevious), pemPKCS8(t, ec))++ srv, store, baseURL := setupTestServer(t, issuer, func(o *Options) {+ o.SigningKeyPath = path+ })+ defer func() { _ = srv.Close() }()++ resp, err := http.Get(baseURL + "/keys")+ if err != nil {+ t.Fatalf("GET /keys: %v", err)+ }+ body, err := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if err != nil {+ t.Fatalf("read /keys: %v", err)+ }+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("GET /keys = %d: %s", resp.StatusCode, body)+ }+ var keys api.KeysResponse+ if err := proto.Unmarshal(body, &keys); err != nil {+ t.Fatalf("Unmarshal KeysResponse: %v", err)+ }+ got := map[string]bool{}+ for _, k := range keys.PublicKeys {+ got[string(k)] = true+ }+ if len(got) != 2 || !got[string(edCurrent.Public().(ed25519.PublicKey))] || !got[string(edPrevious.Public().(ed25519.PublicKey))] {+ t.Fatalf("/keys has %d keys, want exactly the two Ed25519 keys from the file", len(got))+ }+ _, currentPub, err := store.GetCurrentKey(context.Background())+ if err != nil {+ t.Fatalf("GetCurrentKey: %v", err)+ }+ if !currentPub.Equal(edCurrent.Public().(ed25519.PublicKey)) {+ t.Fatal("the signing key is not the first Ed25519 key in the file")+ }++ jwksResp, err := http.Get(baseURL + "/jwks")+ if err != nil {+ t.Fatalf("GET /jwks: %v", err)+ }+ var jwks JSONWebKeySet+ err = json.NewDecoder(jwksResp.Body).Decode(&jwks)+ _ = jwksResp.Body.Close()+ if err != nil {+ t.Fatalf("decode /jwks: %v", err)+ }+ ecPub, err := ec.PublicKey.Bytes()+ if err != nil {+ t.Fatalf("PublicKey.Bytes: %v", err)+ }+ if len(jwks.Keys) != 1 || jwks.Keys[0].Kid != es256Kid(ecPub) {+ t.Fatalf("/jwks = %d keys (kid %s), want the one P-256 key from the file (kid %s)", len(jwks.Keys), firstKid(jwks), es256Kid(ecPub))+ }++ next, err := store.NextKeyRotation(context.Background())+ if err != nil {+ t.Fatalf("NextKeyRotation: %v", err)+ }+ if !next.IsZero() {+ t.Fatalf("automatic rotation scheduled at %s although the operator owns the keys", next)+ }+}++func firstKid(jwks JSONWebKeySet) string {+ if len(jwks.Keys) == 0 {+ return ""+ }+ return jwks.Keys[0].Kid+}diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.goindex 81262125..9be60ee6 100644--- a/internal/controlplane/sts.go+++ b/internal/controlplane/sts.go@@ -181,6 +181,14 @@ func decodeOIDCKeyPair(k storage.OIDCKeyPair) (es256KeyEntry, error) {}, nil}+// es256Kid is the key id of an ES256 key: the first 8 bytes of the SHA-256+// of its uncompressed public point, in hex. A verifier that pinned a kid+// from /jwks matches it whether the control plane or the operator made the key.+func es256Kid(uncompressedPub []byte) string {+ sum := sha256.Sum256(uncompressedPub)+ return hex.EncodeToString(sum[:8])+}+func generateES256KeyEntry() (es256KeyEntry, error) {priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)if err != nil {@@ -192,8 +200,7 @@ func generateES256KeyEntry() (es256KeyEntry, error) {}xBytes := uncompressed[1:33]yBytes := uncompressed[33:65]- sum := sha256.Sum256(uncompressed)- kid := hex.EncodeToString(sum[:8])+ kid := es256Kid(uncompressed)return es256KeyEntry{kid: kid,priv: priv,diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.goindex c52a8287..e8ad74b5 100644--- a/internal/standalone/standalone.go+++ b/internal/standalone/standalone.go@@ -129,6 +129,10 @@ type ControlPlaneTunables struct {KeyRotationInterval time.Duration// KeyGracePeriod keeps rotated-out keys valid for verification.KeyGracePeriod time.Duration+ // SigningKeyPath is the operator's PEM file of signing keys; see+ // controlplane.Options.SigningKeyPath. Empty lets the control plane+ // generate and rotate its own.+ SigningKeyPath string// DHTReplication and DHTReprovideInterval are the mesh's directory// parameters; see controlplane.Options.DHTReplication int@@ -296,6 +300,7 @@ func (s *Server) Start(ctx context.Context) (retErr error) {DHTReplication: s.opts.ControlPlane.DHTReplication,DHTReprovideInterval: s.opts.ControlPlane.DHTReprovideInterval,KeyGracePeriod: s.opts.ControlPlane.KeyGracePeriod,+ SigningKeyPath: s.opts.ControlPlane.SigningKeyPath,BiscuitTTL: s.opts.ControlPlane.BiscuitTTL,WorkloadSessionTTL: s.opts.ControlPlane.WorkloadSessionTTL,NodeRetention: s.opts.ControlPlane.NodeRetention,diff --git a/site/content/docs/guides/cloud-run.md b/site/content/docs/guides/cloud-run.mdindex 3b47c99f..93e2f688 100644--- a/site/content/docs/guides/cloud-run.md+++ b/site/content/docs/guides/cloud-run.md@@ -52,6 +52,43 @@ instance for testing):--args="--data-dir=/data,--port=8080,--db-driver=postgres,--db-dsn=${POSTGRES_DSN}"```+### Operator-owned signing keys and policy without a database++Without PostgreSQL every revision starts from an empty store, which means+new signing keys: every credential the mesh issued stops verifying until each+node re-enrolls. If you want the root of trust in your hands anyway, keep the+keys in Secret Manager (wrap the secret with your own Cloud KMS key if you+want) and mount them as a file; do the same with the policy:++```bash+openssl genpkey -algorithm ed25519 -out ed25519-v1.pem+openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-256 -out p256-v1.pem+cat ed25519-v1.pem p256-v1.pem | gcloud secrets create mesh-signing-keys --data-file=-+gcloud secrets create mesh-policy --data-file=policy.json+SA=$(gcloud projects describe "$PROJECT" --format='value(projectNumber)')[email protected]+for s in mesh-signing-keys mesh-policy; do+ gcloud secrets add-iam-policy-binding "$s" --member="serviceAccount:$SA" --role=roles/secretmanager.secretAccessor+done+gcloud run services update agentmesh-one --project "$PROJECT" --region "$REGION" \+ --set-secrets=/etc/agentmesh/keys/signing-keys.pem=mesh-signing-keys:latest,/etc/agentmesh/policy.json=mesh-policy:latest \+ --args="--data-dir=/data,--port=8080,--signing-key-path=/etc/agentmesh/keys/signing-keys.pem,--policy-file=/etc/agentmesh/policy.json"+```++(Cloud Run mounts one secret per directory, hence the two paths.) The+control plane installs the keys and seeds the policy on every start, so a new+revision changes neither; credentials issued before it keep verifying, and+`/keys` and `/jwks` publish the same keys. Automatic rotation is off while the+file is present; **you rotate**: generate the new key, put it first in the+file, keep the previous one until every credential it signed has expired+(the Biscuit TTL for Ed25519, the STS JWT lifetime for P-256), publish the new+secret version, roll a revision, then drop the old key the same way.++What this does not keep is the enrollment records: a node's own calls to the+control plane (policy sync, catalog, refresh) answer `401 node credential+required` after a revision until the node re-enrolls, which it does at its+next renewal with the bootstrap token. The data plane is unaffected in the+meantime. PostgreSQL keeps the records too.+Why each Cloud Run flag matters:- `--min-instances 1 --max-instances 1`: the router's DHT and relay statediff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.mdindex 30387d15..2f862980 100644--- a/site/content/docs/reference/agentmesh-one.md+++ b/site/content/docs/reference/agentmesh-one.md@@ -43,6 +43,7 @@ downloaded tunnel connector.| `--admin-token-path` | | File containing the admin token. Can also be set with `AGENTMESH_ADMIN_TOKEN`. Generated and saved to the data directory when neither is set. || `--policy-file` | | Protojson `PolicyConfig` that seeds the mesh policy on first boot. Ignored once the database has a policy. Without it, first boot seeds an open development policy and logs a warning. || `--issuer` | | External OIDC issuer(s), comma-separated. Optional. Without an issuer, enrollment works by token only. |+| `--signing-key-path` | | PEM file with operator-owned signing keys (Ed25519 for Biscuits, P-256 for ES256 JWTs; first of each kind signs, later ones verify). Automatic key rotation is off while set; rotate by publishing a new first key and keeping the previous one for the grace period. See the control plane reference. || `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. One that does not answer at start-up is resolved in the background and its tokens refused until then (`agentmesh_control_plane_workload_issuer_ready{issuer}`); a `--issuer` that does not answer is fatal. || `--oidc-client-id` | first audience | Client ID advertised on `/info`. || `--allowed-audiences` | `agentmesh-audience` | Accepted OIDC audiences. |diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex faef6fa6..74274742 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -33,6 +33,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban| `--auto-approve-enrollment` | `false` | Issue credentials for valid bootstrap-token enrollments immediately instead of queueing them for approval. || `--biscuit-ttl` | `24h` | Lifetime of each credential. If the OIDC token expires sooner, the credential expires with it. || `--oidc-session-ttl` | `2160h` (90 days) | How long a human OIDC enrollment may keep refreshing before the identity must log in again. |+| `--signing-key-path` | | PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs behind `/sts/token` and `/jwks`. The first key of each kind signs, later ones only verify. The keys are installed into the store through the normal rotation path, so verifiers learn them from `/keys` and `/jwks` as after any rotation, and a key the control plane generated earlier stays valid for `--key-grace-period`. Turns automatic rotation off: rotate by publishing a new first key and keeping the previous one until every credential it signed has expired. `agentmesh_control_plane_operator_signing_keys{algorithm}` counts the keys loaded. || `--workload-session-ttl` | `48h` | How long a workload OIDC enrollment (`--workload-issuer`) may refresh without presenting a fresh platform JWT in `TokenRefreshRequest.jwt`. || `--key-rotation-interval` | `24h` | How often a new signing key is generated. `0` disables rotation. || `--key-grace-period` | `1h` | How long a rotated-out key stays accepted. Credentials signed by a retired key cannot be verified or refreshed. Nodes and routers must pull `/keys` well within this window (`agentmesh-node --control-plane-sync-interval`, `agentmesh-router --keys-sync-interval`). |