Skip to content

Agent Mesh

controlplane: operator-owned signing keys from a PEM file

Antonio Ojeac79d69b26d71
diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.go
index fcec9af8..17293c4c 100644
--- a/cmd/agentmesh-control-plane/main.go
+++ b/cmd/agentmesh-control-plane/main.go
@@ -47,6 +47,7 @@ var (
allowedAudiencesFlag string
keyRotationInterval time.Duration
keyGracePeriod time.Duration
+ signingKeyPath string
leaseDuration time.Duration
dhtReplication int
dhtReprovideInterval time.Duration
@@ -140,6 +141,7 @@ func main() {
LeaseDuration: leaseDuration,
KeyRotationInterval: keyRotationInterval,
KeyGracePeriod: keyGracePeriod,
+ SigningKeyPath: signingKeyPath,
DHTReplication: dhtReplication,
DHTReprovideInterval: dhtReprovideInterval,
InsecureSkipTLSVerify: insecureSkipTLSVerify,
@@ -205,6 +207,7 @@ func main() {
rootCmd.Flags().StringVar(&allowedAudiencesFlag, "allowed-audiences", api.DefaultAudience, "Comma-separated list of allowed OIDC audiences")
rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 24*time.Hour, "Key rotation interval (e.g. 24h). 0 disables rotation.")
rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 1*time.Hour, "Key grace period for rotated keys.")
+ rootCmd.Flags().StringVar(&signingKeyPath, "signing-key-path", "", "PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs; the first key of each kind signs, later ones only verify. Turns automatic key rotation off: rotate by publishing a new first key and keeping the previous one for the grace period")
rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")
rootCmd.Flags().IntVar(&dhtReplication, "dht-replication", api.DefaultDHTReplication, "How many routers hold each service a member publishes and answer each lookup: the routers closest to the key. Every member holds a session with each of them besides its --routers. More replicas survive more routers down at once; each one is a session more per key on every member. Published on /info for the whole mesh.")
rootCmd.Flags().DurationVar(&dhtReprovideInterval, "dht-reprovide-interval", api.DefaultReprovideInterval, "How often a member re-announces each service it publishes. A provider that stopped without unpublishing stays listed for three intervals; the mesh pays providers / interval announcements a minute. Published on /info for the whole mesh.")
diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.go
index a48728f6..a5d73349 100644
--- a/cmd/agentmesh-one/main.go
+++ b/cmd/agentmesh-one/main.go
@@ -229,6 +229,7 @@ func main() {
rootCmd.Flags().DurationVar(&cpTunables.DHTReprovideInterval, "dht-reprovide-interval", 0, "How often a member re-announces each service it publishes (0 keeps the component default of 5m); a stopped provider stays listed for three intervals; published on /info for the whole mesh")
rootCmd.Flags().DurationVar(&cpTunables.KeyRotationInterval, "control-plane-key-rotation-interval", 0, "Biscuit signing key rotation interval (0 keeps the component default)")
rootCmd.Flags().DurationVar(&cpTunables.KeyGracePeriod, "control-plane-key-grace-period", 0, "How long rotated-out keys stay valid for verification (0 keeps the component default)")
+ rootCmd.Flags().StringVar(&cpTunables.SigningKeyPath, "signing-key-path", "", "PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs; the first key of each kind signs, later ones only verify. Turns automatic key rotation off: rotate by publishing a new first key and keeping the previous one for the grace period")
rootCmd.Flags().DurationVar(&cpTunables.BiscuitTTL, "control-plane-biscuit-ttl", 0, "Lifespan minted into issued biscuits (0 keeps the component default)")
rootCmd.Flags().DurationVar(&cpTunables.WorkloadSessionTTL, "control-plane-workload-session-ttl", 0, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh (0 keeps the component default)")
rootCmd.Flags().DurationVar(&cpTunables.NodeRetention, "control-plane-node-retention", 0, "How long a node record is kept after its session expired, or after its last refresh for a bootstrap enrollment, before it is deleted (0 keeps the component default)")
diff --git a/internal/controlplane/config.go b/internal/controlplane/config.go
index 269ff62f..d50af498 100644
--- a/internal/controlplane/config.go
+++ b/internal/controlplane/config.go
@@ -38,6 +38,13 @@ type Options struct {
LeaseDuration time.Duration
KeyRotationInterval time.Duration
KeyGracePeriod time.Duration
+ // SigningKeyPath names a PEM file with the control plane's private
+ // signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the
+ // ES256 JWTs. For each kind the first key signs and later keys only
+ // verify. When set, the operator owns rotation: the control plane
+ // installs these keys through its normal rotation path at start-up and
+ // never rotates on its own.
+ SigningKeyPath string
// DHTReplication and DHTReprovideInterval are the mesh's directory
// parameters, published on /info for every member and router
// (api.DirectoryParameters); 0 is the api default of each.
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go
index 6f7fb2b0..222baa40 100644
--- a/internal/controlplane/metrics.go
+++ b/internal/controlplane/metrics.go
@@ -57,6 +57,17 @@ var (
},
[]string{"issuer"},
)
+
+ // operatorSigningKeysGauge counts the signing keys the operator supplied
+ // through --signing-key-path, per algorithm; absent when the control
+ // plane generates and rotates its own.
+ operatorSigningKeysGauge = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "agentmesh_control_plane_operator_signing_keys",
+ Help: "Signing keys supplied by the operator (--signing-key-path) by algorithm; automatic rotation is off while any are present",
+ },
+ []string{"algorithm"},
+ )
)
// observeRoute counts and times requests to one registered mux pattern. The
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 7cbd245e..dd4a17f8 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -141,7 +141,10 @@ type Server struct {
// resolving holds the workload issuers a background resolver is working
// on, so that a second pass over the configuration (Init is exported for
// embedders) starts no second resolver for the same issuer.
- resolving map[string]bool
+ resolving map[string]bool
+ // operatorKeys are the signing keys the operator supplied, nil when the
+ // control plane generates and rotates its own.
+ operatorKeys *operatorSigningKeys
providers map[string]*oidc.Provider
workloadIssuers map[string]bool
workloadEmailSuffixes map[string][]string
@@ -191,6 +194,18 @@ func NewServer(config Options, store storage.Store) (*Server, error) {
return nil, err
}
+ var operatorKeys *operatorSigningKeys
+ if config.SigningKeyPath != "" {
+ var err error
+ operatorKeys, err = loadOperatorSigningKeys(config.SigningKeyPath)
+ if err != nil {
+ return nil, err
+ }
+ if err := reconcileES256Keyring(context.Background(), store, operatorKeys.es256, config.KeyGracePeriod); err != nil {
+ return nil, fmt.Errorf("operator signing keys: %w", err)
+ }
+ }
+
signer := config.OIDCSigner
if signer == nil {
var err error
@@ -224,6 +239,7 @@ func NewServer(config Options, store storage.Store) (*Server, error) {
oauthCodes: make(map[string]*oauthAuthCode),
providers: make(map[string]*oidc.Provider),
workloadIssuers: workloadIssuers,
+ operatorKeys: operatorKeys,
workloadEmailSuffixes: workloadEmailSuffixes,
catalog: make(map[string]nodeCatalogEntry),
metricsRegistry: reg,
@@ -292,6 +308,18 @@ func (s *Server) Start() error {
func (s *Server) Init() error {
// Initialize Keyring
ctx := context.Background()
+ if s.operatorKeys != nil {
+ if err := reconcileEd25519Keyring(ctx, s.store, s.operatorKeys.ed25519, s.config.KeyGracePeriod); err != nil {
+ return fmt.Errorf("operator signing keys: %w", err)
+ }
+ operatorSigningKeysGauge.WithLabelValues("ed25519").Set(float64(len(s.operatorKeys.ed25519)))
+ operatorSigningKeysGauge.WithLabelValues("es256").Set(float64(len(s.operatorKeys.es256)))
+ logger.Infow("Signing keys are operator-managed; automatic key rotation is off",
+ "path", s.config.SigningKeyPath,
+ "ed25519_keys", len(s.operatorKeys.ed25519),
+ "es256_keys", len(s.operatorKeys.es256),
+ "rotation", "put the new key first in the file and keep the previous one for the grace period")
+ }
_, _, err := s.store.GetCurrentKey(ctx)
if err == storage.ErrNotFound {
logger.Info("Generating initial control plane signing keys...")
@@ -307,8 +335,9 @@ func (s *Server) Init() error {
}
// The rotation deadline lives in the store, shared by the replicas and
// kept across restarts; a store that never had one gets its first an
- // interval from now.
- if interval := s.config.KeyRotationInterval; interval > 0 {
+ // interval from now. Operator-managed keys rotate when the operator
+ // publishes a new file, never here.
+ if interval := s.config.KeyRotationInterval; interval > 0 && s.operatorKeys == nil {
first := time.Now().Add(interval)
scheduled, err := s.store.ScheduleKeyRotationIfUnset(ctx, first)
if err != nil {
@@ -327,8 +356,10 @@ func (s *Server) Init() error {
}
// Start key rotation routine
- s.wg.Add(1)
- go s.runKeyRotationLoop()
+ if s.operatorKeys == nil {
+ s.wg.Add(1)
+ go s.runKeyRotationLoop()
+ }
s.wg.Add(1)
go s.runNodeGCLoop()
diff --git a/internal/controlplane/signingkeys.go b/internal/controlplane/signingkeys.go
new file mode 100644
index 00000000..29346312
--- /dev/null
+++ b/internal/controlplane/signingkeys.go
@@ -0,0 +1,211 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "crypto/ed25519"
+ "crypto/elliptic"
+ "crypto/sha256"
+ "crypto/x509"
+ "encoding/hex"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/google/agentmesh/internal/storage"
+)
+
+// operatorSigningKeys are the control plane's private signing keys when the
+// operator supplies them (Options.SigningKeyPath) instead of letting the
+// control plane generate and rotate its own. The operator then owns the root
+// of trust: the keys live where the operator keeps them (a secret manager, a
+// KMS-wrapped file) and rotate on the operator's schedule.
+//
+// The file is PEM. Each "PRIVATE KEY" (PKCS#8) or "EC PRIVATE KEY" (SEC 1)
+// block is one key; Ed25519 keys sign Biscuits, P-256 keys sign the ES256
+// JWTs behind /sts/token and /jwks. For each kind, the first key in the file
+// signs and every later one only verifies, which is how a rotation is
+// expressed: put the new key first, keep the previous one until every
+// credential it signed has expired, then drop it.
+type operatorSigningKeys struct {
+ ed25519 []ed25519.PrivateKey
+ es256 []*ecdsa.PrivateKey
+}
+
+// loadOperatorSigningKeys reads and validates the PEM file at path.
+func loadOperatorSigningKeys(path string) (*operatorSigningKeys, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("read signing keys: %w", err)
+ }
+ keys := &operatorSigningKeys{}
+ for block, rest := pem.Decode(data); block != nil; block, rest = pem.Decode(rest) {
+ var parsed any
+ switch block.Type {
+ case "PRIVATE KEY":
+ parsed, err = x509.ParsePKCS8PrivateKey(block.Bytes)
+ case "EC PRIVATE KEY":
+ parsed, err = x509.ParseECPrivateKey(block.Bytes)
+ default:
+ return nil, fmt.Errorf("signing keys %s: unsupported PEM block %q (want PRIVATE KEY or EC PRIVATE KEY)", path, block.Type)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("signing keys %s: %w", path, err)
+ }
+ switch k := parsed.(type) {
+ case ed25519.PrivateKey:
+ keys.ed25519 = append(keys.ed25519, k)
+ case *ecdsa.PrivateKey:
+ if k.Curve != elliptic.P256() {
+ return nil, fmt.Errorf("signing keys %s: EC key on %s, want P-256", path, k.Curve.Params().Name)
+ }
+ keys.es256 = append(keys.es256, k)
+ default:
+ return nil, fmt.Errorf("signing keys %s: unsupported key type %T (want Ed25519 or P-256)", path, parsed)
+ }
+ }
+ if len(keys.ed25519) == 0 && len(keys.es256) == 0 {
+ return nil, fmt.Errorf("signing keys %s: no private key found", path)
+ }
+ return keys, nil
+}
+
+// keyFingerprint identifies a public key in logs without revealing anything
+// a verifier does not already learn from /keys or /jwks.
+func keyFingerprint(pub []byte) string {
+ sum := sha256.Sum256(pub)
+ return hex.EncodeToString(sum[:8])
+}
+
+// reconcileEd25519Keyring makes the store's Biscuit keyring reflect keys:
+// keys[0] current, the rest valid for verification until gracePeriod from
+// now. It goes through the same SaveInitialKey / RotateKeys path as
+// automatic rotation, so a store that already matches is left alone, a store
+// that holds a key the operator does not know (one the control plane
+// generated before the operator took over) keeps it valid for the grace
+// period, and every verifier learns the new set from /keys the usual way.
+func reconcileEd25519Keyring(ctx context.Context, store storage.Store, keys []ed25519.PrivateKey, gracePeriod time.Duration) error {
+ if len(keys) == 0 {
+ return nil
+ }
+ valid, err := store.GetAllValidKeys(ctx)
+ if err != nil && !errors.Is(err, storage.ErrNotFound) {
+ return fmt.Errorf("read keyring: %w", err)
+ }
+ known := func(pub ed25519.PublicKey) bool {
+ for _, k := range valid {
+ if k.Public.Equal(pub) {
+ return true
+ }
+ }
+ return false
+ }
+ // Oldest first, so that the first key in the file ends up current.
+ for i := len(keys) - 1; i >= 0; i-- {
+ priv := keys[i]
+ pub := priv.Public().(ed25519.PublicKey)
+ if known(pub) {
+ continue
+ }
+ if len(valid) == 0 {
+ if err := store.SaveInitialKey(ctx, priv, pub); err != nil {
+ return fmt.Errorf("save signing key %s: %w", keyFingerprint(pub), err)
+ }
+ } else if err := store.RotateKeys(ctx, priv, pub, gracePeriod); err != nil {
+ return fmt.Errorf("install signing key %s: %w", keyFingerprint(pub), err)
+ }
+ valid = append(valid, storage.KeyPair{Private: priv, Public: pub})
+ logger.Infow("Installed operator signing key", "algorithm", "ed25519", "fingerprint", keyFingerprint(pub))
+ }
+ _, currentPub, err := store.GetCurrentKey(ctx)
+ if err != nil {
+ return fmt.Errorf("read current signing key: %w", err)
+ }
+ if wantPub := keys[0].Public().(ed25519.PublicKey); !currentPub.Equal(wantPub) {
+ // The first key in the file is known but not current, which happens
+ // when the operator moves a previous key back to the front; the
+ // store's current key becomes a verification-only key.
+ if err := store.RotateKeys(ctx, keys[0], wantPub, gracePeriod); err != nil {
+ return fmt.Errorf("make signing key %s current: %w", keyFingerprint(wantPub), err)
+ }
+ logger.Infow("Operator signing key made current", "algorithm", "ed25519", "fingerprint", keyFingerprint(wantPub))
+ }
+ return nil
+}
+
+// reconcileES256Keyring is reconcileEd25519Keyring for the ES256 keys behind
+// /sts/token and /jwks. The key id is the one generateES256KeyEntry derives,
+// so a verifier that pinned a kid from /jwks keeps matching.
+func reconcileES256Keyring(ctx context.Context, store storage.Store, keys []*ecdsa.PrivateKey, gracePeriod time.Duration) error {
+ if len(keys) == 0 {
+ return nil
+ }
+ valid, err := store.GetAllValidOIDCKeys(ctx)
+ if err != nil && !errors.Is(err, storage.ErrNotFound) {
+ return fmt.Errorf("read ES256 keyring: %w", err)
+ }
+ known := make(map[string]bool, len(valid))
+ for _, k := range valid {
+ known[k.Kid] = true
+ }
+ encode := func(priv *ecdsa.PrivateKey) (kid string, privBytes, pubBytes []byte, err error) {
+ pubBytes, err = priv.PublicKey.Bytes()
+ if err != nil || len(pubBytes) != 65 {
+ return "", nil, nil, fmt.Errorf("encode P-256 public key: %w", err)
+ }
+ privBytes, err = x509.MarshalECPrivateKey(priv)
+ if err != nil {
+ return "", nil, nil, fmt.Errorf("encode P-256 private key: %w", err)
+ }
+ return es256Kid(pubBytes), privBytes, pubBytes, nil
+ }
+ for i := len(keys) - 1; i >= 0; i-- {
+ kid, privBytes, pubBytes, err := encode(keys[i])
+ if err != nil {
+ return err
+ }
+ if known[kid] {
+ continue
+ }
+ if len(known) == 0 {
+ if err := store.SaveInitialOIDCKey(ctx, kid, privBytes, pubBytes); err != nil {
+ return fmt.Errorf("save ES256 key %s: %w", kid, err)
+ }
+ } else if err := store.RotateOIDCKeys(ctx, kid, privBytes, pubBytes, gracePeriod); err != nil {
+ return fmt.Errorf("install ES256 key %s: %w", kid, err)
+ }
+ known[kid] = true
+ logger.Infow("Installed operator signing key", "algorithm", "es256", "kid", kid)
+ }
+ wantKid, privBytes, pubBytes, err := encode(keys[0])
+ if err != nil {
+ return err
+ }
+ current, err := store.GetCurrentOIDCKey(ctx)
+ if err != nil {
+ return fmt.Errorf("read current ES256 key: %w", err)
+ }
+ if current == nil || current.Kid != wantKid {
+ if err := store.RotateOIDCKeys(ctx, wantKid, privBytes, pubBytes, gracePeriod); err != nil {
+ return fmt.Errorf("make ES256 key %s current: %w", wantKid, err)
+ }
+ logger.Infow("Operator signing key made current", "algorithm", "es256", "kid", wantKid)
+ }
+ return nil
+}
diff --git a/internal/controlplane/signingkeys_test.go b/internal/controlplane/signingkeys_test.go
new file mode 100644
index 00000000..3cc4c867
--- /dev/null
+++ b/internal/controlplane/signingkeys_test.go
@@ -0,0 +1,380 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "crypto/ed25519"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/json"
+ "encoding/pem"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ "github.com/google/agentmesh/internal/storage"
+ "google.golang.org/protobuf/proto"
+)
+
+// pemPKCS8 encodes a private key as one "PRIVATE KEY" block.
+func pemPKCS8(t *testing.T, key any) []byte {
+ t.Helper()
+ der, err := x509.MarshalPKCS8PrivateKey(key)
+ if err != nil {
+ t.Fatalf("MarshalPKCS8PrivateKey: %v", err)
+ }
+ return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})
+}
+
+func newEd25519(t *testing.T) ed25519.PrivateKey {
+ t.Helper()
+ _, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ return priv
+}
+
+func newP256(t *testing.T) *ecdsa.PrivateKey {
+ t.Helper()
+ priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ t.Fatalf("ecdsa.GenerateKey: %v", err)
+ }
+ return priv
+}
+
+func writeKeyFile(t *testing.T, blocks ...[]byte) string {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), "signing-keys.pem")
+ var data []byte
+ for _, b := range blocks {
+ data = append(data, b...)
+ }
+ if err := os.WriteFile(path, data, 0o600); err != nil {
+ t.Fatalf("WriteFile: %v", err)
+ }
+ return path
+}
+
+func TestLoadOperatorSigningKeys(t *testing.T) {
+ edA, edB := newEd25519(t), newEd25519(t)
+ ecA := newP256(t)
+
+ t.Run("pkcs8 ed25519 and p256 in file order", func(t *testing.T) {
+ path := writeKeyFile(t, pemPKCS8(t, edA), pemPKCS8(t, ecA), pemPKCS8(t, edB))
+ keys, err := loadOperatorSigningKeys(path)
+ if err != nil {
+ t.Fatalf("loadOperatorSigningKeys: %v", err)
+ }
+ if len(keys.ed25519) != 2 || !keys.ed25519[0].Equal(edA) || !keys.ed25519[1].Equal(edB) {
+ t.Fatalf("ed25519 keys = %d in wrong order, want [A B]", len(keys.ed25519))
+ }
+ if len(keys.es256) != 1 || !keys.es256[0].Equal(ecA) {
+ t.Fatalf("es256 keys = %d, want [A]", len(keys.es256))
+ }
+ })
+
+ t.Run("sec1 ec block", func(t *testing.T) {
+ der, err := x509.MarshalECPrivateKey(ecA)
+ if err != nil {
+ t.Fatalf("MarshalECPrivateKey: %v", err)
+ }
+ path := writeKeyFile(t, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}))
+ keys, err := loadOperatorSigningKeys(path)
+ if err != nil {
+ t.Fatalf("loadOperatorSigningKeys: %v", err)
+ }
+ if len(keys.es256) != 1 || !keys.es256[0].Equal(ecA) {
+ t.Fatalf("es256 keys = %d, want the SEC 1 key", len(keys.es256))
+ }
+ })
+
+ t.Run("rsa is refused", func(t *testing.T) {
+ rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ t.Fatalf("rsa.GenerateKey: %v", err)
+ }
+ _, err = loadOperatorSigningKeys(writeKeyFile(t, pemPKCS8(t, rsaKey)))
+ if err == nil || !strings.Contains(err.Error(), "unsupported key type") {
+ t.Fatalf("err = %v, want unsupported key type", err)
+ }
+ })
+
+ t.Run("other curve is refused", func(t *testing.T) {
+ p384, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+ if err != nil {
+ t.Fatalf("ecdsa.GenerateKey: %v", err)
+ }
+ _, err = loadOperatorSigningKeys(writeKeyFile(t, pemPKCS8(t, p384)))
+ if err == nil || !strings.Contains(err.Error(), "want P-256") {
+ t.Fatalf("err = %v, want P-256 complaint", err)
+ }
+ })
+
+ t.Run("certificate block is refused", func(t *testing.T) {
+ _, err := loadOperatorSigningKeys(writeKeyFile(t, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte{1}})))
+ if err == nil || !strings.Contains(err.Error(), "unsupported PEM block") {
+ t.Fatalf("err = %v, want unsupported PEM block", err)
+ }
+ })
+
+ t.Run("empty file is refused", func(t *testing.T) {
+ _, err := loadOperatorSigningKeys(writeKeyFile(t, []byte("not pem at all\n")))
+ if err == nil || !strings.Contains(err.Error(), "no private key found") {
+ t.Fatalf("err = %v, want no private key found", err)
+ }
+ })
+
+ t.Run("missing file is refused", func(t *testing.T) {
+ if _, err := loadOperatorSigningKeys(filepath.Join(t.TempDir(), "absent.pem")); err == nil {
+ t.Fatal("loadOperatorSigningKeys on a missing file returned nil")
+ }
+ })
+}
+
+func TestReconcileEd25519Keyring(t *testing.T) {
+ ctx := context.Background()
+ newStore := func(t *testing.T) storage.Store {
+ t.Helper()
+ store, err := storage.NewSQLStore("sqlite", ":memory:")
+ if err != nil {
+ t.Fatalf("NewSQLStore: %v", err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ return store
+ }
+ pubOf := func(k ed25519.PrivateKey) ed25519.PublicKey { return k.Public().(ed25519.PublicKey) }
+ validPubs := func(t *testing.T, store storage.Store) map[string]bool {
+ t.Helper()
+ valid, err := store.GetAllValidKeys(ctx)
+ if err != nil {
+ t.Fatalf("GetAllValidKeys: %v", err)
+ }
+ out := map[string]bool{}
+ for _, k := range valid {
+ out[string(k.Public)] = true
+ }
+ return out
+ }
+ current, previous := newEd25519(t), newEd25519(t)
+
+ t.Run("empty store: first key current, rest valid", func(t *testing.T) {
+ store := newStore(t)
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current, previous}, time.Hour); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ _, gotPub, err := store.GetCurrentKey(ctx)
+ if err != nil {
+ t.Fatalf("GetCurrentKey: %v", err)
+ }
+ if !gotPub.Equal(pubOf(current)) {
+ t.Fatalf("current key = %s, want %s (first in file)", keyFingerprint(gotPub), keyFingerprint(pubOf(current)))
+ }
+ valid := validPubs(t, store)
+ if len(valid) != 2 || !valid[string(pubOf(current))] || !valid[string(pubOf(previous))] {
+ t.Fatalf("valid keys = %d, want both file keys", len(valid))
+ }
+ })
+
+ t.Run("matching store is left alone", func(t *testing.T) {
+ store := newStore(t)
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {
+ t.Fatalf("first reconcile: %v", err)
+ }
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {
+ t.Fatalf("second reconcile: %v", err)
+ }
+ if valid := validPubs(t, store); len(valid) != 1 {
+ t.Fatalf("valid keys = %d after a no-op reconcile, want 1", len(valid))
+ }
+ })
+
+ t.Run("generated key stays valid for the grace period", func(t *testing.T) {
+ store := newStore(t)
+ generated := newEd25519(t)
+ if err := store.SaveInitialKey(ctx, generated, pubOf(generated)); err != nil {
+ t.Fatalf("SaveInitialKey: %v", err)
+ }
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current}, time.Hour); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ _, gotPub, err := store.GetCurrentKey(ctx)
+ if err != nil {
+ t.Fatalf("GetCurrentKey: %v", err)
+ }
+ if !gotPub.Equal(pubOf(current)) {
+ t.Fatalf("current key = %s, want the operator key", keyFingerprint(gotPub))
+ }
+ valid := validPubs(t, store)
+ if !valid[string(pubOf(generated))] {
+ t.Fatal("the key the control plane generated before the operator took over is no longer valid; credentials it signed would break at once")
+ }
+ })
+
+ t.Run("operator rotation: new key first, previous kept", func(t *testing.T) {
+ store := newStore(t)
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{previous}, time.Hour); err != nil {
+ t.Fatalf("reconcile v1: %v", err)
+ }
+ if err := reconcileEd25519Keyring(ctx, store, []ed25519.PrivateKey{current, previous}, time.Hour); err != nil {
+ t.Fatalf("reconcile v2: %v", err)
+ }
+ _, gotPub, err := store.GetCurrentKey(ctx)
+ if err != nil {
+ t.Fatalf("GetCurrentKey: %v", err)
+ }
+ if !gotPub.Equal(pubOf(current)) {
+ t.Fatalf("current key = %s after rotation, want the new first key", keyFingerprint(gotPub))
+ }
+ if valid := validPubs(t, store); len(valid) != 2 || !valid[string(pubOf(previous))] {
+ t.Fatalf("valid keys = %d, want the previous key still valid", len(valid))
+ }
+ })
+}
+
+func TestReconcileES256Keyring(t *testing.T) {
+ ctx := context.Background()
+ store, err := storage.NewSQLStore("sqlite", ":memory:")
+ if err != nil {
+ t.Fatalf("NewSQLStore: %v", err)
+ }
+ defer func() { _ = store.Close() }()
+ kidOf := func(t *testing.T, k *ecdsa.PrivateKey) string {
+ t.Helper()
+ pub, err := k.PublicKey.Bytes()
+ if err != nil {
+ t.Fatalf("PublicKey.Bytes: %v", err)
+ }
+ return es256Kid(pub)
+ }
+ v1, v2 := newP256(t), newP256(t)
+
+ if err := reconcileES256Keyring(ctx, store, []*ecdsa.PrivateKey{v1}, time.Hour); err != nil {
+ t.Fatalf("reconcile v1: %v", err)
+ }
+ cur, err := store.GetCurrentOIDCKey(ctx)
+ if err != nil || cur == nil {
+ t.Fatalf("GetCurrentOIDCKey: %v (%v)", err, cur)
+ }
+ if cur.Kid != kidOf(t, v1) {
+ t.Fatalf("current kid = %s, want %s", cur.Kid, kidOf(t, v1))
+ }
+ if err := reconcileES256Keyring(ctx, store, []*ecdsa.PrivateKey{v2, v1}, time.Hour); err != nil {
+ t.Fatalf("reconcile v2: %v", err)
+ }
+ cur, err = store.GetCurrentOIDCKey(ctx)
+ if err != nil || cur == nil {
+ t.Fatalf("GetCurrentOIDCKey: %v (%v)", err, cur)
+ }
+ if cur.Kid != kidOf(t, v2) {
+ t.Fatalf("current kid after rotation = %s, want %s", cur.Kid, kidOf(t, v2))
+ }
+ all, err := store.GetAllValidOIDCKeys(ctx)
+ if err != nil {
+ t.Fatalf("GetAllValidOIDCKeys: %v", err)
+ }
+ if len(all) != 2 {
+ t.Fatalf("valid ES256 keys = %d after rotation, want 2 (previous kept for the grace period)", len(all))
+ }
+}
+
+// TestServerOperatorSigningKeys starts a control plane on operator keys and
+// checks what verifiers see: /keys carries exactly the file's Ed25519 keys,
+// /jwks the file's P-256 key, and no automatic rotation is scheduled.
+func TestServerOperatorSigningKeys(t *testing.T) {
+ issuer, _ := startCustomMockOIDC(t)
+ edCurrent, edPrevious := newEd25519(t), newEd25519(t)
+ ec := newP256(t)
+ path := writeKeyFile(t, pemPKCS8(t, edCurrent), pemPKCS8(t, edPrevious), pemPKCS8(t, ec))
+
+ srv, store, baseURL := setupTestServer(t, issuer, func(o *Options) {
+ o.SigningKeyPath = path
+ })
+ defer func() { _ = srv.Close() }()
+
+ resp, err := http.Get(baseURL + "/keys")
+ if err != nil {
+ t.Fatalf("GET /keys: %v", err)
+ }
+ body, err := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if err != nil {
+ t.Fatalf("read /keys: %v", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("GET /keys = %d: %s", resp.StatusCode, body)
+ }
+ var keys api.KeysResponse
+ if err := proto.Unmarshal(body, &keys); err != nil {
+ t.Fatalf("Unmarshal KeysResponse: %v", err)
+ }
+ got := map[string]bool{}
+ for _, k := range keys.PublicKeys {
+ got[string(k)] = true
+ }
+ if len(got) != 2 || !got[string(edCurrent.Public().(ed25519.PublicKey))] || !got[string(edPrevious.Public().(ed25519.PublicKey))] {
+ t.Fatalf("/keys has %d keys, want exactly the two Ed25519 keys from the file", len(got))
+ }
+ _, currentPub, err := store.GetCurrentKey(context.Background())
+ if err != nil {
+ t.Fatalf("GetCurrentKey: %v", err)
+ }
+ if !currentPub.Equal(edCurrent.Public().(ed25519.PublicKey)) {
+ t.Fatal("the signing key is not the first Ed25519 key in the file")
+ }
+
+ jwksResp, err := http.Get(baseURL + "/jwks")
+ if err != nil {
+ t.Fatalf("GET /jwks: %v", err)
+ }
+ var jwks JSONWebKeySet
+ err = json.NewDecoder(jwksResp.Body).Decode(&jwks)
+ _ = jwksResp.Body.Close()
+ if err != nil {
+ t.Fatalf("decode /jwks: %v", err)
+ }
+ ecPub, err := ec.PublicKey.Bytes()
+ if err != nil {
+ t.Fatalf("PublicKey.Bytes: %v", err)
+ }
+ if len(jwks.Keys) != 1 || jwks.Keys[0].Kid != es256Kid(ecPub) {
+ t.Fatalf("/jwks = %d keys (kid %s), want the one P-256 key from the file (kid %s)", len(jwks.Keys), firstKid(jwks), es256Kid(ecPub))
+ }
+
+ next, err := store.NextKeyRotation(context.Background())
+ if err != nil {
+ t.Fatalf("NextKeyRotation: %v", err)
+ }
+ if !next.IsZero() {
+ t.Fatalf("automatic rotation scheduled at %s although the operator owns the keys", next)
+ }
+}
+
+func firstKid(jwks JSONWebKeySet) string {
+ if len(jwks.Keys) == 0 {
+ return ""
+ }
+ return jwks.Keys[0].Kid
+}
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index 81262125..9be60ee6 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -181,6 +181,14 @@ func decodeOIDCKeyPair(k storage.OIDCKeyPair) (es256KeyEntry, error) {
}, nil
}
+// es256Kid is the key id of an ES256 key: the first 8 bytes of the SHA-256
+// of its uncompressed public point, in hex. A verifier that pinned a kid
+// from /jwks matches it whether the control plane or the operator made the key.
+func es256Kid(uncompressedPub []byte) string {
+ sum := sha256.Sum256(uncompressedPub)
+ return hex.EncodeToString(sum[:8])
+}
+
func generateES256KeyEntry() (es256KeyEntry, error) {
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
@@ -192,8 +200,7 @@ func generateES256KeyEntry() (es256KeyEntry, error) {
}
xBytes := uncompressed[1:33]
yBytes := uncompressed[33:65]
- sum := sha256.Sum256(uncompressed)
- kid := hex.EncodeToString(sum[:8])
+ kid := es256Kid(uncompressed)
return es256KeyEntry{
kid: kid,
priv: priv,
diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.go
index c52a8287..e8ad74b5 100644
--- a/internal/standalone/standalone.go
+++ b/internal/standalone/standalone.go
@@ -129,6 +129,10 @@ type ControlPlaneTunables struct {
KeyRotationInterval time.Duration
// KeyGracePeriod keeps rotated-out keys valid for verification.
KeyGracePeriod time.Duration
+ // SigningKeyPath is the operator's PEM file of signing keys; see
+ // controlplane.Options.SigningKeyPath. Empty lets the control plane
+ // generate and rotate its own.
+ SigningKeyPath string
// DHTReplication and DHTReprovideInterval are the mesh's directory
// parameters; see controlplane.Options.
DHTReplication int
@@ -296,6 +300,7 @@ func (s *Server) Start(ctx context.Context) (retErr error) {
DHTReplication: s.opts.ControlPlane.DHTReplication,
DHTReprovideInterval: s.opts.ControlPlane.DHTReprovideInterval,
KeyGracePeriod: s.opts.ControlPlane.KeyGracePeriod,
+ SigningKeyPath: s.opts.ControlPlane.SigningKeyPath,
BiscuitTTL: s.opts.ControlPlane.BiscuitTTL,
WorkloadSessionTTL: s.opts.ControlPlane.WorkloadSessionTTL,
NodeRetention: s.opts.ControlPlane.NodeRetention,
diff --git a/site/content/docs/guides/cloud-run.md b/site/content/docs/guides/cloud-run.md
index 3b47c99f..93e2f688 100644
--- a/site/content/docs/guides/cloud-run.md
+++ b/site/content/docs/guides/cloud-run.md
@@ -52,6 +52,43 @@ instance for testing):
--args="--data-dir=/data,--port=8080,--db-driver=postgres,--db-dsn=${POSTGRES_DSN}"
```
+### Operator-owned signing keys and policy without a database
+
+Without PostgreSQL every revision starts from an empty store, which means
+new signing keys: every credential the mesh issued stops verifying until each
+node re-enrolls. If you want the root of trust in your hands anyway, keep the
+keys in Secret Manager (wrap the secret with your own Cloud KMS key if you
+want) and mount them as a file; do the same with the policy:
+
+```bash
+openssl genpkey -algorithm ed25519 -out ed25519-v1.pem
+openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-256 -out p256-v1.pem
+cat ed25519-v1.pem p256-v1.pem | gcloud secrets create mesh-signing-keys --data-file=-
+gcloud secrets create mesh-policy --data-file=policy.json
+SA=$(gcloud projects describe "$PROJECT" --format='value(projectNumber)')[email protected]
+for s in mesh-signing-keys mesh-policy; do
+ gcloud secrets add-iam-policy-binding "$s" --member="serviceAccount:$SA" --role=roles/secretmanager.secretAccessor
+done
+gcloud run services update agentmesh-one --project "$PROJECT" --region "$REGION" \
+ --set-secrets=/etc/agentmesh/keys/signing-keys.pem=mesh-signing-keys:latest,/etc/agentmesh/policy.json=mesh-policy:latest \
+ --args="--data-dir=/data,--port=8080,--signing-key-path=/etc/agentmesh/keys/signing-keys.pem,--policy-file=/etc/agentmesh/policy.json"
+```
+
+(Cloud Run mounts one secret per directory, hence the two paths.) The
+control plane installs the keys and seeds the policy on every start, so a new
+revision changes neither; credentials issued before it keep verifying, and
+`/keys` and `/jwks` publish the same keys. Automatic rotation is off while the
+file is present; **you rotate**: generate the new key, put it first in the
+file, keep the previous one until every credential it signed has expired
+(the Biscuit TTL for Ed25519, the STS JWT lifetime for P-256), publish the new
+secret version, roll a revision, then drop the old key the same way.
+
+What this does not keep is the enrollment records: a node's own calls to the
+control plane (policy sync, catalog, refresh) answer `401 node credential
+required` after a revision until the node re-enrolls, which it does at its
+next renewal with the bootstrap token. The data plane is unaffected in the
+meantime. PostgreSQL keeps the records too.
+
Why each Cloud Run flag matters:
- `--min-instances 1 --max-instances 1`: the router's DHT and relay state
diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.md
index 30387d15..2f862980 100644
--- a/site/content/docs/reference/agentmesh-one.md
+++ b/site/content/docs/reference/agentmesh-one.md
@@ -43,6 +43,7 @@ downloaded tunnel connector.
| `--admin-token-path` | | File containing the admin token. Can also be set with `AGENTMESH_ADMIN_TOKEN`. Generated and saved to the data directory when neither is set. |
| `--policy-file` | | Protojson `PolicyConfig` that seeds the mesh policy on first boot. Ignored once the database has a policy. Without it, first boot seeds an open development policy and logs a warning. |
| `--issuer` | | External OIDC issuer(s), comma-separated. Optional. Without an issuer, enrollment works by token only. |
+| `--signing-key-path` | | PEM file with operator-owned signing keys (Ed25519 for Biscuits, P-256 for ES256 JWTs; first of each kind signs, later ones verify). Automatic key rotation is off while set; rotate by publishing a new first key and keeping the previous one for the grace period. See the control plane reference. |
| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. One that does not answer at start-up is resolved in the background and its tokens refused until then (`agentmesh_control_plane_workload_issuer_ready{issuer}`); a `--issuer` that does not answer is fatal. |
| `--oidc-client-id` | first audience | Client ID advertised on `/info`. |
| `--allowed-audiences` | `agentmesh-audience` | Accepted OIDC audiences. |
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index faef6fa6..74274742 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -33,6 +33,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban
| `--auto-approve-enrollment` | `false` | Issue credentials for valid bootstrap-token enrollments immediately instead of queueing them for approval. |
| `--biscuit-ttl` | `24h` | Lifetime of each credential. If the OIDC token expires sooner, the credential expires with it. |
| `--oidc-session-ttl` | `2160h` (90 days) | How long a human OIDC enrollment may keep refreshing before the identity must log in again. |
+| `--signing-key-path` | | PEM file with the control plane's private signing keys (PKCS#8 or SEC 1): Ed25519 for Biscuits, P-256 for the ES256 JWTs behind `/sts/token` and `/jwks`. The first key of each kind signs, later ones only verify. The keys are installed into the store through the normal rotation path, so verifiers learn them from `/keys` and `/jwks` as after any rotation, and a key the control plane generated earlier stays valid for `--key-grace-period`. Turns automatic rotation off: rotate by publishing a new first key and keeping the previous one until every credential it signed has expired. `agentmesh_control_plane_operator_signing_keys{algorithm}` counts the keys loaded. |
| `--workload-session-ttl` | `48h` | How long a workload OIDC enrollment (`--workload-issuer`) may refresh without presenting a fresh platform JWT in `TokenRefreshRequest.jwt`. |
| `--key-rotation-interval` | `24h` | How often a new signing key is generated. `0` disables rotation. |
| `--key-grace-period` | `1h` | How long a rotated-out key stays accepted. Credentials signed by a retired key cannot be verified or refreshed. Nodes and routers must pull `/keys` well within this window (`agentmesh-node --control-plane-sync-interval`, `agentmesh-router --keys-sync-interval`). |