Skip to content

Agent Mesh

11.7 KB
Raw Blame
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlplane

import (
	"context"
	"crypto/ecdsa"
	"crypto/ed25519"
	"crypto/elliptic"
	"crypto/sha256"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"os"
	"slices"
	"sort"
	"time"

	"github.com/google/agentmesh/api"
	"github.com/google/agentmesh/internal/storage"
)

// operatorSigningKeys are the control plane's private signing keys when the
// operator supplies them (Options.SigningKeyPath) instead of letting the
// control plane generate and rotate its own. The operator then owns the root
// of trust: the keys live where the operator keeps them (a secret manager, a
// KMS-wrapped file) and rotate on the operator's schedule.
//
// The file is PEM. Each "PRIVATE KEY" (PKCS#8) or "EC PRIVATE KEY" (SEC 1)
// block is one key; Ed25519 keys sign Biscuits, P-256 keys sign the ES256
// JWTs behind /sts/token and /jwks. For each kind, the first key in the file
// signs and every later one only verifies, which is how a rotation is
// expressed: put the new key first, keep the previous one until every
// credential it signed has expired, then drop it.
type operatorSigningKeys struct {
	ed25519 []ed25519.PrivateKey
	es256   []*ecdsa.PrivateKey
	// digest of the file content these keys came from, so the watcher
	// reconciles only when the operator published something new.
	digest [sha256.Size]byte
}

// currentEd25519 is the public key the file names as the Biscuit signing
// key, nil when the file carries no Ed25519 key.
func (k *operatorSigningKeys) currentEd25519() ed25519.PublicKey {
	if k == nil || len(k.ed25519) == 0 {
		return nil
	}
	return k.ed25519[0].Public().(ed25519.PublicKey)
}

// loadOperatorSigningKeys reads and validates the PEM file at path.
func loadOperatorSigningKeys(path string) (*operatorSigningKeys, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read signing keys: %w", err)
	}
	keys, err := parseOperatorSigningKeys(path, data)
	if err != nil {
		return nil, err
	}
	keys.digest = sha256.Sum256(data)
	return keys, nil
}

// parseOperatorSigningKeys validates the PEM content of the file at path.
func parseOperatorSigningKeys(path string, data []byte) (*operatorSigningKeys, error) {
	var err error
	keys := &operatorSigningKeys{}
	for block, rest := pem.Decode(data); block != nil; block, rest = pem.Decode(rest) {
		var parsed any
		switch block.Type {
		case "PRIVATE KEY":
			parsed, err = x509.ParsePKCS8PrivateKey(block.Bytes)
		case "EC PRIVATE KEY":
			parsed, err = x509.ParseECPrivateKey(block.Bytes)
		default:
			return nil, fmt.Errorf("signing keys %s: unsupported PEM block %q (want PRIVATE KEY or EC PRIVATE KEY)", path, block.Type)
		}
		if err != nil {
			return nil, fmt.Errorf("signing keys %s: %w", path, err)
		}
		switch k := parsed.(type) {
		case ed25519.PrivateKey:
			keys.ed25519 = append(keys.ed25519, k)
		case *ecdsa.PrivateKey:
			if k.Curve != elliptic.P256() {
				return nil, fmt.Errorf("signing keys %s: EC key on %s, want P-256", path, k.Curve.Params().Name)
			}
			keys.es256 = append(keys.es256, k)
		default:
			return nil, fmt.Errorf("signing keys %s: unsupported key type %T (want Ed25519 or P-256)", path, parsed)
		}
	}
	// Owning the keys means owning both: a file with one kind would leave
	// the other generated per store and no longer rotated by anyone.
	if len(keys.ed25519) == 0 {
		return nil, fmt.Errorf("signing keys %s: no Ed25519 key (the Biscuit signing key); the file must carry at least one key of each kind", path)
	}
	if len(keys.es256) == 0 {
		return nil, fmt.Errorf("signing keys %s: no P-256 key (the ES256 key for /sts/token and /jwks); the file must carry at least one key of each kind", path)
	}
	return keys, nil
}

// keyFingerprint identifies a public key in logs without revealing anything
// a verifier does not already learn from /keys or /jwks.
func keyFingerprint(pub []byte) string {
	sum := sha256.Sum256(pub)
	return hex.EncodeToString(sum[:8])
}

// reconcileEd25519Keyring makes the store's Biscuit keyring reflect keys:
// keys[0] current, the rest valid for verification while the operator lists
// them, every other valid key (one the control plane generated before the
// operator took over, one the operator dropped) kept for gracePeriod so that
// credentials it signed keep verifying meanwhile. One store transaction; a
// set already in place is a no-op, a key moved back to the front becomes
// current again, and an expired row of a listed key is replaced.
func reconcileEd25519Keyring(ctx context.Context, store storage.Store, keys []ed25519.PrivateKey, gracePeriod time.Duration) error {
	if len(keys) == 0 {
		return nil
	}
	before, err := store.GetAllValidKeys(ctx)
	if err != nil && !errors.Is(err, storage.ErrNotFound) {
		return fmt.Errorf("read keyring: %w", err)
	}
	pair := func(priv ed25519.PrivateKey) storage.KeyPair {
		return storage.KeyPair{Private: priv, Public: priv.Public().(ed25519.PublicKey)}
	}
	verifying := make([]storage.KeyPair, 0, len(keys)-1)
	for _, k := range keys[1:] {
		verifying = append(verifying, pair(k))
	}
	current := pair(keys[0])
	if err := store.SetKeyring(ctx, current, verifying, gracePeriod); err != nil {
		return fmt.Errorf("set signing keys: %w", err)
	}
	logKeyringChange("ed25519", keyPairFingerprints(before), func() []string {
		after, err := store.GetAllValidKeys(ctx)
		if err != nil {
			return nil
		}
		return keyPairFingerprints(after)
	}(), keyFingerprint(current.Public))
	return nil
}

// reconcileES256Keyring is reconcileEd25519Keyring for the ES256 keys behind
// /sts/token and /jwks. The key id is the one generateES256KeyEntry derives,
// so a verifier that pinned a kid from /jwks keeps matching.
func reconcileES256Keyring(ctx context.Context, store storage.Store, keys []*ecdsa.PrivateKey, gracePeriod time.Duration) error {
	if len(keys) == 0 {
		return nil
	}
	before, err := store.GetAllValidOIDCKeys(ctx)
	if err != nil && !errors.Is(err, storage.ErrNotFound) {
		return fmt.Errorf("read ES256 keyring: %w", err)
	}
	encode := func(priv *ecdsa.PrivateKey) (storage.OIDCKeyPair, error) {
		pubBytes, err := priv.PublicKey.Bytes()
		if err != nil || len(pubBytes) != 65 {
			return storage.OIDCKeyPair{}, fmt.Errorf("encode P-256 public key: %w", err)
		}
		privBytes, err := x509.MarshalECPrivateKey(priv)
		if err != nil {
			return storage.OIDCKeyPair{}, fmt.Errorf("encode P-256 private key: %w", err)
		}
		return storage.OIDCKeyPair{Kid: es256Kid(pubBytes), PrivateKey: privBytes, PublicKey: pubBytes}, nil
	}
	current, err := encode(keys[0])
	if err != nil {
		return err
	}
	verifying := make([]storage.OIDCKeyPair, 0, len(keys)-1)
	for _, k := range keys[1:] {
		kp, err := encode(k)
		if err != nil {
			return err
		}
		verifying = append(verifying, kp)
	}
	if err := store.SetOIDCKeyring(ctx, current, verifying, gracePeriod); err != nil {
		return fmt.Errorf("set ES256 keys: %w", err)
	}
	logKeyringChange("es256", oidcKids(before), func() []string {
		after, err := store.GetAllValidOIDCKeys(ctx)
		if err != nil {
			return nil
		}
		return oidcKids(after)
	}(), current.Kid)
	return nil
}

func keyPairFingerprints(keys []storage.KeyPair) []string {
	out := make([]string, 0, len(keys))
	for _, k := range keys {
		out = append(out, keyFingerprint(k.Public))
	}
	sort.Strings(out)
	return out
}

func oidcKids(keys []storage.OIDCKeyPair) []string {
	out := make([]string, 0, len(keys))
	for _, k := range keys {
		out = append(out, k.Kid)
	}
	sort.Strings(out)
	return out
}

// logKeyringChange says what the operator's file did to the keyring, in
// public identifiers only, and nothing when it did nothing.
func logKeyringChange(algorithm string, before, after []string, signing string) {
	if slices.Equal(before, after) {
		return
	}
	logger.Infow("Operator signing keys installed", "algorithm", algorithm, "signing", signing, "valid_before", before, "valid_after", after)
}

// installOperatorSigningKeys reconciles both keyrings from keys and reports
// whether the Biscuit signing key changed, so the caller can announce it.
func (s *Server) installOperatorSigningKeys(ctx context.Context, keys *operatorSigningKeys) (bool, error) {
	_, before, err := s.store.GetCurrentKey(ctx)
	if err != nil && !errors.Is(err, storage.ErrNotFound) {
		return false, fmt.Errorf("read current signing key: %w", err)
	}
	if err := reconcileES256Keyring(ctx, s.store, keys.es256, s.config.KeyGracePeriod); err != nil {
		return false, err
	}
	if err := reconcileEd25519Keyring(ctx, s.store, keys.ed25519, s.config.KeyGracePeriod); err != nil {
		return false, err
	}
	operatorSigningKeysGauge.WithLabelValues("ed25519").Set(float64(len(keys.ed25519)))
	operatorSigningKeysGauge.WithLabelValues("es256").Set(float64(len(keys.es256)))
	now := keys.currentEd25519()
	return now != nil && !before.Equal(now), nil
}

// operatorSigningKeysReloadInterval is how often the control plane looks at
// the key file for a change. A package variable so a test can shorten it.
var operatorSigningKeysReloadInterval = 30 * time.Second

// watchOperatorSigningKeys re-reads the key file on a timer and installs
// what changed, so that a rotation is the operator publishing a new file
// and nothing else: no restart, no redeploy. Polling rather than inotify,
// because a Kubernetes Secret volume updates through an atomic symlink swap
// and a Cloud Run secret volume through the platform, neither of which a
// watch on the file itself sees reliably. A file that fails to parse keeps
// the keys in use and is counted and logged; a change of the signing key
// is announced to the mesh the way automatic rotation announces one.
func (s *Server) watchOperatorSigningKeys() {
	defer s.wg.Done()
	ticker := time.NewTicker(operatorSigningKeysReloadInterval)
	defer ticker.Stop()
	for {
		select {
		case <-s.ctx.Done():
			return
		case <-ticker.C:
		}
		s.reloadOperatorSigningKeys()
	}
}

// ReloadOperatorSigningKeys is one pass of the key file watcher, for callers
// that cannot wait for its timer (tests, an operator signal).
func (s *Server) ReloadOperatorSigningKeys() {
	if s.operatorKeys == nil {
		return
	}
	s.reloadOperatorSigningKeys()
}

// reloadOperatorSigningKeys is one pass of watchOperatorSigningKeys.
func (s *Server) reloadOperatorSigningKeys() {
	keys, err := loadOperatorSigningKeys(s.config.SigningKeyPath)
	if err != nil {
		operatorSigningKeysReloadErrors.Inc()
		logger.Warnw("Operator signing keys not reloaded; keeping the keys in use", "path", s.config.SigningKeyPath, "error", err)
		return
	}
	s.operatorKeysMu.Lock()
	unchanged := s.operatorKeys != nil && keys.digest == s.operatorKeys.digest
	s.operatorKeysMu.Unlock()
	if unchanged {
		return
	}
	changed, err := s.installOperatorSigningKeys(s.ctx, keys)
	if err != nil {
		operatorSigningKeysReloadErrors.Inc()
		logger.Errorw("Operator signing keys changed but could not be installed", "path", s.config.SigningKeyPath, "error", err)
		return
	}
	s.operatorKeysMu.Lock()
	s.operatorKeys = keys
	s.operatorKeysMu.Unlock()
	logger.Infow("Operator signing keys reloaded", "path", s.config.SigningKeyPath,
		"ed25519_keys", len(keys.ed25519), "es256_keys", len(keys.es256), "signing_key_changed", changed)
	if changed {
		if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", keys.currentEd25519()); err != nil {
			logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err)
		}
	}
}