Skip to content

Agent Mesh

node, router, controlplane: control-plane transport trust (audit M21)

Antonio Ojea4d92e21bc881
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yaml
index de5b4bef..0247d16b 100644
--- a/.github/k8s/sam-box-canary-template.yaml
+++ b/.github/k8s/sam-box-canary-template.yaml
@@ -78,6 +78,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
# Socket only: with no TCP listener there is no API token to leak, and
# the socket's permissions are the credential.
diff --git a/.github/k8s/sam-node-cop-template.yaml b/.github/k8s/sam-node-cop-template.yaml
index 0d3bcf2d..6a2d7474 100644
--- a/.github/k8s/sam-node-cop-template.yaml
+++ b/.github/k8s/sam-node-cop-template.yaml
@@ -67,6 +67,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
ports:
diff --git a/.github/k8s/sam-node-everything-template.yaml b/.github/k8s/sam-node-everything-template.yaml
index 9f30c3b8..b79ac198 100644
--- a/.github/k8s/sam-node-everything-template.yaml
+++ b/.github/k8s/sam-node-everything-template.yaml
@@ -55,6 +55,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
ports:
diff --git a/.github/k8s/sam-node-openclaw-template.yaml b/.github/k8s/sam-node-openclaw-template.yaml
index 7d69c471..64c65976 100644
--- a/.github/k8s/sam-node-openclaw-template.yaml
+++ b/.github/k8s/sam-node-openclaw-template.yaml
@@ -111,6 +111,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
ports:
diff --git a/.github/k8s/sam-node-openrouter-template.yaml b/.github/k8s/sam-node-openrouter-template.yaml
index 4458ff07..578166a0 100644
--- a/.github/k8s/sam-node-openrouter-template.yaml
+++ b/.github/k8s/sam-node-openrouter-template.yaml
@@ -64,6 +64,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
ports:
diff --git a/.github/k8s/sam-node-template.yaml b/.github/k8s/sam-node-template.yaml
index d41cb8bc..7b3bfdad 100644
--- a/.github/k8s/sam-node-template.yaml
+++ b/.github/k8s/sam-node-template.yaml
@@ -49,6 +49,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
ports:
- containerPort: 8080
diff --git a/.github/k8s/sam-node-vllm-template.yaml b/.github/k8s/sam-node-vllm-template.yaml
index c7e6c1bf..a956e104 100644
--- a/.github/k8s/sam-node-vllm-template.yaml
+++ b/.github/k8s/sam-node-vllm-template.yaml
@@ -104,6 +104,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
ports:
diff --git a/.github/k8s/sam-router-template.yaml b/.github/k8s/sam-router-template.yaml
index 5f56a4ce..681a8a8a 100644
--- a/.github/k8s/sam-router-template.yaml
+++ b/.github/k8s/sam-router-template.yaml
@@ -57,6 +57,7 @@ spec:
name: p2p-udp
args:
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--listen=/ip4/0.0.0.0/tcp/4501"
- "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
- "--external-addr=/dnsaddr/bootstrap.${ENV_NAME}.sam-mesh.dev"
diff --git a/api/sam.pb.go b/api/sam.pb.go
index d9aa4934..3ab2d1fe 100644
--- a/api/sam.pb.go
+++ b/api/sam.pb.go
@@ -1718,8 +1718,16 @@ func (x *PolicyConfigUpdateResponse) GetError() string {
}
type KeysResponse struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- PublicKeys [][]byte `protobuf:"bytes,1,rep,name=public_keys,json=publicKeys,proto3" json:"public_keys,omitempty"`
+ state protoimpl.MessageState `protogen:"open.v1"`
+ PublicKeys [][]byte `protobuf:"bytes,1,rep,name=public_keys,json=publicKeys,proto3" json:"public_keys,omitempty"`
+ // Unix milliseconds at which the set was signed; receivers reject responses
+ // outside a short freshness window so a captured set cannot be replayed.
+ Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+ // One ed25519 signature per entry of public_keys, by that key, over the
+ // deterministic encoding of this message with signatures cleared. A
+ // receiver trusting any key still valid on the control plane can verify
+ // the whole set (see api.VerifyKeysResponse).
+ Signatures [][]byte `protobuf:"bytes,3,rep,name=signatures,proto3" json:"signatures,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1761,6 +1769,20 @@ func (x *KeysResponse) GetPublicKeys() [][]byte {
return nil
}
+func (x *KeysResponse) GetTimestamp() int64 {
+ if x != nil {
+ return x.Timestamp
+ }
+ return 0
+}
+
+func (x *KeysResponse) GetSignatures() [][]byte {
+ if x != nil {
+ return x.Signatures
+ }
+ return nil
+}
+
type TokenRefreshRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Signature with the node key over the UTF-8 bytes of
@@ -3140,10 +3162,14 @@ const file_api_sam_proto_rawDesc = "" +
"\bbindings\x18\x02 \x03(\v2\x15.sam.v1.PolicyBindingR\bbindings\"L\n" +
"\x1aPolicyConfigUpdateResponse\x12\x18\n" +
"\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" +
- "\x05error\x18\x02 \x01(\tR\x05error\"/\n" +
+ "\x05error\x18\x02 \x01(\tR\x05error\"m\n" +
"\fKeysResponse\x12\x1f\n" +
"\vpublic_keys\x18\x01 \x03(\fR\n" +
- "publicKeys\"}\n" +
+ "publicKeys\x12\x1c\n" +
+ "\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12\x1e\n" +
+ "\n" +
+ "signatures\x18\x03 \x03(\fR\n" +
+ "signatures\"}\n" +
"\x13TokenRefreshRequest\x12/\n" +
"\x13challenge_signature\x18\x01 \x01(\fR\x12challengeSignature\x12\x1c\n" +
"\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12\x17\n" +
diff --git a/api/sam.proto b/api/sam.proto
index 8368982b..b89bb3ad 100644
--- a/api/sam.proto
+++ b/api/sam.proto
@@ -249,6 +249,14 @@ message PolicyConfigUpdateResponse {
message KeysResponse {
repeated bytes public_keys = 1;
+ // Unix milliseconds at which the set was signed; receivers reject responses
+ // outside a short freshness window so a captured set cannot be replayed.
+ int64 timestamp = 2;
+ // One ed25519 signature per entry of public_keys, by that key, over the
+ // deterministic encoding of this message with signatures cleared. A
+ // receiver trusting any key still valid on the control plane can verify
+ // the whole set (see api.VerifyKeysResponse).
+ repeated bytes signatures = 3;
}
message TokenRefreshRequest {
diff --git a/api/trust.go b/api/trust.go
new file mode 100644
index 00000000..ac72787f
--- /dev/null
+++ b/api/trust.go
@@ -0,0 +1,142 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "crypto/ed25519"
+ "errors"
+ "fmt"
+ "net"
+ "net/url"
+ "strings"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+)
+
+// ErrInsecureControlPlaneURL marks a plaintext control-plane URL to a host
+// that is not loopback. Whoever answers that URL becomes the trust root
+// (/keys, enrollment, router addresses), so without TLS that is whoever sits
+// on the path.
+var ErrInsecureControlPlaneURL = errors.New("plaintext http:// control plane URL to a non-loopback host")
+
+// ValidateControlPlaneTransport accepts https://, accepts http:// only to a
+// loopback host, and otherwise refuses unless allowInsecure is set by the
+// operator (the --insecure-control-plane flag) for a network they trust.
+func ValidateControlPlaneTransport(rawURL string, allowInsecure bool) error {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return fmt.Errorf("invalid control plane URL %q: %w", rawURL, err)
+ }
+ switch u.Scheme {
+ case "https":
+ return nil
+ case "http":
+ if allowInsecure || isLoopbackHost(u.Hostname()) {
+ return nil
+ }
+ return fmt.Errorf("%w: %q (use https://, or pass --insecure-control-plane to accept plaintext on a network you trust)", ErrInsecureControlPlaneURL, rawURL)
+ default:
+ return fmt.Errorf("control plane URL %q must use http:// or https://", rawURL)
+ }
+}
+
+func isLoopbackHost(host string) bool {
+ if strings.EqualFold(host, "localhost") {
+ return true
+ }
+ ip := net.ParseIP(host)
+ return ip != nil && ip.IsLoopback()
+}
+
+// KeysResponseFreshness bounds how far a signed /keys response's timestamp
+// may be from the receiver's clock: a captured response must not be able to
+// keep a retired key trusted after its grace period.
+const KeysResponseFreshness = 5 * time.Minute
+
+// KeysResponsePayload is the bytes each signature in a KeysResponse covers:
+// the key set and the timestamp, deterministically encoded, signatures
+// cleared.
+func KeysResponsePayload(resp *KeysResponse) ([]byte, error) {
+ unsigned := &KeysResponse{PublicKeys: resp.PublicKeys, Timestamp: resp.Timestamp}
+ return proto.MarshalOptions{Deterministic: true}.Marshal(unsigned)
+}
+
+// SignKeysResponse sets Timestamp and one signature per key pair, so a
+// receiver that trusts any key still valid on the control plane can verify
+// the set. Private keys must be in the same order as resp.PublicKeys.
+func SignKeysResponse(resp *KeysResponse, privateKeys []ed25519.PrivateKey, now time.Time) error {
+ if len(privateKeys) != len(resp.PublicKeys) {
+ return fmt.Errorf("keys response has %d public keys but %d signing keys", len(resp.PublicKeys), len(privateKeys))
+ }
+ resp.Timestamp = now.UnixMilli()
+ payload, err := KeysResponsePayload(resp)
+ if err != nil {
+ return err
+ }
+ resp.Signatures = make([][]byte, len(privateKeys))
+ for i, priv := range privateKeys {
+ resp.Signatures[i] = ed25519.Sign(priv, payload)
+ }
+ return nil
+}
+
+// VerifyKeysResponse returns the key set if it is fresh and at least one
+// listed key is already trusted and its signature verifies. A receiver with
+// nothing trusted yet cannot verify anything and gets an error: enrollment,
+// not /keys, is where the first key comes from.
+//
+// The guarantee is exactly "a key this receiver already trusts vouches for
+// this set". It defends against whoever answers the URL; it cannot defend
+// against the holder of a trusted private key, who is the trust root by
+// definition and could equally mint biscuits or sign events.
+func VerifyKeysResponse(resp *KeysResponse, trusted []ed25519.PublicKey, now time.Time) ([]ed25519.PublicKey, error) {
+ if len(trusted) == 0 {
+ return nil, errors.New("no trusted control plane key to verify /keys against")
+ }
+ if len(resp.Signatures) != len(resp.PublicKeys) {
+ return nil, fmt.Errorf("keys response carries %d signatures for %d keys", len(resp.Signatures), len(resp.PublicKeys))
+ }
+ issued := time.UnixMilli(resp.Timestamp)
+ if now.Sub(issued) > KeysResponseFreshness || issued.Sub(now) > KeysResponseFreshness {
+ return nil, fmt.Errorf("keys response timestamp %s is outside the freshness window", issued.UTC().Format(time.RFC3339))
+ }
+ payload, err := KeysResponsePayload(resp)
+ if err != nil {
+ return nil, err
+ }
+ var keys []ed25519.PublicKey
+ verified := false
+ for i, kb := range resp.PublicKeys {
+ if len(kb) != ed25519.PublicKeySize {
+ continue
+ }
+ pub := ed25519.PublicKey(kb)
+ keys = append(keys, pub)
+ if verified {
+ continue
+ }
+ for _, t := range trusted {
+ if t.Equal(pub) && ed25519.Verify(pub, payload, resp.Signatures[i]) {
+ verified = true
+ break
+ }
+ }
+ }
+ if !verified {
+ return nil, errors.New("keys response is not signed by any trusted control plane key")
+ }
+ return keys, nil
+}
diff --git a/api/trust_test.go b/api/trust_test.go
new file mode 100644
index 00000000..d98aa98c
--- /dev/null
+++ b/api/trust_test.go
@@ -0,0 +1,157 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "crypto/ed25519"
+ "crypto/rand"
+ "errors"
+ "testing"
+ "time"
+)
+
+func TestValidateControlPlaneTransport(t *testing.T) {
+ tests := []struct {
+ url string
+ insecure bool
+ wantErr bool
+ wantKind error
+ }{
+ {"https://hub.sam-mesh.dev", false, false, nil},
+ {"https://10.0.0.5:8443", false, false, nil},
+ {"http://127.0.0.1:8080", false, false, nil},
+ {"http://localhost:8080", false, false, nil},
+ {"http://[::1]:8080", false, false, nil},
+ {"http://[::ffff:127.0.0.1]:8080", false, false, nil},
+ // Fail closed on anything that is not literally loopback. "127.1" is
+ // loopback to the resolver but not to net.ParseIP; refusing it costs
+ // the operator a clearer spelling, accepting a lookalike would cost
+ // the trust root.
+ {"http://127.1:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://localhost.:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://0.0.0.0:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://sam-control-plane:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://10.0.0.5:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://127.0.0.1.evil.example:8080", false, true, ErrInsecureControlPlaneURL},
+ {"http://sam-control-plane:8080", true, false, nil},
+ {"ftp://hub.sam-mesh.dev", false, true, nil},
+ {"hub.sam-mesh.dev", false, true, nil},
+ }
+ for _, tt := range tests {
+ t.Run(tt.url, func(t *testing.T) {
+ err := ValidateControlPlaneTransport(tt.url, tt.insecure)
+ if (err != nil) != tt.wantErr {
+ t.Fatalf("ValidateControlPlaneTransport(%q, insecure=%v) = %v, wantErr %v", tt.url, tt.insecure, err, tt.wantErr)
+ }
+ if tt.wantKind != nil && !errors.Is(err, tt.wantKind) {
+ t.Errorf("error %v is not %v", err, tt.wantKind)
+ }
+ })
+ }
+}
+
+func genKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) {
+ t.Helper()
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return pub, priv
+}
+
+// A key set is only as good as the signature a receiver can already check:
+// trusting the retiring key must be enough to learn the new one, and a set
+// signed by nobody the receiver trusts must not replace anything.
+func TestKeysResponseSignatureChain(t *testing.T) {
+ oldPub, oldPriv := genKey(t)
+ newPub, newPriv := genKey(t)
+ now := time.Now()
+
+ signed := func() *KeysResponse {
+ resp := &KeysResponse{PublicKeys: [][]byte{oldPub, newPub}}
+ if err := SignKeysResponse(resp, []ed25519.PrivateKey{oldPriv, newPriv}, now); err != nil {
+ t.Fatal(err)
+ }
+ return resp
+ }
+
+ t.Run("receiver on the retiring key learns the new one", func(t *testing.T) {
+ keys, err := VerifyKeysResponse(signed(), []ed25519.PublicKey{oldPub}, now)
+ if err != nil {
+ t.Fatalf("verify: %v", err)
+ }
+ if len(keys) != 2 || !keys[1].Equal(newPub) {
+ t.Errorf("keys = %d entries, want old and new", len(keys))
+ }
+ })
+
+ t.Run("receiver on the new key verifies too", func(t *testing.T) {
+ if _, err := VerifyKeysResponse(signed(), []ed25519.PublicKey{newPub}, now); err != nil {
+ t.Fatalf("verify: %v", err)
+ }
+ })
+
+ t.Run("set signed by a stranger is refused", func(t *testing.T) {
+ strangerPub, strangerPriv := genKey(t)
+ resp := &KeysResponse{PublicKeys: [][]byte{strangerPub}}
+ if err := SignKeysResponse(resp, []ed25519.PrivateKey{strangerPriv}, now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := VerifyKeysResponse(resp, []ed25519.PublicKey{oldPub}, now); err == nil {
+ t.Fatal("a set signed by an untrusted key must not be accepted")
+ }
+ })
+
+ t.Run("listing a trusted key without its signature is refused", func(t *testing.T) {
+ // The attacker knows the victim trusts oldPub and lists it, but can
+ // only sign with their own key.
+ attackerPub, attackerPriv := genKey(t)
+ resp := &KeysResponse{PublicKeys: [][]byte{oldPub, attackerPub}}
+ if err := SignKeysResponse(resp, []ed25519.PrivateKey{attackerPriv, attackerPriv}, now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := VerifyKeysResponse(resp, []ed25519.PublicKey{oldPub}, now); err == nil {
+ t.Fatal("a listed-but-not-signing trusted key must not vouch for the set")
+ }
+ })
+
+ t.Run("tampered set is refused", func(t *testing.T) {
+ resp := signed()
+ resp.PublicKeys = append(resp.PublicKeys, make([]byte, ed25519.PublicKeySize))
+ resp.Signatures = append(resp.Signatures, resp.Signatures[0])
+ if _, err := VerifyKeysResponse(resp, []ed25519.PublicKey{oldPub}, now); err == nil {
+ t.Fatal("appending a key must break every signature")
+ }
+ })
+
+ t.Run("replayed set outside the freshness window is refused", func(t *testing.T) {
+ if _, err := VerifyKeysResponse(signed(), []ed25519.PublicKey{oldPub}, now.Add(KeysResponseFreshness+time.Minute)); err == nil {
+ t.Fatal("a stale set must not be accepted")
+ }
+ })
+
+ t.Run("nothing trusted verifies nothing", func(t *testing.T) {
+ if _, err := VerifyKeysResponse(signed(), nil, now); err == nil {
+ t.Fatal("with no trusted key there is nothing to verify against")
+ }
+ })
+
+ t.Run("unsigned legacy response is refused", func(t *testing.T) {
+ resp := &KeysResponse{PublicKeys: [][]byte{oldPub}}
+ if _, err := VerifyKeysResponse(resp, []ed25519.PublicKey{oldPub}, now); err == nil {
+ t.Fatal("a response without signatures must not be accepted")
+ }
+ })
+}
diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml
index 21f4353b..ff7c9e88 100644
--- a/charts/sam-mesh/templates/router-statefulset.yaml
+++ b/charts/sam-mesh/templates/router-statefulset.yaml
@@ -107,6 +107,9 @@ spec:
{{- end }}
args:
- "--control-plane=http://{{ include "sam-mesh.fullname" . }}-control-plane:{{ .Values.controlPlane.service.port }}"
+ # In-cluster plaintext to the chart's own control-plane Service; the
+ # cluster network is the trust boundary here.
+ - "--insecure-control-plane"
- "--listen=/ip4/0.0.0.0/tcp/4501"
- "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
{{- if .Values.router.externalAddrs }}
diff --git a/charts/sam-mesh/tests/router-statefulset_test.yaml b/charts/sam-mesh/tests/router-statefulset_test.yaml
index 781aca87..72636ddc 100644
--- a/charts/sam-mesh/tests/router-statefulset_test.yaml
+++ b/charts/sam-mesh/tests/router-statefulset_test.yaml
@@ -28,6 +28,9 @@ tests:
- contains:
path: spec.template.spec.containers[0].args
content: --external-addr=/dns4/sam-mesh-router/tcp/4501
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --insecure-control-plane
- notExists:
path: spec.template.spec.containers[0].ports[0].hostPort
diff --git a/charts/sam-node/templates/deployment.yaml b/charts/sam-node/templates/deployment.yaml
index bac61bcd..76eaf769 100644
--- a/charts/sam-node/templates/deployment.yaml
+++ b/charts/sam-node/templates/deployment.yaml
@@ -30,6 +30,11 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane={{ required "controlPlaneUrl is required" .Values.controlPlaneUrl }}"
+ {{- if hasPrefix "http://" .Values.controlPlaneUrl }}
+ # Plaintext to a non-loopback control plane is refused unless opted
+ # in; an in-cluster Service URL is the expected case.
+ - "--insecure-control-plane"
+ {{- end }}
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr={{ .Values.bindAddr }}"
{{- range .Values.extraArgs }}
diff --git a/charts/sam-node/tests/deployment_test.yaml b/charts/sam-node/tests/deployment_test.yaml
index 9ea039b1..7a0f6c6f 100644
--- a/charts/sam-node/tests/deployment_test.yaml
+++ b/charts/sam-node/tests/deployment_test.yaml
@@ -23,6 +23,9 @@ tests:
- contains:
path: spec.template.spec.containers[0].args
content: "--control-plane=http://sam-mesh-control-plane:8080"
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--insecure-control-plane"
- contains:
path: spec.template.spec.containers[0].args
content: "--bind-addr=127.0.0.1:8080"
@@ -30,6 +33,18 @@ tests:
path: spec.template.spec.volumes[1].projected.sources[0].serviceAccountToken.audience
value: sam-mesh-audience
+ - it: does not opt into plaintext for an https control plane
+ template: templates/deployment.yaml
+ set:
+ controlPlaneUrl: https://hub.example.com
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--control-plane=https://hub.example.com"
+ - notContains:
+ path: spec.template.spec.containers[0].args
+ content: "--insecure-control-plane"
+
- it: renders the service container when service.image is set
template: templates/deployment.yaml
set:
diff --git a/charts/sam-node/values.yaml b/charts/sam-node/values.yaml
index 63bff0e6..8579c8ab 100644
--- a/charts/sam-node/values.yaml
+++ b/charts/sam-node/values.yaml
@@ -6,7 +6,9 @@ image:
replicaCount: 1
# Required: control plane URL the node enrolls with,
-# e.g. http://sam-mesh-control-plane:8080
+# e.g. http://sam-mesh-control-plane:8080. A plaintext http:// URL makes the
+# chart pass --insecure-control-plane: whoever answers it becomes the node's
+# trust root, so use https:// for anything beyond the cluster network.
controlPlaneUrl: ""
# Audience of the projected ServiceAccount token; must be one of the control
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 13848a5c..318e3cdf 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -74,6 +74,7 @@ var (
dataDirFlag string
headlessFlag bool
authModeFlag string
+ insecureControlPlaneFlag bool
daemonizeFlag bool
resetAllFlag bool
assumeYesFlag bool
@@ -263,6 +264,15 @@ func main() {
rootCmd := &cobra.Command{
Use: "sam-node",
Short: "Sovereign Agent Mesh Node",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+ node.SetAllowInsecureControlPlane(insecureControlPlaneFlag)
+ // The stored URL is checked at request time by the same policy;
+ // this only turns a flag-supplied URL into an immediate error.
+ if controlPlaneAddr != "" {
+ return api.ValidateControlPlaneTransport(normalizeControlPlaneURL(controlPlaneAddr), insecureControlPlaneFlag)
+ }
+ return nil
+ },
}
// RUN COMMAND: Start the Mesh
@@ -858,6 +868,7 @@ func main() {
runCmd.Flags().DurationVar(&policySyncIntervalFlag, "policy-sync-interval", 1*time.Hour, "Interval for syncing mesh policy from the control plane")
runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")
rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")
+ rootCmd.PersistentFlags().BoolVar(&insecureControlPlaneFlag, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this node's trust root; only for networks you already trust)")
rootCmd.PersistentFlags().StringVar(&configFile, "config", node.DefaultConfigFile, "Path to sam-node.yaml configuration file")
rootCmd.PersistentFlags().StringVar(&oidcIssuerFlag, "oidc-issuer", "", "OIDC Issuer URL")
rootCmd.PersistentFlags().StringVar(&deviceAuthURLFlag, "device-auth-url", "", "OIDC Device Authorization URL")
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.go
index acb4d4df..ff8ca0a3 100644
--- a/cmd/sam-router/main.go
+++ b/cmd/sam-router/main.go
@@ -27,23 +27,24 @@ import (
)
var (
- controlPlaneURL string
- listenAddrs []string
- externalAddrs []string
- keysSyncInterval time.Duration
- leaseRenewInterval time.Duration
- oidcToken string
- bootstrapToken string
- bootstrapTokenPath string
- jwtPath string
- keysPath string
- allowLoopback bool
- connsPerSourceIP int
- logLevel string
- dhtProviderAddrTTL time.Duration
- dhtMaxRecordAge time.Duration
- lowWaterMark int
- highWaterMark int
+ controlPlaneURL string
+ insecureControlPlane bool
+ listenAddrs []string
+ externalAddrs []string
+ keysSyncInterval time.Duration
+ leaseRenewInterval time.Duration
+ oidcToken string
+ bootstrapToken string
+ bootstrapTokenPath string
+ jwtPath string
+ keysPath string
+ allowLoopback bool
+ connsPerSourceIP int
+ logLevel string
+ dhtProviderAddrTTL time.Duration
+ dhtMaxRecordAge time.Duration
+ lowWaterMark int
+ highWaterMark int
)
var logger = golog.Logger("sam-router-cli")
@@ -66,22 +67,23 @@ func main() {
}
opts := router.Options{
- ControlPlaneURL: controlPlaneURL,
- ListenAddrs: listenAddrs,
- ExternalAddrs: externalAddrs,
- KeysSyncInterval: keysSyncInterval,
- LeaseRenewInterval: leaseRenewInterval,
- OIDCToken: oidcToken,
- BootstrapToken: bootstrapToken,
- BootstrapTokenPath: bootstrapTokenPath,
- JWTPath: jwtPath,
- KeysDBPath: keysPath,
- AllowLoopback: allowLoopback,
- ConnsPerSourceIP: connsPerSourceIP,
- DHTProviderAddrTTL: dhtProviderAddrTTL,
- DHTMaxRecordAge: dhtMaxRecordAge,
- LowWaterMark: lowWaterMark,
- HighWaterMark: highWaterMark,
+ ControlPlaneURL: controlPlaneURL,
+ AllowInsecureControlPlane: insecureControlPlane,
+ ListenAddrs: listenAddrs,
+ ExternalAddrs: externalAddrs,
+ KeysSyncInterval: keysSyncInterval,
+ LeaseRenewInterval: leaseRenewInterval,
+ OIDCToken: oidcToken,
+ BootstrapToken: bootstrapToken,
+ BootstrapTokenPath: bootstrapTokenPath,
+ JWTPath: jwtPath,
+ KeysDBPath: keysPath,
+ AllowLoopback: allowLoopback,
+ ConnsPerSourceIP: connsPerSourceIP,
+ DHTProviderAddrTTL: dhtProviderAddrTTL,
+ DHTMaxRecordAge: dhtMaxRecordAge,
+ LowWaterMark: lowWaterMark,
+ HighWaterMark: highWaterMark,
}
r, err := router.NewRouter(cmd.Context(), opts)
@@ -103,6 +105,7 @@ func main() {
}
rootCmd.Flags().StringVar(&controlPlaneURL, "control-plane", "http://127.0.0.1:8080", "Control Plane web service URL")
+ rootCmd.Flags().BoolVar(&insecureControlPlane, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this router's trust root; only for networks you already trust)")
rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{"/ip4/0.0.0.0/tcp/5001", "/ip6/::/tcp/5001"}, "libp2p Listen Addresses")
rootCmd.Flags().StringSliceVar(&externalAddrs, "external-addr", []string{}, "External addresses to announce to control plane")
rootCmd.Flags().DurationVar(&keysSyncInterval, "keys-sync-interval", 5*time.Minute, "Key synchronization polling interval")
diff --git a/internal/controlplane/mesh.go b/internal/controlplane/mesh.go
index 5d8d0cff..e43e7ca8 100644
--- a/internal/controlplane/mesh.go
+++ b/internal/controlplane/mesh.go
@@ -15,6 +15,7 @@
package controlplane
import (
+ "bytes"
"context"
"crypto/ed25519"
"fmt"
@@ -130,7 +131,7 @@ func (p *P2PMeshAdapter) PublishEvent(ctx context.Context, eventType api.MeshEve
var privKey ed25519.PrivateKey
if p.store != nil {
- key, _, err := p.store.GetCurrentKey(ctx)
+ key, err := p.signingKeyFor(ctx, eventType, payload)
if err != nil {
return fmt.Errorf("failed to retrieve signing key for event publishing: %w", err)
}
@@ -175,6 +176,37 @@ func (p *P2PMeshAdapter) PublishEvent(ctx context.Context, eventType api.MeshEve
return nil
}
+// signingKeyFor picks the key receivers can verify with. A KEY_ROTATION
+// announces newPub, which nobody trusts yet, so it is signed by the key just
+// retired into its grace period (the one with the latest expiration that is
+// not newPub); every other event is signed by the current key.
+func (p *P2PMeshAdapter) signingKeyFor(ctx context.Context, eventType api.MeshEvent_Type, newPub []byte) (ed25519.PrivateKey, error) {
+ if eventType != api.MeshEvent_KEY_ROTATION {
+ key, _, err := p.store.GetCurrentKey(ctx)
+ return key, err
+ }
+ pairs, err := p.store.GetAllValidKeys(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var retiring *storage.KeyPair
+ for i := range pairs {
+ kp := &pairs[i]
+ if kp.Expiration.IsZero() || bytes.Equal(kp.Public, newPub) {
+ continue
+ }
+ if retiring == nil || kp.Expiration.After(retiring.Expiration) {
+ retiring = kp
+ }
+ }
+ if retiring == nil {
+ // First key ever: there is no previous key and nobody to convince.
+ key, _, err := p.store.GetCurrentKey(ctx)
+ return key, err
+ }
+ return retiring.Private, nil
+}
+
func (p *P2PMeshAdapter) DiscoverServices(ctx context.Context, serviceType string) ([]*ServiceAnnouncement, error) {
return nil, nil
}
diff --git a/internal/controlplane/mesh_test.go b/internal/controlplane/mesh_test.go
index 7a0a9165..d6439262 100644
--- a/internal/controlplane/mesh_test.go
+++ b/internal/controlplane/mesh_test.go
@@ -15,6 +15,7 @@
package controlplane
import (
+ "bytes"
"context"
"crypto/ed25519"
"crypto/rand"
@@ -159,3 +160,49 @@ func TestP2PMeshAdapter_PublishAndSubscribe(t *testing.T) {
t.Fatalf("ed25519 signature verification failed for published MeshEvent")
}
}
+
+// A KEY_ROTATION announces a key nobody trusts yet, so it has to be signed by
+// the key being retired: that is the only signature a node holding the old
+// key can check. Signing with the new key (the store's current key after the
+// rotation) made every node drop the announcement as a spoofing attempt.
+func TestKeyRotationEventIsSignedByTheRetiringKey(t *testing.T) {
+ ctx := context.Background()
+ store, err := storage.NewSQLStore("sqlite", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ oldPub, oldPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveInitialKey(ctx, oldPriv, oldPub); err != nil {
+ t.Fatal(err)
+ }
+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.RotateKeys(ctx, newPriv, newPub, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+
+ adapter := &P2PMeshAdapter{store: store}
+ signer, err := adapter.signingKeyFor(ctx, api.MeshEvent_KEY_ROTATION, newPub)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(signer, oldPriv) {
+ t.Error("KEY_ROTATION must be signed by the retiring key, which is the one receivers still trust")
+ }
+
+ // Every other event is signed by the current key.
+ signer, err = adapter.signingKeyFor(ctx, api.MeshEvent_BANNED, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(signer, newPriv) {
+ t.Error("a BANNED event after rotation must be signed by the current key")
+ }
+}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 75ffa531..b136959d 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -957,20 +957,25 @@ func (s *Server) HandleKeys(w http.ResponseWriter, r *http.Request) {
return
}
- validKeys, err := s.store.GetAllValidPublicKeys(r.Context())
+ validKeys, err := s.store.GetAllValidKeys(r.Context())
if err != nil {
logger.Errorf("Failed to retrieve valid keys: %v", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
- var pubKeys [][]byte
+ resp := &api.KeysResponse{}
+ privs := make([]ed25519.PrivateKey, 0, len(validKeys))
for _, k := range validKeys {
- pubKeys = append(pubKeys, k)
+ resp.PublicKeys = append(resp.PublicKeys, k.Public)
+ privs = append(privs, k.Private)
}
-
- resp := &api.KeysResponse{
- PublicKeys: pubKeys,
+ // Signed by every valid key: a receiver still on a key in its grace
+ // period verifies with that one and learns the new one from the set.
+ if err := api.SignKeysResponse(resp, privs, time.Now()); err != nil {
+ logger.Errorf("Failed to sign keys response: %v", err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
}
respData, err := proto.Marshal(resp)
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index e0f85ae1..a36407a0 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -267,6 +267,14 @@ func TestControlPlaneBasic(t *testing.T) {
if len(keys.PublicKeys) != 1 {
t.Errorf("expected 1 valid public key, got %d", len(keys.PublicKeys))
}
+ // The set is self-certifying for anyone already holding a listed key.
+ verified, err := api.VerifyKeysResponse(&keys, []ed25519.PublicKey{ed25519.PublicKey(keys.PublicKeys[0])}, time.Now())
+ if err != nil {
+ t.Fatalf("/keys response does not verify under its own listed key: %v", err)
+ }
+ if len(verified) != 1 {
+ t.Errorf("verified key set has %d keys, want 1", len(verified))
+ }
}
func TestNodeAndRouterRegistrationFlow(t *testing.T) {
@@ -2059,8 +2067,16 @@ func TestNodeProactiveTokenRefresh(t *testing.T) {
t.Fatalf("failed to save initial expiration: %v", err)
}
- n := &node.SamNode{
- Store: nStore,
+ // The node trusts the key enrollment handed it, as a real node would; the
+ // refreshed token is verified against that key before it is adopted.
+ n, err := node.NewSamNode(node.Options{
+ PrivKey: privNode,
+ Store: nStore,
+ ControlPlanePubKey: enrollNodeResp.ControlPlanePublicKey,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ })
+ if err != nil {
+ t.Fatalf("NewSamNode: %v", err)
}
// Trigger proactive refresh
diff --git a/internal/node/controlplane.go b/internal/node/controlplane.go
index 0c8b78a1..f5bfb310 100644
--- a/internal/node/controlplane.go
+++ b/internal/node/controlplane.go
@@ -48,9 +48,7 @@ func FetchControlPlaneInfo(ctx context.Context, controlPlaneURL string) (*api.Co
return nil, fmt.Errorf("failed to create HTTP request: %w", err)
}
- client := &http.Client{
- Timeout: 10 * time.Second,
- }
+ client := controlPlaneHTTPClient(10 * time.Second)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
@@ -78,8 +76,9 @@ func FetchControlPlaneInfo(ctx context.Context, controlPlaneURL string) (*api.Co
// plane public keys from the /keys endpoint — the same catch-up path routers
// use. Enrollment only hands out the newest key, so this is how a node
// learns keys still in their rotation grace period, or rotations it missed
-// while offline.
-func FetchControlPlaneKeys(ctx context.Context, controlPlaneURL string) ([]ed25519.PublicKey, error) {
+// while offline. The set is accepted only if signed by a key in trusted:
+// whoever answers /keys must already be the control plane, not become it.
+func FetchControlPlaneKeys(ctx context.Context, controlPlaneURL string, trusted []ed25519.PublicKey) ([]ed25519.PublicKey, error) {
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
controlPlaneURL = "https://" + controlPlaneURL
}
@@ -90,7 +89,7 @@ func FetchControlPlaneKeys(ctx context.Context, controlPlaneURL string) ([]ed255
return nil, fmt.Errorf("failed to create HTTP request: %w", err)
}
- client := &http.Client{Timeout: 10 * time.Second}
+ client := controlPlaneHTTPClient(10 * time.Second)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
@@ -110,11 +109,9 @@ func FetchControlPlaneKeys(ctx context.Context, controlPlaneURL string) ([]ed255
return nil, fmt.Errorf("failed to decode /keys response: %w", err)
}
- var keys []ed25519.PublicKey
- for _, kb := range keysResp.PublicKeys {
- if len(kb) == ed25519.PublicKeySize {
- keys = append(keys, ed25519.PublicKey(kb))
- }
+ keys, err := api.VerifyKeysResponse(&keysResp, trusted, time.Now())
+ if err != nil {
+ return nil, fmt.Errorf("/keys response rejected: %w", err)
}
return keys, nil
}
@@ -198,15 +195,18 @@ func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiadd
}
}
- // Catch up on the valid key set: an empty result would wipe the trust
- // set, so it is ignored like any fetch failure.
- if keys, keysErr := FetchControlPlaneKeys(ctx, controlPlaneURL); keysErr != nil {
+ // Catch up on the valid key set. Verified against what is already
+ // trusted, so with nothing stored yet there is nothing to do: the
+ // first key comes from enrollment. An empty result would wipe the
+ // trust set, so it is ignored like any fetch failure.
+ existing, loadErr := s.LoadTrustedKeys()
+ if loadErr != nil {
+ logger.Warnf("Failed to load stored trusted keys, skipping key sync: %v", loadErr)
+ } else if len(existing) == 0 {
+ logger.Debugf("No trusted control plane keys stored yet; skipping /keys sync until enrolled")
+ } else if keys, keysErr := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing)); keysErr != nil {
logger.Warnf("Failed to fetch control plane keys via HTTP (using cached): %v", keysErr)
} else if len(keys) > 0 {
- existing, loadErr := s.LoadTrustedKeys()
- if loadErr != nil {
- logger.Warnf("Failed to load stored trusted keys, replacing: %v", loadErr)
- }
if saveErr := s.SaveTrustedKeys(mergeTrustedKeys(existing, keys, time.Now())); saveErr != nil {
logger.Errorf("Failed to save trusted keys to store: %v", saveErr)
}
@@ -216,6 +216,14 @@ func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiadd
return pubKey, routerAddrs, bannedPeerIDs, nil
}
+func publicKeysOf(keys []TrustedKey) []ed25519.PublicKey {
+ out := make([]ed25519.PublicKey, 0, len(keys))
+ for _, tk := range keys {
+ out = append(out, tk.Key)
+ }
+ return out
+}
+
// FetchMeshPolicy retrieves the latest mesh policy from the control plane's /policies endpoint using a biscuit token.
func FetchMeshPolicy(ctx context.Context, controlPlaneURL string, biscuitToken []byte) (*api.PolicyConfigGetResponse, error) {
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
@@ -231,9 +239,7 @@ func FetchMeshPolicy(ctx context.Context, controlPlaneURL string, biscuitToken [
req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuitToken))
- client := &http.Client{
- Timeout: 10 * time.Second,
- }
+ client := controlPlaneHTTPClient(10 * time.Second)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
@@ -280,7 +286,7 @@ func ReportNodeCatalog(ctx context.Context, controlPlaneURL string, biscuitToken
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuitToken))
- client := &http.Client{Timeout: 10 * time.Second}
+ client := controlPlaneHTTPClient(10 * time.Second)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HTTP request failed: %w", err)
diff --git a/internal/node/controlplane_client.go b/internal/node/controlplane_client.go
new file mode 100644
index 00000000..8411e89a
--- /dev/null
+++ b/internal/node/controlplane_client.go
@@ -0,0 +1,57 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "net/http"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/sam/api"
+)
+
+// allowInsecureControlPlane is process-wide because the control-plane URL
+// reaches the node from several places (flag, store, mobile FFI) and every
+// request to it must be held to the same transport policy.
+var allowInsecureControlPlane atomic.Bool
+
+// SetAllowInsecureControlPlane records the operator's --insecure-control-plane
+// choice: plaintext http:// to a non-loopback control plane is then accepted.
+func SetAllowInsecureControlPlane(allow bool) {
+ allowInsecureControlPlane.Store(allow)
+}
+
+// controlPlaneTransport applies api.ValidateControlPlaneTransport to every
+// request, including redirects, so a plaintext hop is refused wherever the
+// URL came from.
+type controlPlaneTransport struct {
+ base http.RoundTripper
+}
+
+func (t controlPlaneTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ if err := api.ValidateControlPlaneTransport(req.URL.String(), allowInsecureControlPlane.Load()); err != nil {
+ return nil, err
+ }
+ return t.base.RoundTrip(req)
+}
+
+// controlPlaneHTTPClient is the client for every request the node makes to
+// its control plane.
+func controlPlaneHTTPClient(timeout time.Duration) *http.Client {
+ return &http.Client{
+ Timeout: timeout,
+ Transport: controlPlaneTransport{base: http.DefaultTransport},
+ }
+}
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.go
index 38469539..d40c13ed 100644
--- a/internal/node/controlplane_test.go
+++ b/internal/node/controlplane_test.go
@@ -18,6 +18,7 @@ import (
"context"
"crypto/ed25519"
"encoding/base64"
+ "errors"
"io"
"net/http"
"net/http/httptest"
@@ -139,9 +140,13 @@ func TestSyncMeshConfig(t *testing.T) {
t.Fatalf("Failed to marshal info: %v", err)
}
- cpPub, _, _ := ed25519.GenerateKey(nil)
- gracePub, _, _ := ed25519.GenerateKey(nil)
- keysBody, err := proto.Marshal(&api.KeysResponse{PublicKeys: [][]byte{cpPub, gracePub}})
+ cpPub, cpPriv, _ := ed25519.GenerateKey(nil)
+ gracePub, gracePriv, _ := ed25519.GenerateKey(nil)
+ keysResp := &api.KeysResponse{PublicKeys: [][]byte{cpPub, gracePub}}
+ if err := api.SignKeysResponse(keysResp, []ed25519.PrivateKey{cpPriv, gracePriv}, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ keysBody, err := proto.Marshal(keysResp)
if err != nil {
t.Fatalf("Failed to marshal keys: %v", err)
}
@@ -175,7 +180,9 @@ func TestSyncMeshConfig(t *testing.T) {
t.Errorf("Expected empty result for empty store, got pubKey=%v, addrs=%v", pubKey, addrs)
}
- // Save initial config with explicit control plane URL
+ // Save initial config with explicit control plane URL. The node trusts
+ // only cpPub, as after enrollment; the grace key must be learned through
+ // cpPub's signature on the set.
testPubKey := []byte("test-pub-key")
if err := store.SaveMeshConfig(testPubKey, []string{"/ip4/1.2.3.4/tcp/1234"}); err != nil {
t.Fatalf("Failed to save mesh config: %v", err)
@@ -183,6 +190,9 @@ func TestSyncMeshConfig(t *testing.T) {
if err := store.SaveControlPlaneURL(server.URL); err != nil {
t.Fatalf("Failed to save control plane URL: %v", err)
}
+ if err := store.SaveTrustedKeys([]TrustedKey{{Key: cpPub, ReceivedAt: time.Now()}}); err != nil {
+ t.Fatalf("SaveTrustedKeys: %v", err)
+ }
// Call SyncMeshConfig, it should fetch new addrs from server
pubKey, addrs, _, err = SyncMeshConfig(context.Background(), store)
@@ -223,6 +233,103 @@ func TestSyncMeshConfig(t *testing.T) {
}
}
+// Whoever answers /keys must already be the control plane: a set that is not
+// signed by a key the node trusts leaves the trust set untouched.
+func TestSyncMeshConfigRefusesUntrustedKeySet(t *testing.T) {
+ cpPub, _, _ := ed25519.GenerateKey(nil)
+ attackerPub, attackerPriv, _ := ed25519.GenerateKey(nil)
+
+ infoBody, err := proto.Marshal(&api.ControlPlaneInfoResponse{RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4001"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for name, keysResp := range map[string]*api.KeysResponse{
+ "unsigned set": {PublicKeys: [][]byte{cpPub, attackerPub}},
+ "set signed only by the attacker": func() *api.KeysResponse {
+ r := &api.KeysResponse{PublicKeys: [][]byte{cpPub, attackerPub}}
+ if err := api.SignKeysResponse(r, []ed25519.PrivateKey{attackerPriv, attackerPriv}, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ return r
+ }(),
+ } {
+ t.Run(name, func(t *testing.T) {
+ keysBody, err := proto.Marshal(keysResp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/keys" {
+ _, _ = w.Write(keysBody)
+ return
+ }
+ _, _ = w.Write(infoBody)
+ }))
+ defer server.Close()
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.Close() //nolint:errcheck
+ if err := store.SaveMeshConfig(cpPub, nil); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveControlPlaneURL(server.URL); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveTrustedKeys([]TrustedKey{{Key: cpPub, ReceivedAt: time.Now()}}); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, _, _, err := SyncMeshConfig(context.Background(), store); err != nil {
+ t.Fatalf("SyncMeshConfig: %v", err)
+ }
+
+ trusted, err := store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(trusted) != 1 || !trusted[0].Key.Equal(cpPub) {
+ t.Fatalf("trust set was replaced by an unverified /keys answer: %d keys", len(trusted))
+ }
+ })
+ }
+}
+
+// The control plane is the trust root, so a plaintext hop to it is refused
+// unless the operator opted in; loopback is the standalone case and is fine.
+func TestControlPlaneClientRefusesPlaintextToNonLoopback(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, _ := proto.Marshal(&api.ControlPlaneInfoResponse{})
+ _, _ = w.Write(body)
+ }))
+ defer server.Close()
+ // httptest binds 127.0.0.1; spell it as a non-loopback name that the
+ // transport must refuse before any connection is attempted.
+ nonLoopbackURL := strings.Replace(server.URL, "127.0.0.1", "sam-control-plane.invalid", 1)
+
+ t.Cleanup(func() { SetAllowInsecureControlPlane(false) })
+
+ if _, err := FetchControlPlaneInfo(context.Background(), server.URL); err != nil {
+ t.Fatalf("loopback plaintext must be accepted: %v", err)
+ }
+
+ _, err := FetchControlPlaneInfo(context.Background(), nonLoopbackURL)
+ if !errors.Is(err, api.ErrInsecureControlPlaneURL) {
+ t.Fatalf("plaintext to a non-loopback host: err = %v, want %v", err, api.ErrInsecureControlPlaneURL)
+ }
+
+ // With the opt-in the request is attempted; the name does not resolve,
+ // which is a dial error, not the policy error.
+ SetAllowInsecureControlPlane(true)
+ _, err = FetchControlPlaneInfo(context.Background(), nonLoopbackURL)
+ if err == nil || errors.Is(err, api.ErrInsecureControlPlaneURL) {
+ t.Fatalf("with --insecure-control-plane the policy must not be what fails: %v", err)
+ }
+}
+
func TestReportNodeCatalog(t *testing.T) {
services := []*api.ServiceInfo{
{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "stvv-compliance-docs", Description: "doc lookup"},
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 5d212a92..bbed85eb 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -112,7 +112,7 @@ func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, p
}
httpReq.Header.Set("Content-Type", "application/x-protobuf")
- client := &http.Client{Timeout: 30 * time.Second}
+ client := controlPlaneHTTPClient(30 * time.Second)
resp, err := client.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %v", err)
@@ -255,7 +255,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
enrollURL := controlPlaneURL + "/enroll"
logger.Infof("Enrolling via Bootstrap token at %s", enrollURL)
- client := &http.Client{Timeout: 30 * time.Second}
+ client := controlPlaneHTTPClient(30 * time.Second)
httpReq, err := http.NewRequestWithContext(ctx, "POST", enrollURL, bytes.NewReader(data))
if err != nil {
return fmt.Errorf("failed to create HTTP request: %w", err)
diff --git a/internal/node/node.go b/internal/node/node.go
index 6ac1cb65..6c6bad98 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -1127,7 +1127,7 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
b64Biscuit := base64.StdEncoding.EncodeToString(currentBiscuit)
httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
- client := &http.Client{Timeout: 30 * time.Second}
+ client := controlPlaneHTTPClient(30 * time.Second)
resp, err := client.Do(httpReq)
if err != nil {
return fmt.Errorf("http request failed: %w", err)
@@ -1135,11 +1135,14 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusForbidden {
- logger.Errorf("Refresh rejected: Node is banned (403 Forbidden). Initiating hard-kill.")
- if n.Host != nil {
- _ = n.Host.Close()
+ // Not fatal on its own: a 403 is a claim by whoever answered, and
+ // only a verified MeshEvent_BANNED is the control plane's word. The
+ // node keeps serving on its current biscuit until that expires.
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
+ return &RefreshError{
+ StatusCode: resp.StatusCode,
+ Message: fmt.Sprintf("refresh refused (403 Forbidden): %s", string(body)),
}
- os.Exit(1)
}
if resp.StatusCode != http.StatusOK {
@@ -1164,6 +1167,13 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
return fmt.Errorf("refresh error: %s", refreshResp.ErrorMessage)
}
+ // Same checks as enrollment: the token must be signed by a key this node
+ // already trusts and carry the role it runs as, or a response from the
+ // wrong party would replace a good identity with a useless one.
+ if err := n.verifyOwnBiscuit(refreshResp.BiscuitToken); err != nil {
+ return fmt.Errorf("refreshed biscuit rejected: %w", err)
+ }
+
// Save new biscuit and its expiration
if err := n.Store.SaveIdentity(refreshResp.BiscuitToken); err != nil {
return fmt.Errorf("failed to save refreshed identity: %w", err)
@@ -1176,6 +1186,44 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
return nil
}
+// verifyOwnBiscuit checks a token the control plane handed this node: signed
+// by a trusted key, bound to this peer, and carrying the configured role.
+func (n *SamNode) verifyOwnBiscuit(token []byte) error {
+ if len(token) == 0 {
+ return errors.New("empty biscuit token")
+ }
+ n.keysMu.RLock()
+ trusted := publicKeysOf(n.trustedKeys)
+ n.keysMu.RUnlock()
+ if len(trusted) == 0 {
+ return errors.New("no trusted control plane keys loaded")
+ }
+ var peerID peer.ID
+ if n.Host != nil {
+ peerID = n.Host.ID()
+ } else {
+ privBytes, err := n.Store.LoadKey()
+ if err != nil {
+ return fmt.Errorf("load node key: %w", err)
+ }
+ priv, err := crypto.UnmarshalPrivateKey(privBytes)
+ if err != nil {
+ return fmt.Errorf("corrupted node key: %w", err)
+ }
+ if peerID, err = peer.IDFromPrivateKey(priv); err != nil {
+ return err
+ }
+ }
+ _, key, err := identity.VerifyBiscuitAndGetKey(token, peerID, trusted, n.BiscuitTimeout)
+ if err != nil {
+ return err
+ }
+ if n.config.RequiredRole == "" {
+ return nil
+ }
+ return identity.VerifyBiscuitRole(token, key, n.config.RequiredRole, n.BiscuitTimeout)
+}
+
func (n *SamNode) renewWithRefreshToken(ctx context.Context, clientSecret string) (string, error) {
if n.Store == nil {
return "", fmt.Errorf("store is not initialized")
diff --git a/internal/node/refresh_test.go b/internal/node/refresh_test.go
new file mode 100644
index 00000000..85dc31bb
--- /dev/null
+++ b/internal/node/refresh_test.go
@@ -0,0 +1,195 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/biscuit-auth/biscuit-go/v2"
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "google.golang.org/protobuf/proto"
+)
+
+// refreshHarness is a node with a stored identity and a mock control plane
+// whose /refresh answer the test chooses.
+type refreshHarness struct {
+ node *SamNode
+ cpPub ed25519.PublicKey
+ cpPriv ed25519.PrivateKey
+ peerID peer.ID
+ identity []byte
+}
+
+func mintRoleBiscuit(t *testing.T, priv ed25519.PrivateKey, peerID peer.ID, role string) []byte {
+ t.Helper()
+ builder := biscuit.NewBuilder(priv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID.String())}}},
+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(role)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}},
+ } {
+ if err := builder.AddAuthorityFact(f); err != nil {
+ t.Fatal(err)
+ }
+ }
+ tok, err := builder.Build()
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := tok.Serialize()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func newRefreshHarness(t *testing.T, refresh http.HandlerFunc) *refreshHarness {
+ t.Helper()
+ cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+
+ privKey := GetOrGenerateKey(store)
+ peerID, err := peer.IDFromPublicKey(privKey.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+ current := mintRoleBiscuit(t, cpPriv, peerID, api.RoleNode)
+ if err := store.SaveIdentity(current); err != nil {
+ t.Fatal(err)
+ }
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/refresh", refresh)
+ srv := httptest.NewServer(mux)
+ t.Cleanup(srv.Close)
+ if err := store.SaveControlPlaneURL(srv.URL); err != nil {
+ t.Fatal(err)
+ }
+
+ node := &SamNode{
+ Store: store,
+ trustedKeys: []TrustedKey{{Key: cpPub, ReceivedAt: time.Now()}},
+ BiscuitTimeout: 500 * time.Millisecond,
+ config: Options{RequiredRole: api.RoleNode},
+ }
+ node.SetIdentityCache(current)
+ return &refreshHarness{node: node, cpPub: cpPub, cpPriv: cpPriv, peerID: peerID, identity: current}
+}
+
+func writeRefreshResponse(t *testing.T, w http.ResponseWriter, token []byte) {
+ t.Helper()
+ data, err := proto.Marshal(&api.TokenRefreshResponse{BiscuitToken: token, ExpiresAt: time.Now().Add(24 * time.Hour).Unix()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ _, _ = w.Write(data)
+}
+
+func (h *refreshHarness) storedIdentityUnchanged(t *testing.T) {
+ t.Helper()
+ stored, err := h.node.Store.LoadIdentity()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(stored, h.identity) {
+ t.Error("stored identity was replaced by a response that must not have been accepted")
+ }
+ if !bytes.Equal(h.node.GetIdentity(), h.identity) {
+ t.Error("cached identity was replaced by a response that must not have been accepted")
+ }
+}
+
+// A 403 from whoever answers /refresh is not the control plane's word that
+// this node is banned; that is a verified MeshEvent_BANNED. The node reports
+// the refusal and keeps its current identity instead of exiting.
+func TestRefreshEnrollmentForbiddenDoesNotExit(t *testing.T) {
+ h := newRefreshHarness(t, func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "node banned", http.StatusForbidden)
+ })
+
+ err := h.node.RefreshEnrollment(context.Background())
+ var rerr *RefreshError
+ if !errors.As(err, &rerr) || rerr.StatusCode != http.StatusForbidden {
+ t.Fatalf("RefreshEnrollment = %v, want *RefreshError with 403", err)
+ }
+ h.storedIdentityUnchanged(t)
+}
+
+// The refreshed token is held to the same bar as the enrolled one: signed by
+// a key this node already trusts, bound to this peer, carrying its role.
+func TestRefreshEnrollmentRejectsUntrustworthyToken(t *testing.T) {
+ _, strangerPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ otherPeer := randomPeerID(t)
+
+ tests := []struct {
+ name string
+ mint func(h *refreshHarness) []byte
+ }{
+ {"signed by an untrusted key", func(h *refreshHarness) []byte {
+ return mintRoleBiscuit(t, strangerPriv, h.peerID, api.RoleNode)
+ }},
+ {"bound to another peer", func(h *refreshHarness) []byte {
+ return mintRoleBiscuit(t, h.cpPriv, otherPeer, api.RoleNode)
+ }},
+ {"wrong role", func(h *refreshHarness) []byte {
+ return mintRoleBiscuit(t, h.cpPriv, h.peerID, api.RoleRouter)
+ }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var h *refreshHarness
+ h = newRefreshHarness(t, func(w http.ResponseWriter, r *http.Request) {
+ writeRefreshResponse(t, w, tt.mint(h))
+ })
+ if err := h.node.RefreshEnrollment(context.Background()); err == nil {
+ t.Fatal("RefreshEnrollment accepted a token it must have rejected")
+ }
+ h.storedIdentityUnchanged(t)
+ })
+ }
+
+ t.Run("a good token is adopted", func(t *testing.T) {
+ var h *refreshHarness
+ h = newRefreshHarness(t, func(w http.ResponseWriter, r *http.Request) {
+ writeRefreshResponse(t, w, mintRoleBiscuit(t, h.cpPriv, h.peerID, api.RoleNode))
+ })
+ if err := h.node.RefreshEnrollment(context.Background()); err != nil {
+ t.Fatalf("RefreshEnrollment: %v", err)
+ }
+ if bytes.Equal(h.node.GetIdentity(), h.identity) {
+ t.Error("a valid refreshed token must replace the current identity")
+ }
+ })
+}
diff --git a/internal/router/config.go b/internal/router/config.go
index 73ef9327..7debe25f 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -53,6 +53,10 @@ type Options struct {
// cap (default: 8) when > 0. Raise it when the listener sits behind a
// TLS-terminating proxy or NAT, where many peers share a few source IPs.
ConnsPerSourceIP int
+ // AllowInsecureControlPlane accepts a plaintext http:// ControlPlaneURL
+ // to a non-loopback host. Off by default: whoever answers that URL is
+ // the trust root.
+ AllowInsecureControlPlane bool
}
// Default sets default values for options.
@@ -91,5 +95,5 @@ func (o *Options) Validate() error {
if o.ControlPlaneURL == "" {
return fmt.Errorf("ControlPlaneURL must be specified")
}
- return nil
+ return api.ValidateControlPlaneTransport(o.ControlPlaneURL, o.AllowInsecureControlPlane)
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 85c80a43..02899279 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -420,7 +420,7 @@ func (r *Router) enroll(peerID peer.ID) error {
return err
}
- client := &http.Client{Timeout: 30 * time.Second}
+ client := r.controlPlaneClient(30 * time.Second)
resp, err := client.Post(r.config.ControlPlaneURL+"/register", "application/x-protobuf", bytes.NewReader(data))
if err != nil {
return err
@@ -481,7 +481,7 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
return err
}
- client := &http.Client{Timeout: 30 * time.Second}
+ client := r.controlPlaneClient(30 * time.Second)
resp, err := client.Post(r.config.ControlPlaneURL+"/enroll", "application/x-protobuf", bytes.NewReader(data))
if err != nil {
return err
@@ -668,7 +668,7 @@ func (r *Router) recoverAfterLease401() error {
}
func (r *Router) syncKeys() error {
- client := &http.Client{Timeout: 10 * time.Second}
+ client := r.controlPlaneClient(10 * time.Second)
resp, err := client.Get(r.config.ControlPlaneURL + "/keys")
if err != nil {
return err
@@ -689,13 +689,14 @@ func (r *Router) syncKeys() error {
return err
}
- r.keysMu.Lock()
- var newKeys []ed25519.PublicKey
- for _, kb := range keysResp.PublicKeys {
- if len(kb) == ed25519.PublicKeySize {
- newKeys = append(newKeys, ed25519.PublicKey(kb))
- }
+ // Only a set signed by a key this router already trusts may replace
+ // the trust set; anything else is whoever answered the URL.
+ newKeys, err := api.VerifyKeysResponse(&keysResp, r.getTrustedPublicKeys(), time.Now())
+ if err != nil {
+ return fmt.Errorf("/keys response rejected: %w", err)
}
+
+ r.keysMu.Lock()
r.trustedPublicKeys = newKeys
r.keysMu.Unlock()
@@ -703,6 +704,24 @@ func (r *Router) syncKeys() error {
return nil
}
+// controlPlaneClient is the client for every request to the control plane;
+// its transport re-checks the plaintext policy on each hop, redirects included.
+func (r *Router) controlPlaneClient(timeout time.Duration) *http.Client {
+ return &http.Client{
+ Timeout: timeout,
+ Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ if err := api.ValidateControlPlaneTransport(req.URL.String(), r.config.AllowInsecureControlPlane); err != nil {
+ return nil, err
+ }
+ return http.DefaultTransport.RoundTrip(req)
+ }),
+ }
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
+
func (r *Router) getTrustedPublicKeys() []ed25519.PublicKey {
r.keysMu.RLock()
defer r.keysMu.RUnlock()
@@ -890,7 +909,7 @@ func (r *Router) renewLease() {
}
data, _ := proto.Marshal(req)
- client := &http.Client{Timeout: 10 * time.Second}
+ client := r.controlPlaneClient(10 * time.Second)
resp, err := client.Post(r.config.ControlPlaneURL+"/routers/lease", "application/x-protobuf", bytes.NewReader(data))
if err != nil {
logger.Errorf("Failed to renew lease with control plane: %v", err)
@@ -1009,7 +1028,7 @@ func (r *Router) runFederationLoop() {
}
func (r *Router) connectBootstrapRouters() {
- client := &http.Client{Timeout: 10 * time.Second}
+ client := r.controlPlaneClient(10 * time.Second)
// Taken before the request: anything banned after this point cannot be
// reflected in the answer, so reconciliation must not read its absence as
// an unban (see reconcileBannedPeers).
@@ -1333,7 +1352,7 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
b64Biscuit := base64.StdEncoding.EncodeToString(currentBiscuit)
httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
- client := &http.Client{Timeout: 10 * time.Second}
+ client := r.controlPlaneClient(10 * time.Second)
resp, err := client.Do(httpReq)
if err != nil {
return fmt.Errorf("http request failed: %w", err)
@@ -1347,11 +1366,11 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
}
if resp.StatusCode == http.StatusForbidden {
- logger.Errorf("Refresh rejected: Router is banned (403 Forbidden). Hard-killing router.")
- if r.Host != nil {
- _ = r.Host.Close()
- }
- os.Exit(1)
+ // A 403 is a claim by whoever answered; only a verified
+ // MeshEvent_BANNED is the control plane's word. Keep serving on the
+ // current biscuit until it expires.
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("refresh refused (403 Forbidden): %s", string(body))
}
if resp.StatusCode != http.StatusOK {
@@ -1373,6 +1392,12 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
return fmt.Errorf("refresh error: %s", refreshResp.ErrorMessage)
}
+ // Same checks as enrollment: signed by a trusted key, bound to this
+ // router, carrying the router role.
+ if err := r.verifyOwnBiscuit(refreshResp.BiscuitToken, peerID); err != nil {
+ return fmt.Errorf("refreshed biscuit rejected: %w", err)
+ }
+
// Update local biscuit token and expiration under lock
r.keysMu.Lock()
r.biscuitToken = refreshResp.BiscuitToken
@@ -1383,6 +1408,18 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
return nil
}
+func (r *Router) verifyOwnBiscuit(token []byte, peerID peer.ID) error {
+ trusted := r.getTrustedPublicKeys()
+ if len(trusted) == 0 {
+ return fmt.Errorf("no trusted control plane keys loaded")
+ }
+ _, key, err := identity.VerifyBiscuitAndGetKey(token, peerID, trusted, r.config.BiscuitTimeout)
+ if err != nil {
+ return err
+ }
+ return identity.VerifyBiscuitRole(token, key, r.config.RequiredRole, r.config.BiscuitTimeout)
+}
+
func (r *Router) runBiscuitRenewalLoop() {
defer r.wg.Done()
ticker := time.NewTicker(api.TokenRefreshCheckInterval)
diff --git a/internal/router/trust_test.go b/internal/router/trust_test.go
new file mode 100644
index 00000000..a2ef260b
--- /dev/null
+++ b/internal/router/trust_test.go
@@ -0,0 +1,264 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/biscuit-auth/biscuit-go/v2"
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "google.golang.org/protobuf/proto"
+)
+
+// A router pointed at a plaintext control plane on another host has no way
+// to know who it is talking to, so Options refuse it unless the operator
+// says the network is trusted. Loopback is the standalone case.
+func TestOptionsValidateControlPlaneTransport(t *testing.T) {
+ tests := []struct {
+ url string
+ insecure bool
+ wantErr bool
+ }{
+ {"http://127.0.0.1:8080", false, false},
+ {"https://cp.example.com", false, false},
+ {"http://sam-mesh-control-plane:8080", false, true},
+ {"http://sam-mesh-control-plane:8080", true, false},
+ }
+ for _, tt := range tests {
+ o := Options{ControlPlaneURL: tt.url, AllowInsecureControlPlane: tt.insecure}
+ o.Default()
+ err := o.Validate()
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Validate(%q, insecure=%v) = %v, wantErr %v", tt.url, tt.insecure, err, tt.wantErr)
+ }
+ if tt.wantErr && !errors.Is(err, api.ErrInsecureControlPlaneURL) {
+ t.Errorf("Validate(%q) = %v, want %v", tt.url, err, api.ErrInsecureControlPlaneURL)
+ }
+ }
+}
+
+// The transport re-checks the policy on every hop, so a stored or redirected
+// plaintext URL is refused just like a flag-supplied one.
+func TestControlPlaneClientRefusesPlaintextHop(t *testing.T) {
+ r := &Router{config: Options{}}
+ req, err := http.NewRequest(http.MethodGet, "http://sam-mesh-control-plane:8080/keys", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = r.controlPlaneClient(time.Second).Do(req)
+ if !errors.Is(err, api.ErrInsecureControlPlaneURL) {
+ t.Fatalf("Do = %v, want %v", err, api.ErrInsecureControlPlaneURL)
+ }
+}
+
+// /keys may only replace the trust set when signed by a key already in it.
+func TestSyncKeysRequiresTrustedSignature(t *testing.T) {
+ oldPub, oldPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ attackerPub, attackerPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var body []byte
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write(body)
+ }))
+ defer srv.Close()
+
+ newRouter := func() *Router {
+ return &Router{config: Options{ControlPlaneURL: srv.URL}, trustedPublicKeys: []ed25519.PublicKey{oldPub}}
+ }
+ marshal := func(resp *api.KeysResponse) []byte {
+ data, err := proto.Marshal(resp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+ }
+
+ t.Run("rotation signed by the retiring key is adopted", func(t *testing.T) {
+ resp := &api.KeysResponse{PublicKeys: [][]byte{oldPub, newPub}}
+ if err := api.SignKeysResponse(resp, []ed25519.PrivateKey{oldPriv, newPriv}, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ body = marshal(resp)
+ r := newRouter()
+ if err := r.syncKeys(); err != nil {
+ t.Fatalf("syncKeys: %v", err)
+ }
+ if keys := r.getTrustedPublicKeys(); len(keys) != 2 || !keys[1].Equal(newPub) {
+ t.Errorf("trusted keys = %d, want old and new", len(keys))
+ }
+ })
+
+ t.Run("unsigned set is refused", func(t *testing.T) {
+ body = marshal(&api.KeysResponse{PublicKeys: [][]byte{oldPub, attackerPub}})
+ r := newRouter()
+ if err := r.syncKeys(); err == nil {
+ t.Fatal("an unsigned /keys answer must not be adopted")
+ }
+ if keys := r.getTrustedPublicKeys(); len(keys) != 1 || !keys[0].Equal(oldPub) {
+ t.Errorf("trust set changed on a refused answer: %d keys", len(keys))
+ }
+ })
+
+ t.Run("set signed only by a stranger is refused", func(t *testing.T) {
+ resp := &api.KeysResponse{PublicKeys: [][]byte{oldPub, attackerPub}}
+ if err := api.SignKeysResponse(resp, []ed25519.PrivateKey{attackerPriv, attackerPriv}, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ body = marshal(resp)
+ r := newRouter()
+ if err := r.syncKeys(); err == nil {
+ t.Fatal("a /keys answer signed by an untrusted key must not be adopted")
+ }
+ if keys := r.getTrustedPublicKeys(); len(keys) != 1 || !keys[0].Equal(oldPub) {
+ t.Errorf("trust set changed on a refused answer: %d keys", len(keys))
+ }
+ })
+}
+
+func mintRouterBiscuit(t *testing.T, priv ed25519.PrivateKey, peerID peer.ID, role string) []byte {
+ t.Helper()
+ builder := biscuit.NewBuilder(priv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID.String())}}},
+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(role)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}},
+ } {
+ if err := builder.AddAuthorityFact(f); err != nil {
+ t.Fatal(err)
+ }
+ }
+ tok, err := builder.Build()
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := tok.Serialize()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+// A 403 from /refresh is reported, not obeyed with an exit; and a refreshed
+// token is held to the enrollment bar (trusted signer, this peer, router role).
+func TestRouterRefreshEnrollmentHardening(t *testing.T) {
+ cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, strangerPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ peerID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ current := mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)
+
+ var respond func(w http.ResponseWriter)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/refresh" {
+ t.Errorf("unexpected request to %s", r.URL.Path)
+ http.Error(w, "unexpected", http.StatusInternalServerError)
+ return
+ }
+ respond(w)
+ }))
+ defer srv.Close()
+
+ newRouter := func() *Router {
+ return &Router{
+ privKey: priv,
+ biscuitToken: current,
+ trustedPublicKeys: []ed25519.PublicKey{cpPub},
+ config: Options{ControlPlaneURL: srv.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},
+ }
+ }
+ tokenResponse := func(token []byte) func(http.ResponseWriter) {
+ return func(w http.ResponseWriter) {
+ data, err := proto.Marshal(&api.TokenRefreshResponse{BiscuitToken: token, ExpiresAt: time.Now().Add(time.Hour).Unix()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, _ = w.Write(data)
+ }
+ }
+
+ t.Run("403 is an error, not an exit", func(t *testing.T) {
+ respond = func(w http.ResponseWriter) { http.Error(w, "router banned", http.StatusForbidden) }
+ r := newRouter()
+ err := r.RefreshEnrollment(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "403") {
+ t.Fatalf("RefreshEnrollment = %v, want a 403 error", err)
+ }
+ if !bytes.Equal(r.biscuitToken, current) {
+ t.Error("the current biscuit must be kept on a refused refresh")
+ }
+ })
+
+ for name, token := range map[string][]byte{
+ "token signed by an untrusted key": mintRouterBiscuit(t, strangerPriv, peerID, api.RoleRouter),
+ "token bound to another peer": mintRouterBiscuit(t, cpPriv, newTestPeerID(t), api.RoleRouter),
+ "token without the router role": mintRouterBiscuit(t, cpPriv, peerID, api.RoleNode),
+ } {
+ t.Run(name+" is refused", func(t *testing.T) {
+ respond = tokenResponse(token)
+ r := newRouter()
+ if err := r.RefreshEnrollment(context.Background()); err == nil {
+ t.Fatal("RefreshEnrollment adopted a token it must have refused")
+ }
+ if !bytes.Equal(r.biscuitToken, current) {
+ t.Error("the current biscuit must be kept when the refreshed one is refused")
+ }
+ })
+ }
+
+ t.Run("a good token is adopted", func(t *testing.T) {
+ fresh := mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)
+ respond = tokenResponse(fresh)
+ r := newRouter()
+ if err := r.RefreshEnrollment(context.Background()); err != nil {
+ t.Fatalf("RefreshEnrollment: %v", err)
+ }
+ if !bytes.Equal(r.biscuitToken, fresh) {
+ t.Error("a valid refreshed token must replace the current one")
+ }
+ })
+}
diff --git a/mobile/mobile_e2e.sh b/mobile/mobile_e2e.sh
index 6b962f33..6d4f7b30 100755
--- a/mobile/mobile_e2e.sh
+++ b/mobile/mobile_e2e.sh
@@ -105,6 +105,7 @@ docker run --name sam-router \
-d --rm \
sam-router:local \
--control-plane http://sam-control-plane:37001 \
+ --insecure-control-plane \
--listen /ip4/0.0.0.0/tcp/37002 \
--listen /ip4/0.0.0.0/udp/37002/quic-v1 \
--external-addr /ip4/10.0.2.2/tcp/37002 \
@@ -250,6 +251,7 @@ docker run --name host-node \
run \
--data-dir /data \
--control-plane http://sam-control-plane:37001 \
+ --insecure-control-plane \
--jwt "$HOST_JWT" \
--bind-addr 0.0.0.0:8081 \
--allow-loopback \
diff --git a/site/content/docs/development/kubernetes-deployment.md b/site/content/docs/development/kubernetes-deployment.md
index c54f7862..c9a3bcfa 100644
--- a/site/content/docs/development/kubernetes-deployment.md
+++ b/site/content/docs/development/kubernetes-deployment.md
@@ -245,6 +245,7 @@ If you are using the **Mock OIDC Provider**, the node can fetch the token using
```bash
sam-node run \
--control-plane "http://$CONTROL_PLANE_IP:9090" \
+ --insecure-control-plane \
--oidc-issuer "http://$MOCK_IP:18080" \
--client-id "sam-mesh-audience" \
# client secret via SAM_CLIENT_SECRET env or --client-secret-path
@@ -254,6 +255,7 @@ If you are using **Google OIDC**, you must obtain a valid Google ID token for yo
```bash
sam-node run \
--control-plane "http://$CONTROL_PLANE_IP:9090" \
+ --insecure-control-plane \
--jwt "<your-google-id-token>"
```
@@ -322,7 +324,7 @@ The SAM project supports three primary flows for acquiring a JWT token to enroll
* **Example:**
```bash
sam-node run \
- --control-plane "http://control-plane.example.com:9090" \
+ --control-plane "https://control-plane.example.com:9090" \
--oidc-issuer "https://accounts.google.com" \
--client-id "$SAM_OIDC_ID" \
--client-secret "$SAM_OIDC_SECRET"
@@ -349,7 +351,7 @@ sam-node run
* **Example:**
```bash
sam-node run \
- --control-plane "http://control-plane.example.com:9090" \
+ --control-plane "https://control-plane.example.com:9090" \
--jwt-path "/var/run/secrets/kubernetes.io/serviceaccount/token"
```
> [!NOTE]
diff --git a/site/content/docs/manifests/sam-router.yaml b/site/content/docs/manifests/sam-router.yaml
index 0d75b0ee..5567a6dd 100644
--- a/site/content/docs/manifests/sam-router.yaml
+++ b/site/content/docs/manifests/sam-router.yaml
@@ -40,6 +40,7 @@ spec:
imagePullPolicy: IfNotPresent
args:
- "--control-plane=http://sam-control-plane:8080"
+ - "--insecure-control-plane"
- "--listen=/ip4/0.0.0.0/tcp/4501"
- "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
diff --git a/site/content/docs/user/control-plane-configuration.md b/site/content/docs/user/control-plane-configuration.md
index ac7510e8..7fd625ed 100644
--- a/site/content/docs/user/control-plane-configuration.md
+++ b/site/content/docs/user/control-plane-configuration.md
@@ -39,13 +39,14 @@ The Router is a dedicated GossipSub helper that maintains stable network address
| CLI Flag | Default Value | Description |
| :--- | :--- | :--- |
-| `--control-plane` | `http://127.0.0.1:8080` | Control Plane web service URL. |
+| `--control-plane` | `http://127.0.0.1:8080` | Control Plane web service URL. Plaintext `http://` is accepted only for a loopback host. |
+| `--insecure-control-plane` | `false` | Accept a plaintext `http://` control-plane URL to a non-loopback host (e.g. an in-cluster Service). Whoever answers that URL becomes the router's trust root, so leave this off outside a network you already trust. The same flag exists on `sam-node`. |
| `--listen` | `/ip4/0.0.0.0/tcp/5001`, `/ip6/::/tcp/5001` | Comma-separated libp2p multiaddrs to listen on. |
| `--external-addr` | *None* | External multiaddrs to announce to control plane. |
| `--keys-path` | `router.key` | Path to save/load persistent private key (determines Peer ID). |
| `--jwt-path` | *None* | Path to file containing OIDC JWT token for enrollment. |
| `--oidc-token` | *None* | Direct OIDC ID token or bootstrap secret token for enrollment. |
-| `--keys-sync-interval` | `5m` | Key synchronization polling interval. |
+| `--keys-sync-interval` | `5m` | Key synchronization polling interval. `GET /keys` returns the valid key set signed by every key in it; the router (and nodes, at start-up) accept the set only if one of those signatures verifies under a key they already trust, so the first key always comes from enrollment and a rotation is learned from the retiring key. |
| `--lease-renew-interval` | `300s` | Lease renewal registration interval. |
| `--allow-loopback` | `false` | Allow loopback and link-local addresses for discovery (development only). |
diff --git a/site/content/docs/user/kubernetes-deployment.md b/site/content/docs/user/kubernetes-deployment.md
index 68c8fba1..e7d53e77 100644
--- a/site/content/docs/user/kubernetes-deployment.md
+++ b/site/content/docs/user/kubernetes-deployment.md
@@ -197,6 +197,8 @@ spec:
name: p2p-udp
args:
- "--control-plane=http://sam-control-plane.sam.svc.cluster.local:8080"
+ # In-cluster plaintext; the cluster network is the trust boundary.
+ - "--insecure-control-plane"
- "--listen=/ip4/0.0.0.0/tcp/4501"
- "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
@@ -304,6 +306,7 @@ spec:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- "--control-plane=http://sam-control-plane.sam.svc.cluster.local:8080"
+ - "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--api-token-path=/var/run/secrets/sam/api-token"
volumeMounts:
diff --git a/tests/e2e/auth_flows.bats b/tests/e2e/auth_flows.bats
index eeea49f5..730f4436 100644
--- a/tests/e2e/auth_flows.bats
+++ b/tests/e2e/auth_flows.bats
@@ -85,7 +85,8 @@ teardown() {
"sam-node:local" \
run \
--data-dir /data \
- --control-plane "http://sam-control-plane:8080"
+ --control-plane "http://sam-control-plane:8080" \
+ --insecure-control-plane
MESH_CONTAINERS+=("${node_name}")
mesh_wait_for_log "${node_name}" "Using stored identity." 20
@@ -152,6 +153,7 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
"sam-node:local" \
run \
--control-plane "http://sam-control-plane:8080" \
+ --insecure-control-plane \
--jwt-path "/var/run/secrets/tokens/sa-token"
MESH_CONTAINERS+=("${node_name}")
@@ -216,7 +218,8 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
"sam-node:local" \
run \
--data-dir /data \
- --control-plane "http://sam-control-plane:8080"
+ --control-plane "http://sam-control-plane:8080" \
+ --insecure-control-plane
MESH_CONTAINERS+=("${node_name}")
mesh_wait_for_log "${node_name}" "Using stored identity." 20
diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash
index 5caba6c7..e0a21e97 100644
--- a/tests/e2e/lib/container_mesh.bash
+++ b/tests/e2e/lib/container_mesh.bash
@@ -497,6 +497,7 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
--log-level debug \
--discovery-interval 2s \
--control-plane "http://sam-control-plane:8080" \
+ --insecure-control-plane \
--client-id "sam-mesh-audience" \
--oidc-issuer "http://mock-oidc:18080" \
--listen "/ip4/0.0.0.0/udp/5001/quic-v1" \