Skip to content

Agent Mesh

Fix postgres volume mounting on GKE and switch E2E tests to dynamic bootstrap token API

Antonio Ojea0f888bfb4409
diff --git a/.github/k8s/sam-control-plane-template.yaml b/.github/k8s/sam-control-plane-template.yaml
index ba57bf93..7c792dae 100644
--- a/.github/k8s/sam-control-plane-template.yaml
+++ b/.github/k8s/sam-control-plane-template.yaml
@@ -41,6 +41,8 @@ spec:
labels:
app: sam-db-${ENV_NAME}
spec:
+ securityContext:
+ fsGroup: 999
containers:
- name: postgres
image: postgres:16-alpine
@@ -51,6 +53,8 @@ spec:
value: sam
- name: POSTGRES_PASSWORD
value: sam-secret-password
+ - name: PGDATA
+ value: /var/lib/postgresql/data/pgdata
ports:
- containerPort: 5432
protocol: TCP
@@ -81,9 +85,20 @@ spec:
labels:
app: sam-control-plane-${ENV_NAME}
spec:
+ initContainers:
+ - name: wait-for-db
+ image: busybox:latest
+ command: ['sh', '-c', 'until nc -z -w3 sam-db-${ENV_NAME} 5432; do echo waiting for db; sleep 1; done']
containers:
- name: sam-control-plane
image: ghcr.io/google/sam-control-plane:${IMAGE_TAG}
+ env:
+ - name: ADMIN_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: sam-control-plane-secret-${ENV_NAME}
+ key: admin-token
+ optional: true
ports:
- containerPort: 8080
protocol: TCP
@@ -109,6 +124,7 @@ spec:
- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"
- "--policy-file=/etc/sam/policies/policies.yaml"
+ - "--admin-token=$(ADMIN_TOKEN)"
resources:
requests:
cpu: 100m
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 7cc63e59..141119bb 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -222,6 +222,20 @@ jobs:
+ - name: Provision Control Plane Secrets
+ run: |
+ export ENV_NAME="${{ vars.ENV_NAME }}"
+ export NAMESPACE="sam-${ENV_NAME}"
+ if ! kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
+ echo "Generating a new random 32-byte hex key for admin-token..."
+ ADMIN_TOKEN=$(openssl rand -hex 32)
+ kubectl create secret generic sam-control-plane-secret-${ENV_NAME} \
+ --namespace=${NAMESPACE} \
+ --from-literal=admin-token="${ADMIN_TOKEN}"
+ else
+ echo "sam-control-plane-secret-${ENV_NAME} already exists in namespace ${NAMESPACE}. Skipping generation."
+ fi
+
- name: Deploy Control Plane and Routers
run: |
print_rollout_diagnostics() {
diff --git a/cmd/sam-control-plane/main.go b/cmd/sam-control-plane/main.go
index 23e4cd22..acf46d06 100644
--- a/cmd/sam-control-plane/main.go
+++ b/cmd/sam-control-plane/main.go
@@ -38,7 +38,6 @@ var (
keyGracePeriod time.Duration
leaseDuration time.Duration
adminToken string
- bootstrapToken string
insecureSkipTLSVerify bool
logLevel string
autoApproveEnrollment bool
@@ -101,7 +100,6 @@ func main() {
InsecureSkipTLSVerify: insecureSkipTLSVerify,
PolicyPath: policyFile,
BiscuitTimeout: 10 * time.Second,
- BootstrapToken: bootstrapToken,
AdminToken: adminToken,
AutoApproveEnrollment: autoApproveEnrollment,
}
@@ -135,7 +133,6 @@ func main() {
rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 1*time.Hour, "Key grace period for rotated keys.")
rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")
rootCmd.Flags().StringVar(&adminToken, "admin-token", "", "Token for authenticating policy REST API requests")
- rootCmd.Flags().StringVar(&bootstrapToken, "bootstrap-token", "", "Pre-shared bootstrap token for bypassing OIDC verification")
rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC providers")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
rootCmd.Flags().BoolVar(&autoApproveEnrollment, "auto-approve-enrollment", false, "Auto-approve valid bootstrap token enrollment requests")
diff --git a/internal/controlplane/config.go b/internal/controlplane/config.go
index 9ebe46eb..1a5bd4de 100644
--- a/internal/controlplane/config.go
+++ b/internal/controlplane/config.go
@@ -32,7 +32,6 @@ type Options struct {
InsecureSkipTLSVerify bool
BiscuitTimeout time.Duration
PolicyPath string // Optional: path to bootstrap policy configuration
- BootstrapToken string // Optional: pre-shared bootstrap token for bypassing OIDC verification (e.g. for router enrollment)
AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs
AutoApproveEnrollment bool // If true, valid bootstrap token enrollment requests are immediately approved without administrative manual gate
}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index c12a3f9f..d9cd82fc 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -123,30 +123,7 @@ func (s *Server) Start() error {
}
}
- // Seed the initial command-line bootstrap token if provided
- if s.config.BootstrapToken != "" {
- hash := sha256.Sum256([]byte(s.config.BootstrapToken))
- tokenID := fmt.Sprintf("%x", hash)
- _, err := s.store.GetBootstrapToken(ctx, tokenID)
- if err == storage.ErrNotFound {
- logger.Infof("Seeding initial bootstrap token into database (ID: %s)...", tokenID)
- tokenRecord := &storage.BootstrapToken{
- ID: tokenID,
- TokenHash: tokenID,
- Role: api.RoleRouter,
- MaxUsages: 10000,
- UsagesCount: 0,
- Description: "Command Line Seeded Token",
- CreatedAt: time.Now(),
- ExpiresAt: time.Now().Add(365 * 24 * time.Hour),
- }
- if err := s.store.SaveBootstrapToken(ctx, tokenRecord); err != nil {
- return fmt.Errorf("failed to save initial bootstrap token: %w", err)
- }
- } else if err != nil {
- return fmt.Errorf("failed to check initial bootstrap token state: %w", err)
- }
- }
+
// Initialize OIDC Providers
if err := s.discoverProviders(); err != nil {
diff --git a/tests/e2e/fixtures/control-plane-router.yaml b/tests/e2e/fixtures/control-plane.yaml
similarity index 64%
rename from tests/e2e/fixtures/control-plane-router.yaml
rename to tests/e2e/fixtures/control-plane.yaml
index 1802d107..682a8517 100644
--- a/tests/e2e/fixtures/control-plane-router.yaml
+++ b/tests/e2e/fixtures/control-plane.yaml
@@ -138,7 +138,6 @@ spec:
- "--insecure-skip-tls-verify"
- "--lease-duration=15s"
- "--admin-token=super-secret-admin-token"
- - "--bootstrap-token=super-secret-bootstrap-token"
- "--auto-approve-enrollment"
- "--log-level=debug"
volumeMounts:
@@ -149,80 +148,3 @@ spec:
- name: config-volume
configMap:
name: sam-hub-config
----
-apiVersion: v1
-kind: Service
-metadata:
- name: sam-router
-spec:
- clusterIP: None
- ports:
- - port: 4501
- targetPort: 4501
- protocol: TCP
- name: p2p-tcp
- - port: 4501
- targetPort: 4501
- protocol: UDP
- name: p2p-udp
- selector:
- app: sam-router
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: sam-router
-spec:
- serviceName: "sam-router"
- replicas: 1
- selector:
- matchLabels:
- app: sam-router
- template:
- metadata:
- labels:
- app: sam-router
- spec:
- containers:
- - name: sam-router
- image: sam-router:local
- ports:
- - containerPort: 4501
- hostPort: 4501
- protocol: TCP
- - containerPort: 4501
- hostPort: 4501
- protocol: UDP
- env:
- - name: ROUTER_NODE_IP
- valueFrom:
- fieldRef:
- fieldPath: status.hostIP
- args:
- - "--control-plane=http://sam-control-plane:8080"
- - "--listen=/ip4/0.0.0.0/tcp/4501"
- - "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
- - "--external-addr=/dns4/sam-router/tcp/4501"
- - "--external-addr=/dns4/sam-router/udp/4501/quic-v1"
- - "--external-addr=/ip4/$(ROUTER_NODE_IP)/tcp/4501"
- - "--external-addr=/ip4/$(ROUTER_NODE_IP)/udp/4501/quic-v1"
- - "--bootstrap-token-path=/var/run/secrets/tokens/sa-token"
- - "--keys-path=/data/router.key"
- - "--allow-loopback"
- - "--lease-renew-interval=5s"
- - "--log-level=debug"
- volumeMounts:
- - name: router-data
- mountPath: /data
- - name: router-token-vol
- mountPath: /var/run/secrets/tokens
- readOnly: true
- volumes:
- - name: router-data
- emptyDir: {}
- - name: router-token-vol
- secret:
- secretName: sam-router-token
- items:
- - key: token
- path: sa-token
diff --git a/tests/e2e/fixtures/router.yaml b/tests/e2e/fixtures/router.yaml
new file mode 100644
index 00000000..d11bca9a
--- /dev/null
+++ b/tests/e2e/fixtures/router.yaml
@@ -0,0 +1,76 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: sam-router
+spec:
+ clusterIP: None
+ ports:
+ - port: 4501
+ targetPort: 4501
+ protocol: TCP
+ name: p2p-tcp
+ - port: 4501
+ targetPort: 4501
+ protocol: UDP
+ name: p2p-udp
+ selector:
+ app: sam-router
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: sam-router
+spec:
+ serviceName: "sam-router"
+ replicas: 1
+ selector:
+ matchLabels:
+ app: sam-router
+ template:
+ metadata:
+ labels:
+ app: sam-router
+ spec:
+ containers:
+ - name: sam-router
+ image: sam-router:local
+ ports:
+ - containerPort: 4501
+ hostPort: 4501
+ protocol: TCP
+ - containerPort: 4501
+ hostPort: 4501
+ protocol: UDP
+ env:
+ - name: ROUTER_NODE_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.hostIP
+ args:
+ - "--control-plane=http://sam-control-plane:8080"
+ - "--listen=/ip4/0.0.0.0/tcp/4501"
+ - "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
+ - "--external-addr=/dns4/sam-router/tcp/4501"
+ - "--external-addr=/dns4/sam-router/udp/4501/quic-v1"
+ - "--external-addr=/ip4/$(ROUTER_NODE_IP)/tcp/4501"
+ - "--external-addr=/ip4/$(ROUTER_NODE_IP)/udp/4501/quic-v1"
+ - "--bootstrap-token-path=/var/run/secrets/tokens/sa-token"
+ - "--keys-path=/data/router.key"
+ - "--allow-loopback"
+ - "--lease-renew-interval=5s"
+ - "--log-level=debug"
+ volumeMounts:
+ - name: router-data
+ mountPath: /data
+ - name: router-token-vol
+ mountPath: /var/run/secrets/tokens
+ readOnly: true
+ volumes:
+ - name: router-data
+ emptyDir: {}
+ - name: router-token-vol
+ secret:
+ secretName: sam-router-token
+ items:
+ - key: token
+ path: sa-token
diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash
index cddf6324..b1515c20 100644
--- a/tests/e2e/lib/container_mesh.bash
+++ b/tests/e2e/lib/container_mesh.bash
@@ -263,15 +263,28 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
local oidc_node_ip
oidc_node_ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"${MESH_NETWORK:-kind}\").IPAddress}}" "${oidc_node}")
- local router_token="super-secret-bootstrap-token"
- kubectl --context="${KUBECONTEXT}" create secret generic sam-router-token --from-literal=token="${router_token}" --dry-run=client -o yaml | kubectl --context="${KUBECONTEXT}" apply -f -
-
- envsubst '$ISSUERS' < tests/e2e/fixtures/control-plane-router.yaml | kubectl --context="${KUBECONTEXT}" apply -f -
- kubectl --context="${KUBECONTEXT}" rollout restart deployment/sam-db || true
+ envsubst '$ISSUERS' < tests/e2e/fixtures/control-plane.yaml | kubectl --context="${KUBECONTEXT}" apply -f -
kubectl --context="${KUBECONTEXT}" rollout status deployment/sam-db --timeout=60s
- kubectl --context="${KUBECONTEXT}" rollout restart deployment/sam-control-plane
- kubectl --context="${KUBECONTEXT}" rollout restart statefulset/sam-router
kubectl --context="${KUBECONTEXT}" rollout status deployment/sam-control-plane --timeout=60s
+
+ local cp_pod
+ cp_pod=$(kubectl --context="${KUBECONTEXT}" get pod -l app=sam-control-plane -o jsonpath='{.items[0].metadata.name}')
+
+ local token_json
+ token_json=$(kubectl --context="${KUBECONTEXT}" exec "${cp_pod}" -c sam-control-plane -- \
+ wget -qO- \
+ --post-data='{"role": "sam:role:router", "max_usages": 999999}' \
+ --header="Content-Type: application/json" \
+ --header="Authorization: Bearer super-secret-admin-token" \
+ http://localhost:8080/admin/bootstrap-tokens)
+
+ local router_token
+ router_token=$(echo "${token_json}" | jq -r .token)
+ [[ -n "${router_token}" && "${router_token}" != "null" ]]
+
+ kubectl --context="${KUBECONTEXT}" create secret generic sam-router-token --from-literal=token="${router_token}" --dry-run=client -o yaml | kubectl --context="${KUBECONTEXT}" apply -f -
+
+ kubectl --context="${KUBECONTEXT}" apply -f tests/e2e/fixtures/router.yaml
kubectl --context="${KUBECONTEXT}" rollout status statefulset/sam-router --timeout=60s
local i