Skip to content

Agent Mesh

fix(storage,deploy): migrate persisted sam: prefixes and support in-place GKE testnet upgrade

Antonio Ojeab8386f1ce043
diff --git a/.github/k8s/agentmesh-control-plane-template.yaml b/.github/k8s/agentmesh-control-plane-template.yaml
index 5767b6b4..54190984 100644
--- a/.github/k8s/agentmesh-control-plane-template.yaml
+++ b/.github/k8s/agentmesh-control-plane-template.yaml
@@ -129,7 +129,7 @@ spec:
- "--bind-address=0.0.0.0:8080"
- "--db-driver=postgres"
- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
- - "--allowed-audiences=agentmesh-audience,agentmesh-control-plane-audience"
+ - "--allowed-audiences=agentmesh-audience,agentmesh-control-plane-audience,sam-mesh-audience,sam-control-plane-audience"
- "--auto-approve-enrollment"
- "--key-grace-period=48h"
resources:
@@ -163,7 +163,7 @@ spec:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
- name: agentmesh-control-plane-gateway-${ENV_NAME}
+ name: sam-control-plane-gateway-${ENV_NAME}
namespace: ${NAMESPACE}
annotations:
networking.gke.io/certmap: sam-hub-cert-map-${ENV_NAME}
@@ -183,11 +183,11 @@ spec:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
- name: agentmesh-control-plane-route-${ENV_NAME}
+ name: sam-control-plane-route-${ENV_NAME}
namespace: ${NAMESPACE}
spec:
parentRefs:
- - name: agentmesh-control-plane-gateway-${ENV_NAME}
+ - name: sam-control-plane-gateway-${ENV_NAME}
hostnames:
- "${ENV_NAME}.sam-mesh.dev"
rules:
diff --git a/.github/k8s/dex-config.yaml b/.github/k8s/dex-config.yaml
index 7bac3da5..83f5a2d2 100644
--- a/.github/k8s/dex-config.yaml
+++ b/.github/k8s/dex-config.yaml
@@ -42,7 +42,7 @@ data:
redirectURI: https://auth.sam-mesh.dev/callback
staticClients:
- # This ID becomes the 'aud' claim in the JWT consumed by sam-hub
+ # This ID becomes the 'aud' claim in the JWT consumed by agentmesh-control-plane
- id: agentmesh-audience
name: 'Agent Mesh CLI'
public: true # Allows PKCE flow for CLIs without exposing the secret
@@ -56,4 +56,18 @@ data:
- 'https://hub.sam-mesh.dev/console/auth/callback'
- 'https://auth.sam-mesh.dev/device/callback'
- 'urn:ietf:wg:oauth:2.0:oob'
+ - '/device/callback'
+ - id: sam-mesh-audience
+ name: 'Agent Mesh CLI (Legacy)'
+ public: true
+ redirectURIs:
+ - 'http://127.0.0.1:13000/callback'
+ - 'http://127.0.0.1:13001/callback'
+ - 'http://127.0.0.1:13002/callback'
+ - 'http://127.0.0.1/callback'
+ - 'http://localhost/callback'
+ - 'https://bananas.sam-mesh.dev/console/auth/callback'
+ - 'https://hub.sam-mesh.dev/console/auth/callback'
+ - 'https://auth.sam-mesh.dev/device/callback'
+ - 'urn:ietf:wg:oauth:2.0:oob'
- '/device/callback'
\ No newline at end of file
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index e2b1071b..197b14a9 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -261,6 +261,11 @@ jobs:
export ENV_NAME="${VAR_ENV_NAME}"
export NAMESPACE="sam-${ENV_NAME}"
export CANARY_NAMESPACE="agentmesh-canary-${ENV_NAME}"
+
+ # Retire the legacy sam-canary namespace so its workloads (including the
+ # single-TPU vllm-canary pod) release cluster resources before rolling out
+ # agentmesh-canary.
+ kubectl delete namespace sam-canary-${ENV_NAME} --ignore-not-found
kubectl create namespace dex --dry-run=client -o yaml | kubectl apply -f -
kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -
@@ -349,10 +354,19 @@ jobs:
export ENV_NAME="${VAR_ENV_NAME}"
export NAMESPACE="sam-${ENV_NAME}"
if ! kubectl get secret agentmesh-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
- echo "Generating new random secrets for admin-token and the DB..."
- ADMIN_TOKEN=$(openssl rand -hex 32)
- DB_PASSWORD=$(openssl rand -hex 32)
- DB_DSN="postgres://agentmesh:${DB_PASSWORD}@sam-db-${ENV_NAME}:5432/agentmesh?sslmode=disable"
+ if kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
+ echo "Migrating existing credentials from sam-control-plane-secret-${ENV_NAME}..."
+ ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d)
+ DB_PASSWORD=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.db-password}' | base64 -d)
+ DB_DSN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.db-dsn}' | base64 -d)
+ else
+ echo "Generating new random secrets for admin-token and the DB..."
+ ADMIN_TOKEN=$(openssl rand -hex 32)
+ DB_PASSWORD=$(openssl rand -hex 32)
+ DB_DSN="postgres://sam:${DB_PASSWORD}@sam-db-${ENV_NAME}:5432/sam_mesh?sslmode=disable"
+ fi
+ echo "::add-mask::${ADMIN_TOKEN}"
+ echo "::add-mask::${DB_PASSWORD}"
kubectl create secret generic agentmesh-control-plane-secret-${ENV_NAME} \
--namespace=${NAMESPACE} \
--from-literal=admin-token="${ADMIN_TOKEN}" \
@@ -402,6 +416,10 @@ jobs:
export CLUSTER_REGION="${VAR_CLUSTER_REGION}"
export CLUSTER_NAME="${VAR_CLUSTER_NAME}"
+ # Remove legacy sam-router StatefulSet before applying agentmesh-router so
+ # hostPort 4501 (TCP/UDP) on the cluster nodes is released for the new pods.
+ kubectl delete statefulset/sam-router-${ENV_NAME} service/sam-router-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found
+
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/agentmesh-control-plane-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/agentmesh-router-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/agentmesh-console-template.yaml | kubectl apply -f -
@@ -409,6 +427,7 @@ jobs:
# Managed Prometheus ships with GKE; a cluster without it should not block the mesh rollout.
if kubectl get crd podmonitorings.monitoring.googleapis.com >/dev/null 2>&1; then
envsubst '${ENV_NAME} ${NAMESPACE}' < .github/k8s/agentmesh-monitoring-template.yaml | kubectl apply -f -
+ kubectl delete podmonitoring/sam-control-plane-${ENV_NAME} podmonitoring/sam-router-${ENV_NAME} rules/sam-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found || true
else
echo "::warning::PodMonitoring CRD not found; managed collection is off on this cluster, metrics will not be scraped"
fi
@@ -468,6 +487,10 @@ jobs:
exit 1
}
+ # Retire legacy sam-control-plane and sam-console resources now that the
+ # Gateway points at agentmesh-control-plane and agentmesh-console.
+ kubectl delete deployment/sam-control-plane-${ENV_NAME} service/sam-control-plane-${ENV_NAME} healthcheckpolicy/sam-control-plane-healthcheck-${ENV_NAME} deployment/sam-console-${ENV_NAME} service/sam-console-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found || true
+
# Three routers, each held 90s after it is ready (minReadySeconds)
# so members re-attach before the next one goes.
kubectl rollout status statefulset/agentmesh-router-${ENV_NAME} -n ${NAMESPACE} --timeout=600s || {
@@ -545,7 +568,8 @@ jobs:
echo "Successfully generated bootstrap token."
echo "Provisioning GCE VM Router instance..."
- # Delete existing VM if it exists to ensure fresh deployment with the new token and image
+ # Delete existing VM (including legacy sam-router-*-vm) if it exists to ensure fresh deployment with the new token and image
+ gcloud compute instances delete sam-router-${ENV_NAME}-vm --zone=${ZONE} --project=${GCP_PROJECT_ID} --quiet || true
gcloud compute instances delete agentmesh-router-${ENV_NAME}-vm --zone=${ZONE} --project=${GCP_PROJECT_ID} --quiet || true
# Create a new COS VM instance running the agentmesh-router
@@ -555,7 +579,7 @@ jobs:
--machine-type=e2-micro \
--image-family=cos-stable \
--image-project=cos-cloud \
- --tags=agentmesh-router \
+ --tags=sam-router,agentmesh-router \
--metadata=startup-script='#!/bin/bash
set -euo pipefail
EXTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
diff --git a/internal/storage/migration_test.go b/internal/storage/migration_test.go
index 9d8dd604..ea4c43d9 100644
--- a/internal/storage/migration_test.go
+++ b/internal/storage/migration_test.go
@@ -181,3 +181,94 @@ func TestMigrationCreatesIndexes(t *testing.T) {
t.Fatalf("indexes = %v, want %v", got, want)
}
}
+
+func TestMigrationRenamesSamPrefixesToMesh(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "cp.db")
+ now := time.Now().UnixMilli()
+
+ db := openAtSchemaVersion(t, path, 16)
+ exec := func(q string, args ...any) {
+ t.Helper()
+ if _, err := db.Exec(q, args...); err != nil {
+ t.Fatalf("%s: %v", q, err)
+ }
+ }
+
+ exec(`INSERT INTO roles (name, description, created_at) VALUES ('sam:role:node', '', ?), ('sam:role:router', '', ?)`, now, now)
+ exec(`INSERT INTO role_permissions (role_name, resource_type, resource_value) VALUES
+ ('sam:role:node', 'service', 'system://sam.catalog'),
+ ('sam:role:router', 'service', '*')`)
+ exec(`INSERT INTO role_bindings (role_name, member) VALUES
+ ('sam:role:node', 'sam:system:authenticated'),
+ ('sam:role:router', 'group:routers')`)
+ exec(`INSERT INTO nodes (peer_id, public_key, biscuit_token, role, enrollment_type, enrolled_at, expires_at, banned)
+ VALUES ('12D3KooWNode', X'01', X'02', 'sam:role:node', 'OIDC', ?, ?, 0)`, now, now+3600000)
+ exec(`INSERT INTO bootstrap_tokens (id, token_hash, role, max_usages, usages_count, description, created_at, expires_at)
+ VALUES ('tok1', 'tok1', 'sam:role:router', 1, 0, 'legacy router token', ?, ?)`, now, now+3600000)
+ exec(`INSERT INTO egress_destinations (name, target_url, credential, served_by, config_json)
+ VALUES ('api.example.com', 'https://api.example.com', 'TOKEN', '["sam:role:node"]', '{"name":"api.example.com","target_url":"https://api.example.com","credential":"TOKEN","served_by":["sam:role:node"]}')`)
+ if err := db.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ store, err := NewSQLStore("sqlite", path)
+ if err != nil {
+ t.Fatalf("NewSQLStore: %v", err)
+ }
+ defer func() { _ = store.Close() }()
+ ctx := context.Background()
+
+ roles, bindings, err := store.GetMeshPolicy(ctx)
+ if err != nil {
+ t.Fatalf("GetMeshPolicy: %v", err)
+ }
+ roleMap := make(map[string]*api.PolicyRole)
+ for _, r := range roles {
+ roleMap[r.Name] = r
+ }
+ if _, ok := roleMap["sam:role:node"]; ok {
+ t.Errorf("legacy sam:role:node still present in roles")
+ }
+ nodeRole, ok := roleMap["mesh:role:node"]
+ if !ok {
+ t.Fatalf("mesh:role:node missing from migrated roles: %+v", roleMap)
+ }
+ if !reflect.DeepEqual(nodeRole.AllowedServices, []string{"system://mesh.catalog"}) {
+ t.Errorf("mesh:role:node AllowedServices = %v, want [system://mesh.catalog]", nodeRole.AllowedServices)
+ }
+
+ bindingMap := make(map[string][]string)
+ for _, b := range bindings {
+ bindingMap[b.Role] = b.Members
+ }
+ if !reflect.DeepEqual(bindingMap["mesh:role:node"], []string{"mesh:system:authenticated"}) {
+ t.Errorf("mesh:role:node members = %v, want [mesh:system:authenticated]", bindingMap["mesh:role:node"])
+ }
+ if !reflect.DeepEqual(bindingMap["mesh:role:router"], []string{"group:routers"}) {
+ t.Errorf("mesh:role:router members = %v, want [group:routers]", bindingMap["mesh:role:router"])
+ }
+
+ node, err := store.GetNode(ctx, "12D3KooWNode")
+ if err != nil {
+ t.Fatalf("GetNode: %v", err)
+ }
+ if node.Role != "mesh:role:node" {
+ t.Errorf("node.Role = %q, want mesh:role:node", node.Role)
+ }
+
+ tok, err := store.GetBootstrapToken(ctx, "tok1")
+ if err != nil {
+ t.Fatalf("GetBootstrapToken: %v", err)
+ }
+ if tok.Role != "mesh:role:router" {
+ t.Errorf("tok.Role = %q, want mesh:role:router", tok.Role)
+ }
+
+ egress, err := store.GetEgressDestinations(ctx)
+ if err != nil {
+ t.Fatalf("GetEgressDestinations: %v", err)
+ }
+ if len(egress) != 1 || !reflect.DeepEqual(egress[0].GetServedBy(), []string{"mesh:role:node"}) {
+ t.Errorf("egress ServedBy = %+v, want [mesh:role:node]", egress)
+ }
+}
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index b4000d7c..2360f3a7 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -535,6 +535,29 @@ var migrations = []migration{
postgres: indexMigration,
sqlite: indexMigration,
},
+ {
+ // Migrate persisted roles, bindings, catalog service grants, enrolled
+ // nodes, bootstrap tokens, and egress selectors from the legacy "sam:"
+ // wire prefix to "mesh:".
+ version: 17,
+ postgres: meshPrefixMigration,
+ sqlite: meshPrefixMigration,
+ },
+}
+
+var meshPrefixMigration = []string{
+ `INSERT INTO roles (name, description, created_at)
+ SELECT 'mesh:role:' || SUBSTR(name, 10), description, created_at
+ FROM roles WHERE name LIKE 'sam:role:%'
+ ON CONFLICT (name) DO NOTHING`,
+ `UPDATE role_permissions SET role_name = 'mesh:role:' || SUBSTR(role_name, 10) WHERE role_name LIKE 'sam:role:%'`,
+ `UPDATE role_bindings SET role_name = 'mesh:role:' || SUBSTR(role_name, 10) WHERE role_name LIKE 'sam:role:%'`,
+ `DELETE FROM roles WHERE name LIKE 'sam:role:%'`,
+ `UPDATE role_bindings SET member = 'mesh:system:authenticated' WHERE member = 'sam:system:authenticated'`,
+ `UPDATE role_permissions SET resource_value = 'system://mesh.catalog' WHERE resource_type = 'service' AND resource_value = 'system://sam.catalog'`,
+ `UPDATE nodes SET role = 'mesh:role:' || SUBSTR(role, 10) WHERE role LIKE 'sam:role:%'`,
+ `UPDATE bootstrap_tokens SET role = 'mesh:role:' || SUBSTR(role, 10) WHERE role LIKE 'sam:role:%'`,
+ `UPDATE egress_destinations SET served_by = REPLACE(served_by, '"sam:role:', '"mesh:role:'), config_json = REPLACE(config_json, '"sam:role:', '"mesh:role:') WHERE served_by LIKE '%"sam:role:%' OR config_json LIKE '%"sam:role:%'`,
}
var indexMigration = []string{