Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
Configure control plane with PostgreSQL backend and multi-master replication
Antonio Ojeacee1b5d37781
diff --git a/.github/k8s/sam-control-plane-template.yaml b/.github/k8s/sam-control-plane-template.yamlindex 542a6df6..a98c9e1a 100644--- a/.github/k8s/sam-control-plane-template.yaml+++ b/.github/k8s/sam-control-plane-template.yaml@@ -12,13 +12,67 @@ spec:selector:app: sam-control-plane-${ENV_NAME}---+apiVersion: v1+kind: Service+metadata:+ name: sam-db-${ENV_NAME}+ namespace: ${NAMESPACE}+spec:+ ports:+ - name: postgres+ port: 5432+ targetPort: 5432+ selector:+ app: sam-db-${ENV_NAME}+---+apiVersion: apps/v1+kind: StatefulSet+metadata:+ name: sam-db-${ENV_NAME}+ namespace: ${NAMESPACE}+spec:+ serviceName: sam-db-${ENV_NAME}+ replicas: 1+ selector:+ matchLabels:+ app: sam-db-${ENV_NAME}+ template:+ metadata:+ labels:+ app: sam-db-${ENV_NAME}+ spec:+ containers:+ - name: postgres+ image: postgres:16-alpine+ env:+ - name: POSTGRES_DB+ value: sam_mesh+ - name: POSTGRES_USER+ value: sam+ - name: POSTGRES_PASSWORD+ value: sam-secret-password+ ports:+ - containerPort: 5432+ protocol: TCP+ volumeMounts:+ - name: db-data+ mountPath: /var/lib/postgresql/data+ volumeClaimTemplates:+ - metadata:+ name: db-data+ spec:+ accessModes: [ "ReadWriteOnce" ]+ resources:+ requests:+ storage: 5Gi+---apiVersion: apps/v1kind: Deploymentmetadata:name: sam-control-plane-${ENV_NAME}namespace: ${NAMESPACE}spec:- replicas: 1+ replicas: 2selector:matchLabels:app: sam-control-plane-${ENV_NAME}@@ -50,8 +104,8 @@ spec:periodSeconds: 20args:- "--bind-address=0.0.0.0:8080"- - "--db-driver=sqlite"- - "--db-dsn=/data/control-plane.db"+ - "--db-driver=postgres"+ - "--db-dsn=postgres://sam:sam-secret-password@sam-db-${ENV_NAME}:5432/sam_mesh?sslmode=disable"- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"- "--policy-file=/etc/sam/policies/policies.yaml"@@ -66,14 +120,10 @@ spec:- name: policies-volumemountPath: /etc/sam/policiesreadOnly: true- - name: db-data- mountPath: /datavolumes:- name: policies-volumeconfigMap:name: sam-control-plane-policies-${ENV_NAME}- - name: db-data- emptyDir: {}---apiVersion: v1kind: ConfigMapdiff --git a/cmd/sam-control-plane/main.go b/cmd/sam-control-plane/main.goindex 9f80bbb5..db8dbeb7 100644--- a/cmd/sam-control-plane/main.go+++ b/cmd/sam-control-plane/main.go@@ -128,7 +128,7 @@ func main() {rootCmd.Flags().StringVar(&policyFile, "policy-file", "policies.yaml", "Path to policies.yaml (bootstrapping only)")rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 24*time.Hour, "Key rotation interval (e.g. 24h). 0 disables rotation.")rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 1*time.Hour, "Key grace period for rotated keys.")- rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 60*time.Second, "Router lease registration TTL.")+ rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")rootCmd.Flags().StringVar(&adminToken, "admin-token", "", "Token for authenticating policy REST API requests")rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC providers")rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex 25d7a63f..0b484afe 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -538,7 +538,7 @@ func main() {runCmd.Flags().StringVar(&tlsCertFlag, "tls-cert", "", "Path to TLS certificate for sidecar API")runCmd.Flags().StringVar(&tlsKeyFlag, "tls-key", "", "Path to TLS key for sidecar API")runCmd.Flags().StringVar(&tlsCAFlag, "tls-ca", "", "Path to TLS CA for sidecar API mTLS")- rootCmd.PersistentFlags().StringVar(&hubAddr, "hub", node.DefaultHubURL, "Hub URL")+ rootCmd.PersistentFlags().StringVar(&hubAddr, "hub", "", "Hub URL")rootCmd.PersistentFlags().StringVar(&configFile, "config", node.DefaultConfigFile, "Path to sam-node.yaml configuration file")rootCmd.PersistentFlags().StringVar(&oidcIssuerFlag, "oidc-issuer", "", "OIDC Issuer URL")rootCmd.PersistentFlags().StringVar(&deviceAuthURLFlag, "device-auth-url", "", "OIDC Device Authorization URL")diff --git a/development/kind/10-control-plane.yaml b/development/kind/10-control-plane.yamlindex 08e447d8..b99a7031 100644--- a/development/kind/10-control-plane.yaml+++ b/development/kind/10-control-plane.yaml@@ -10,13 +10,66 @@ spec:ports:- { name: http, port: 8080, targetPort: 8080, protocol: TCP }---+apiVersion: v1+kind: Service+metadata:+ name: sam-db+ namespace: ${NAMESPACE}+spec:+ type: ClusterIP+ selector:+ app: sam-db+ ports:+ - { name: postgres, port: 5432, targetPort: 5432, protocol: TCP }+---+apiVersion: apps/v1+kind: StatefulSet+metadata:+ name: sam-db+ namespace: ${NAMESPACE}+spec:+ serviceName: sam-db+ replicas: 1+ selector:+ matchLabels:+ app: sam-db+ template:+ metadata:+ labels:+ app: sam-db+ spec:+ containers:+ - name: postgres+ image: postgres:16-alpine+ env:+ - name: POSTGRES_DB+ value: sam_mesh+ - name: POSTGRES_USER+ value: sam+ - name: POSTGRES_PASSWORD+ value: sam-secret-password+ ports:+ - containerPort: 5432+ protocol: TCP+ volumeMounts:+ - name: db-data+ mountPath: /var/lib/postgresql/data+ volumeClaimTemplates:+ - metadata:+ name: db-data+ spec:+ accessModes: [ "ReadWriteOnce" ]+ resources:+ requests:+ storage: 1Gi+---apiVersion: apps/v1kind: Deploymentmetadata:name: sam-control-planenamespace: ${NAMESPACE}spec:- replicas: 1+ replicas: 2selector:matchLabels:app: sam-control-plane@@ -33,8 +86,8 @@ spec:imagePullPolicy: IfNotPresentargs:- "--bind-address=0.0.0.0:8080"- - "--db-driver=sqlite"- - "--db-dsn=/data/control-plane.db"+ - "--db-driver=postgres"+ - "--db-dsn=postgres://sam:sam-secret-password@sam-db:5432/sam_mesh?sslmode=disable"- "--issuer=${ISSUER}"- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"- "--policy-file=/etc/sam/policies/policies.yaml"@@ -49,14 +102,10 @@ spec:- name: policies-volumemountPath: /etc/sam/policiesreadOnly: true- - name: db-data- mountPath: /datavolumes:- name: policies-volumeconfigMap:name: sam-control-plane-policies- - name: db-data- emptyDir: {}---# Host-facing access for a locally-built sam-node. Reachable on the host at# 127.0.0.1:9090 via the control-plane extraPortMappings.diff --git a/development/kind/run.sh b/development/kind/run.shindex 5743ec59..71e7742c 100755--- a/development/kind/run.sh+++ b/development/kind/run.sh@@ -123,6 +123,8 @@ for f in "${SCRIPT_DIR}"/00-*.yaml "${SCRIPT_DIR}"/10-*.yaml "${SCRIPT_DIR}"/11-envsubst '${NAMESPACE} ${ISSUER} ${IMAGE_TAG}' < "$f" | kubectl --context "${KCTX}" apply -f -done+echo "== Waiting for database to be ready =="+kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-dbecho "== Waiting for control plane to be ready =="kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-control-planeecho "== Waiting for router to be ready =="diff --git a/internal/controlplane/config.go b/internal/controlplane/config.goindex 3ae9479b..a9b52f3c 100644--- a/internal/controlplane/config.go+++ b/internal/controlplane/config.go@@ -44,7 +44,7 @@ func (o *Options) Default() {o.DataSourceName = "control-plane.db"}if o.LeaseDuration <= 0 {- o.LeaseDuration = 60 * time.Second+ o.LeaseDuration = 15 * time.Minute}if o.KeyRotationInterval <= 0 {o.KeyRotationInterval = 24 * time.Hourdiff --git a/internal/node/node.go b/internal/node/node.goindex 0ccaf3f3..c12252df 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -244,7 +244,6 @@ func NewSamNode(cfg Options) (*SamNode, error) {reprovideTrigger: make(chan struct{}, 1),logger: golog.Logger("sam-node"),}- node.BiscuitTimeout = 2 * time.Secondvar err errornode.rateLimiter, err = NewPeerRateLimiter(RateLimiterSize)diff --git a/internal/node/options.go b/internal/node/options.goindex 994e8ea7..573b4e2d 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -26,7 +26,6 @@ import (const (DefaultMeshName = "public-mesh"DefaultDiscoveryInterval = "30s"- DefaultHubURL = "https://bananas.sam-mesh.dev"DefaultConfigFile = "sam-node.yaml"DefaultHubConnectTimeout = 5 * time.Second)diff --git a/tests/e2e/fixtures/sam-hub.yaml b/tests/e2e/fixtures/sam-hub.yamlindex b97b7e68..efb95a3d 100644--- a/tests/e2e/fixtures/sam-hub.yaml+++ b/tests/e2e/fixtures/sam-hub.yaml@@ -136,6 +136,7 @@ spec:- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"- "--policy-file=/etc/sam/policy.yaml"- "--insecure-skip-tls-verify"+ - "--lease-duration=15s"- "--log-level=debug"volumeMounts:- name: config-volume@@ -205,6 +206,7 @@ spec:- "--jwt-path=/var/run/secrets/tokens/sa-token"- "--keys-path=/data/router.key"- "--allow-loopback"+ - "--lease-renew-interval=5s"- "--log-level=debug"volumeMounts:- name: router-datadiff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.batsindex 07bd5517..72b3d43c 100644--- a/tests/e2e/key_rotation_test.bats+++ b/tests/e2e/key_rotation_test.bats@@ -22,7 +22,7 @@ teardown() {mesh_start_standalone_hub "${hub_name}" "5s" "2s"- mesh_wait_for_log "${hub_name}" "PeerID:" 20+ mesh_wait_for_log "${hub_name}" "Lease renewed successfully" 20local hub_peer_idhub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.batsindex 9702bb16..af781c52 100644--- a/tests/e2e/revocation_test.bats+++ b/tests/e2e/revocation_test.bats@@ -21,7 +21,7 @@ teardown() {mesh_start_standalone_hub "${hub_name}"- mesh_wait_for_log "${hub_name}" "PeerID:" 20+ mesh_wait_for_log "${hub_name}" "Lease renewed successfully" 20local hub_peer_idhub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)