Skip to content

Agent Mesh

Configure control plane with PostgreSQL backend and multi-master replication

Antonio Ojeacee1b5d37781
diff --git a/.github/k8s/sam-control-plane-template.yaml b/.github/k8s/sam-control-plane-template.yaml
index 542a6df6..a98c9e1a 100644
--- a/.github/k8s/sam-control-plane-template.yaml
+++ b/.github/k8s/sam-control-plane-template.yaml
@@ -12,13 +12,67 @@ spec:
selector:
app: sam-control-plane-${ENV_NAME}
---
+apiVersion: v1
+kind: Service
+metadata:
+ name: sam-db-${ENV_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ ports:
+ - name: postgres
+ port: 5432
+ targetPort: 5432
+ selector:
+ app: sam-db-${ENV_NAME}
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: sam-db-${ENV_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ serviceName: sam-db-${ENV_NAME}
+ replicas: 1
+ selector:
+ matchLabels:
+ app: sam-db-${ENV_NAME}
+ template:
+ metadata:
+ labels:
+ app: sam-db-${ENV_NAME}
+ spec:
+ containers:
+ - name: postgres
+ image: postgres:16-alpine
+ env:
+ - name: POSTGRES_DB
+ value: sam_mesh
+ - name: POSTGRES_USER
+ value: sam
+ - name: POSTGRES_PASSWORD
+ value: sam-secret-password
+ ports:
+ - containerPort: 5432
+ protocol: TCP
+ volumeMounts:
+ - name: db-data
+ mountPath: /var/lib/postgresql/data
+ volumeClaimTemplates:
+ - metadata:
+ name: db-data
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 5Gi
+---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sam-control-plane-${ENV_NAME}
namespace: ${NAMESPACE}
spec:
- replicas: 1
+ replicas: 2
selector:
matchLabels:
app: sam-control-plane-${ENV_NAME}
@@ -50,8 +104,8 @@ spec:
periodSeconds: 20
args:
- "--bind-address=0.0.0.0:8080"
- - "--db-driver=sqlite"
- - "--db-dsn=/data/control-plane.db"
+ - "--db-driver=postgres"
+ - "--db-dsn=postgres://sam:sam-secret-password@sam-db-${ENV_NAME}:5432/sam_mesh?sslmode=disable"
- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"
- "--policy-file=/etc/sam/policies/policies.yaml"
@@ -66,14 +120,10 @@ spec:
- name: policies-volume
mountPath: /etc/sam/policies
readOnly: true
- - name: db-data
- mountPath: /data
volumes:
- name: policies-volume
configMap:
name: sam-control-plane-policies-${ENV_NAME}
- - name: db-data
- emptyDir: {}
---
apiVersion: v1
kind: ConfigMap
diff --git a/cmd/sam-control-plane/main.go b/cmd/sam-control-plane/main.go
index 9f80bbb5..db8dbeb7 100644
--- a/cmd/sam-control-plane/main.go
+++ b/cmd/sam-control-plane/main.go
@@ -128,7 +128,7 @@ func main() {
rootCmd.Flags().StringVar(&policyFile, "policy-file", "policies.yaml", "Path to policies.yaml (bootstrapping only)")
rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 24*time.Hour, "Key rotation interval (e.g. 24h). 0 disables rotation.")
rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 1*time.Hour, "Key grace period for rotated keys.")
- rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 60*time.Second, "Router lease registration TTL.")
+ rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")
rootCmd.Flags().StringVar(&adminToken, "admin-token", "", "Token for authenticating policy REST API requests")
rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC providers")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 25d7a63f..0b484afe 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -538,7 +538,7 @@ func main() {
runCmd.Flags().StringVar(&tlsCertFlag, "tls-cert", "", "Path to TLS certificate for sidecar API")
runCmd.Flags().StringVar(&tlsKeyFlag, "tls-key", "", "Path to TLS key for sidecar API")
runCmd.Flags().StringVar(&tlsCAFlag, "tls-ca", "", "Path to TLS CA for sidecar API mTLS")
- rootCmd.PersistentFlags().StringVar(&hubAddr, "hub", node.DefaultHubURL, "Hub URL")
+ rootCmd.PersistentFlags().StringVar(&hubAddr, "hub", "", "Hub URL")
rootCmd.PersistentFlags().StringVar(&configFile, "config", node.DefaultConfigFile, "Path to sam-node.yaml configuration file")
rootCmd.PersistentFlags().StringVar(&oidcIssuerFlag, "oidc-issuer", "", "OIDC Issuer URL")
rootCmd.PersistentFlags().StringVar(&deviceAuthURLFlag, "device-auth-url", "", "OIDC Device Authorization URL")
diff --git a/development/kind/10-control-plane.yaml b/development/kind/10-control-plane.yaml
index 08e447d8..b99a7031 100644
--- a/development/kind/10-control-plane.yaml
+++ b/development/kind/10-control-plane.yaml
@@ -10,13 +10,66 @@ spec:
ports:
- { name: http, port: 8080, targetPort: 8080, protocol: TCP }
---
+apiVersion: v1
+kind: Service
+metadata:
+ name: sam-db
+ namespace: ${NAMESPACE}
+spec:
+ type: ClusterIP
+ selector:
+ app: sam-db
+ ports:
+ - { name: postgres, port: 5432, targetPort: 5432, protocol: TCP }
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: sam-db
+ namespace: ${NAMESPACE}
+spec:
+ serviceName: sam-db
+ replicas: 1
+ selector:
+ matchLabels:
+ app: sam-db
+ template:
+ metadata:
+ labels:
+ app: sam-db
+ spec:
+ containers:
+ - name: postgres
+ image: postgres:16-alpine
+ env:
+ - name: POSTGRES_DB
+ value: sam_mesh
+ - name: POSTGRES_USER
+ value: sam
+ - name: POSTGRES_PASSWORD
+ value: sam-secret-password
+ ports:
+ - containerPort: 5432
+ protocol: TCP
+ volumeMounts:
+ - name: db-data
+ mountPath: /var/lib/postgresql/data
+ volumeClaimTemplates:
+ - metadata:
+ name: db-data
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 1Gi
+---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sam-control-plane
namespace: ${NAMESPACE}
spec:
- replicas: 1
+ replicas: 2
selector:
matchLabels:
app: sam-control-plane
@@ -33,8 +86,8 @@ spec:
imagePullPolicy: IfNotPresent
args:
- "--bind-address=0.0.0.0:8080"
- - "--db-driver=sqlite"
- - "--db-dsn=/data/control-plane.db"
+ - "--db-driver=postgres"
+ - "--db-dsn=postgres://sam:sam-secret-password@sam-db:5432/sam_mesh?sslmode=disable"
- "--issuer=${ISSUER}"
- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"
- "--policy-file=/etc/sam/policies/policies.yaml"
@@ -49,14 +102,10 @@ spec:
- name: policies-volume
mountPath: /etc/sam/policies
readOnly: true
- - name: db-data
- mountPath: /data
volumes:
- name: policies-volume
configMap:
name: sam-control-plane-policies
- - name: db-data
- emptyDir: {}
---
# Host-facing access for a locally-built sam-node. Reachable on the host at
# 127.0.0.1:9090 via the control-plane extraPortMappings.
diff --git a/development/kind/run.sh b/development/kind/run.sh
index 5743ec59..71e7742c 100755
--- a/development/kind/run.sh
+++ b/development/kind/run.sh
@@ -123,6 +123,8 @@ for f in "${SCRIPT_DIR}"/00-*.yaml "${SCRIPT_DIR}"/10-*.yaml "${SCRIPT_DIR}"/11-
envsubst '${NAMESPACE} ${ISSUER} ${IMAGE_TAG}' < "$f" | kubectl --context "${KCTX}" apply -f -
done
+echo "== Waiting for database to be ready =="
+kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-db
echo "== Waiting for control plane to be ready =="
kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-control-plane
echo "== Waiting for router to be ready =="
diff --git a/internal/controlplane/config.go b/internal/controlplane/config.go
index 3ae9479b..a9b52f3c 100644
--- a/internal/controlplane/config.go
+++ b/internal/controlplane/config.go
@@ -44,7 +44,7 @@ func (o *Options) Default() {
o.DataSourceName = "control-plane.db"
}
if o.LeaseDuration <= 0 {
- o.LeaseDuration = 60 * time.Second
+ o.LeaseDuration = 15 * time.Minute
}
if o.KeyRotationInterval <= 0 {
o.KeyRotationInterval = 24 * time.Hour
diff --git a/internal/node/node.go b/internal/node/node.go
index 0ccaf3f3..c12252df 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -244,7 +244,6 @@ func NewSamNode(cfg Options) (*SamNode, error) {
reprovideTrigger: make(chan struct{}, 1),
logger: golog.Logger("sam-node"),
}
- node.BiscuitTimeout = 2 * time.Second
var err error
node.rateLimiter, err = NewPeerRateLimiter(RateLimiterSize)
diff --git a/internal/node/options.go b/internal/node/options.go
index 994e8ea7..573b4e2d 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -26,7 +26,6 @@ import (
const (
DefaultMeshName = "public-mesh"
DefaultDiscoveryInterval = "30s"
- DefaultHubURL = "https://bananas.sam-mesh.dev"
DefaultConfigFile = "sam-node.yaml"
DefaultHubConnectTimeout = 5 * time.Second
)
diff --git a/tests/e2e/fixtures/sam-hub.yaml b/tests/e2e/fixtures/sam-hub.yaml
index b97b7e68..efb95a3d 100644
--- a/tests/e2e/fixtures/sam-hub.yaml
+++ b/tests/e2e/fixtures/sam-hub.yaml
@@ -136,6 +136,7 @@ spec:
- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"
- "--policy-file=/etc/sam/policy.yaml"
- "--insecure-skip-tls-verify"
+ - "--lease-duration=15s"
- "--log-level=debug"
volumeMounts:
- name: config-volume
@@ -205,6 +206,7 @@ spec:
- "--jwt-path=/var/run/secrets/tokens/sa-token"
- "--keys-path=/data/router.key"
- "--allow-loopback"
+ - "--lease-renew-interval=5s"
- "--log-level=debug"
volumeMounts:
- name: router-data
diff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.bats
index 07bd5517..72b3d43c 100644
--- a/tests/e2e/key_rotation_test.bats
+++ b/tests/e2e/key_rotation_test.bats
@@ -22,7 +22,7 @@ teardown() {
mesh_start_standalone_hub "${hub_name}" "5s" "2s"
- mesh_wait_for_log "${hub_name}" "PeerID:" 20
+ mesh_wait_for_log "${hub_name}" "Lease renewed successfully" 20
local hub_peer_id
hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)
diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.bats
index 9702bb16..af781c52 100644
--- a/tests/e2e/revocation_test.bats
+++ b/tests/e2e/revocation_test.bats
@@ -21,7 +21,7 @@ teardown() {
mesh_start_standalone_hub "${hub_name}"
- mesh_wait_for_log "${hub_name}" "PeerID:" 20
+ mesh_wait_for_log "${hub_name}" "Lease renewed successfully" 20
local hub_peer_id
hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)