Skip to content

Agent Mesh

use statefulset

Antonio Ojea632da1807353
diff --git a/.github/k8s/sam-hub-template.yaml b/.github/k8s/sam-hub-template.yaml
index 7b69573f..b1c16651 100644
--- a/.github/k8s/sam-hub-template.yaml
+++ b/.github/k8s/sam-hub-template.yaml
@@ -4,42 +4,26 @@ metadata:
name: sam-hub-${ENV_NAME}
namespace: ${NAMESPACE}
spec:
- type: LoadBalancer
- loadBalancerClass: "networking.gke.io/l4-regional-external"
- loadBalancerIP: ${LOAD_BALANCER_IP}
- allocateLoadBalancerNodePorts: false
- sessionAffinity: ClientIP
- externalTrafficPolicy: Local
+ clusterIP: None
ports:
- - port: 443
- targetPort: 9090
+ - port: 4501
+ targetPort: 4501
protocol: TCP
- name: register-https
- selector:
- app: sam-hub-${ENV_NAME}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: sam-hub-internal-${ENV_NAME}
- namespace: ${NAMESPACE}
-spec:
- type: ClusterIP
- sessionAffinity: ClientIP
- ports:
- - port: 443
- targetPort: 9090
- protocol: TCP
- name: https
+ name: p2p-tcp
+ - port: 4501
+ targetPort: 4501
+ protocol: UDP
+ name: p2p-udp
selector:
app: sam-hub-${ENV_NAME}
---
apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
metadata:
name: sam-hub-${ENV_NAME}
namespace: ${NAMESPACE}
spec:
+ serviceName: "sam-hub-${ENV_NAME}"
replicas: 3
selector:
matchLabels:
@@ -53,6 +37,10 @@ spec:
- name: sam-hub
image: ghcr.io/google/sam-hub:${IMAGE_TAG}
env:
+ - name: HOST_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.hostIP
- name: SAM_HUB_KEY
valueFrom:
secretKeyRef:
@@ -63,28 +51,38 @@ spec:
- containerPort: 9090
protocol: TCP
name: metrics
+ - containerPort: 4501
+ hostPort: 4501
+ protocol: TCP
+ name: p2p-tcp
+ - containerPort: 4501
+ hostPort: 4501
+ protocol: UDP
+ name: p2p-udp
readinessProbe:
httpGet:
path: /healthz
port: metrics
- scheme: HTTPS
+ scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /healthz
port: metrics
- scheme: HTTPS
+ scheme: HTTP
initialDelaySeconds: 15
periodSeconds: 20
args:
- - "--external-multiaddr=/dns4/${ENV_NAME}.sam-mesh.dev/tcp/443/wss"
+ - "--listen=/ip4/0.0.0.0/tcp/4501"
+ - "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
+ - "--external-multiaddr=/ip4/$(HOST_IP)/tcp/4501"
+ - "--external-multiaddr=/ip4/$(HOST_IP)/udp/4501/quic-v1"
+ - "--external-multiaddr=/ip4/$(HOST_IP)/tcp/4501/wss"
# 1. Trust Dex for humans/Google/GitHub. 2. Trust GKE for internal Canaries.
- "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
# 1. Dex CLI client ID. 2. Canary projected volume audience.
- "--allowed-audiences=sam-mesh-audience,sam-hub-audience"
- - "--tls-cert-file=/etc/sam/certs/tls.crt"
- - "--tls-key-file=/etc/sam/certs/tls.key"
- "--policy-file=/etc/sam/policies/policies.yaml"
resources:
requests:
@@ -94,16 +92,10 @@ spec:
cpu: 500m
memory: 512Mi
volumeMounts:
- - name: tls-certs
- mountPath: /etc/sam/certs
- readOnly: true
- name: policies-volume
mountPath: /etc/sam/policies
readOnly: true
volumes:
- - name: tls-certs
- secret:
- secretName: hub-sam-mesh-tls
- name: policies-volume
configMap:
name: sam-hub-policies-${ENV_NAME}
@@ -139,6 +131,4 @@ spec:
- port: metrics
interval: 30s
path: /metrics
- scheme: https
- tls:
- insecureSkipVerify: true
+ scheme: http
diff --git a/.github/k8s/sam-node-openclaw-template.yaml b/.github/k8s/sam-node-openclaw-template.yaml
index 305ad3c2..27c1656e 100644
--- a/.github/k8s/sam-node-openclaw-template.yaml
+++ b/.github/k8s/sam-node-openclaw-template.yaml
@@ -85,7 +85,7 @@ spec:
args:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- - "--hub=https://${ENV_NAME}.sam-mesh.dev"
+ - "--hub=http://sam-hub-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:9090"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--api-token=secret-token"
- "--bind-addr=127.0.0.1:8080"
diff --git a/.github/k8s/sam-node-template.yaml b/.github/k8s/sam-node-template.yaml
index 4a4c0ce8..40ad62ed 100644
--- a/.github/k8s/sam-node-template.yaml
+++ b/.github/k8s/sam-node-template.yaml
@@ -45,7 +45,7 @@ spec:
args:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- - "--hub=https://${ENV_NAME}.sam-mesh.dev"
+ - "--hub=http://sam-hub-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:9090"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--api-token=secret-token"
ports:
diff --git a/.github/k8s/sam-node-vllm-template.yaml b/.github/k8s/sam-node-vllm-template.yaml
index f6b99c1d..fba091a2 100644
--- a/.github/k8s/sam-node-vllm-template.yaml
+++ b/.github/k8s/sam-node-vllm-template.yaml
@@ -96,7 +96,7 @@ spec:
args:
- "run"
- "--config=/etc/sam/sam-node.yaml"
- - "--hub=https://${ENV_NAME}.sam-mesh.dev"
+ - "--hub=http://sam-hub-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:9090"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--api-token=secret-token"
- "--bind-addr=127.0.0.1:8080"
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 70b15ce1..ce3336f5 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -198,27 +198,18 @@ jobs:
echo "sam-hub-secret already exists in namespace ${NAMESPACE}. Skipping generation."
fi
- - name: Copy Wildcard TLS Secret to sam-hub
- run: |
- export ENV_NAME="${{ vars.ENV_NAME }}"
- export NAMESPACE="sam-${ENV_NAME}"
- if [[ "${NAMESPACE}" == "sam-hub" ]]; then
- kubectl delete secret hub-sam-mesh-tls -n sam-hub --ignore-not-found
- kubectl get secret hub-sam-mesh-tls -n sam-bananas -o yaml | \
- sed 's/namespace: sam-bananas/namespace: sam-hub/' | \
- kubectl create -f -
- fi
+
- name: Deploy Hub
run: |
print_rollout_diagnostics() {
local namespace="$1"
- local deployment="$2"
+ local resource="$2"
local selector="$3"
- echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..."
+ echo "Collecting diagnostics for ${resource} in namespace ${namespace}..."
- kubectl describe deployment/${deployment} -n ${namespace} || true
+ kubectl describe ${resource} -n ${namespace} || true
kubectl get pods -n ${namespace} -l "${selector}" -o wide || true
kubectl describe pods -n ${namespace} -l "${selector}" || true
kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true
@@ -247,13 +238,13 @@ jobs:
envsubst < .github/k8s/sam-hub-template.yaml | kubectl apply -f -
- kubectl rollout status deployment/sam-hub-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {
+ kubectl rollout status statefulset/sam-hub-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {
echo "Hub Deployment failed!"
- print_rollout_diagnostics "${NAMESPACE}" "sam-hub-${ENV_NAME}" "app=sam-hub-${ENV_NAME}"
+ print_rollout_diagnostics "${NAMESPACE}" "statefulset/sam-hub-${ENV_NAME}" "app=sam-hub-${ENV_NAME}"
echo "Rolling back Hub deployment..."
- kubectl rollout undo deployment/sam-hub-${ENV_NAME} -n ${NAMESPACE}
- kubectl rollout status deployment/sam-hub-${ENV_NAME} -n ${NAMESPACE}
+ kubectl rollout undo statefulset/sam-hub-${ENV_NAME} -n ${NAMESPACE}
+ kubectl rollout status statefulset/sam-hub-${ENV_NAME} -n ${NAMESPACE}
exit 1
}
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index eb81ba04..edb88b51 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -710,12 +710,10 @@ func main() {
// Start key rotation if enabled
h.startRotation(ctx)
- logger.Infof("SAM Hub Online (QUIC + TCP + WS on %s)", bindAddress)
+ logger.Infof("SAM Hub Online (HTTP on %s, P2P on %v)", bindAddress, listenAddrs)
isHubReady.Store(true)
logger.Infof("MeshID: %s", h.MeshID)
logger.Infof("PeerID: %s", h.Host.ID())
-
- logger.Infof("Hub running on P2P transports only.")
<-ctx.Done()
},
}
@@ -727,6 +725,7 @@ func main() {
rootCmd.Flags().StringVar(&oidcIssuer, "issuer", defIssuer, "OIDC Issuer URL")
rootCmd.Flags().StringVar(&clientID, "client-id", os.Getenv("SAM_OIDC_ID"), "OIDC Client ID")
rootCmd.Flags().StringVar(&biscuitHex, "key", os.Getenv("SAM_HUB_KEY"), "Hub Private Key (32-byte Hex)")
+ rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{}, "libp2p Listen Addrs")
rootCmd.Flags().StringSliceVar(&externalMultiaddrs, "external-multiaddr", []string{}, "External multiaddrs to announce")
rootCmd.Flags().StringVar(&meshName, "mesh", DefaultMeshName, "Mesh federation name")
rootCmd.Flags().StringVar(&allowedAudiencesFlag, "allowed-audiences", api.DefaultAudience, "Comma-separated list of allowed OIDC audiences")