Skip to content

Agent Mesh

deploy vllm

Antonio Ojea55ce0fae8641
diff --git a/.github/k8s/sam-node-vllm-template.yaml b/.github/k8s/sam-node-vllm-template.yaml
new file mode 100644
index 00000000..aa8547c3
--- /dev/null
+++ b/.github/k8s/sam-node-vllm-template.yaml
@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sam-canary-vllm-config-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+data:
+ sam-node.yaml: |
+ version: "v1alpha1"
+ attenuation:
+ policies: []
+ checks: []
+ rules: []
+ services:
+ - type: inference
+ name: vllm-tpu
+ description: "vLLM TPU Inference Service"
+ target_url: "http://localhost:8000/v1"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: vllm-canary-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: vllm-canary-${ENV_NAME}
+ template:
+ metadata:
+ labels:
+ app: vllm-canary-${ENV_NAME}
+ spec:
+ serviceAccountName: sam-node-sa
+ nodeSelector:
+ cloud.google.com/gke-tpu-accelerator: tpu-v5-lite-podslice
+ tolerations:
+ - key: "google.com/tpu"
+ operator: "Exists"
+ effect: "NoSchedule"
+ containers:
+ - name: vllm-tpu
+ image: vllm/vllm-tpu:latest
+ args: ["--model", "google/gemma-4-12B-it", "--port", "8000"]
+ ports:
+ - containerPort: 8000
+ name: http
+ env:
+ - name: HF_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: vllm-secret-${ENV_NAME}
+ key: hf-token
+ resources:
+ requests:
+ cpu: 100m
+ memory: 2Gi
+ google.com/tpu: "1"
+ limits:
+ cpu: 1000m
+ memory: 8Gi
+ google.com/tpu: "1"
+ volumeMounts:
+ - name: dshm
+ mountPath: /dev/shm
+ - name: sam-node
+ image: ghcr.io/google/sam-node:${IMAGE_TAG}
+ args:
+ - "run"
+ - "--config=/etc/sam/sam-node.yaml"
+ - "--hub=https://${ENV_NAME}.sam-mesh.dev"
+ - "--jwt-path=/var/run/secrets/tokens/sam-token"
+ - "--api-token=secret-token"
+ - "--bind-addr=127.0.0.1:8080"
+ ports:
+ - containerPort: 8080
+ resources:
+ requests:
+ cpu: 50m
+ memory: 64Mi
+ limits:
+ cpu: 200m
+ memory: 256Mi
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/sam
+ - name: sam-token
+ mountPath: /var/run/secrets/tokens
+ readOnly: true
+ volumes:
+ - name: config-volume
+ configMap:
+ name: sam-canary-vllm-config-${ENV_NAME}
+ - name: dshm
+ emptyDir:
+ medium: Memory
+ sizeLimit: 10Gi
+ - name: sam-token
+ projected:
+ sources:
+ - serviceAccountToken:
+ path: sam-token
+ expirationSeconds: 3600
+ audience: "sam-hub-audience"
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index dcc8d382..70b15ce1 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -286,6 +286,7 @@ jobs:
export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
export IMAGE_TAG="${{ env.IMAGE_TAG }}"
export GEMINI_API_KEY="${{ secrets.GEMINI_API_KEY }}"
+ export HF_TOKEN="${{ secrets.HF_TOKEN }}"
# Provision the openclaw-secret, preserving the gateway-token if it exists
GATEWAY_TOKEN=$(kubectl get secret openclaw-secret-${ENV_NAME} -n ${CANARY_NAMESPACE} -o jsonpath="{.data.gateway-token}" 2>/dev/null | base64 -d || openssl rand -hex 16)
@@ -295,8 +296,14 @@ jobs:
--from-literal=gemini-api-key="${GEMINI_API_KEY}" \
--dry-run=client -o yaml | kubectl apply -f -
+ kubectl create secret generic vllm-secret-${ENV_NAME} \
+ --namespace=${CANARY_NAMESPACE} \
+ --from-literal=hf-token="${HF_TOKEN}" \
+ --dry-run=client -o yaml | kubectl apply -f -
+
envsubst < .github/k8s/sam-node-template.yaml | kubectl apply -f -
envsubst < .github/k8s/sam-node-openclaw-template.yaml | kubectl apply -f -
+ envsubst < .github/k8s/sam-node-vllm-template.yaml | kubectl apply -f -
kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
@@ -318,3 +325,13 @@ jobs:
kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
exit 1
}
+
+ kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || {
+ echo "vLLM Canary Deployment failed!"
+ print_rollout_diagnostics "${CANARY_NAMESPACE}" "vllm-canary-${ENV_NAME}" "app=vllm-canary-${ENV_NAME}"
+
+ echo "Rolling back vLLM Canary deployment..."
+ kubectl rollout undo deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
+ kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
+ exit 1
+ }