Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

ci(deploy): allow ai-gateway deploys to prod and skip rollout when the ECS service is missing (#19470)

Hassieb Pakzada7cfd8c25dfb
diff --git a/.github/workflows/_deploy_ecs_service.yml b/.github/workflows/_deploy_ecs_service.yml
index f4255cce13..5185fe517c 100644
--- a/.github/workflows/_deploy_ecs_service.yml
+++ b/.github/workflows/_deploy_ecs_service.yml
@@ -25,7 +25,6 @@ on:
required: true
jobs:
ecs-deploy:
- if: ${{ inputs.service != 'ai-gateway' || inputs.environment == 'staging' }}
runs-on: blacksmith-4vcpu-ubuntu-2404
environment: ${{ inputs.environment }}
permissions:
@@ -124,14 +123,46 @@ jobs:
--build-arg NEXT_PUBLIC_ASSET_PREFIX=${VARS_NEXT_PUBLIC_ASSET_PREFIX} \
.
docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+ # Temporary until the prod ai-gateway ECS services are bootstrapped by infra
+ # from an image pushed above: a missing ai-gateway service skips the rollout
+ # instead of failing the deploy. Other services always roll out and fail
+ # loudly when their service is missing.
+ - name: Check AWS ECS Service exists
+ id: ecs-service
+ if: ${{ inputs.service == 'ai-gateway' }}
+ env:
+ CLUSTER: ${{ inputs.environment }}-cluster
+ SERVICE: ${{ inputs.environment }}-${{ inputs.service }}
+ IMAGE_TAG: ${{ github.sha }}
+ run: |
+ status="$(aws ecs describe-services \
+ --cluster "$CLUSTER" \
+ --services "$SERVICE" \
+ --query 'services[0].status' \
+ --output text)"
+ if [ "$status" = "ACTIVE" ]; then
+ echo "exists=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ if [ "$status" != "None" ] && [ "$status" != "INACTIVE" ]; then
+ echo "::error::ECS service $SERVICE in cluster $CLUSTER has unexpected status $status."
+ exit 1
+ fi
+ echo "exists=false" >> "$GITHUB_OUTPUT"
+ message="ECS service $SERVICE in cluster $CLUSTER does not exist (status: $status). Pushed image tag $IMAGE_TAG and skipped the ECS rollout."
+ echo "::notice title=ECS rollout skipped::$message"
+ echo "### ECS rollout skipped" >> "$GITHUB_STEP_SUMMARY"
+ echo "$message" >> "$GITHUB_STEP_SUMMARY"
- name: Render AWS ECS Task Definition
id: render-task-definition
+ if: ${{ inputs.service != 'ai-gateway' || steps.ecs-service.outputs.exists == 'true' }}
uses: aws-actions/amazon-ecs-render-task-definition@138c24f321fdbdf7edee4a685519d253cae2cdea # v1.9.0
with:
container-name: ${{ inputs.service }}
image: ${{ steps.login-ecr.outputs.registry }}/${{ inputs.service == 'ai-gateway' && 'ai-gateway' || steps.split.outputs._0 }}:${{ github.sha }}
task-definition-family: ${{ inputs.environment }}-${{ inputs.service }}
- name: Update AWS ECS Service
+ if: ${{ inputs.service != 'ai-gateway' || steps.ecs-service.outputs.exists == 'true' }}
uses: aws-actions/amazon-ecs-deploy-task-definition@c465972ecbd160473f22e683363b422a5412a3de # v2.6.3
with:
task-definition: ${{ steps.render-task-definition.outputs.task-definition }}
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 5656619195..8239fbedc9 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -32,7 +32,8 @@ permissions: {}
concurrency:
# Support concurrent `push` and `workflow_dispatch`` actions
- group: deploy-${{ github.event_name }}-${{ github.ref }}
+ # Dispatches to different environments/services must not cancel each other.
+ group: deploy-${{ github.event_name }}-${{ github.ref }}${{ github.event_name == 'workflow_dispatch' && format('-{0}-{1}', inputs.environment, inputs.service) || '' }}
cancel-in-progress: true
name: Deploy to ECS
@@ -48,6 +49,8 @@ jobs:
with:
script: |
const services = ["web", "web-ingestion", "web-iso", "worker", "worker-cpu"];
+ // Prod ai-gateway deploys run only via explicit dispatch until its
+ // prod ECS services are bootstrapped.
const isStaging = context.eventName === "workflow_dispatch"
? context.payload.inputs.environment === "staging"
: context.ref === "refs/heads/main";
@@ -55,9 +58,6 @@ jobs:
if (context.eventName === "workflow_dispatch") {
const service = context.payload.inputs.service;
- if (service === "ai-gateway" && !isStaging) {
- throw new Error("AI Gateway deployments are only enabled in staging.");
- }
return JSON.stringify(service === "all" ? services : [service]);
}
if (context.eventName === "push") {