Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(ecs): track deployment health and roll back failures (#1783)

Johannes Grumböckf79600537d39
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index bbdd256a..04f6b3ba 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -7036,10 +7036,18 @@ def _ecs_task_def_delete(physical_id, props):
_ecs._task_defs.pop(td_key, None)
+def _ecs_deployment_configuration(props):
+ """Translate AWS::ECS::Service deployment configuration to ECS API form."""
+ configuration = props.get("DeploymentConfiguration")
+ if configuration is None:
+ return None
+ return _pascal_to_camel(configuration)
+
+
def _ecs_service_create(logical_id, props, stack_name):
name = props.get("ServiceName", f"{stack_name}-{logical_id}")
cluster = props.get("Cluster", "default")
- _ecs._create_service({
+ request = {
"serviceName": name,
"cluster": cluster,
"taskDefinition": props.get("TaskDefinition", ""),
@@ -7048,11 +7056,54 @@ def _ecs_service_create(logical_id, props, stack_name):
"loadBalancers": props.get("LoadBalancers", []),
"networkConfiguration": props.get("NetworkConfiguration", {}),
"tags": [{"key": t["Key"], "value": t["Value"]} for t in props.get("Tags", [])],
- })
+ }
+ deployment_configuration = _ecs_deployment_configuration(props)
+ if deployment_configuration is not None:
+ request["deploymentConfiguration"] = deployment_configuration
+ _ecs._create_service(request)
arn = f"arn:aws:ecs:{get_region()}:{get_account_id()}:service/{cluster}/{name}"
return arn, {"ServiceArn": arn, "Name": name}
+def _ecs_service_update(physical_id, old_props, new_props, stack_name):
+ """Apply mutable AWS::ECS::Service properties through UpdateService."""
+ cluster = new_props.get("Cluster", old_props.get("Cluster", "default"))
+ name = new_props.get("ServiceName") or physical_id.rsplit("/", 1)[-1]
+ request = {"cluster": cluster, "service": name}
+ property_map = {
+ "TaskDefinition": "taskDefinition",
+ "DesiredCount": "desiredCount",
+ "NetworkConfiguration": "networkConfiguration",
+ "LoadBalancers": "loadBalancers",
+ "HealthCheckGracePeriodSeconds": "healthCheckGracePeriodSeconds",
+ "EnableExecuteCommand": "enableExecuteCommand",
+ "PlatformVersion": "platformVersion",
+ }
+ for cf_property, ecs_property in property_map.items():
+ if new_props.get(cf_property) != old_props.get(cf_property):
+ request[ecs_property] = new_props.get(cf_property)
+
+ if (new_props.get("DeploymentConfiguration")
+ != old_props.get("DeploymentConfiguration")):
+ deployment_configuration = _ecs_deployment_configuration(new_props)
+ if deployment_configuration is None:
+ deployment_configuration = {
+ "maximumPercent": 200,
+ "minimumHealthyPercent": 100,
+ "deploymentCircuitBreaker": {"enable": False, "rollback": False},
+ }
+ request["deploymentConfiguration"] = deployment_configuration
+
+ # The resource engine only calls us when properties changed. Some ECS
+ # properties are replacement-only and are intentionally left to their
+ # existing semantics; these are the fields UpdateService can apply in
+ # place and that MiniStack currently models.
+ response = _ecs._update_service(request)
+ if response[0] >= 400:
+ raise ValueError(f"AWS::ECS::Service update failed: {response[2]!r}")
+ return physical_id, {"ServiceArn": physical_id, "Name": name}
+
+
def _ecs_service_delete(physical_id, props):
cluster = props.get("Cluster", "default")
name = props.get("ServiceName", "")
@@ -10895,7 +10946,11 @@ _RESOURCE_HANDLERS = {
"AWS::EC2::SubnetRouteTableAssociation": {"create": _ec2_subnet_rtb_assoc_create, "delete": _ec2_subnet_rtb_assoc_delete},
"AWS::ECS::Cluster": {"create": _ecs_cluster_create, "delete": _ecs_cluster_delete},
"AWS::ECS::TaskDefinition": {"create": _ecs_task_def_create, "delete": _ecs_task_def_delete},
- "AWS::ECS::Service": {"create": _ecs_service_create, "delete": _ecs_service_delete},
+ "AWS::ECS::Service": {
+ "create": _ecs_service_create,
+ "update": _ecs_service_update,
+ "delete": _ecs_service_delete,
+ },
"AWS::EC2::LaunchTemplate": {"create": _ec2_launch_template_create, "delete": _ec2_launch_template_delete},
"AWS::ElasticLoadBalancingV2::LoadBalancer": {
"create": _elbv2_load_balancer_create,
diff --git a/ministack/services/ecs.py b/ministack/services/ecs.py
index 49c11116..414e989e 100644
--- a/ministack/services/ecs.py
+++ b/ministack/services/ecs.py
@@ -28,6 +28,7 @@ Container execution: if Docker socket is available, RunTask actually runs contai
import copy
import json
import logging
+import math
import os
import secrets
import threading
@@ -35,7 +36,7 @@ import time
from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.concurrency import resource_lock, run_reentrant
+from ministack.core.concurrency import resource_lock, run_reentrant, spawn_background
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
@@ -97,6 +98,11 @@ _ECS_REAP_INTERVAL = int(os.environ.get("ECS_REAP_INTERVAL_SECONDS", "60"))
# containers for restored services.
_ECS_RESTORE_RECONCILE_DELAY = float(
os.environ.get("ECS_RESTORE_RECONCILE_DELAY_SECONDS", "2"))
+# A container that merely reaches RUNNING is not immediately a steady service
+# deployment: it can exit during its process startup. Keep a short window
+# before completing a rollout so the lifecycle watcher can report that exit.
+_ECS_DEPLOYMENT_STEADY_DELAY = float(
+ os.environ.get("ECS_DEPLOYMENT_STEADY_DELAY_SECONDS", "1"))
_ecs_reaper_started = False
_ecs_reaper_lock = threading.Lock()
@@ -879,82 +885,266 @@ def _refresh_service_state(cluster_name, group):
cluster_arn = svc.get("clusterArn", "")
running = 0
pending = 0
+ deployment_counts = {
+ dep["id"]: {"running": 0, "pending": 0}
+ for dep in svc.get("deployments", [])
+ }
for task in _tasks.values():
if task.get("group") != group or task.get("clusterArn") != cluster_arn:
continue
if task.get("lastStatus") == "RUNNING":
running += 1
+ state = "running"
elif task.get("lastStatus") == "PENDING":
pending += 1
+ state = "pending"
+ else:
+ continue
+ deployment_id = task.get("_deployment_id")
+ if deployment_id in deployment_counts:
+ deployment_counts[deployment_id][state] += 1
svc["runningCount"] = running
svc["pendingCount"] = pending
- _refresh_service_deployments(cluster_name, svc)
+ for dep in svc.get("deployments", []):
+ counts = deployment_counts.get(dep["id"], {})
+ dep["runningCount"] = counts.get("running", 0)
+ dep["pendingCount"] = counts.get("pending", 0)
+ primary = _primary_deployment(svc)
+ if primary and primary.get("rolloutState") == "COMPLETED":
+ # A nested _stop_task refresh runs while a successful rollout is
+ # draining its old tasks. Collapse old deployments here as well as in
+ # the completion path so the final response cannot retain a drained
+ # ACTIVE deployment merely because that nested refresh won the race.
+ svc["deployments"] = [
+ dep for dep in svc.get("deployments", [])
+ if dep.get("status") == "PRIMARY"
+ or dep.get("rolloutState") == "FAILED"
+ or deployment_counts.get(dep.get("id"), {}).get("running", 0)
+ or deployment_counts.get(dep.get("id"), {}).get("pending", 0)
+ ]
_sync_service_targets(cluster_name, svc)
-def _refresh_service_deployments(cluster_name, svc):
- """Reconcile per-deployment counts and retire a completed old deployment."""
- deployments = svc.get("deployments") or []
- if not deployments:
- return
+def _primary_deployment(svc):
+ return next(
+ (dep for dep in svc.get("deployments", [])
+ if dep.get("status") == "PRIMARY"),
+ None,
+ )
- cluster_arn = svc.get("clusterArn", "")
- group = f"service:{svc.get('serviceName', '')}"
- service_tasks = [
- task for task in _tasks.values()
- if task.get("group") == group
- and task.get("clusterArn") == cluster_arn
- ]
- for deployment in deployments:
- td_arn = deployment.get("taskDefinition")
- matching = [task for task in service_tasks
- if task.get("taskDefinitionArn") == td_arn]
- deployment["runningCount"] = sum(
- task.get("lastStatus") == "RUNNING" for task in matching
- )
- deployment["pendingCount"] = sum(
- task.get("lastStatus") == "PENDING" for task in matching
+
+def _deployment_for_task(svc, task):
+ deployment_id = task.get("_deployment_id")
+ if deployment_id:
+ return next(
+ (dep for dep in svc.get("deployments", [])
+ if dep.get("id") == deployment_id),
+ None,
)
+ # Tasks persisted before deployment IDs were recorded still need to be
+ # attributed when they stop after a restart.
+ return next(
+ (dep for dep in svc.get("deployments", [])
+ if dep.get("taskDefinition") == task.get("taskDefinitionArn")),
+ None,
+ )
- primary = deployments[0]
- primary_desired = primary.get("desiredCount", svc.get("desiredCount", 0))
- healthy = primary.get("runningCount", 0) >= primary_desired
- if len(deployments) > 1 and not healthy:
- primary["rolloutState"] = "IN_PROGRESS"
- primary["rolloutStateReason"] = ""
+
+def _circuit_breaker_threshold(svc):
+ """Return the ECS failure threshold for the service's current size."""
+ breaker = (svc.get("deploymentConfiguration") or {}).get(
+ "deploymentCircuitBreaker") or {}
+ config = breaker.get("thresholdConfiguration") or {}
+ threshold_type = config.get("type", "BOUNDED_PERCENT")
+ value = config.get("value", 50)
+ if threshold_type == "COUNT":
+ return max(1, int(value))
+ calculated = math.ceil(float(value) * int(svc.get("desiredCount", 0)) / 100)
+ if threshold_type == "UNBOUNDED_PERCENT":
+ return max(1, calculated)
+ return min(200, max(3, calculated))
+
+
+def _complete_service_deployment(cluster_name, svc_key):
+ """Complete a stable PRIMARY rollout, then drain its old deployment.
+
+ A completed deployment is the rollback point. Therefore old tasks stay
+ alive while the new deployment is IN_PROGRESS; only a stable new deployment
+ drains them. This is also what makes a circuit-breaker rollback able to
+ restore the previous revision rather than re-create deleted deployment data.
+ """
+ svc = _services.get(svc_key)
+ if not svc or svc.get("status") != "ACTIVE":
return
- if not healthy:
+
+ svc_name = svc["serviceName"]
+ with resource_lock("ecs-service", svc_key):
+ primary = _primary_deployment(svc)
+ if (not primary or primary.get("rolloutState") != "IN_PROGRESS"
+ or primary.get("runningCount", 0) < svc.get("desiredCount", 0)):
+ return
+ primary["rolloutState"] = "COMPLETED"
+ primary["rolloutStateReason"] = "ECS deployment completed."
+ primary["updatedAt"] = _iso()
+ stale_task_arns = [
+ arn for arn, task in _tasks.items()
+ if task.get("group") == f"service:{svc['serviceName']}"
+ and task.get("clusterArn") == svc.get("clusterArn")
+ and task.get("_deployment_id") != primary.get("id")
+ and task.get("lastStatus") in _PRE_STOP_STATUSES
+ ]
+
+ for task_arn in stale_task_arns:
+ _stop_task({"task": task_arn, "cluster": cluster_name,
+ "reason": "Deployment completed"})
+
+ with resource_lock("ecs-service", svc_key):
+ # ECS no longer returns a deployment once it has no live tasks. Keep
+ # a FAILED deployment, however: callers need its failure diagnostics.
+ svc = _services.get(svc_key)
+ if svc:
+ svc["deployments"] = [
+ dep for dep in svc.get("deployments", [])
+ if dep.get("status") == "PRIMARY"
+ or dep.get("rolloutState") == "FAILED"
+ or any(task.get("_deployment_id") == dep.get("id")
+ and task.get("lastStatus") in _PRE_STOP_STATUSES
+ for task in _tasks.values())
+ ]
+ _refresh_service_state(cluster_name, f"service:{svc_name}")
+
+
+def _record_service_task_healthy(svc_key, task):
+ """Apply ``resetOnHealthyTask`` after a task survives startup.
+
+ A Docker container can report RUNNING and then exit a few milliseconds
+ later, so RUNNING by itself is not the circuit breaker's healthy signal in
+ MiniStack. The caller invokes this only after the same stability window
+ used to complete a rollout.
+ """
+ with resource_lock("ecs-service", svc_key):
+ svc = _services.get(svc_key)
+ if not svc:
+ return
+ deployment = _deployment_for_task(svc, task)
+ primary = _primary_deployment(svc)
+ breaker = (svc.get("deploymentConfiguration") or {}).get(
+ "deploymentCircuitBreaker") or {}
+ if (deployment is primary and deployment
+ and deployment.get("rolloutState") == "IN_PROGRESS"
+ and breaker.get("enable", False)
+ and breaker.get("resetOnHealthyTask", True)):
+ deployment["failedTasks"] = 0
+ deployment["updatedAt"] = _iso()
+
+
+def _schedule_service_deployment_completion(
+ cluster_name, svc_key, account_id, region, healthy_task=None):
+ """Check a Docker-backed service rollout after its startup grace window."""
+ if not cluster_name or "/" not in svc_key:
return
+ group = f"service:{svc_key.split('/', 1)[1]}"
+
+ def _check():
+ time.sleep(_ECS_DEPLOYMENT_STEADY_DELAY)
+ with request_scope(account_id, region):
+ if _services.get(svc_key, {}).get("status") == "ACTIVE":
+ if (healthy_task and healthy_task.get("lastStatus") == "RUNNING"):
+ _record_service_task_healthy(svc_key, healthy_task)
+ _refresh_service_state(cluster_name, group)
+ _complete_service_deployment(cluster_name, svc_key)
+
+ spawn_background(
+ _check,
+ thread_name=("ministack-ecs-deployment-"
+ f"{svc_key.rsplit('/', 1)[-1][:8]}"),
+ )
- primary["rolloutState"] = "COMPLETED"
- primary["rolloutStateReason"] = "ECS deployment completed."
- primary_td = primary.get("taskDefinition")
- for deployment in deployments[1:]:
- deployment["desiredCount"] = 0
- stale = [
- task for task in service_tasks
- if task.get("taskDefinitionArn") != primary_td
- and task.get("lastStatus") in _PRE_STOP_STATUSES
- ]
- for task in stale:
- _stop_task({
- "task": task["taskArn"],
- "cluster": cluster_name,
- "reason": "ECS deployment completed",
- })
- # _stop_task refreshes this service recursively, so only live tasks can
- # keep an old deployment in the response.
- for deployment in list(deployments[1:]):
- td_arn = deployment.get("taskDefinition")
- if not any(
- task.get("taskDefinitionArn") == td_arn
- and task.get("group") == group
- and task.get("clusterArn") == cluster_arn
- and task.get("lastStatus") in _PRE_STOP_STATUSES
- for task in _tasks.values()
- ):
- deployments.remove(deployment)
+def _record_service_task_failure(task):
+ """Feed a natural task/startup failure into its deployment circuit breaker."""
+ group = task.get("group", "")
+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))
+ if not cluster_name or not group.startswith("service:"):
+ return
+ svc_key = f"{cluster_name}/{group.split(':', 1)[1]}"
+ should_reconcile = False
+ schedule_completion = False
+
+ with resource_lock("ecs-service", svc_key):
+ svc = _services.get(svc_key)
+ if not svc or svc.get("status") != "ACTIVE":
+ return
+ deployment = _deployment_for_task(svc, task)
+ primary = _primary_deployment(svc)
+ breaker = (svc.get("deploymentConfiguration") or {}).get(
+ "deploymentCircuitBreaker") or {}
+ is_rolling = (svc.get("deploymentController") or {}).get("type", "ECS") == "ECS"
+
+ # Only the deployment being evaluated is allowed to accumulate task
+ # failures. An old task stopped during a successful drain must never
+ # poison the replacement deployment.
+ if (deployment is not primary
+ or not deployment
+ or deployment.get("rolloutState") != "IN_PROGRESS"
+ or not is_rolling
+ or not breaker.get("enable", False)):
+ should_reconcile = deployment is primary and bool(deployment)
+ else:
+ deployment["failedTasks"] = int(deployment.get("failedTasks", 0)) + 1
+ deployment["updatedAt"] = _iso()
+ if deployment["failedTasks"] < _circuit_breaker_threshold(svc):
+ should_reconcile = True
+ else:
+ deployment["rolloutState"] = "FAILED"
+ deployment["rolloutStateReason"] = (
+ "ECS deployment circuit breaker was triggered after "
+ f"{deployment['failedTasks']} consecutive task failures."
+ )
+ deployment["desiredCount"] = 0
+ deployment["updatedAt"] = _iso()
+ svc["events"].insert(0, {
+ "id": new_uuid(),
+ "createdAt": _iso(),
+ "message": (
+ f"(service {svc['serviceName']}) deployment failed: "
+ "deployment circuit breaker was triggered."
+ ),
+ })
+
+ if breaker.get("rollback", False):
+ rollback = next(
+ (candidate for candidate in svc.get("deployments", [])
+ if candidate is not deployment
+ and candidate.get("rolloutState") == "COMPLETED"),
+ None,
+ )
+ if rollback:
+ deployment["status"] = "ACTIVE"
+ rollback["status"] = "PRIMARY"
+ rollback["desiredCount"] = svc.get("desiredCount", 0)
+ rollback["rolloutState"] = "IN_PROGRESS"
+ rollback["rolloutStateReason"] = (
+ "ECS deployment rollback in progress."
+ )
+ rollback["updatedAt"] = _iso()
+ svc["taskDefinition"] = rollback["taskDefinition"]
+ schedule_completion = True
+ svc["events"].insert(0, {
+ "id": new_uuid(),
+ "createdAt": _iso(),
+ "message": (
+ f"(service {svc['serviceName']}) rolled back to "
+ "the last completed deployment."
+ ),
+ })
+ should_reconcile = True
+
+ if should_reconcile:
+ _reconcile_service_tasks(cluster_name, svc_key)
+ if schedule_completion:
+ _schedule_service_deployment_completion(
+ cluster_name, svc_key, get_account_id(), get_region())
def _reconcile_service_tasks(cluster_name, svc_key):
@@ -989,21 +1179,37 @@ def _reconcile_service_tasks(cluster_name, svc_key):
else:
stale_tasks.append((arn, t))
+ primary = _primary_deployment(svc)
+ # Keep old tasks alive while a replacement deployment establishes itself.
+ # They are drained by _complete_service_deployment after the replacement is
+ # stable, or retained for _rollback_failed_deployment if it fails.
+ drain_stale = not primary or primary.get("rolloutState") != "IN_PROGRESS"
+ if drain_stale:
+ for task_arn, _ in stale_tasks:
+ _stop_task({"task": task_arn, "cluster": cluster_name,
+ "reason": "Task definition updated"})
+
# Scale up: spawn tasks to reach desiredCount
to_spawn = desired - len(current_tasks)
+ if primary and primary.get("rolloutState") == "FAILED":
+ # A failed deployment must not keep launching replacements. With
+ # rollback enabled _record_service_task_failure makes the old
+ # deployment primary before reconciliation reaches this branch.
+ to_spawn = 0
if to_spawn > 0:
if not td:
svc["runningCount"] = desired
if svc["deployments"]:
svc["deployments"][0]["runningCount"] = desired
return
+ # Preserve #1779's rolling-deployment capacity guard. New tasks may
+ # overlap old ones up to maximumPercent; at 100%, drain enough stale
+ # capacity before starting the replacement.
deployment_config = svc.get("deploymentConfiguration") or {}
maximum_percent = int(deployment_config.get("maximumPercent", 200))
max_running = max(1, desired * maximum_percent // 100)
capacity = max_running - len(current_tasks) - len(stale_tasks)
if capacity < to_spawn:
- # A maximumPercent of 100 requires old tasks to drain before
- # replacement tasks can be started.
for task_arn, _ in stale_tasks[:to_spawn - max(0, capacity)]:
_stop_task({
"task": task_arn,
@@ -1019,6 +1225,7 @@ def _reconcile_service_tasks(cluster_name, svc_key):
"launchType": launch_type,
"networkConfiguration": network_cfg,
"enableExecuteCommand": svc.get("enableExecuteCommand", False),
+ "_deploymentId": primary.get("id") if primary else None,
})
elif to_spawn < 0:
# Scale down: stop newest tasks first
@@ -1031,6 +1238,11 @@ def _reconcile_service_tasks(cluster_name, svc_key):
_refresh_service_state(cluster_name, f"service:{svc_name}")
_recount_cluster(cluster_name)
+ # In the no-Docker fallback tasks are already RUNNING when registered, so
+ # there is no lifecycle watcher to schedule the steady-state transition.
+ if primary and primary.get("rolloutState") == "IN_PROGRESS" and not _get_docker():
+ _complete_service_deployment(cluster_name, svc_key)
+
def _create_service(data):
cluster_name = _resolve_cluster_name(data.get("cluster", "default"))
@@ -1083,8 +1295,6 @@ def _create_service(data):
"healthCheckGracePeriodSeconds": data.get("healthCheckGracePeriodSeconds", 0),
"schedulingStrategy": data.get("schedulingStrategy", "REPLICA"),
"deploymentController": data.get("deploymentController", {"type": "ECS"}),
- # deploymentCircuitBreaker and alarms are accepted for API
- # compatibility but are not evaluated by the emulator.
"deploymentConfiguration": data.get("deploymentConfiguration", {
"maximumPercent": 200,
"minimumHealthyPercent": 100,
@@ -1225,6 +1435,12 @@ def _update_service(data):
if dep["status"] == "PRIMARY":
dep["status"] = "ACTIVE"
new_dep = _make_deployment(td_arn, svc["desiredCount"])
+ # Creation's first deployment is immediately PRIMARY/COMPLETED for
+ # compatibility with the existing service shape. A replacement,
+ # however, must remain observable as IN_PROGRESS until its tasks
+ # are stable (or its circuit breaker fails it).
+ new_dep["rolloutState"] = "IN_PROGRESS"
+ new_dep["rolloutStateReason"] = ""
svc["deployments"].insert(0, new_dep)
svc["taskDefinition"] = td_arn
changed = True
@@ -1730,7 +1946,7 @@ def _mark_task_stopped(task_arn, task, reason, stop_code, exit_code=None):
task["stopCode"] = stop_code
for container in task.get("containers", []):
container["lastStatus"] = "STOPPED"
- if exit_code is not None:
+ if exit_code is not None and container.get("exitCode") is None:
container["exitCode"] = exit_code
ecs_metadata.set_task_status(
@@ -1742,6 +1958,8 @@ def _mark_task_stopped(task_arn, task, reason, stop_code, exit_code=None):
if cluster_name:
_recount_cluster(cluster_name)
_refresh_service_state(cluster_name, task.get("group", ""))
+ if stop_code in ("EssentialContainerExited", "TaskFailedToStart"):
+ _record_service_task_failure(task)
return True
@@ -1992,6 +2210,8 @@ def _run_task_worker(task, td, container_overrides, docker_client, account_id, r
with request_scope(account_id, region):
try:
_start_task_worker(task, td, container_overrides, docker_client)
+ if task.get("lastStatus") == "RUNNING":
+ _watch_task_lifecycle(task, docker_client, account_id, region)
except _SecretResolutionError as exc:
_fail_task_start(
task, docker_client,
@@ -2008,6 +2228,38 @@ def _run_task_worker(task, td, container_overrides, docker_client, account_id, r
)
+def _watch_task_lifecycle(task, docker_client, account_id, region):
+ """Watch real Docker containers after launch, independent of API polls.
+
+ The older on-demand inspection in DescribeTasks/ListTasks is retained as a
+ recovery path, but a service scheduler cannot depend on a client polling a
+ task that it owns. This lightweight inspect loop gives a natural exit the
+ same task/service transition path as an explicit StopTask.
+ """
+ task_arn = task["taskArn"]
+
+ def _watch():
+ with request_scope(account_id, region):
+ while _task_is_active(task_arn, task):
+ _maybe_mark_stopped(task, docker_client)
+ if task.get("lastStatus") == "STOPPED":
+ return
+ time.sleep(0.1)
+
+ spawn_background(
+ _watch,
+ thread_name=("ministack-ecs-watch-"
+ f"{task_arn.rsplit('/', 1)[-1][:8]}"),
+ )
+ group = task.get("group", "")
+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))
+ if cluster_name and group.startswith("service:"):
+ _schedule_service_deployment_completion(
+ cluster_name, f"{cluster_name}/{group.split(':', 1)[1]}",
+ account_id, region, healthy_task=task,
+ )
+
+
def _run_task(data):
account_id = get_account_id()
region = get_region()
@@ -2034,6 +2286,7 @@ def _run_task(data):
started_by = data.get("startedBy", "")
enable_exec = data.get("enableExecuteCommand", False)
req_tags = data.get("tags", [])
+ deployment_id = data.get("_deploymentId")
docker_client = _get_docker()
docker_backed = bool(docker_client)
# Read when the container comes up, so both ride on the record until then.
@@ -2102,6 +2355,11 @@ def _run_task(data):
"healthStatus": "UNKNOWN",
"ephemeralStorage": td.get("ephemeralStorage", {"sizeInGiB": 20}),
"_docker_ids": [],
+ "_deployment_id": deployment_id,
+ "_container_essentials": [
+ bool(container.get("essential", True))
+ for container in td.get("containerDefinitions", [])
+ ],
}
_tasks[task_arn] = task
@@ -2182,8 +2440,14 @@ def _describe_tasks(data):
return json_response({"tasks": result, "failures": failures})
-def _maybe_mark_stopped(task):
- """Check Docker containers and transition task to STOPPED if all have exited."""
+def _maybe_mark_stopped(task, docker_client=None):
+ """Reflect an essential container exit in the task lifecycle.
+
+ ECS stops the task when an essential container exits; a non-essential
+ container can exit without taking its task down. Docker has no ECS task
+ primitive, so inspect each container and map that outcome back to the ECS
+ task and container fields.
+ """
task_arn = task.get("taskArn", "")
with resource_lock("ecs-task", task_arn):
if (not _task_is_active(task_arn, task)
@@ -2191,14 +2455,17 @@ def _maybe_mark_stopped(task):
or not task.get("_docker_ids")):
return
docker_ids = list(task["_docker_ids"])
+ essentials = list(task.get("_container_essentials") or [])
- docker_client = _get_docker()
+ docker_client = docker_client or _get_docker()
if not docker_client:
return
all_stopped = True
exit_code = 0
- for docker_id in docker_ids:
+ essential_exited = False
+ for index, docker_id in enumerate(docker_ids):
+ container_exit_code = None
try:
container = docker_client.containers.get(docker_id)
# docker SDK caches status; refresh before checking lifecycle
@@ -2208,14 +2475,26 @@ def _maybe_mark_stopped(task):
pass
if getattr(container, "status", None) != "exited":
all_stopped = False
- break
+ continue
result = container.wait()
- exit_code = max(exit_code, result.get("StatusCode", 0))
+ container_exit_code = result.get("StatusCode", 0)
except Exception:
- # Container removed or unreachable — treat as stopped
- pass
+ # An inspect failure (daemon unavailable, transient transport
+ # error) is not evidence that the process exited. Keep the ECS
+ # task live until Docker positively reports an exit; otherwise a
+ # temporary Docker outage would manufacture task failures.
+ all_stopped = False
+ continue
+
+ exit_code = max(exit_code, container_exit_code or 0)
+ with resource_lock("ecs-task", task_arn):
+ if index < len(task.get("containers", [])):
+ task["containers"][index]["lastStatus"] = "STOPPED"
+ task["containers"][index]["exitCode"] = container_exit_code
+ if index >= len(essentials) or essentials[index]:
+ essential_exited = True
- if not all_stopped:
+ if not essential_exited and not all_stopped:
return
stopped = _mark_task_stopped(
diff --git a/tests/test_ecs.py b/tests/test_ecs.py
index 931092f5..9101bd7b 100644
--- a/tests/test_ecs.py
+++ b/tests/test_ecs.py
@@ -32,6 +32,20 @@ def _wait_until(predicate, timeout=5):
assert predicate(), "condition did not become true before timeout"
+def _ecs_docker_reachable():
+ """Whether this end-to-end ECS lifecycle test can start real containers."""
+ try:
+ import docker
+ docker.from_env(timeout=2).ping()
+ except Exception:
+ return False
+ return True
+
+
+requires_ecs_docker = pytest.mark.skipif(
+ not _ecs_docker_reachable(), reason="requires a reachable Docker daemon")
+
+
def _replace_arn_region(arn):
return _replace_arn_section(arn, 3, _different_region(arn.split(":", 5)[3]))
@@ -1197,10 +1211,10 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
containerDefinitions=[{"name": "app", "image": "nginx:latest", "cpu": 64, "memory": 128}],
)
ecs.create_service(
- cluster=cluster, serviceName="tdu-svc", taskDefinition="tdu-td:1", desiredCount=1,
+ cluster=cluster, serviceName="tdu-svc", taskDefinition="tdu-td:1", desiredCount=2,
)
old_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")
- assert len(old_tasks["taskArns"]) == 1
+ assert len(old_tasks["taskArns"]) == 2
# Register new revision and update service
resp2 = ecs.register_task_definition(
@@ -1210,59 +1224,188 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
new_td_arn = resp2["taskDefinition"]["taskDefinitionArn"]
ecs.update_service(cluster=cluster, service="tdu-svc", taskDefinition="tdu-td:2")
- # ECS keeps both deployments while the replacement becomes healthy, then
- # drains the old deployment and collapses the service to the new one.
+ # A rolling deployment keeps the old tasks until the replacement is
+ # healthy. Wait specifically for two RUNNING tasks on the new revision
+ # instead of treating the still-running old tasks as replacements.
_wait_until(
- lambda: (
- len(ecs.describe_services(
- cluster=cluster, services=["tdu-svc"]
- )["services"][0]["deployments"]) == 1
- and ecs.describe_services(
- cluster=cluster, services=["tdu-svc"]
- )["services"][0]["deployments"][0]["rolloutState"] == "COMPLETED"
- ),
+ lambda: len([
+ task for task in ecs.describe_tasks(
+ cluster=cluster,
+ tasks=ecs.list_tasks(
+ cluster=cluster, serviceName="tdu-svc"
+ )["taskArns"],
+ )["tasks"]
+ if task["taskDefinitionArn"] == new_td_arn
+ and task["lastStatus"] == "RUNNING"
+ ]) == 2,
timeout=30,
)
- # New tasks should be on the new TD
- new_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")
- assert len(new_tasks["taskArns"]) == 1
-
- # Verify all running tasks use the new task definition
+ # Old tasks should be stopped
_wait_until(
lambda: all(
- task["lastStatus"] == "RUNNING"
+ task["lastStatus"] == "STOPPED"
for task in ecs.describe_tasks(
- cluster=cluster, tasks=new_tasks["taskArns"]
+ cluster=cluster, tasks=old_tasks["taskArns"]
)["tasks"]
),
timeout=30,
)
- desc = ecs.describe_tasks(cluster=cluster, tasks=new_tasks["taskArns"])
- for t in desc["tasks"]:
- assert t["taskDefinitionArn"] == new_td_arn, \
- f"Task still on old TD: {t['taskDefinitionArn']}"
- assert t["lastStatus"] == "RUNNING"
- # Old tasks should be stopped
- old_desc = ecs.describe_tasks(cluster=cluster, tasks=old_tasks["taskArns"])
- for t in old_desc["tasks"]:
- assert t["lastStatus"] == "STOPPED"
+ new_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")
+ assert len(new_tasks["taskArns"]) == 2
+ desc = ecs.describe_tasks(cluster=cluster, tasks=new_tasks["taskArns"])
+ assert all(t["taskDefinitionArn"] == new_td_arn for t in desc["tasks"])
# Service should reflect correct counts
_wait_until(
lambda: ecs.describe_services(
cluster=cluster, services=["tdu-svc"]
- )["services"][0]["runningCount"] == 1,
+ )["services"][0]["runningCount"] == 2,
timeout=30,
)
svc = ecs.describe_services(cluster=cluster, services=["tdu-svc"])
- service = svc["services"][0]
- assert service["runningCount"] == 1
- assert len(service["deployments"]) == 1
- assert service["deployments"][0]["taskDefinition"] == new_td_arn
- assert service["deployments"][0]["status"] == "PRIMARY"
- assert service["deployments"][0]["rolloutState"] == "COMPLETED"
+ assert svc["services"][0]["runningCount"] == 2
+ deployments = svc["services"][0]["deployments"]
+ assert len(deployments) == 1
+ assert deployments[0]["taskDefinition"] == new_td_arn
+ assert deployments[0]["status"] == "PRIMARY"
+ assert deployments[0]["rolloutState"] == "COMPLETED"
+
+
+@requires_ecs_docker
+def test_ecs_service_circuit_breaker_rolls_back_crashing_revision(ecs):
+ """A real container exit fails the new deployment and restores the old one.
+
+ This deliberately observes only DescribeServices after UpdateService. The
+ service's background lifecycle watcher, rather than an incidental
+ DescribeTasks request, must discover every ``exit 1`` and give the circuit
+ breaker the failure signal.
+ """
+ cluster = "circuit-breaker-c"
+ family = "circuit-breaker-td"
+ service = "circuit-breaker-svc"
+ ecs.create_cluster(clusterName=cluster)
+ healthy = ecs.register_task_definition(
+ family=family,
+ requiresCompatibilities=["FARGATE"],
+ networkMode="awsvpc",
+ containerDefinitions=[{
+ "name": "app",
+ "image": "alpine:latest",
+ "command": ["sh", "-c", "sleep 600"],
+ }],
+ )["taskDefinition"]["taskDefinitionArn"]
+ ecs.create_service(
+ cluster=cluster,
+ serviceName=service,
+ taskDefinition=healthy,
+ desiredCount=1,
+ launchType="FARGATE",
+ networkConfiguration={"awsvpcConfiguration": {"subnets": ["subnet-test"]}},
+ )
+ _wait_until(
+ lambda: ecs.describe_services(cluster=cluster, services=[service])
+ ["services"][0]["runningCount"] == 1,
+ timeout=30,
+ )
+
+ crashing = ecs.register_task_definition(
+ family=family,
+ requiresCompatibilities=["FARGATE"],
+ networkMode="awsvpc",
+ containerDefinitions=[{
+ "name": "app",
+ "image": "alpine:latest",
+ "command": ["sh", "-c", "exit 1"],
+ }],
+ )["taskDefinition"]["taskDefinitionArn"]
+ ecs.update_service(
+ cluster=cluster,
+ service=service,
+ taskDefinition=crashing,
+ deploymentConfiguration={
+ "deploymentCircuitBreaker": {"enable": True, "rollback": True},
+ },
+ )
+
+ final_service = None
+
+ def rolled_back():
+ nonlocal final_service
+ final_service = ecs.describe_services(
+ cluster=cluster, services=[service]
+ )["services"][0]
+ deployments = final_service["deployments"]
+ failed = next(
+ (deployment for deployment in deployments
+ if deployment["taskDefinition"] == crashing
+ and deployment.get("rolloutState") == "FAILED"),
+ None,
+ )
+ primary = next(
+ (deployment for deployment in deployments
+ if deployment.get("status") == "PRIMARY"),
+ None,
+ )
+ return bool(
+ failed and failed.get("rolloutStateReason")
+ and primary and primary["taskDefinition"] == healthy
+ and primary.get("rolloutState") == "COMPLETED"
+ and final_service["taskDefinition"] == healthy
+ and final_service["runningCount"] == 1
+ )
+
+ _wait_until(rolled_back, timeout=30)
+ failed = next(
+ deployment for deployment in final_service["deployments"]
+ if deployment["taskDefinition"] == crashing
+ )
+ assert failed["failedTasks"] == 3
+ assert failed["rolloutState"] == "FAILED"
+ assert "circuit breaker" in failed["rolloutStateReason"]
+
+
+ ("reset_on_healthy", "expected_failures"),
+ [(None, 0), (True, 0), (False, 2)],
+)
+def test_ecs_circuit_breaker_reset_on_healthy_task(
+ reset_on_healthy, expected_failures):
+ """A stable task resets failures unless cumulative mode is requested."""
+ from ministack.services import ecs as _ecs
+
+ cluster = f"reset-healthy-{_uuid_mod.uuid4().hex[:8]}"
+ service = "svc"
+ svc_key = f"{cluster}/{service}"
+ deployment = _ecs._make_deployment("reset-healthy-td:2", 2)
+ deployment["rolloutState"] = "IN_PROGRESS"
+ deployment["rolloutStateReason"] = ""
+ deployment["failedTasks"] = 2
+ breaker = {"enable": True, "rollback": True}
+ if reset_on_healthy is not None:
+ breaker["resetOnHealthyTask"] = reset_on_healthy
+ svc = {
+ "serviceName": service,
+ "clusterArn": (
+ f"arn:aws:ecs:us-east-1:000000000000:cluster/{cluster}"
+ ),
+ "status": "ACTIVE",
+ "deploymentConfiguration": {"deploymentCircuitBreaker": breaker},
+ "deployments": [deployment],
+ }
+ task = {
+ "taskDefinitionArn": "reset-healthy-td:2",
+ "_deployment_id": deployment["id"],
+ "lastStatus": "RUNNING",
+ }
+ _ecs._services[svc_key] = svc
+ try:
+ _ecs._record_service_task_healthy(svc_key, task)
+ assert deployment["failedTasks"] == expected_failures
+ finally:
+ _ecs._services.pop(svc_key, None)
def test_ecs_service_delete_stops_tasks(ecs):
@@ -1388,6 +1531,148 @@ def test_ecs_cfn_service_visible(ecs, cfn):
cfn.delete_stack(StackName=stack_name)
+def test_ecs_cfn_service_deployment_circuit_breaker_create_and_update(ecs, cfn):
+ """CloudFormation maps circuit-breaker fields and updates them in place."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name = f"ecs-cfn-circuit-{suffix}"
+ cluster_name = f"ecs-cfn-circuit-c-{suffix}"
+ service_name = f"ecs-cfn-circuit-s-{suffix}"
+ first_family = f"ecs-cfn-circuit-one-{suffix}"
+ second_family = f"ecs-cfn-circuit-two-{suffix}"
+
+ def template(task_definition, configuration):
+ return {
+ "AWSTemplateFormatVersion": "2010-09-09",
+ "Resources": {
+ "Cluster": {
+ "Type": "AWS::ECS::Cluster",
+ "Properties": {"ClusterName": cluster_name},
+ },
+ "FirstTaskDefinition": {
+ "Type": "AWS::ECS::TaskDefinition",
+ "Properties": {
+ "Family": first_family,
+ "ContainerDefinitions": [{"Name": "app", "Image": "busybox"}],
+ },
+ },
+ "SecondTaskDefinition": {
+ "Type": "AWS::ECS::TaskDefinition",
+ "Properties": {
+ "Family": second_family,
+ "ContainerDefinitions": [{"Name": "app", "Image": "busybox"}],
+ },
+ },
+ "Service": {
+ "Type": "AWS::ECS::Service",
+ "DependsOn": [
+ "Cluster", "FirstTaskDefinition", "SecondTaskDefinition",
+ ],
+ "Properties": {
+ "Cluster": {"Ref": "Cluster"},
+ "ServiceName": service_name,
+ "TaskDefinition": {"Ref": task_definition},
+ "DesiredCount": 0,
+ "DeploymentConfiguration": configuration,
+ },
+ },
+ },
+ }
+
+ initial = {
+ "MaximumPercent": 150,
+ "MinimumHealthyPercent": 50,
+ "DeploymentCircuitBreaker": {
+ "Enable": True,
+ "Rollback": True,
+ "ResetOnHealthyTask": False,
+ "ThresholdConfiguration": {"Type": "COUNT", "Value": 5},
+ },
+ }
+ cfn.create_stack(
+ StackName=stack_name,
+ TemplateBody=json.dumps(template("FirstTaskDefinition", initial)),
+ )
+ _wait_until(
+ lambda: cfn.describe_stacks(StackName=stack_name)["Stacks"][0]["StackStatus"]
+ == "CREATE_COMPLETE",
+ timeout=30,
+ )
+ service = ecs.describe_services(
+ cluster=cluster_name, services=[service_name]
+ )["services"][0]
+ assert service["deploymentConfiguration"] == {
+ "maximumPercent": 150,
+ "minimumHealthyPercent": 50,
+ "deploymentCircuitBreaker": {
+ "enable": True,
+ "rollback": True,
+ "resetOnHealthyTask": False,
+ "thresholdConfiguration": {"type": "COUNT", "value": 5},
+ },
+ }
+
+ updated = {
+ "DeploymentCircuitBreaker": {
+ "Enable": True,
+ "Rollback": False,
+ "ResetOnHealthyTask": True,
+ "ThresholdConfiguration": {
+ "Type": "UNBOUNDED_PERCENT", "Value": 25,
+ },
+ },
+ }
+ cfn.update_stack(
+ StackName=stack_name,
+ TemplateBody=json.dumps(template("SecondTaskDefinition", updated)),
+ )
+ _wait_until(
+ lambda: cfn.describe_stacks(StackName=stack_name)["Stacks"][0]["StackStatus"]
+ == "UPDATE_COMPLETE",
+ timeout=30,
+ )
+ service = ecs.describe_services(
+ cluster=cluster_name, services=[service_name]
+ )["services"][0]
+ second_arn = ecs.describe_task_definition(
+ taskDefinition=second_family
+ )["taskDefinition"]["taskDefinitionArn"]
+ assert service["taskDefinition"] == second_arn
+ assert service["deploymentConfiguration"] == {
+ "deploymentCircuitBreaker": {
+ "enable": True,
+ "rollback": False,
+ "resetOnHealthyTask": True,
+ "thresholdConfiguration": {
+ "type": "UNBOUNDED_PERCENT", "value": 25,
+ },
+ },
+ }
+ cfn.delete_stack(StackName=stack_name)
+
+
+def test_ecs_cfn_service_update_propagates_ecs_errors(monkeypatch):
+ """A rejected ECS UpdateService must fail the CloudFormation update."""
+ from ministack.services.cloudformation import provisioners
+
+ monkeypatch.setattr(
+ ecs_service,
+ "_update_service",
+ lambda _request: (
+ 400,
+ {"Content-Type": "application/x-amz-json-1.0"},
+ b'{"__type":"ClientException","message":"task definition not found"}',
+ ),
+ )
+
+ with pytest.raises(ValueError, match="AWS::ECS::Service update failed"):
+ provisioners._ecs_service_update(
+ "arn:aws:ecs:us-east-1:000000000000:service/default/example",
+ {"TaskDefinition": "example:1"},
+ {"TaskDefinition": "example:99"},
+ "stack",
+ )
+
+
def test_ecs_cfn_taskdef_populates_registered_fields(ecs, cfn):
"""CFN-created TaskDefinitions must surface registeredAt/registeredBy/compatibilities,
matching what RegisterTaskDefinition emits. Workloads like Go-SDK reconcilers fall