Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(ecs): track deployment health and roll back failures (#1783)
Johannes Grumböckf79600537d39
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex bbdd256a..04f6b3ba 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -7036,10 +7036,18 @@ def _ecs_task_def_delete(physical_id, props):_ecs._task_defs.pop(td_key, None)+def _ecs_deployment_configuration(props):+ """Translate AWS::ECS::Service deployment configuration to ECS API form."""+ configuration = props.get("DeploymentConfiguration")+ if configuration is None:+ return None+ return _pascal_to_camel(configuration)++def _ecs_service_create(logical_id, props, stack_name):name = props.get("ServiceName", f"{stack_name}-{logical_id}")cluster = props.get("Cluster", "default")- _ecs._create_service({+ request = {"serviceName": name,"cluster": cluster,"taskDefinition": props.get("TaskDefinition", ""),@@ -7048,11 +7056,54 @@ def _ecs_service_create(logical_id, props, stack_name):"loadBalancers": props.get("LoadBalancers", []),"networkConfiguration": props.get("NetworkConfiguration", {}),"tags": [{"key": t["Key"], "value": t["Value"]} for t in props.get("Tags", [])],- })+ }+ deployment_configuration = _ecs_deployment_configuration(props)+ if deployment_configuration is not None:+ request["deploymentConfiguration"] = deployment_configuration+ _ecs._create_service(request)arn = f"arn:aws:ecs:{get_region()}:{get_account_id()}:service/{cluster}/{name}"return arn, {"ServiceArn": arn, "Name": name}+def _ecs_service_update(physical_id, old_props, new_props, stack_name):+ """Apply mutable AWS::ECS::Service properties through UpdateService."""+ cluster = new_props.get("Cluster", old_props.get("Cluster", "default"))+ name = new_props.get("ServiceName") or physical_id.rsplit("/", 1)[-1]+ request = {"cluster": cluster, "service": name}+ property_map = {+ "TaskDefinition": "taskDefinition",+ "DesiredCount": "desiredCount",+ "NetworkConfiguration": "networkConfiguration",+ "LoadBalancers": "loadBalancers",+ "HealthCheckGracePeriodSeconds": "healthCheckGracePeriodSeconds",+ "EnableExecuteCommand": "enableExecuteCommand",+ "PlatformVersion": "platformVersion",+ }+ for cf_property, ecs_property in property_map.items():+ if new_props.get(cf_property) != old_props.get(cf_property):+ request[ecs_property] = new_props.get(cf_property)++ if (new_props.get("DeploymentConfiguration")+ != old_props.get("DeploymentConfiguration")):+ deployment_configuration = _ecs_deployment_configuration(new_props)+ if deployment_configuration is None:+ deployment_configuration = {+ "maximumPercent": 200,+ "minimumHealthyPercent": 100,+ "deploymentCircuitBreaker": {"enable": False, "rollback": False},+ }+ request["deploymentConfiguration"] = deployment_configuration++ # The resource engine only calls us when properties changed. Some ECS+ # properties are replacement-only and are intentionally left to their+ # existing semantics; these are the fields UpdateService can apply in+ # place and that MiniStack currently models.+ response = _ecs._update_service(request)+ if response[0] >= 400:+ raise ValueError(f"AWS::ECS::Service update failed: {response[2]!r}")+ return physical_id, {"ServiceArn": physical_id, "Name": name}++def _ecs_service_delete(physical_id, props):cluster = props.get("Cluster", "default")name = props.get("ServiceName", "")@@ -10895,7 +10946,11 @@ _RESOURCE_HANDLERS = {"AWS::EC2::SubnetRouteTableAssociation": {"create": _ec2_subnet_rtb_assoc_create, "delete": _ec2_subnet_rtb_assoc_delete},"AWS::ECS::Cluster": {"create": _ecs_cluster_create, "delete": _ecs_cluster_delete},"AWS::ECS::TaskDefinition": {"create": _ecs_task_def_create, "delete": _ecs_task_def_delete},- "AWS::ECS::Service": {"create": _ecs_service_create, "delete": _ecs_service_delete},+ "AWS::ECS::Service": {+ "create": _ecs_service_create,+ "update": _ecs_service_update,+ "delete": _ecs_service_delete,+ },"AWS::EC2::LaunchTemplate": {"create": _ec2_launch_template_create, "delete": _ec2_launch_template_delete},"AWS::ElasticLoadBalancingV2::LoadBalancer": {"create": _elbv2_load_balancer_create,diff --git a/ministack/services/ecs.py b/ministack/services/ecs.pyindex 49c11116..414e989e 100644--- a/ministack/services/ecs.py+++ b/ministack/services/ecs.py@@ -28,6 +28,7 @@ Container execution: if Docker socket is available, RunTask actually runs contaiimport copyimport jsonimport logging+import mathimport osimport secretsimport threading@@ -35,7 +36,7 @@ import timefrom ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.concurrency import resource_lock, run_reentrant+from ministack.core.concurrency import resource_lock, run_reentrant, spawn_backgroundfrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,@@ -97,6 +98,11 @@ _ECS_REAP_INTERVAL = int(os.environ.get("ECS_REAP_INTERVAL_SECONDS", "60"))# containers for restored services._ECS_RESTORE_RECONCILE_DELAY = float(os.environ.get("ECS_RESTORE_RECONCILE_DELAY_SECONDS", "2"))+# A container that merely reaches RUNNING is not immediately a steady service+# deployment: it can exit during its process startup. Keep a short window+# before completing a rollout so the lifecycle watcher can report that exit.+_ECS_DEPLOYMENT_STEADY_DELAY = float(+ os.environ.get("ECS_DEPLOYMENT_STEADY_DELAY_SECONDS", "1"))_ecs_reaper_started = False_ecs_reaper_lock = threading.Lock()@@ -879,82 +885,266 @@ def _refresh_service_state(cluster_name, group):cluster_arn = svc.get("clusterArn", "")running = 0pending = 0+ deployment_counts = {+ dep["id"]: {"running": 0, "pending": 0}+ for dep in svc.get("deployments", [])+ }for task in _tasks.values():if task.get("group") != group or task.get("clusterArn") != cluster_arn:continueif task.get("lastStatus") == "RUNNING":running += 1+ state = "running"elif task.get("lastStatus") == "PENDING":pending += 1+ state = "pending"+ else:+ continue+ deployment_id = task.get("_deployment_id")+ if deployment_id in deployment_counts:+ deployment_counts[deployment_id][state] += 1svc["runningCount"] = runningsvc["pendingCount"] = pending- _refresh_service_deployments(cluster_name, svc)+ for dep in svc.get("deployments", []):+ counts = deployment_counts.get(dep["id"], {})+ dep["runningCount"] = counts.get("running", 0)+ dep["pendingCount"] = counts.get("pending", 0)+ primary = _primary_deployment(svc)+ if primary and primary.get("rolloutState") == "COMPLETED":+ # A nested _stop_task refresh runs while a successful rollout is+ # draining its old tasks. Collapse old deployments here as well as in+ # the completion path so the final response cannot retain a drained+ # ACTIVE deployment merely because that nested refresh won the race.+ svc["deployments"] = [+ dep for dep in svc.get("deployments", [])+ if dep.get("status") == "PRIMARY"+ or dep.get("rolloutState") == "FAILED"+ or deployment_counts.get(dep.get("id"), {}).get("running", 0)+ or deployment_counts.get(dep.get("id"), {}).get("pending", 0)+ ]_sync_service_targets(cluster_name, svc)-def _refresh_service_deployments(cluster_name, svc):- """Reconcile per-deployment counts and retire a completed old deployment."""- deployments = svc.get("deployments") or []- if not deployments:- return+def _primary_deployment(svc):+ return next(+ (dep for dep in svc.get("deployments", [])+ if dep.get("status") == "PRIMARY"),+ None,+ )- cluster_arn = svc.get("clusterArn", "")- group = f"service:{svc.get('serviceName', '')}"- service_tasks = [- task for task in _tasks.values()- if task.get("group") == group- and task.get("clusterArn") == cluster_arn- ]- for deployment in deployments:- td_arn = deployment.get("taskDefinition")- matching = [task for task in service_tasks- if task.get("taskDefinitionArn") == td_arn]- deployment["runningCount"] = sum(- task.get("lastStatus") == "RUNNING" for task in matching- )- deployment["pendingCount"] = sum(- task.get("lastStatus") == "PENDING" for task in matching++def _deployment_for_task(svc, task):+ deployment_id = task.get("_deployment_id")+ if deployment_id:+ return next(+ (dep for dep in svc.get("deployments", [])+ if dep.get("id") == deployment_id),+ None,)+ # Tasks persisted before deployment IDs were recorded still need to be+ # attributed when they stop after a restart.+ return next(+ (dep for dep in svc.get("deployments", [])+ if dep.get("taskDefinition") == task.get("taskDefinitionArn")),+ None,+ )- primary = deployments[0]- primary_desired = primary.get("desiredCount", svc.get("desiredCount", 0))- healthy = primary.get("runningCount", 0) >= primary_desired- if len(deployments) > 1 and not healthy:- primary["rolloutState"] = "IN_PROGRESS"- primary["rolloutStateReason"] = ""++def _circuit_breaker_threshold(svc):+ """Return the ECS failure threshold for the service's current size."""+ breaker = (svc.get("deploymentConfiguration") or {}).get(+ "deploymentCircuitBreaker") or {}+ config = breaker.get("thresholdConfiguration") or {}+ threshold_type = config.get("type", "BOUNDED_PERCENT")+ value = config.get("value", 50)+ if threshold_type == "COUNT":+ return max(1, int(value))+ calculated = math.ceil(float(value) * int(svc.get("desiredCount", 0)) / 100)+ if threshold_type == "UNBOUNDED_PERCENT":+ return max(1, calculated)+ return min(200, max(3, calculated))+++def _complete_service_deployment(cluster_name, svc_key):+ """Complete a stable PRIMARY rollout, then drain its old deployment.++ A completed deployment is the rollback point. Therefore old tasks stay+ alive while the new deployment is IN_PROGRESS; only a stable new deployment+ drains them. This is also what makes a circuit-breaker rollback able to+ restore the previous revision rather than re-create deleted deployment data.+ """+ svc = _services.get(svc_key)+ if not svc or svc.get("status") != "ACTIVE":return- if not healthy:++ svc_name = svc["serviceName"]+ with resource_lock("ecs-service", svc_key):+ primary = _primary_deployment(svc)+ if (not primary or primary.get("rolloutState") != "IN_PROGRESS"+ or primary.get("runningCount", 0) < svc.get("desiredCount", 0)):+ return+ primary["rolloutState"] = "COMPLETED"+ primary["rolloutStateReason"] = "ECS deployment completed."+ primary["updatedAt"] = _iso()+ stale_task_arns = [+ arn for arn, task in _tasks.items()+ if task.get("group") == f"service:{svc['serviceName']}"+ and task.get("clusterArn") == svc.get("clusterArn")+ and task.get("_deployment_id") != primary.get("id")+ and task.get("lastStatus") in _PRE_STOP_STATUSES+ ]++ for task_arn in stale_task_arns:+ _stop_task({"task": task_arn, "cluster": cluster_name,+ "reason": "Deployment completed"})++ with resource_lock("ecs-service", svc_key):+ # ECS no longer returns a deployment once it has no live tasks. Keep+ # a FAILED deployment, however: callers need its failure diagnostics.+ svc = _services.get(svc_key)+ if svc:+ svc["deployments"] = [+ dep for dep in svc.get("deployments", [])+ if dep.get("status") == "PRIMARY"+ or dep.get("rolloutState") == "FAILED"+ or any(task.get("_deployment_id") == dep.get("id")+ and task.get("lastStatus") in _PRE_STOP_STATUSES+ for task in _tasks.values())+ ]+ _refresh_service_state(cluster_name, f"service:{svc_name}")+++def _record_service_task_healthy(svc_key, task):+ """Apply ``resetOnHealthyTask`` after a task survives startup.++ A Docker container can report RUNNING and then exit a few milliseconds+ later, so RUNNING by itself is not the circuit breaker's healthy signal in+ MiniStack. The caller invokes this only after the same stability window+ used to complete a rollout.+ """+ with resource_lock("ecs-service", svc_key):+ svc = _services.get(svc_key)+ if not svc:+ return+ deployment = _deployment_for_task(svc, task)+ primary = _primary_deployment(svc)+ breaker = (svc.get("deploymentConfiguration") or {}).get(+ "deploymentCircuitBreaker") or {}+ if (deployment is primary and deployment+ and deployment.get("rolloutState") == "IN_PROGRESS"+ and breaker.get("enable", False)+ and breaker.get("resetOnHealthyTask", True)):+ deployment["failedTasks"] = 0+ deployment["updatedAt"] = _iso()+++def _schedule_service_deployment_completion(+ cluster_name, svc_key, account_id, region, healthy_task=None):+ """Check a Docker-backed service rollout after its startup grace window."""+ if not cluster_name or "/" not in svc_key:return+ group = f"service:{svc_key.split('/', 1)[1]}"++ def _check():+ time.sleep(_ECS_DEPLOYMENT_STEADY_DELAY)+ with request_scope(account_id, region):+ if _services.get(svc_key, {}).get("status") == "ACTIVE":+ if (healthy_task and healthy_task.get("lastStatus") == "RUNNING"):+ _record_service_task_healthy(svc_key, healthy_task)+ _refresh_service_state(cluster_name, group)+ _complete_service_deployment(cluster_name, svc_key)++ spawn_background(+ _check,+ thread_name=("ministack-ecs-deployment-"+ f"{svc_key.rsplit('/', 1)[-1][:8]}"),+ )- primary["rolloutState"] = "COMPLETED"- primary["rolloutStateReason"] = "ECS deployment completed."- primary_td = primary.get("taskDefinition")- for deployment in deployments[1:]:- deployment["desiredCount"] = 0- stale = [- task for task in service_tasks- if task.get("taskDefinitionArn") != primary_td- and task.get("lastStatus") in _PRE_STOP_STATUSES- ]- for task in stale:- _stop_task({- "task": task["taskArn"],- "cluster": cluster_name,- "reason": "ECS deployment completed",- })- # _stop_task refreshes this service recursively, so only live tasks can- # keep an old deployment in the response.- for deployment in list(deployments[1:]):- td_arn = deployment.get("taskDefinition")- if not any(- task.get("taskDefinitionArn") == td_arn- and task.get("group") == group- and task.get("clusterArn") == cluster_arn- and task.get("lastStatus") in _PRE_STOP_STATUSES- for task in _tasks.values()- ):- deployments.remove(deployment)+def _record_service_task_failure(task):+ """Feed a natural task/startup failure into its deployment circuit breaker."""+ group = task.get("group", "")+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))+ if not cluster_name or not group.startswith("service:"):+ return+ svc_key = f"{cluster_name}/{group.split(':', 1)[1]}"+ should_reconcile = False+ schedule_completion = False++ with resource_lock("ecs-service", svc_key):+ svc = _services.get(svc_key)+ if not svc or svc.get("status") != "ACTIVE":+ return+ deployment = _deployment_for_task(svc, task)+ primary = _primary_deployment(svc)+ breaker = (svc.get("deploymentConfiguration") or {}).get(+ "deploymentCircuitBreaker") or {}+ is_rolling = (svc.get("deploymentController") or {}).get("type", "ECS") == "ECS"++ # Only the deployment being evaluated is allowed to accumulate task+ # failures. An old task stopped during a successful drain must never+ # poison the replacement deployment.+ if (deployment is not primary+ or not deployment+ or deployment.get("rolloutState") != "IN_PROGRESS"+ or not is_rolling+ or not breaker.get("enable", False)):+ should_reconcile = deployment is primary and bool(deployment)+ else:+ deployment["failedTasks"] = int(deployment.get("failedTasks", 0)) + 1+ deployment["updatedAt"] = _iso()+ if deployment["failedTasks"] < _circuit_breaker_threshold(svc):+ should_reconcile = True+ else:+ deployment["rolloutState"] = "FAILED"+ deployment["rolloutStateReason"] = (+ "ECS deployment circuit breaker was triggered after "+ f"{deployment['failedTasks']} consecutive task failures."+ )+ deployment["desiredCount"] = 0+ deployment["updatedAt"] = _iso()+ svc["events"].insert(0, {+ "id": new_uuid(),+ "createdAt": _iso(),+ "message": (+ f"(service {svc['serviceName']}) deployment failed: "+ "deployment circuit breaker was triggered."+ ),+ })++ if breaker.get("rollback", False):+ rollback = next(+ (candidate for candidate in svc.get("deployments", [])+ if candidate is not deployment+ and candidate.get("rolloutState") == "COMPLETED"),+ None,+ )+ if rollback:+ deployment["status"] = "ACTIVE"+ rollback["status"] = "PRIMARY"+ rollback["desiredCount"] = svc.get("desiredCount", 0)+ rollback["rolloutState"] = "IN_PROGRESS"+ rollback["rolloutStateReason"] = (+ "ECS deployment rollback in progress."+ )+ rollback["updatedAt"] = _iso()+ svc["taskDefinition"] = rollback["taskDefinition"]+ schedule_completion = True+ svc["events"].insert(0, {+ "id": new_uuid(),+ "createdAt": _iso(),+ "message": (+ f"(service {svc['serviceName']}) rolled back to "+ "the last completed deployment."+ ),+ })+ should_reconcile = True++ if should_reconcile:+ _reconcile_service_tasks(cluster_name, svc_key)+ if schedule_completion:+ _schedule_service_deployment_completion(+ cluster_name, svc_key, get_account_id(), get_region())def _reconcile_service_tasks(cluster_name, svc_key):@@ -989,21 +1179,37 @@ def _reconcile_service_tasks(cluster_name, svc_key):else:stale_tasks.append((arn, t))+ primary = _primary_deployment(svc)+ # Keep old tasks alive while a replacement deployment establishes itself.+ # They are drained by _complete_service_deployment after the replacement is+ # stable, or retained for _rollback_failed_deployment if it fails.+ drain_stale = not primary or primary.get("rolloutState") != "IN_PROGRESS"+ if drain_stale:+ for task_arn, _ in stale_tasks:+ _stop_task({"task": task_arn, "cluster": cluster_name,+ "reason": "Task definition updated"})+# Scale up: spawn tasks to reach desiredCountto_spawn = desired - len(current_tasks)+ if primary and primary.get("rolloutState") == "FAILED":+ # A failed deployment must not keep launching replacements. With+ # rollback enabled _record_service_task_failure makes the old+ # deployment primary before reconciliation reaches this branch.+ to_spawn = 0if to_spawn > 0:if not td:svc["runningCount"] = desiredif svc["deployments"]:svc["deployments"][0]["runningCount"] = desiredreturn+ # Preserve #1779's rolling-deployment capacity guard. New tasks may+ # overlap old ones up to maximumPercent; at 100%, drain enough stale+ # capacity before starting the replacement.deployment_config = svc.get("deploymentConfiguration") or {}maximum_percent = int(deployment_config.get("maximumPercent", 200))max_running = max(1, desired * maximum_percent // 100)capacity = max_running - len(current_tasks) - len(stale_tasks)if capacity < to_spawn:- # A maximumPercent of 100 requires old tasks to drain before- # replacement tasks can be started.for task_arn, _ in stale_tasks[:to_spawn - max(0, capacity)]:_stop_task({"task": task_arn,@@ -1019,6 +1225,7 @@ def _reconcile_service_tasks(cluster_name, svc_key):"launchType": launch_type,"networkConfiguration": network_cfg,"enableExecuteCommand": svc.get("enableExecuteCommand", False),+ "_deploymentId": primary.get("id") if primary else None,})elif to_spawn < 0:# Scale down: stop newest tasks first@@ -1031,6 +1238,11 @@ def _reconcile_service_tasks(cluster_name, svc_key):_refresh_service_state(cluster_name, f"service:{svc_name}")_recount_cluster(cluster_name)+ # In the no-Docker fallback tasks are already RUNNING when registered, so+ # there is no lifecycle watcher to schedule the steady-state transition.+ if primary and primary.get("rolloutState") == "IN_PROGRESS" and not _get_docker():+ _complete_service_deployment(cluster_name, svc_key)+def _create_service(data):cluster_name = _resolve_cluster_name(data.get("cluster", "default"))@@ -1083,8 +1295,6 @@ def _create_service(data):"healthCheckGracePeriodSeconds": data.get("healthCheckGracePeriodSeconds", 0),"schedulingStrategy": data.get("schedulingStrategy", "REPLICA"),"deploymentController": data.get("deploymentController", {"type": "ECS"}),- # deploymentCircuitBreaker and alarms are accepted for API- # compatibility but are not evaluated by the emulator."deploymentConfiguration": data.get("deploymentConfiguration", {"maximumPercent": 200,"minimumHealthyPercent": 100,@@ -1225,6 +1435,12 @@ def _update_service(data):if dep["status"] == "PRIMARY":dep["status"] = "ACTIVE"new_dep = _make_deployment(td_arn, svc["desiredCount"])+ # Creation's first deployment is immediately PRIMARY/COMPLETED for+ # compatibility with the existing service shape. A replacement,+ # however, must remain observable as IN_PROGRESS until its tasks+ # are stable (or its circuit breaker fails it).+ new_dep["rolloutState"] = "IN_PROGRESS"+ new_dep["rolloutStateReason"] = ""svc["deployments"].insert(0, new_dep)svc["taskDefinition"] = td_arnchanged = True@@ -1730,7 +1946,7 @@ def _mark_task_stopped(task_arn, task, reason, stop_code, exit_code=None):task["stopCode"] = stop_codefor container in task.get("containers", []):container["lastStatus"] = "STOPPED"- if exit_code is not None:+ if exit_code is not None and container.get("exitCode") is None:container["exitCode"] = exit_codeecs_metadata.set_task_status(@@ -1742,6 +1958,8 @@ def _mark_task_stopped(task_arn, task, reason, stop_code, exit_code=None):if cluster_name:_recount_cluster(cluster_name)_refresh_service_state(cluster_name, task.get("group", ""))+ if stop_code in ("EssentialContainerExited", "TaskFailedToStart"):+ _record_service_task_failure(task)return True@@ -1992,6 +2210,8 @@ def _run_task_worker(task, td, container_overrides, docker_client, account_id, rwith request_scope(account_id, region):try:_start_task_worker(task, td, container_overrides, docker_client)+ if task.get("lastStatus") == "RUNNING":+ _watch_task_lifecycle(task, docker_client, account_id, region)except _SecretResolutionError as exc:_fail_task_start(task, docker_client,@@ -2008,6 +2228,38 @@ def _run_task_worker(task, td, container_overrides, docker_client, account_id, r)+def _watch_task_lifecycle(task, docker_client, account_id, region):+ """Watch real Docker containers after launch, independent of API polls.++ The older on-demand inspection in DescribeTasks/ListTasks is retained as a+ recovery path, but a service scheduler cannot depend on a client polling a+ task that it owns. This lightweight inspect loop gives a natural exit the+ same task/service transition path as an explicit StopTask.+ """+ task_arn = task["taskArn"]++ def _watch():+ with request_scope(account_id, region):+ while _task_is_active(task_arn, task):+ _maybe_mark_stopped(task, docker_client)+ if task.get("lastStatus") == "STOPPED":+ return+ time.sleep(0.1)++ spawn_background(+ _watch,+ thread_name=("ministack-ecs-watch-"+ f"{task_arn.rsplit('/', 1)[-1][:8]}"),+ )+ group = task.get("group", "")+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))+ if cluster_name and group.startswith("service:"):+ _schedule_service_deployment_completion(+ cluster_name, f"{cluster_name}/{group.split(':', 1)[1]}",+ account_id, region, healthy_task=task,+ )++def _run_task(data):account_id = get_account_id()region = get_region()@@ -2034,6 +2286,7 @@ def _run_task(data):started_by = data.get("startedBy", "")enable_exec = data.get("enableExecuteCommand", False)req_tags = data.get("tags", [])+ deployment_id = data.get("_deploymentId")docker_client = _get_docker()docker_backed = bool(docker_client)# Read when the container comes up, so both ride on the record until then.@@ -2102,6 +2355,11 @@ def _run_task(data):"healthStatus": "UNKNOWN","ephemeralStorage": td.get("ephemeralStorage", {"sizeInGiB": 20}),"_docker_ids": [],+ "_deployment_id": deployment_id,+ "_container_essentials": [+ bool(container.get("essential", True))+ for container in td.get("containerDefinitions", [])+ ],}_tasks[task_arn] = task@@ -2182,8 +2440,14 @@ def _describe_tasks(data):return json_response({"tasks": result, "failures": failures})-def _maybe_mark_stopped(task):- """Check Docker containers and transition task to STOPPED if all have exited."""+def _maybe_mark_stopped(task, docker_client=None):+ """Reflect an essential container exit in the task lifecycle.++ ECS stops the task when an essential container exits; a non-essential+ container can exit without taking its task down. Docker has no ECS task+ primitive, so inspect each container and map that outcome back to the ECS+ task and container fields.+ """task_arn = task.get("taskArn", "")with resource_lock("ecs-task", task_arn):if (not _task_is_active(task_arn, task)@@ -2191,14 +2455,17 @@ def _maybe_mark_stopped(task):or not task.get("_docker_ids")):returndocker_ids = list(task["_docker_ids"])+ essentials = list(task.get("_container_essentials") or [])- docker_client = _get_docker()+ docker_client = docker_client or _get_docker()if not docker_client:returnall_stopped = Trueexit_code = 0- for docker_id in docker_ids:+ essential_exited = False+ for index, docker_id in enumerate(docker_ids):+ container_exit_code = Nonetry:container = docker_client.containers.get(docker_id)# docker SDK caches status; refresh before checking lifecycle@@ -2208,14 +2475,26 @@ def _maybe_mark_stopped(task):passif getattr(container, "status", None) != "exited":all_stopped = False- break+ continueresult = container.wait()- exit_code = max(exit_code, result.get("StatusCode", 0))+ container_exit_code = result.get("StatusCode", 0)except Exception:- # Container removed or unreachable — treat as stopped- pass+ # An inspect failure (daemon unavailable, transient transport+ # error) is not evidence that the process exited. Keep the ECS+ # task live until Docker positively reports an exit; otherwise a+ # temporary Docker outage would manufacture task failures.+ all_stopped = False+ continue++ exit_code = max(exit_code, container_exit_code or 0)+ with resource_lock("ecs-task", task_arn):+ if index < len(task.get("containers", [])):+ task["containers"][index]["lastStatus"] = "STOPPED"+ task["containers"][index]["exitCode"] = container_exit_code+ if index >= len(essentials) or essentials[index]:+ essential_exited = True- if not all_stopped:+ if not essential_exited and not all_stopped:returnstopped = _mark_task_stopped(diff --git a/tests/test_ecs.py b/tests/test_ecs.pyindex 931092f5..9101bd7b 100644--- a/tests/test_ecs.py+++ b/tests/test_ecs.py@@ -32,6 +32,20 @@ def _wait_until(predicate, timeout=5):assert predicate(), "condition did not become true before timeout"+def _ecs_docker_reachable():+ """Whether this end-to-end ECS lifecycle test can start real containers."""+ try:+ import docker+ docker.from_env(timeout=2).ping()+ except Exception:+ return False+ return True+++requires_ecs_docker = pytest.mark.skipif(+ not _ecs_docker_reachable(), reason="requires a reachable Docker daemon")++def _replace_arn_region(arn):return _replace_arn_section(arn, 3, _different_region(arn.split(":", 5)[3]))@@ -1197,10 +1211,10 @@ def test_ecs_service_td_update_replaces_tasks(ecs):containerDefinitions=[{"name": "app", "image": "nginx:latest", "cpu": 64, "memory": 128}],)ecs.create_service(- cluster=cluster, serviceName="tdu-svc", taskDefinition="tdu-td:1", desiredCount=1,+ cluster=cluster, serviceName="tdu-svc", taskDefinition="tdu-td:1", desiredCount=2,)old_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")- assert len(old_tasks["taskArns"]) == 1+ assert len(old_tasks["taskArns"]) == 2# Register new revision and update serviceresp2 = ecs.register_task_definition(@@ -1210,59 +1224,188 @@ def test_ecs_service_td_update_replaces_tasks(ecs):new_td_arn = resp2["taskDefinition"]["taskDefinitionArn"]ecs.update_service(cluster=cluster, service="tdu-svc", taskDefinition="tdu-td:2")- # ECS keeps both deployments while the replacement becomes healthy, then- # drains the old deployment and collapses the service to the new one.+ # A rolling deployment keeps the old tasks until the replacement is+ # healthy. Wait specifically for two RUNNING tasks on the new revision+ # instead of treating the still-running old tasks as replacements._wait_until(- lambda: (- len(ecs.describe_services(- cluster=cluster, services=["tdu-svc"]- )["services"][0]["deployments"]) == 1- and ecs.describe_services(- cluster=cluster, services=["tdu-svc"]- )["services"][0]["deployments"][0]["rolloutState"] == "COMPLETED"- ),+ lambda: len([+ task for task in ecs.describe_tasks(+ cluster=cluster,+ tasks=ecs.list_tasks(+ cluster=cluster, serviceName="tdu-svc"+ )["taskArns"],+ )["tasks"]+ if task["taskDefinitionArn"] == new_td_arn+ and task["lastStatus"] == "RUNNING"+ ]) == 2,timeout=30,)- # New tasks should be on the new TD- new_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")- assert len(new_tasks["taskArns"]) == 1-- # Verify all running tasks use the new task definition+ # Old tasks should be stopped_wait_until(lambda: all(- task["lastStatus"] == "RUNNING"+ task["lastStatus"] == "STOPPED"for task in ecs.describe_tasks(- cluster=cluster, tasks=new_tasks["taskArns"]+ cluster=cluster, tasks=old_tasks["taskArns"])["tasks"]),timeout=30,)- desc = ecs.describe_tasks(cluster=cluster, tasks=new_tasks["taskArns"])- for t in desc["tasks"]:- assert t["taskDefinitionArn"] == new_td_arn, \- f"Task still on old TD: {t['taskDefinitionArn']}"- assert t["lastStatus"] == "RUNNING"- # Old tasks should be stopped- old_desc = ecs.describe_tasks(cluster=cluster, tasks=old_tasks["taskArns"])- for t in old_desc["tasks"]:- assert t["lastStatus"] == "STOPPED"+ new_tasks = ecs.list_tasks(cluster=cluster, serviceName="tdu-svc")+ assert len(new_tasks["taskArns"]) == 2+ desc = ecs.describe_tasks(cluster=cluster, tasks=new_tasks["taskArns"])+ assert all(t["taskDefinitionArn"] == new_td_arn for t in desc["tasks"])# Service should reflect correct counts_wait_until(lambda: ecs.describe_services(cluster=cluster, services=["tdu-svc"]- )["services"][0]["runningCount"] == 1,+ )["services"][0]["runningCount"] == 2,timeout=30,)svc = ecs.describe_services(cluster=cluster, services=["tdu-svc"])- service = svc["services"][0]- assert service["runningCount"] == 1- assert len(service["deployments"]) == 1- assert service["deployments"][0]["taskDefinition"] == new_td_arn- assert service["deployments"][0]["status"] == "PRIMARY"- assert service["deployments"][0]["rolloutState"] == "COMPLETED"+ assert svc["services"][0]["runningCount"] == 2+ deployments = svc["services"][0]["deployments"]+ assert len(deployments) == 1+ assert deployments[0]["taskDefinition"] == new_td_arn+ assert deployments[0]["status"] == "PRIMARY"+ assert deployments[0]["rolloutState"] == "COMPLETED"+++@requires_ecs_docker+def test_ecs_service_circuit_breaker_rolls_back_crashing_revision(ecs):+ """A real container exit fails the new deployment and restores the old one.++ This deliberately observes only DescribeServices after UpdateService. The+ service's background lifecycle watcher, rather than an incidental+ DescribeTasks request, must discover every ``exit 1`` and give the circuit+ breaker the failure signal.+ """+ cluster = "circuit-breaker-c"+ family = "circuit-breaker-td"+ service = "circuit-breaker-svc"+ ecs.create_cluster(clusterName=cluster)+ healthy = ecs.register_task_definition(+ family=family,+ requiresCompatibilities=["FARGATE"],+ networkMode="awsvpc",+ containerDefinitions=[{+ "name": "app",+ "image": "alpine:latest",+ "command": ["sh", "-c", "sleep 600"],+ }],+ )["taskDefinition"]["taskDefinitionArn"]+ ecs.create_service(+ cluster=cluster,+ serviceName=service,+ taskDefinition=healthy,+ desiredCount=1,+ launchType="FARGATE",+ networkConfiguration={"awsvpcConfiguration": {"subnets": ["subnet-test"]}},+ )+ _wait_until(+ lambda: ecs.describe_services(cluster=cluster, services=[service])+ ["services"][0]["runningCount"] == 1,+ timeout=30,+ )++ crashing = ecs.register_task_definition(+ family=family,+ requiresCompatibilities=["FARGATE"],+ networkMode="awsvpc",+ containerDefinitions=[{+ "name": "app",+ "image": "alpine:latest",+ "command": ["sh", "-c", "exit 1"],+ }],+ )["taskDefinition"]["taskDefinitionArn"]+ ecs.update_service(+ cluster=cluster,+ service=service,+ taskDefinition=crashing,+ deploymentConfiguration={+ "deploymentCircuitBreaker": {"enable": True, "rollback": True},+ },+ )++ final_service = None++ def rolled_back():+ nonlocal final_service+ final_service = ecs.describe_services(+ cluster=cluster, services=[service]+ )["services"][0]+ deployments = final_service["deployments"]+ failed = next(+ (deployment for deployment in deployments+ if deployment["taskDefinition"] == crashing+ and deployment.get("rolloutState") == "FAILED"),+ None,+ )+ primary = next(+ (deployment for deployment in deployments+ if deployment.get("status") == "PRIMARY"),+ None,+ )+ return bool(+ failed and failed.get("rolloutStateReason")+ and primary and primary["taskDefinition"] == healthy+ and primary.get("rolloutState") == "COMPLETED"+ and final_service["taskDefinition"] == healthy+ and final_service["runningCount"] == 1+ )++ _wait_until(rolled_back, timeout=30)+ failed = next(+ deployment for deployment in final_service["deployments"]+ if deployment["taskDefinition"] == crashing+ )+ assert failed["failedTasks"] == 3+ assert failed["rolloutState"] == "FAILED"+ assert "circuit breaker" in failed["rolloutStateReason"]+++ ("reset_on_healthy", "expected_failures"),+ [(None, 0), (True, 0), (False, 2)],+)+def test_ecs_circuit_breaker_reset_on_healthy_task(+ reset_on_healthy, expected_failures):+ """A stable task resets failures unless cumulative mode is requested."""+ from ministack.services import ecs as _ecs++ cluster = f"reset-healthy-{_uuid_mod.uuid4().hex[:8]}"+ service = "svc"+ svc_key = f"{cluster}/{service}"+ deployment = _ecs._make_deployment("reset-healthy-td:2", 2)+ deployment["rolloutState"] = "IN_PROGRESS"+ deployment["rolloutStateReason"] = ""+ deployment["failedTasks"] = 2+ breaker = {"enable": True, "rollback": True}+ if reset_on_healthy is not None:+ breaker["resetOnHealthyTask"] = reset_on_healthy+ svc = {+ "serviceName": service,+ "clusterArn": (+ f"arn:aws:ecs:us-east-1:000000000000:cluster/{cluster}"+ ),+ "status": "ACTIVE",+ "deploymentConfiguration": {"deploymentCircuitBreaker": breaker},+ "deployments": [deployment],+ }+ task = {+ "taskDefinitionArn": "reset-healthy-td:2",+ "_deployment_id": deployment["id"],+ "lastStatus": "RUNNING",+ }+ _ecs._services[svc_key] = svc+ try:+ _ecs._record_service_task_healthy(svc_key, task)+ assert deployment["failedTasks"] == expected_failures+ finally:+ _ecs._services.pop(svc_key, None)def test_ecs_service_delete_stops_tasks(ecs):@@ -1388,6 +1531,148 @@ def test_ecs_cfn_service_visible(ecs, cfn):cfn.delete_stack(StackName=stack_name)+def test_ecs_cfn_service_deployment_circuit_breaker_create_and_update(ecs, cfn):+ """CloudFormation maps circuit-breaker fields and updates them in place."""+ suffix = _uuid_mod.uuid4().hex[:8]+ stack_name = f"ecs-cfn-circuit-{suffix}"+ cluster_name = f"ecs-cfn-circuit-c-{suffix}"+ service_name = f"ecs-cfn-circuit-s-{suffix}"+ first_family = f"ecs-cfn-circuit-one-{suffix}"+ second_family = f"ecs-cfn-circuit-two-{suffix}"++ def template(task_definition, configuration):+ return {+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "Cluster": {+ "Type": "AWS::ECS::Cluster",+ "Properties": {"ClusterName": cluster_name},+ },+ "FirstTaskDefinition": {+ "Type": "AWS::ECS::TaskDefinition",+ "Properties": {+ "Family": first_family,+ "ContainerDefinitions": [{"Name": "app", "Image": "busybox"}],+ },+ },+ "SecondTaskDefinition": {+ "Type": "AWS::ECS::TaskDefinition",+ "Properties": {+ "Family": second_family,+ "ContainerDefinitions": [{"Name": "app", "Image": "busybox"}],+ },+ },+ "Service": {+ "Type": "AWS::ECS::Service",+ "DependsOn": [+ "Cluster", "FirstTaskDefinition", "SecondTaskDefinition",+ ],+ "Properties": {+ "Cluster": {"Ref": "Cluster"},+ "ServiceName": service_name,+ "TaskDefinition": {"Ref": task_definition},+ "DesiredCount": 0,+ "DeploymentConfiguration": configuration,+ },+ },+ },+ }++ initial = {+ "MaximumPercent": 150,+ "MinimumHealthyPercent": 50,+ "DeploymentCircuitBreaker": {+ "Enable": True,+ "Rollback": True,+ "ResetOnHealthyTask": False,+ "ThresholdConfiguration": {"Type": "COUNT", "Value": 5},+ },+ }+ cfn.create_stack(+ StackName=stack_name,+ TemplateBody=json.dumps(template("FirstTaskDefinition", initial)),+ )+ _wait_until(+ lambda: cfn.describe_stacks(StackName=stack_name)["Stacks"][0]["StackStatus"]+ == "CREATE_COMPLETE",+ timeout=30,+ )+ service = ecs.describe_services(+ cluster=cluster_name, services=[service_name]+ )["services"][0]+ assert service["deploymentConfiguration"] == {+ "maximumPercent": 150,+ "minimumHealthyPercent": 50,+ "deploymentCircuitBreaker": {+ "enable": True,+ "rollback": True,+ "resetOnHealthyTask": False,+ "thresholdConfiguration": {"type": "COUNT", "value": 5},+ },+ }++ updated = {+ "DeploymentCircuitBreaker": {+ "Enable": True,+ "Rollback": False,+ "ResetOnHealthyTask": True,+ "ThresholdConfiguration": {+ "Type": "UNBOUNDED_PERCENT", "Value": 25,+ },+ },+ }+ cfn.update_stack(+ StackName=stack_name,+ TemplateBody=json.dumps(template("SecondTaskDefinition", updated)),+ )+ _wait_until(+ lambda: cfn.describe_stacks(StackName=stack_name)["Stacks"][0]["StackStatus"]+ == "UPDATE_COMPLETE",+ timeout=30,+ )+ service = ecs.describe_services(+ cluster=cluster_name, services=[service_name]+ )["services"][0]+ second_arn = ecs.describe_task_definition(+ taskDefinition=second_family+ )["taskDefinition"]["taskDefinitionArn"]+ assert service["taskDefinition"] == second_arn+ assert service["deploymentConfiguration"] == {+ "deploymentCircuitBreaker": {+ "enable": True,+ "rollback": False,+ "resetOnHealthyTask": True,+ "thresholdConfiguration": {+ "type": "UNBOUNDED_PERCENT", "value": 25,+ },+ },+ }+ cfn.delete_stack(StackName=stack_name)+++def test_ecs_cfn_service_update_propagates_ecs_errors(monkeypatch):+ """A rejected ECS UpdateService must fail the CloudFormation update."""+ from ministack.services.cloudformation import provisioners++ monkeypatch.setattr(+ ecs_service,+ "_update_service",+ lambda _request: (+ 400,+ {"Content-Type": "application/x-amz-json-1.0"},+ b'{"__type":"ClientException","message":"task definition not found"}',+ ),+ )++ with pytest.raises(ValueError, match="AWS::ECS::Service update failed"):+ provisioners._ecs_service_update(+ "arn:aws:ecs:us-east-1:000000000000:service/default/example",+ {"TaskDefinition": "example:1"},+ {"TaskDefinition": "example:99"},+ "stack",+ )++def test_ecs_cfn_taskdef_populates_registered_fields(ecs, cfn):"""CFN-created TaskDefinitions must surface registeredAt/registeredBy/compatibilities,matching what RegisterTaskDefinition emits. Workloads like Go-SDK reconcilers fall