Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(ecs): return RunTask before image pulls complete (#1687)

po-luka-mileticbb7a3114c488
diff --git a/ministack/services/ecs.py b/ministack/services/ecs.py
index fb957661..5ab74a89 100644
--- a/ministack/services/ecs.py
+++ b/ministack/services/ecs.py
@@ -105,6 +105,8 @@ def _live_container_ids():
"""
ids = set()
for _key, task in _tasks.all_items():
+ if task.get("lastStatus") == "STOPPED":
+ continue
for cid in task.get("_docker_ids") or []:
if cid:
ids.add(cid)
@@ -383,9 +385,9 @@ async def handle_request(method, path, headers, body, query_params):
target = headers.get("x-amz-target", "") or headers.get("X-Amz-Target", "")
if target:
action = target.split(".")[-1]
- # Docker work (RunTask/StopTask/service updates) blocks for as long as
- # the daemon takes: a cached nginx start already holds the loop >7s,
- # measured, during which no service in the process can be served.
+ # ECS mutations such as StopTask and service reconciliation still make
+ # blocking Docker calls. RunTask registers first and hands its startup
+ # work to a task worker, but this dispatcher must remain off the loop.
#
# run_reentrant, not the shared pool: every task container is handed
# AWS_ENDPOINT_URL pointing back here (see _task_env), so a starting
@@ -801,8 +803,33 @@ def _sync_service_targets(cluster_name, svc):
logger.debug("ECS: %s -> %d target(s) in %s", svc_name, len(targets), tg_arn)
+def _refresh_service_state(cluster_name, group):
+ if not group or not group.startswith("service:"):
+ return
+ svc_name = group.split(":", 1)[1]
+ svc = _services.get(f"{cluster_name}/{svc_name}")
+ if not svc:
+ return
+ cluster_arn = svc.get("clusterArn", "")
+ running = 0
+ pending = 0
+ for task in _tasks.values():
+ if task.get("group") != group or task.get("clusterArn") != cluster_arn:
+ continue
+ if task.get("lastStatus") == "RUNNING":
+ running += 1
+ elif task.get("lastStatus") == "PENDING":
+ pending += 1
+ svc["runningCount"] = running
+ svc["pendingCount"] = pending
+ if svc.get("deployments"):
+ svc["deployments"][0]["runningCount"] = running
+ svc["deployments"][0]["pendingCount"] = pending
+ _sync_service_targets(cluster_name, svc)
+
+
def _reconcile_service_tasks(cluster_name, svc_key):
- """Spawn or stop tasks so running tasks match desiredCount and task definition."""
+ """Spawn or stop active tasks so desiredCount and task definition match."""
svc = _services.get(svc_key)
if not svc or svc["status"] != "ACTIVE":
return
@@ -819,13 +846,15 @@ def _reconcile_service_tasks(cluster_name, svc_key):
td = _task_defs.get(td_key)
target_td_arn = td["taskDefinitionArn"] if td else td_arn
- # Partition running service tasks into current-TD and stale-TD
+ # Partition active service tasks into current-TD and stale-TD. A PENDING
+ # task already counts toward desired capacity, so reconciliation does not
+ # launch duplicates while its worker is pulling the image.
current_tasks = []
stale_tasks = []
for arn, t in _tasks.items():
if (t.get("group") == f"service:{svc_name}"
and t.get("clusterArn") == cluster_arn
- and t.get("lastStatus") == "RUNNING"):
+ and t.get("lastStatus") in ("PENDING", "RUNNING")):
if t.get("taskDefinitionArn") == target_td_arn:
current_tasks.append((arn, t))
else:
@@ -862,21 +891,9 @@ def _reconcile_service_tasks(cluster_name, svc_key):
_stop_task({"task": task_arn, "cluster": cluster_name,
"reason": "Service scaling down"})
- # Recount actual running tasks
- running = sum(
- 1 for t in _tasks.values()
- if t.get("group") == f"service:{svc_name}"
- and t.get("clusterArn") == cluster_arn
- and t.get("lastStatus") == "RUNNING"
- )
- svc["runningCount"] = running
- if svc["deployments"]:
- svc["deployments"][0]["runningCount"] = running
+ _refresh_service_state(cluster_name, f"service:{svc_name}")
_recount_cluster(cluster_name)
- # Membership of the service's target groups follows its running tasks.
- _sync_service_targets(cluster_name, svc)
-
def _create_service(data):
cluster_name = _resolve_cluster_name(data.get("cluster", "default"))
@@ -986,7 +1003,7 @@ def _delete_service(data):
a for a, t in _tasks.items()
if t.get("group") == f"service:{svc_name}"
and t.get("clusterArn") == cluster_arn
- and t.get("lastStatus") == "RUNNING"
+ and t.get("lastStatus") in ("PENDING", "RUNNING")
]:
_stop_task({"task": task_arn, "cluster": cluster_name, "reason": "Service deleted"})
@@ -1360,7 +1377,291 @@ def _resolve_container_secrets(cdef):
return resolved
+def _task_is_active(task_arn, task):
+ return (
+ _tasks.get(task_arn) is task
+ and not task.get("_ecs_resetting")
+ and task.get("lastStatus") != "STOPPED"
+ and task.get("desiredStatus") != "STOPPED"
+ )
+
+
+def _remove_docker_container(docker_client, container_or_id):
+ if not docker_client or container_or_id is None:
+ return
+ container = container_or_id
+ if isinstance(container_or_id, str):
+ try:
+ container = docker_client.containers.get(container_or_id)
+ except Exception:
+ return
+ try:
+ container.stop(timeout=5)
+ except Exception:
+ pass
+ try:
+ container.remove(v=True)
+ except Exception:
+ try:
+ container.remove(v=True, force=True)
+ except Exception:
+ pass
+
+
+def _cleanup_task_resources(task, docker_client, extra_container=None):
+ task_arn = task.get("taskArn", "")
+ with resource_lock("ecs-task", task_arn):
+ docker_ids = list(task.get("_docker_ids") or [])
+ metadata_tokens = list(task.get("_metadata_tokens") or [])
+ task["_docker_ids"] = []
+ task["_metadata_tokens"] = []
+
+ for token in metadata_tokens:
+ ecs_metadata.unregister_token(token)
+
+ seen = set()
+ if extra_container is not None:
+ extra_id = getattr(extra_container, "id", None)
+ if extra_id:
+ seen.add(extra_id)
+ _remove_docker_container(docker_client, extra_container)
+ for docker_id in docker_ids:
+ if docker_id not in seen:
+ _remove_docker_container(docker_client, docker_id)
+
+
+def _mark_task_stopped(task_arn, task, reason, stop_code, exit_code=None):
+ with resource_lock("ecs-task", task_arn):
+ if not _task_is_active(task_arn, task):
+ return False
+ now = _iso()
+ task["lastStatus"] = "STOPPED"
+ task["desiredStatus"] = "STOPPED"
+ task["stoppingAt"] = task.get("stoppingAt") or now
+ task["stoppedAt"] = now
+ task["stoppedReason"] = reason
+ task["stopCode"] = stop_code
+ for container in task.get("containers", []):
+ container["lastStatus"] = "STOPPED"
+ if exit_code is not None:
+ container["exitCode"] = exit_code
+
+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))
+ if cluster_name:
+ _recount_cluster(cluster_name)
+ _refresh_service_state(cluster_name, task.get("group", ""))
+ return True
+
+
+def _mark_task_running(task_arn, task):
+ with resource_lock("ecs-task", task_arn):
+ if not _task_is_active(task_arn, task):
+ return False
+ now = _iso()
+ task["lastStatus"] = "RUNNING"
+ task["pullStoppedAt"] = task.get("pullStoppedAt") or now
+ task["startedAt"] = task.get("startedAt") or now
+
+ cluster_name = _cluster_name_from_arn(task.get("clusterArn", ""))
+ if cluster_name:
+ _recount_cluster(cluster_name)
+ _refresh_service_state(cluster_name, task.get("group", ""))
+ return True
+
+
+def _attach_started_container(task, container, index, metadata_token, ecs_network):
+ task_arn = task["taskArn"]
+ container_id = container.id
+ with resource_lock("ecs-task", task_arn):
+ if not _task_is_active(task_arn, task):
+ return False
+ task.setdefault("_docker_ids", []).append(container_id)
+ if index < len(task.get("containers", [])):
+ task["containers"][index]["runtimeId"] = container_id[:12]
+ task["containers"][index]["lastStatus"] = "RUNNING"
+
+ ecs_metadata.set_docker_id(metadata_token, container_id)
+ _record_task_ip(task, container, ecs_network)
+ logger.info("ECS: started container %s for task %s", container_id, task_arn[:8])
+ return True
+
+
+def _run_docker_container(docker_client, cdef, run_kwargs):
+ container = None
+ try:
+ try:
+ container = docker_client.containers.run(cdef["image"], **run_kwargs)
+ except Exception as exc:
+ # Best-effort platform pin: a host that cannot run the declared
+ # runtimePlatform can still use the host architecture, as before.
+ pinned = run_kwargs.pop("platform", None)
+ if not pinned:
+ raise
+ logger.warning(
+ "ECS: task definition declares %s but this host cannot "
+ "run it (%s); running %s on the host architecture "
+ "instead. Install a binfmt/qemu handler for real "
+ "cross-architecture execution.",
+ pinned, exc, cdef.get("image"))
+ container = docker_client.containers.run(cdef["image"], **run_kwargs)
+
+ pinned = run_kwargs.get("platform")
+ if pinned:
+ # Without emulation, Docker can create the container successfully
+ # and then fail on its first instruction. Only that failure gets
+ # the architecture fallback; other exits belong to the task.
+ time.sleep(0.25)
+ exec_dead = False
+ try:
+ container.reload()
+ exec_dead = (container.status == "exited"
+ and b"exec format error"
+ in (container.logs(tail=5) or b""))
+ except Exception:
+ exec_dead = False
+ if exec_dead:
+ logger.warning(
+ "ECS: task definition declares %s but this host cannot "
+ "execute it; running %s on the host architecture "
+ "instead. Install a binfmt/qemu handler for real "
+ "cross-architecture execution.",
+ pinned, cdef.get("image"))
+ try:
+ container.remove(force=True)
+ except Exception:
+ _remove_docker_container(docker_client, container)
+ run_kwargs.pop("platform", None)
+ container = docker_client.containers.run(
+ cdef["image"], **run_kwargs)
+ return container
+ except Exception:
+ if container is not None:
+ _remove_docker_container(docker_client, container)
+ raise
+
+
+def _fail_task_start(task, docker_client, reason):
+ try:
+ _mark_task_stopped(task["taskArn"], task, reason, "TaskFailedToStart")
+ finally:
+ _cleanup_task_resources(task, docker_client)
+
+
+def _start_task_worker(task, td, container_overrides, docker_client):
+ task_arn = task["taskArn"]
+ task_id = task_arn.rsplit("/", 1)[-1]
+ cluster_arn = task["clusterArn"]
+ launch_type = task.get("launchType", "EC2")
+
+ with resource_lock("ecs-task", task_arn):
+ active = _task_is_active(task_arn, task)
+ if active:
+ task["pullStartedAt"] = task.get("pullStartedAt") or _iso()
+ if not active:
+ return
+
+ ecs_network = None
+ ministack_net_ip = None
+ try:
+ self_container = docker_client.containers.get(os.environ.get("HOSTNAME", ""))
+ nets = self_container.attrs["NetworkSettings"]["Networks"]
+ if nets:
+ ecs_network = next(iter(nets))
+ ministack_net_ip = nets[ecs_network].get("IPAddress") or None
+ logger.debug(
+ "ECS: detected Ministack network=%s ip=%s",
+ ecs_network, ministack_net_ip,
+ )
+ except Exception:
+ logger.debug("ECS: could not detect Ministack network, using default")
+
+ for i, cdef in enumerate(td.get("containerDefinitions", [])):
+ with resource_lock("ecs-task", task_arn):
+ active = _task_is_active(task_arn, task)
+ if not active:
+ _cleanup_task_resources(task, docker_client)
+ return
+
+ container_override = _container_override_for(
+ container_overrides, cdef["name"]
+ )
+ env_override = {
+ e["name"]: e["value"]
+ for e in container_override.get("environment", [])
+ }
+ env = {e["name"]: e["value"] for e in cdef.get("environment", [])}
+ env.update(_resolve_container_secrets(cdef))
+ env.update(env_override)
+
+ effective_cdef = dict(cdef)
+ if "command" in container_override:
+ effective_cdef["command"] = container_override["command"]
+
+ port_bindings = {}
+ if td.get("networkMode") != "awsvpc":
+ for pm in cdef.get("portMappings", []):
+ host_port = pm.get("hostPort", pm.get("containerPort"))
+ port_bindings[f"{pm['containerPort']}/tcp"] = host_port
+
+ host_mode = td.get("networkMode") == "host"
+ metadata_token = _register_metadata(
+ task_arn, cluster_arn, td, cdef, launch_type, env,
+ host_mode, ministack_net_ip,
+ )
+ with resource_lock("ecs-task", task_arn):
+ active = _task_is_active(task_arn, task)
+ if active:
+ task.setdefault("_metadata_tokens", []).append(metadata_token)
+ if not active:
+ ecs_metadata.unregister_token(metadata_token)
+ _cleanup_task_resources(task, docker_client)
+ return
+
+ run_kwargs = _build_run_kwargs(
+ effective_cdef, td, env, port_bindings, ecs_network,
+ host_mode, task_id, task_arn, ministack_net_ip, cluster_arn,
+ )
+
+ with resource_lock("ecs-task", task_arn):
+ active = _task_is_active(task_arn, task)
+ if not active:
+ _cleanup_task_resources(task, docker_client)
+ return
+
+ container = _run_docker_container(docker_client, effective_cdef, run_kwargs)
+
+ if not _attach_started_container(
+ task, container, i, metadata_token, ecs_network):
+ _cleanup_task_resources(task, docker_client, container)
+ return
+
+ if not _mark_task_running(task_arn, task):
+ _cleanup_task_resources(task, docker_client)
+
+
+def _run_task_worker(task, td, container_overrides, docker_client, account_id, region):
+ with request_scope(account_id, region):
+ try:
+ _start_task_worker(task, td, container_overrides, docker_client)
+ except _SecretResolutionError as exc:
+ _fail_task_start(
+ task, docker_client,
+ "ResourceInitializationError: unable to pull secrets or "
+ "registry auth: execution resource retrieval failed: "
+ + str(exc),
+ )
+ except Exception as exc:
+ logger.warning("ECS: Docker start failed for %s: %s", task["taskArn"], exc)
+ _fail_task_start(
+ task, docker_client,
+ "ResourceInitializationError: unable to pull image or start "
+ "container: " + str(exc),
+ )
+
+
def _run_task(data):
+ account_id = get_account_id()
+ region = get_region()
cluster_name = _resolve_cluster_name(data.get("cluster", "default"))
if cluster_name is None:
return error_response_json("ClusterNotFoundException", "Cluster not found.", 400)
@@ -1375,33 +1676,49 @@ def _run_task(data):
f"Unable to find task definition: {td_ref}", 400)
count = data.get("count", 1)
- container_overrides = data.get("overrides", {}).get("containerOverrides", [])
+ overrides = data.get(
+ "overrides", {"containerOverrides": [], "inferenceAcceleratorOverrides": []}
+ )
+ container_overrides = copy.deepcopy(overrides.get("containerOverrides", []))
launch_type = data.get("launchType", "EC2")
group = data.get("group", "")
started_by = data.get("startedBy", "")
enable_exec = data.get("enableExecuteCommand", False)
- network_cfg = data.get("networkConfiguration", {})
req_tags = data.get("tags", [])
+ docker_client = _get_docker()
+ docker_backed = bool(docker_client)
+ initial_status = "PENDING" if docker_backed else "RUNNING"
tasks = []
failures = []
+ worker_args = []
+ task_definition = copy.deepcopy(td) if docker_backed else td
+ if docker_backed:
+ _ensure_ecs_reaper_thread()
for _ in range(count):
task_id = new_uuid()
- task_arn = f"arn:aws:ecs:{get_region()}:{get_account_id()}:task/{cluster_name}/{task_id}"
+ task_arn = f"arn:aws:ecs:{region}:{account_id}:task/{cluster_name}/{task_id}"
now = _iso()
containers = _build_task_containers(td, container_overrides)
- for c in containers:
- c["taskArn"] = task_arn
+ if docker_backed:
+ for container in containers:
+ container["lastStatus"] = "PENDING"
+ container["runtimeId"] = None
+ for container in containers:
+ container["taskArn"] = task_arn
task = {
"taskArn": task_arn,
"clusterArn": _clusters[cluster_name]["clusterArn"],
"taskDefinitionArn": td["taskDefinitionArn"],
- "containerInstanceArn": f"arn:aws:ecs:{get_region()}:{get_account_id()}:container-instance/{cluster_name}/{new_uuid()}",
- "overrides": data.get("overrides", {"containerOverrides": [], "inferenceAcceleratorOverrides": []}),
- "lastStatus": "RUNNING",
+ "containerInstanceArn": (
+ f"arn:aws:ecs:{region}:{account_id}:container-instance/"
+ f"{cluster_name}/{new_uuid()}"
+ ),
+ "overrides": overrides,
+ "lastStatus": initial_status,
"desiredStatus": "RUNNING",
"launchType": launch_type,
"cpu": td.get("cpu", "256"),
@@ -1410,10 +1727,10 @@ def _run_task(data):
"platformFamily": "",
"connectivity": "CONNECTED",
"connectivityAt": now,
- "pullStartedAt": now,
- "pullStoppedAt": now,
+ "pullStartedAt": now if not docker_backed else None,
+ "pullStoppedAt": now if not docker_backed else None,
"createdAt": now,
- "startedAt": now,
+ "startedAt": now if not docker_backed else None,
"stoppingAt": None,
"stoppedAt": None,
"stoppedReason": "",
@@ -1423,7 +1740,7 @@ def _run_task(data):
"version": 1,
"containers": containers,
"attachments": [],
- "availabilityZone": f"{get_region()}a",
+ "availabilityZone": f"{region}a",
"enableExecuteCommand": enable_exec,
"tags": req_tags,
"healthStatus": "UNKNOWN",
@@ -1431,152 +1748,33 @@ def _run_task(data):
"_docker_ids": [],
}
- docker_client = _get_docker()
- if docker_client and td:
- _ensure_ecs_reaper_thread()
- # Detect the Docker network Ministack is running on,
- # so ECS containers can reach sibling services (S3, etc.)
- ecs_network = None
- ministack_net_ip = None
- try:
- self_container = docker_client.containers.get(os.environ.get("HOSTNAME", ""))
- nets = self_container.attrs["NetworkSettings"]["Networks"]
- if nets:
- ecs_network = next(iter(nets))
- # IP that ECS containers should use to reach Ministack.
- # Using ministack's own address on the shared network is
- # the only thing that works reliably on Linux Docker —
- # `host-gateway` magic resolves to 172.17.0.1 (docker0),
- # which iptables typically blocks from a sibling bridge.
- ministack_net_ip = nets[ecs_network].get("IPAddress") or None
- logger.debug(
- "ECS: detected Ministack network=%s ip=%s",
- ecs_network, ministack_net_ip,
- )
- except Exception:
- logger.debug("ECS: could not detect Ministack network, using default")
-
- for i, cdef in enumerate(td.get("containerDefinitions", [])):
- container_override = _container_override_for(
- container_overrides, cdef["name"]
- )
- env_override = {}
- for e in container_override.get("environment", []):
- env_override[e["name"]] = e["value"]
-
- env = {e["name"]: e["value"] for e in cdef.get("environment", [])}
- try:
- env.update(_resolve_container_secrets(cdef))
- except _SecretResolutionError as exc:
- # AWS fails the whole task to start (ResourceInitializationError)
- # when a secret can't be retrieved — don't launch any container.
- now = _iso()
- task["lastStatus"] = "STOPPED"
- task["desiredStatus"] = "STOPPED"
- task["stoppingAt"] = now
- task["stoppedAt"] = now
- task["stopCode"] = "TaskFailedToStart"
- task["stoppedReason"] = (
- "ResourceInitializationError: unable to pull secrets or "
- "registry auth: execution resource retrieval failed: "
- + str(exc))
- for c in task.get("containers", []):
- c["lastStatus"] = "STOPPED"
- break
- env.update(env_override)
- effective_cdef = dict(cdef)
- if "command" in container_override:
- effective_cdef["command"] = container_override["command"]
-
- # awsvpc tasks get their own network namespace — an ENI in AWS — so
- # container ports are never published on the host. Publishing them
- # makes two tasks that share a container port collide on the host,
- # which cannot happen on Fargate. Only bridge/host mode publishes.
- port_bindings = {}
- if td.get("networkMode") != "awsvpc":
- for pm in cdef.get("portMappings", []):
- host_port = pm.get("hostPort", pm.get("containerPort"))
- port_bindings[f"{pm['containerPort']}/tcp"] = host_port
-
- host_mode = td.get("networkMode") == "host"
- metadata_token = _register_metadata(
- task_arn, _clusters[cluster_name]["clusterArn"],
- td, cdef, launch_type, env, host_mode, ministack_net_ip,
- )
- run_kwargs = _build_run_kwargs(
- effective_cdef, td, env, port_bindings, ecs_network,
- host_mode, task_id, task_arn, ministack_net_ip,
- _clusters[cluster_name]["clusterArn"],
- )
-
- try:
- try:
- container = docker_client.containers.run(cdef["image"], **run_kwargs)
- except Exception as exc:
- # Best-effort platform pin: a host that cannot run the
- # declared runtimePlatform (no emulation handler, or a
- # cached image of the other architecture) logs the
- # mismatch and runs on its own architecture, as every
- # setup did before the pin existed.
- pinned = run_kwargs.pop("platform", None)
- if not pinned:
- raise
- logger.warning(
- "ECS: task definition declares %s but this host cannot "
- "run it (%s); running %s on the host architecture "
- "instead. Install a binfmt/qemu handler for real "
- "cross-architecture execution.",
- pinned, exc, cdef.get("image"))
- container = docker_client.containers.run(cdef["image"], **run_kwargs)
- pinned = run_kwargs.get("platform")
- if pinned:
- # No emulation handler makes the container start cleanly
- # and die on its first instruction — docker raises
- # nothing. Only an exec-format death triggers the
- # fallback: an instant exit for any other reason is the
- # task's own business.
- time.sleep(0.25)
- exec_dead = False
- try:
- container.reload()
- exec_dead = (container.status == "exited"
- and b"exec format error"
- in (container.logs(tail=5) or b""))
- except Exception:
- # The death-check is best-effort; if it cannot even
- # be performed, keep the container that started.
- exec_dead = False
- if exec_dead:
- logger.warning(
- "ECS: task definition declares %s but this host "
- "cannot execute it; running %s on the host "
- "architecture instead. Install a binfmt/qemu "
- "handler for real cross-architecture execution.",
- pinned, cdef.get("image"))
- try:
- container.remove(force=True)
- except Exception:
- pass
- run_kwargs.pop("platform", None)
- container = docker_client.containers.run(
- cdef["image"], **run_kwargs)
- task["_docker_ids"].append(container.id)
- ecs_metadata.set_docker_id(metadata_token, container.id)
- task.setdefault("_metadata_tokens", []).append(metadata_token)
- if i < len(task["containers"]):
- task["containers"][i]["runtimeId"] = container.id[:12]
- _record_task_ip(task, container, ecs_network)
- logger.info("ECS: started container %s for task %s", cdef['image'], task_id[:8])
- except Exception as e:
- ecs_metadata.unregister_token(metadata_token)
- logger.warning("ECS: Docker run failed for %s: %s", cdef.get('image'), e)
-
_tasks[task_arn] = task
if req_tags:
_tags[task_arn] = list(req_tags)
tasks.append(_sanitize(task))
+ if docker_backed:
+ worker_args.append((task, task_definition, container_overrides))
_recount_cluster(cluster_name)
+ for task, task_definition, overrides in worker_args:
+ worker = threading.Thread(
+ target=_run_task_worker,
+ args=(
+ task, task_definition, overrides, docker_client,
+ account_id, region,
+ ),
+ daemon=True,
+ name=f"ministack-ecs-task-{task['taskArn'].rsplit('/', 1)[-1][:8]}",
+ )
+ try:
+ worker.start()
+ except RuntimeError as exc:
+ with request_scope(account_id, region):
+ _fail_task_start(
+ task, docker_client,
+ "ResourceInitializationError: unable to start task worker: "
+ + str(exc),
+ )
return json_response({"tasks": tasks, "failures": failures})
@@ -1596,32 +1794,10 @@ def _stop_task(data):
return json_response({"task": _sanitize(task)})
docker_client = _get_docker()
- if docker_client:
- for docker_id in task.get("_docker_ids", []):
- try:
- c = docker_client.containers.get(docker_id)
- c.stop(timeout=5)
- c.remove(v=True)
- except Exception as e:
- logger.warning("ECS: failed to stop container %s: %s", docker_id, e)
-
- for tok in task.get("_metadata_tokens", []):
- ecs_metadata.unregister_token(tok)
-
- now = _iso()
- task["lastStatus"] = "STOPPED"
- task["desiredStatus"] = "STOPPED"
- task["stoppingAt"] = now
- task["stoppedAt"] = now
- task["stoppedReason"] = reason
- task["stopCode"] = "UserInitiated"
- for c in task.get("containers", []):
- c["lastStatus"] = "STOPPED"
- c["exitCode"] = 0
-
- cname = _cluster_name_from_arn(task.get("clusterArn", ""))
- if cname:
- _recount_cluster(cname)
+ _mark_task_stopped(
+ task["taskArn"], task, reason, "UserInitiated", exit_code=0
+ )
+ _cleanup_task_resources(task, docker_client)
return json_response({"task": _sanitize(task)})
@@ -1638,7 +1814,8 @@ def _describe_tasks(data):
task = _resolve_task(ref, cluster_name)
if task:
_maybe_mark_stopped(task)
- t = _sanitize(task)
+ with resource_lock("ecs-task", task.get("taskArn", "")):
+ t = _sanitize(task)
if "TAGS" in include:
t["tags"] = _tags.get(task["taskArn"], [])
result.append(t)
@@ -1651,8 +1828,13 @@ def _describe_tasks(data):
def _maybe_mark_stopped(task):
"""Check Docker containers and transition task to STOPPED if all have exited."""
- if task.get("lastStatus") != "RUNNING" or not task.get("_docker_ids"):
- return
+ task_arn = task.get("taskArn", "")
+ with resource_lock("ecs-task", task_arn):
+ if (not _task_is_active(task_arn, task)
+ or task.get("lastStatus") != "RUNNING"
+ or not task.get("_docker_ids")):
+ return
+ docker_ids = list(task["_docker_ids"])
docker_client = _get_docker()
if not docker_client:
@@ -1660,7 +1842,7 @@ def _maybe_mark_stopped(task):
all_stopped = True
exit_code = 0
- for docker_id in task["_docker_ids"]:
+ for docker_id in docker_ids:
try:
container = docker_client.containers.get(docker_id)
# docker SDK caches status; refresh before checking lifecycle
@@ -1680,20 +1862,12 @@ def _maybe_mark_stopped(task):
if not all_stopped:
return
- now = _iso()
- task["lastStatus"] = "STOPPED"
- task["desiredStatus"] = "STOPPED"
- task["stoppingAt"] = task.get("stoppingAt") or now
- task["stoppedAt"] = now
- task["stoppedReason"] = "Essential container exited"
- task["stopCode"] = "EssentialContainerExited"
- for c in task.get("containers", []):
- c["lastStatus"] = "STOPPED"
- c["exitCode"] = exit_code
-
- cname = _cluster_name_from_arn(task.get("clusterArn", ""))
- if cname:
- _recount_cluster(cname)
+ stopped = _mark_task_stopped(
+ task_arn, task, "Essential container exited",
+ "EssentialContainerExited", exit_code=exit_code,
+ )
+ if stopped:
+ _cleanup_task_resources(task, None)
def _list_tasks(data):
@@ -2398,6 +2572,10 @@ _ACTION_MAP = {
def reset():
+ for _key, task in _tasks.all_items():
+ with resource_lock("ecs-task", task.get("taskArn", "")):
+ task["_ecs_resetting"] = True
+
docker_client = _get_docker()
if docker_client:
# Stop + remove every ministack=ecs container (running OR exited).
@@ -2423,3 +2601,4 @@ def reset():
_account_settings.clear()
_capacity_providers.clear()
_attributes.clear()
+ ecs_metadata.reset()
diff --git a/tests/conftest.py b/tests/conftest.py
index bbd70eb2..8a613d31 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -193,6 +193,16 @@ _SERIAL_TESTS = {
# STOPPED, so list_tasks (RUNNING-only) sees fewer than desiredCount. Passes
# serially; run it in the serial phase.
"tests/test_ecs.py::test_ecs_service_spawns_tasks",
+ # Cluster task counts use the same Docker-backed service startup and can
+ # remain PENDING behind unrelated container churn. Run serially as well.
+ "tests/test_ecs.py::test_ecs_cluster_task_counts",
+ # SSM Run Command provisions real EC2-agent containers. Keeping these
+ # together avoids Docker API timeouts after the parallel container-heavy
+ # phase.
+ "tests/test_ssm.py::test_ssm_run_command_health_probe",
+ "tests/test_ssm.py::test_ssm_run_command_probe_can_fail",
+ "tests/test_ssm.py::test_ssm_send_command_accepts_a_managed_instance",
+ "tests/test_ssm.py::test_ssm_command_lookup_filters_and_errors",
# WS/MQTT-broker tests (MQTT-over-WebSocket connect/publish/subscribe,
# device shadows over MQTT, fleet-index connectivity). They drive the
# single-event-loop broker over real WebSocket connections with tight
diff --git a/tests/test_ecs.py b/tests/test_ecs.py
index 20074495..16ef90a1 100644
--- a/tests/test_ecs.py
+++ b/tests/test_ecs.py
@@ -23,6 +23,15 @@ def _different_region(region):
return "us-west-2" if region != "us-west-2" else "us-east-1"
+def _wait_until(predicate, timeout=5):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if predicate():
+ return
+ time.sleep(0.01)
+ assert predicate(), "condition did not become true before timeout"
+
+
def _replace_arn_region(arn):
return _replace_arn_section(arn, 3, _different_region(arn.split(":", 5)[3]))
@@ -71,7 +80,7 @@ def test_ecs_run_task_stops_after_exit(ecs):
)
resp = ecs.run_task(cluster="task-lifecycle", taskDefinition="short-lived")
task_arn = resp["tasks"][0]["taskArn"]
- assert resp["tasks"][0]["lastStatus"] == "RUNNING"
+ assert resp["tasks"][0]["lastStatus"] in ("PENDING", "RUNNING")
# Poll until STOPPED (container exits almost immediately)
stopped = False
@@ -161,7 +170,7 @@ def test_ecs_run_task_network_connectivity(ecs):
)
resp = ecs.run_task(cluster="net-test", taskDefinition="net-probe")
task_arn = resp["tasks"][0]["taskArn"]
- assert resp["tasks"][0]["lastStatus"] == "RUNNING"
+ assert resp["tasks"][0]["lastStatus"] in ("PENDING", "RUNNING")
# Poll until STOPPED — wget should succeed (exit 0) if network is correct
success = False
@@ -209,7 +218,7 @@ def test_ecs_run_task_metadata_v4(ecs):
)
resp = ecs.run_task(cluster="metadata-test", taskDefinition="metadata-probe")
task_arn = resp["tasks"][0]["taskArn"]
- assert resp["tasks"][0]["lastStatus"] == "RUNNING"
+ assert resp["tasks"][0]["lastStatus"] in ("PENDING", "RUNNING")
success = False
for _ in range(30):
@@ -277,6 +286,7 @@ def test_ecs_run_task_applies_container_command_overrides(monkeypatch):
},
})
+ _wait_until(lambda: len(fake_containers.calls) == 2)
calls_by_name = {
kwargs["labels"]["com.amazonaws.ecs.container-name"]: kwargs
for _image, kwargs in fake_containers.calls
@@ -331,6 +341,7 @@ def test_ecs_run_task_command_override_allows_empty_command(monkeypatch):
},
})
+ _wait_until(lambda: fake_containers.calls)
assert fake_containers.calls[0][1]["command"] == []
def test_ecs_run_task_injects_secrets_manager_secrets(monkeypatch):
@@ -381,6 +392,7 @@ def test_ecs_run_task_injects_secrets_manager_secrets(monkeypatch):
_ecs._run_task({"cluster": "secrets-c", "taskDefinition": "secrets-td"})
+ _wait_until(lambda: fake_containers.calls)
env = fake_containers.calls[0][1]["environment"]
assert env["FOO"] == "bar"
assert env["SECRET_VAL"] == "s3cr3t"
@@ -858,6 +870,15 @@ def test_ecs_service_spawns_tasks(ecs):
assert len(tasks["taskArns"]) == 2
# Verify describe_tasks returns correct metadata
+ _wait_until(
+ lambda: all(
+ task["lastStatus"] == "RUNNING"
+ for task in ecs.describe_tasks(
+ cluster=cluster, tasks=tasks["taskArns"]
+ )["tasks"]
+ ),
+ timeout=30,
+ )
desc = ecs.describe_tasks(cluster=cluster, tasks=tasks["taskArns"])
for t in desc["tasks"]:
assert t["lastStatus"] == "RUNNING"
@@ -897,6 +918,12 @@ def test_ecs_service_running_count(ecs):
ecs.create_service(
cluster=cluster, serviceName="rc-svc", taskDefinition="rc-td", desiredCount=3,
)
+ _wait_until(
+ lambda: ecs.describe_services(
+ cluster=cluster, services=["rc-svc"]
+ )["services"][0]["runningCount"] == 3,
+ timeout=30,
+ )
resp = ecs.describe_services(cluster=cluster, services=["rc-svc"])
svc = resp["services"][0]
assert svc["runningCount"] == 3
@@ -921,6 +948,12 @@ def test_ecs_service_scale_up(ecs):
tasks_after = ecs.list_tasks(cluster=cluster, serviceName="su-svc")
assert len(tasks_after["taskArns"]) == 3
+ _wait_until(
+ lambda: ecs.describe_services(
+ cluster=cluster, services=["su-svc"]
+ )["services"][0]["runningCount"] == 3,
+ timeout=30,
+ )
resp = ecs.describe_services(cluster=cluster, services=["su-svc"])
assert resp["services"][0]["runningCount"] == 3
@@ -943,6 +976,12 @@ def test_ecs_service_scale_down(ecs):
tasks_after = ecs.list_tasks(cluster=cluster, serviceName="sd-svc")
assert len(tasks_after["taskArns"]) == 1
+ _wait_until(
+ lambda: ecs.describe_services(
+ cluster=cluster, services=["sd-svc"]
+ )["services"][0]["runningCount"] == 1,
+ timeout=30,
+ )
resp = ecs.describe_services(cluster=cluster, services=["sd-svc"])
assert resp["services"][0]["runningCount"] == 1
@@ -953,7 +992,7 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="tdu-td",
- containerDefinitions=[{"name": "app", "image": "nginx:1.0", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "nginx:latest", "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="tdu-svc", taskDefinition="tdu-td:1", desiredCount=2,
@@ -964,7 +1003,7 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
# Register new revision and update service
resp2 = ecs.register_task_definition(
family="tdu-td",
- containerDefinitions=[{"name": "app", "image": "nginx:2.0", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "nginx:alpine", "cpu": 64, "memory": 128}],
)
new_td_arn = resp2["taskDefinition"]["taskDefinitionArn"]
ecs.update_service(cluster=cluster, service="tdu-svc", taskDefinition="tdu-td:2")
@@ -974,6 +1013,15 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
assert len(new_tasks["taskArns"]) == 2
# Verify all running tasks use the new task definition
+ _wait_until(
+ lambda: all(
+ task["lastStatus"] == "RUNNING"
+ for task in ecs.describe_tasks(
+ cluster=cluster, tasks=new_tasks["taskArns"]
+ )["tasks"]
+ ),
+ timeout=30,
+ )
desc = ecs.describe_tasks(cluster=cluster, tasks=new_tasks["taskArns"])
for t in desc["tasks"]:
assert t["taskDefinitionArn"] == new_td_arn, \
@@ -986,6 +1034,12 @@ def test_ecs_service_td_update_replaces_tasks(ecs):
assert t["lastStatus"] == "STOPPED"
# Service should reflect correct counts
+ _wait_until(
+ lambda: ecs.describe_services(
+ cluster=cluster, services=["tdu-svc"]
+ )["services"][0]["runningCount"] == 2,
+ timeout=30,
+ )
svc = ecs.describe_services(cluster=cluster, services=["tdu-svc"])
assert svc["services"][0]["runningCount"] == 2
@@ -1054,6 +1108,12 @@ def test_ecs_cluster_task_counts(ecs):
ecs.create_service(
cluster=cluster, serviceName="ct-svc", taskDefinition="ct-td", desiredCount=3,
)
+ _wait_until(
+ lambda: ecs.describe_clusters(
+ clusters=[cluster]
+ )["clusters"][0]["runningTasksCount"] == 3,
+ timeout=30,
+ )
resp = ecs.describe_clusters(clusters=[cluster])
cl = resp["clusters"][0]
assert cl["runningTasksCount"] == 3
@@ -1415,12 +1475,356 @@ def _fake_docker_recorder(run_impl=None):
self.calls.append((image, kwargs))
if run_impl is not None:
return run_impl(image, kwargs)
- return SimpleNamespace(id=f"container-{len(self.calls):012d}")
+ return SimpleNamespace(id=f"container{len(self.calls):03d}")
fake_containers = FakeContainers()
return fake_containers, SimpleNamespace(containers=fake_containers)
+def test_ecs_run_task_returns_pending_before_docker_start(monkeypatch):
+ """Registration is visible while a slow Docker start is still blocked."""
+ import threading
+
+ from ministack.services import ecs as _ecs
+
+ started = threading.Event()
+ release = threading.Event()
+ containers = {}
+
+ class FakeContainer:
+ def __init__(self, cid):
+ self.id = cid
+ self.status = "running"
+ self.attrs = {"NetworkSettings": {"Networks": {}}}
+ self.removed = False
+
+ def reload(self):
+ pass
+
+ def wait(self):
+ return {"StatusCode": 0}
+
+ def stop(self, timeout=5):
+ self.status = "exited"
+
+ def remove(self, **kwargs):
+ self.removed = True
+
+ class FakeContainers:
+ def get(self, name):
+ if name in containers:
+ return containers[name]
+ raise Exception("not found")
+
+ def list(self, *args, **kwargs):
+ return list(containers.values())
+
+ def run(self, image, **kwargs):
+ started.set()
+ assert release.wait(timeout=5)
+ container = FakeContainer("pending-test-container")
+ containers[container.id] = container
+ return container
+
+ monkeypatch.setattr(
+ _ecs, "_get_docker", lambda: SimpleNamespace(containers=FakeContainers())
+ )
+ _ecs._register_task_definition({
+ "family": "pending-test-td",
+ "containerDefinitions": [{"name": "app", "image": "busybox"}],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "pending-test-c",
+ "taskDefinition": "pending-test-td",
+ })
+ task = json.loads(response[2])["tasks"][0]
+ assert task["lastStatus"] == "PENDING"
+ assert task["containers"][0]["lastStatus"] == "PENDING"
+ assert started.wait(timeout=2)
+ assert _ecs._tasks[task["taskArn"]]["lastStatus"] == "PENDING"
+
+ release.set()
+ _wait_until(lambda: _ecs._tasks[task["taskArn"]]["lastStatus"] == "RUNNING")
+
+ containers["pending-test-container"].status = "exited"
+ described = json.loads(_ecs._describe_tasks({
+ "cluster": "pending-test-c",
+ "tasks": [task["taskArn"]],
+ })[2])["tasks"][0]
+ assert described["lastStatus"] == "STOPPED"
+ assert described["containers"][0]["exitCode"] == 0
+
+
+def test_ecs_run_task_startup_failure_is_a_stopped_task(monkeypatch):
+ from ministack.services import ecs as _ecs
+
+ class FakeContainers:
+ def get(self, _name):
+ raise Exception("not found")
+
+ def run(self, image, **kwargs):
+ raise RuntimeError("image pull failed")
+
+ monkeypatch.setattr(
+ _ecs, "_get_docker", lambda: SimpleNamespace(containers=FakeContainers())
+ )
+ _ecs._register_task_definition({
+ "family": "startup-failure-td",
+ "containerDefinitions": [{"name": "app", "image": "missing"}],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "startup-failure-c",
+ "taskDefinition": "startup-failure-td",
+ })
+ task = json.loads(response[2])["tasks"][0]
+ _wait_until(lambda: _ecs._tasks[task["taskArn"]]["lastStatus"] == "STOPPED")
+ stopped = _ecs._tasks[task["taskArn"]]
+ assert stopped["stopCode"] == "TaskFailedToStart"
+ assert "image pull failed" in stopped["stoppedReason"]
+ assert stopped["containers"][0]["lastStatus"] == "STOPPED"
+
+
+def test_ecs_stop_task_during_pending_start_cleans_late_container(monkeypatch):
+ import threading
+
+ from ministack.services import ecs as _ecs
+
+ started = threading.Event()
+ release = threading.Event()
+ containers = {}
+
+ class FakeContainer:
+ def __init__(self):
+ self.id = "late-container"
+ self.status = "running"
+ self.attrs = {"NetworkSettings": {"Networks": {}}}
+ self.removed = False
+
+ def reload(self):
+ pass
+
+ def stop(self, timeout=5):
+ self.status = "exited"
+
+ def remove(self, **kwargs):
+ self.removed = True
+
+ class FakeContainers:
+ def get(self, name):
+ if name in containers:
+ return containers[name]
+ raise Exception("not found")
+
+ def run(self, image, **kwargs):
+ started.set()
+ assert release.wait(timeout=5)
+ container = FakeContainer()
+ containers[container.id] = container
+ return container
+
+ fake_docker = SimpleNamespace(containers=FakeContainers())
+ monkeypatch.setattr(_ecs, "_get_docker", lambda: fake_docker)
+ _ecs._register_task_definition({
+ "family": "stop-pending-td",
+ "containerDefinitions": [{"name": "app", "image": "busybox"}],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "stop-pending-c",
+ "taskDefinition": "stop-pending-td",
+ })
+ task = json.loads(response[2])["tasks"][0]
+ assert started.wait(timeout=2)
+ stopped = json.loads(_ecs._stop_task({
+ "cluster": "stop-pending-c",
+ "task": task["taskArn"],
+ })[2])["task"]
+ assert stopped["lastStatus"] == "STOPPED"
+
+ release.set()
+ _wait_until(lambda: "late-container" in containers)
+ _wait_until(lambda: containers["late-container"].removed)
+ assert _ecs._tasks[task["taskArn"]]["lastStatus"] == "STOPPED"
+
+
+def test_ecs_reset_during_pending_start_cleans_late_container(monkeypatch):
+ import threading
+
+ from ministack.services import ecs as _ecs
+
+ started = threading.Event()
+ release = threading.Event()
+ containers = {}
+
+ class FakeContainer:
+ id = "reset-late-container"
+ status = "running"
+ attrs = {"NetworkSettings": {"Networks": {}}}
+
+ def __init__(self):
+ self.removed = False
+
+ def reload(self):
+ pass
+
+ def stop(self, timeout=5):
+ self.status = "exited"
+
+ def remove(self, **kwargs):
+ self.removed = True
+
+ class FakeContainers:
+ def get(self, name):
+ if name in containers:
+ return containers[name]
+ raise Exception("not found")
+
+ def list(self, *args, **kwargs):
+ return list(containers.values())
+
+ def run(self, image, **kwargs):
+ started.set()
+ assert release.wait(timeout=5)
+ container = FakeContainer()
+ containers[container.id] = container
+ return container
+
+ fake_docker = SimpleNamespace(containers=FakeContainers())
+ monkeypatch.setattr(_ecs, "_get_docker", lambda: fake_docker)
+ _ecs.reset()
+ _ecs._register_task_definition({
+ "family": "reset-pending-td",
+ "containerDefinitions": [{"name": "app", "image": "busybox"}],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "reset-pending-c",
+ "taskDefinition": "reset-pending-td",
+ })
+ task_arn = json.loads(response[2])["tasks"][0]["taskArn"]
+ assert started.wait(timeout=2)
+
+ _ecs.reset()
+ release.set()
+ _wait_until(lambda: "reset-late-container" in containers)
+ _wait_until(lambda: containers["reset-late-container"].removed)
+ assert task_arn not in _ecs._tasks
+
+
+def test_ecs_secret_resolution_failure_stops_before_docker_run(monkeypatch):
+ from ministack.services import ecs as _ecs
+
+ fake_containers, fake_docker = _fake_docker_recorder()
+ monkeypatch.setattr(_ecs, "_get_docker", lambda: fake_docker)
+ _ecs._register_task_definition({
+ "family": "secret-failure-td",
+ "containerDefinitions": [{
+ "name": "app",
+ "image": "busybox",
+ "secrets": [{
+ "name": "MISSING",
+ "valueFrom": "arn:aws:secretsmanager:us-east-1:000000000000:secret:missing",
+ }],
+ }],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "secret-failure-c",
+ "taskDefinition": "secret-failure-td",
+ })
+ task = json.loads(response[2])["tasks"][0]
+ _wait_until(lambda: _ecs._tasks[task["taskArn"]]["lastStatus"] == "STOPPED")
+ stopped = _ecs._tasks[task["taskArn"]]
+ assert not fake_containers.calls
+ assert stopped["stopCode"] == "TaskFailedToStart"
+ assert "unable to retrieve secret" in stopped["stoppedReason"]
+
+
+def test_ecs_run_task_count_and_multi_container_startup_are_independent(monkeypatch):
+ from ministack.services import ecs as _ecs
+
+ fake_containers, fake_docker = _fake_docker_recorder()
+ monkeypatch.setattr(_ecs, "_get_docker", lambda: fake_docker)
+ _ecs._register_task_definition({
+ "family": "multi-start-td",
+ "containerDefinitions": [
+ {"name": "app", "image": "busybox"},
+ {"name": "sidecar", "image": "busybox"},
+ ],
+ })
+
+ response = _ecs._run_task({
+ "cluster": "multi-start-c",
+ "taskDefinition": "multi-start-td",
+ "count": 2,
+ })
+ tasks = json.loads(response[2])["tasks"]
+ assert all(task["lastStatus"] == "PENDING" for task in tasks)
+ _wait_until(lambda: len(fake_containers.calls) == 4)
+ _wait_until(
+ lambda: all(
+ _ecs._tasks[task["taskArn"]]["lastStatus"] == "RUNNING"
+ for task in tasks
+ )
+ )
+ runtime_ids = {
+ container["runtimeId"]
+ for task in tasks
+ for container in _ecs._tasks[task["taskArn"]]["containers"]
+ }
+ assert len(runtime_ids) == 4
+
+
+def test_ecs_run_task_preserves_nonzero_exit_code(monkeypatch):
+ from ministack.services import ecs as _ecs
+
+ class FakeContainer:
+ id = "nonzero-container"
+ status = "exited"
+ attrs = {"NetworkSettings": {"Networks": {}}}
+
+ def reload(self):
+ pass
+
+ def wait(self):
+ return {"StatusCode": 17}
+
+ container = FakeContainer()
+
+ class FakeContainers:
+ def get(self, name):
+ if name == container.id:
+ return container
+ raise Exception("not found")
+
+ def run(self, image, **kwargs):
+ return container
+
+ monkeypatch.setattr(
+ _ecs, "_get_docker", lambda: SimpleNamespace(containers=FakeContainers())
+ )
+ _ecs._register_task_definition({
+ "family": "nonzero-td",
+ "containerDefinitions": [{"name": "app", "image": "busybox"}],
+ })
+ response = _ecs._run_task({
+ "cluster": "nonzero-c",
+ "taskDefinition": "nonzero-td",
+ })
+ task = json.loads(response[2])["tasks"][0]
+ _wait_until(
+ lambda: json.loads(_ecs._describe_tasks({
+ "cluster": "nonzero-c", "tasks": [task["taskArn"]],
+ })[2])["tasks"][0]["lastStatus"] == "STOPPED"
+ )
+ stopped = json.loads(_ecs._describe_tasks({
+ "cluster": "nonzero-c", "tasks": [task["taskArn"]],
+ })[2])["tasks"][0]
+ assert stopped["containers"][0]["exitCode"] == 17
+
+
def test_ecs_awsvpc_task_does_not_publish_host_ports(monkeypatch):
"""awsvpc tasks get their own ENI, so container ports are not bound on the host.
@@ -1443,6 +1847,7 @@ def test_ecs_awsvpc_task_does_not_publish_host_ports(monkeypatch):
})
_ecs._run_task({"cluster": "awsvpc-ports-c", "taskDefinition": "awsvpc-ports-td"})
+ _wait_until(lambda: fake_containers.calls)
assert fake_containers.calls, "expected the container to be launched"
_image, kwargs = fake_containers.calls[0]
assert not kwargs.get("ports"), (
@@ -1468,6 +1873,7 @@ def test_ecs_bridge_task_still_publishes_host_ports(monkeypatch):
})
_ecs._run_task({"cluster": "bridge-ports-c", "taskDefinition": "bridge-ports-td"})
+ _wait_until(lambda: fake_containers.calls)
_image, kwargs = fake_containers.calls[0]
assert kwargs.get("ports") == {"80/tcp": 8080}
@@ -1525,6 +1931,7 @@ def test_ecs_service_registers_tasks_in_target_group(monkeypatch):
],
})
+ _wait_until(lambda: _alb._targets.get(tg_arn) == [{"Id": task_ip, "Port": 80}])
registered = _alb._targets.get(tg_arn, [])
assert registered == [{"Id": task_ip, "Port": 80}], registered
@@ -1565,6 +1972,8 @@ def test_ecs_task_records_private_ipv4_attachment(monkeypatch):
_status, _headers, raw = resp
task = json.loads(raw)["tasks"][0]
+ _wait_until(lambda: _ecs._task_ip(_ecs._tasks.get(task["taskArn"])) == "172.30.0.9")
+ task = _ecs._tasks[task["taskArn"]]
assert _ecs._task_ip(task) == "172.30.0.9"
assert task["attachmentsStatus"] == "ATTACHED"
assert task["attachments"][0]["type"] == "ElasticNetworkInterface"
@@ -1840,6 +2249,12 @@ def test_ecs_service_reconcile_spares_foreign_targets(monkeypatch):
{"targetGroupArn": tg_arn, "containerName": "web", "containerPort": 80},
],
})
+ _wait_until(
+ lambda: _alb._targets.get(tg_arn) == [
+ {"Id": "10.9.9.9", "Port": 80},
+ {"Id": task_ip, "Port": 80},
+ ]
+ )
registered = sorted(t["Id"] for t in _alb._targets.get(tg_arn, []))
assert registered == ["10.9.9.9", task_ip], registered
diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.py
index b5afefac..8d11bd79 100644
--- a/tests/test_stepfunctions.py
+++ b/tests/test_stepfunctions.py
@@ -3694,6 +3694,39 @@ def test_sfn_integration_ecs_run_task(sfn, ecs):
output = json.loads(desc["output"])
assert "tasks" in output
+
+def test_sfn_ecs_sync_poll_waits_for_stopped_and_preserves_result(monkeypatch):
+ """The sync integration must wait past PENDING/RUNNING states."""
+ from ministack.services import ecs as ecs_service
+ from ministack.services import stepfunctions as sfn_service
+
+ responses = [
+ {"tasks": [{"lastStatus": "PENDING"}], "failures": []},
+ {"tasks": [{"lastStatus": "RUNNING"}], "failures": []},
+ {
+ "tasks": [{
+ "lastStatus": "STOPPED",
+ "containers": [{"exitCode": 17}],
+ }],
+ "failures": [],
+ },
+ ]
+ seen = []
+
+ def describe(data):
+ seen.append(data)
+ return 200, {}, json.dumps(responses.pop(0))
+
+ monkeypatch.setattr(ecs_service, "_describe_tasks", describe)
+ monkeypatch.setattr(sfn_service, "_scaled_sleep", lambda _seconds: None)
+
+ result = sfn_service._poll_ecs_tasks("cluster", ["task-arn"])
+
+ assert len(seen) == 3
+ assert result["tasks"][0]["lastStatus"] == "STOPPED"
+ assert result["tasks"][0]["containers"][0]["exitCode"] == 17
+
+
def test_sfn_integration_ecs_run_task_sync_success(sfn, ecs):
"""ecs:runTask.sync waits for task STOPPED, then returns task result."""
import threading
@@ -3816,7 +3849,7 @@ def test_sfn_integration_ecs_run_task_output_contains_status(sfn, ecs):
assert "taskArn" in task_out
assert "containers" in task_out
assert task_out["containers"][0]["name"] == "app"
- assert task_out["lastStatus"] == "RUNNING"
+ assert task_out["lastStatus"] in ("PENDING", "RUNNING")
assert "failures" in output
def test_sfn_integration_ecs_run_task_container_overrides_reach_the_task(sfn, ecs):