Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(iot): run a CONTINUOUS job again for a thing that rejoins its group (#2003)
iot-rocketc3f67beee4b8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 997c8e1e..90fe13ad 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed+- **IoT — a CONTINUOUS job runs again for a thing that rejoins its target group** — executions were kept one per thing and job, so a thing that finished the job, left the group and was added back never got another execution. Joining a target group now queues the next execution number when the thing's newest execution is finished, leaving the group moves a QUEUED execution to `REMOVED`, both publish `jobs/notify(-next)`, the `notify-next` execution leaves out `thingName` and an empty `statusDetails`, and `ListJobExecutionsForThing`, `DescribeJobExecution` with `executionNumber` and `jobProcessDetails` include the earlier executions, as AWS does. Contributed by @iot-rocket.- **IoT — a `#` region no longer widens topic filters** — a WebSocket session whose credential names `#` as its region received every message published under that region, whatever its filter, because filters were matched with the account and region prefix in front of them. Contributed by @iot-rocket.- **Cognito — app client secret hashes** — clients created with `GenerateSecret=true` require a valid `SecretHash` on signup, confirmation and password recovery, and `SECRET_HASH` on authentication and challenge responses. Missing or incorrect hashes return `NotAuthorizedException` before user changes, email delivery or Lambda triggers. Clients without a secret reject a supplied hash on self-service and initial authentication calls, while refresh and challenge responses ignore it, as on AWS. Refresh authentication verifies the token owner's username, or `sub` when the pool uses `UsernameAttributes`, and custom challenge sessions cannot be answered through a different client. The checks apply in both `AUTH` modes.### Addeddiff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex e1f99c24..e74dfcab 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -527,9 +527,9 @@ async def _route_request(# ThingGroups — special add/remove paths must come BEFORE the# generic ``/thing-groups/{name}`` handler.if path == "/thing-groups/addThingToThingGroup" and method in ("PUT", "POST"):- return _add_thing_to_group(_parse_body(body))+ return await _add_thing_to_group(_parse_body(body))if path == "/thing-groups/removeThingFromThingGroup" and method in ("PUT", "POST"):- return _remove_thing_from_group(_parse_body(body))+ return await _remove_thing_from_group(_parse_body(body))if path == "/thing-groups" and method == "GET":return _list_thing_groups(qp)if path.startswith("/thing-groups/") and path.endswith("/things") and method == "GET":@@ -1173,7 +1173,9 @@ def _delete_thing_group(name: str) -> tuple:return json_response({})-def _add_thing_to_group(payload: dict) -> tuple:+async def _add_thing_to_group(payload: dict) -> tuple:+ """Async because a CONTINUOUS job the thing joins publishes+ jobs/notify(-next) through the broker."""gname = payload.get("thingGroupName")tname = payload.get("thingName")if not gname or not tname:@@ -1186,17 +1188,23 @@ def _add_thing_to_group(payload: dict) -> tuple:thing = _things.get(tname)if thing is None:return _error_not_found("Thing", tname)- if tname not in group.get("things", []):+ joined = tname not in group.get("things", [])+ if joined:+ prev_next = jobs_first_pending_job_id(tname)group.setdefault("things", []).append(tname)_thing_groups[gname] = group_membership_event("ADDED", group, thing)if gname not in thing.get("thingGroupNames", []):thing.setdefault("thingGroupNames", []).append(gname)_things[tname] = thing+ if joined and _jobs_thing_joined_group(tname, _thing_group_arn(gname)):+ await jobs_notify_thing(get_account_id(), get_region(), tname, prev_next)return json_response({})-def _remove_thing_from_group(payload: dict) -> tuple:+async def _remove_thing_from_group(payload: dict) -> tuple:+ """Async because a QUEUED CONTINUOUS execution the removal retires+ publishes jobs/notify(-next) through the broker."""gname = payload.get("thingGroupName")tname = payload.get("thingName")group = _thing_groups.get(gname) if gname else None@@ -1205,6 +1213,7 @@ def _remove_thing_from_group(payload: dict) -> tuple:return _error_not_found("ThingGroup", gname or "")if thing is None:return _error_not_found("Thing", tname or "")+ prev_next = jobs_first_pending_job_id(tname)if tname in group.get("things", []):group["things"].remove(tname)_thing_groups[gname] = group@@ -1212,6 +1221,8 @@ def _remove_thing_from_group(payload: dict) -> tuple:if gname in thing.get("thingGroupNames", []):thing["thingGroupNames"].remove(gname)_things[tname] = thing+ if _jobs_thing_left_group(tname):+ await jobs_notify_thing(get_account_id(), get_region(), tname, prev_next)return json_response({})@@ -4043,13 +4054,15 @@ def _list_topic_rules(qp: dict) -> tuple:## `versionNumber` gives optimistic concurrency: every transition bumps it, and# a stale `expectedVersion` is rejected with a 409. `executionNumber` is NOT a-# concurrency token here — an execution is created once at 1 and nothing-# re-queues it, so it never changes (AWS increments it when a job execution is-# retried, which MiniStack does not model).+# concurrency token here. It counts a thing's executions of one job: 1 for the+# first, one more each time the thing joins a CONTINUOUS job's target group+# again after its last execution finished (AWS also increments it when a job+# execution is retried, which MiniStack does not model). `_job_executions`+# holds the newest execution per (thing, job); the older, terminal ones ride+# along newest first under its internal `history` key.#-# TIMED_OUT and REMOVED are recognized as terminal (so a restored record in-# either state behaves, and `jobProcessDetails` counts them) but nothing sets-# them: there are no execution timeouts, and deleting a thing DELETES its+# A thing leaving every target of a CONTINUOUS job moves its QUEUED execution+# to REMOVED (see `_jobs_thing_left_group`). Deleting a thing DELETES its# executions (see `_delete_thing`) rather than marking them REMOVED.# Job ids are stricter than thing names, and identically so in both service@@ -4170,21 +4183,97 @@ def _jobs_materialize_executions(job_id: str) -> None:key = (thing, job_id)if key in _job_executions:continue- _job_executions[key] = {- "jobId": job_id,- "thingName": thing,- "status": "QUEUED",- "statusDetails": {},- "queuedAt": now,- "startedAt": None,- "lastUpdatedAt": now,- "executionNumber": 1,- "versionNumber": 1,- }+ _jobs_queue_execution(thing, job_id, now)if job.get("targetSelection") == "SNAPSHOT":job["snapshotted"] = True+def _jobs_queue_execution(thing: str, job_id: str, now: int) -> None:+ """Queue the thing's next execution of a job, at version 1.++ The first one is number 1. A later one takes the next number and keeps+ the finished one before it under `history`.+ """+ previous = _job_executions.get((thing, job_id))+ execution = {+ "jobId": job_id,+ "thingName": thing,+ "status": "QUEUED",+ "statusDetails": {},+ "queuedAt": now,+ "startedAt": None,+ "lastUpdatedAt": now,+ "executionNumber": 1,+ "versionNumber": 1,+ }+ if previous is not None:+ execution["executionNumber"] = previous["executionNumber"] + 1+ execution["history"] = [+ {k: v for k, v in previous.items() if k != "history"},+ *previous.get("history", []),+ ]+ _job_executions[(thing, job_id)] = execution+++def _jobs_thing_joined_group(thing: str, group_arn: str) -> bool:+ """Run every CONTINUOUS job targeting a group for a thing that just+ joined it.++ The thing gets a new QUEUED execution unless its newest one is still+ QUEUED or IN_PROGRESS. That covers a thing rejoining after its execution+ finished, and a thing that finished the job joining another target group+ of it (measured eu-central-1 2026-10-05). Returns whether anything was+ queued, i.e. whether the thing's jobs/notify topics have news.+ """+ now = _jobs_now_ms()+ queued = False+ for job_id, job in _jobs.items():+ if (+ job.get("targetSelection") != "CONTINUOUS"+ or job.get("status") == "CANCELED"+ or group_arn not in (job.get("targets") or [])+ ):+ continue+ previous = _job_executions.get((thing, job_id))+ if previous is not None and previous["status"] not in _JOB_EXECUTION_TERMINAL:+ continue+ _jobs_queue_execution(thing, job_id, now)+ queued = True+ return queued+++def _jobs_thing_left_group(thing: str) -> bool:+ """Retire a thing's QUEUED CONTINUOUS executions after it left a group.++ Call after the membership change. For every CONTINUOUS job the thing no+ longer resolves into, a QUEUED execution becomes REMOVED with its+ `versionNumber` unchanged. IN_PROGRESS, SUCCEEDED and FAILED executions+ keep their status (measured eu-central-1 2026-10-05). Returns whether a+ pending execution went away, i.e. whether jobs/notify has news.+ """+ now = _jobs_now_ms()+ removed = False+ for job_id, job in _jobs.items():+ execution = _job_executions.get((thing, job_id))+ if (+ execution is None+ or execution["status"] != "QUEUED"+ or job.get("targetSelection") != "CONTINUOUS"+ or job.get("status") == "CANCELED"+ or thing in _job_target_things(job.get("targets"))+ ):+ continue+ execution["status"] = "REMOVED"+ execution["lastUpdatedAt"] = now+ removed = True+ return removed+++def _jobs_with_history(execution: dict) -> list[dict]:+ """A thing's executions of one job, newest first."""+ return [execution, *execution.get("history", [])]++def _jobs_materialize_all() -> None:for job_id in list(_jobs.keys()):_jobs_materialize_executions(job_id)@@ -4512,10 +4601,13 @@ def _describe_job(job_id: str) -> tuple:if job is None:return _error_not_found("Job", job_id)_jobs_materialize_executions(job_id)+ # Every execution counts, a rejoined thing's earlier ones included+ # (measured eu-central-1 2026-10-05).counts = {status: 0 for status in _JOB_EXECUTION_STATUSES}- for execution in _job_executions.values():- if execution["jobId"] == job_id:- counts[execution["status"]] += 1+ for latest in _job_executions.values():+ if latest["jobId"] == job_id:+ for execution in _jobs_with_history(latest):+ counts[execution["status"]] += 1job_doc = {**_job_summary(job),"targets": list(job.get("targets") or []),@@ -4695,7 +4787,7 @@ async def _handle_thing_jobs(method: str, path: str, body: bytes, qp: dict) -> tthing, tail[:-len("/cancel")], _parse_body(body), qp)if "/" not in tail and method == "GET":- return _describe_job_execution(thing, tail)+ return _describe_job_execution(thing, tail, qp)return error_response_json("InvalidRequestException", f"Unsupported IoT path: {method} {path}", 400)@@ -4709,7 +4801,13 @@ def _list_job_executions_for_thing(thing: str, qp: dict) -> tuple:# The wire shape nests the detail under `jobExecutionSummary`, with only# the jobId beside it (JobExecutionSummaryForThing).summaries = []- for execution in _job_executions.values():+ # A rejoined thing's executions of one job come newest first, as on AWS.+ executions = [+ execution+ for latest in _job_executions.values()+ for execution in _jobs_with_history(latest)+ ]+ for execution in executions:if execution["thingName"] != thing:continueif wanted_status and execution["status"] != wanted_status:@@ -4731,9 +4829,22 @@ def _list_job_executions_for_thing(thing: str, qp: dict) -> tuple:return json_response({"executionSummaries": summaries})-def _describe_job_execution(thing: str, job_id: str) -> tuple:+def _describe_job_execution(thing: str, job_id: str, qp: dict) -> tuple:+ """Without `executionNumber` the newest execution, with it that one."""_jobs_materialize_executions(job_id)execution = _job_executions.get((thing, job_id))+ number = qp.get("executionNumber")+ if execution is not None and number is not None:+ try:+ number = int(number)+ except (TypeError, ValueError):+ return error_response_json(+ "InvalidRequestException", f"Invalid executionNumber: {number!r}", 400+ )+ execution = next(+ (e for e in _jobs_with_history(execution) if e["executionNumber"] == number),+ None,+ )if execution is None:return _error_not_found("Job execution", f"{thing}/{job_id}")view = {@@ -5025,12 +5136,13 @@ async def jobs_notify_thing(Payloads (all timestamps epoch seconds): ``notify`` carries``{"timestamp", "jobs": {<STATUS>: [summaries]}}`` with empty statuslists omitted (``{}`` once nothing is pending); ``notify-next`` carries- the full execution view — status and ``jobDocument`` as a JSON object- included — or a bare ``{"timestamp"}`` when the queue emptied.+ the execution view (``jobDocument`` as a JSON object, no ``thingName``)+ or a bare ``{"timestamp"}`` when the queue emptied.- Known boundaries: a CONTINUOUS job's execution materialized lazily on a- *read* (a thing that joined the target group later) does not notify —- the materialization sites are synchronous read paths — and DeleteThing's+ Known boundaries: an execution materialized lazily on a *read* does not+ notify — the materialization sites are synchronous read paths (a thing+ joining a CONTINUOUS job's target group is queued eagerly by+ AddThingToThingGroup, which does notify) — and DeleteThing'sexecution sweep does not notify the deleted thing (that sweep is itselfa divergence: AWS keeps a deleted thing's executions, so it publishesnothing there either)."""@@ -5042,6 +5154,12 @@ async def jobs_notify_thing(if nxt is not Noneelse None)+ if next_view is not None:+ # AWS sends no thingName and no empty statusDetails here (measured+ # eu-central-1 2026-10-05).+ del next_view["thingName"]+ if not next_view["statusDetails"]:+ del next_view["statusDetails"]ts = int(time.time())jobs_by_status: dict = {}for execution in pending:diff --git a/ministack/services/iot_jobs_data.py b/ministack/services/iot_jobs_data.pyindex a8decb4e..72f2edb7 100644--- a/ministack/services/iot_jobs_data.py+++ b/ministack/services/iot_jobs_data.py@@ -199,7 +199,8 @@ def _describe_execution(thing: str, job_id: str, qp: dict) -> tuple:async def _update_execution(thing: str, job_id: str, payload: dict) -> tuple:- # Nothing re-queues here, so only execution number 1 exists.+ # Only the newest execution takes updates; a rejoined thing's older ones+ # are terminal.execution_number = payload.get("executionNumber")if execution_number is not None:try:diff --git a/tests/test_iot_jobs_data.py b/tests/test_iot_jobs_data.pyindex 8f4f6156..84b7fa3d 100644--- a/tests/test_iot_jobs_data.py+++ b/tests/test_iot_jobs_data.py@@ -728,6 +728,250 @@ def test_iot_jobs_continuous_late_thing_first_call_is_update(_cleanup(iot_client, jobs=[job_id], things=[late], groups=[group])+def _executions_for(iot_client, thing, job_id):+ """(executionNumber, status) pairs ListJobExecutionsForThing lists."""+ summaries = iot_client.list_job_executions_for_thing(+ thingName=thing, jobId=job_id+ )["executionSummaries"]+ return [+ (s["jobExecutionSummary"]["executionNumber"], s["jobExecutionSummary"]["status"])+ for s in summaries+ ]+++def _drive(iot_jobs_data, thing, job_id, status):+ """Start the thing's pending execution, then report `status` unless it+ is IN_PROGRESS."""+ iot_jobs_data.start_next_pending_job_execution(thingName=thing)+ if status != "IN_PROGRESS":+ iot_jobs_data.update_job_execution(+ thingName=thing, jobId=job_id, status=status+ )+++def test_iot_jobs_continuous_thing_rejoining_group_runs_job_again(+ iot_client, iot_jobs_data+):+ """A thing that leaves a CONTINUOUS job's target group and rejoins it+ after its execution finished gets the next execution number, QUEUED.+ Leaving moves a QUEUED execution to REMOVED; IN_PROGRESS, SUCCEEDED and+ FAILED keep their status, and a thing rejoining while IN_PROGRESS gets+ nothing new (measured eu-central-1 2026-10-05)."""+ group = _unique("jobs-group")+ job_id = _unique("job-cont")+ cases = ["queued", "inprogress", "succeeded", "failed", "stays"]+ things = {case: _unique(f"jobs-{case}") for case in cases}+ try:+ group_arn = iot_client.create_thing_group(thingGroupName=group)[+ "thingGroupArn"+ ]+ iot_client.create_job(+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,+ targetSelection="CONTINUOUS",+ )+ for thing in things.values():+ _create_thing(iot_client, thing)+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ _drive(iot_jobs_data, things["inprogress"], job_id, "IN_PROGRESS")+ _drive(iot_jobs_data, things["succeeded"], job_id, "SUCCEEDED")+ _drive(iot_jobs_data, things["failed"], job_id, "FAILED")+ _drive(iot_jobs_data, things["stays"], job_id, "SUCCEEDED")++ movers = [things[case] for case in cases if case != "stays"]+ for thing in movers:+ iot_client.remove_thing_from_thing_group(+ thingGroupName=group, thingName=thing+ )+ removed = iot_client.describe_job_execution(+ jobId=job_id, thingName=things["queued"]+ )["execution"]+ assert (removed["status"], removed["versionNumber"]) == ("REMOVED", 1)+ pending = iot_jobs_data.get_pending_job_executions(thingName=things["queued"])+ assert pending["queuedJobs"] == []+ assert _executions_for(iot_client, things["inprogress"], job_id) == [+ (1, "IN_PROGRESS")+ ]+ assert _executions_for(iot_client, things["failed"], job_id) == [(1, "FAILED")]++ for thing in movers:+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ assert _executions_for(iot_client, things["queued"], job_id) == [+ (2, "QUEUED"), (1, "REMOVED"),+ ]+ assert _executions_for(iot_client, things["succeeded"], job_id) == [+ (2, "QUEUED"), (1, "SUCCEEDED"),+ ]+ assert _executions_for(iot_client, things["failed"], job_id) == [+ (2, "QUEUED"), (1, "FAILED"),+ ]+ assert _executions_for(iot_client, things["inprogress"], job_id) == [+ (1, "IN_PROGRESS")+ ]+ assert _executions_for(iot_client, things["stays"], job_id) == [+ (1, "SUCCEEDED")+ ]++ # The device sees the new execution; the newest one answers a+ # describe without executionNumber, a number picks an older one.+ pending = iot_jobs_data.get_pending_job_executions(+ thingName=things["succeeded"]+ )+ assert [(q["jobId"], q["executionNumber"]) for q in pending["queuedJobs"]] == [+ (job_id, 2)+ ]+ newest = iot_client.describe_job_execution(+ jobId=job_id, thingName=things["succeeded"]+ )["execution"]+ assert (newest["executionNumber"], newest["status"]) == (2, "QUEUED")+ assert newest["versionNumber"] == 1+ first = iot_client.describe_job_execution(+ jobId=job_id, thingName=things["succeeded"], executionNumber=1+ )["execution"]+ assert (first["executionNumber"], first["status"]) == (1, "SUCCEEDED")+ with pytest.raises(ClientError) as exc:+ iot_client.describe_job_execution(+ jobId=job_id, thingName=things["succeeded"], executionNumber=3+ )+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"++ # Every execution counts, the earlier ones of a rejoined thing too.+ details = iot_client.describe_job(jobId=job_id)["job"]["jobProcessDetails"]+ assert details["numberOfQueuedThings"] == 3+ assert details["numberOfInProgressThings"] == 1+ assert details["numberOfRemovedThings"] == 1+ assert details["numberOfSucceededThings"] == 2+ assert details["numberOfFailedThings"] == 1++ # The IN_PROGRESS execution that rejoined finishes as a member: no new+ # execution follows. A second leave and rejoin runs the job again.+ _drive(iot_jobs_data, things["inprogress"], job_id, "SUCCEEDED")+ assert _executions_for(iot_client, things["inprogress"], job_id) == [+ (1, "SUCCEEDED")+ ]+ _drive(iot_jobs_data, things["succeeded"], job_id, "SUCCEEDED")+ for thing in (things["inprogress"], things["succeeded"]):+ iot_client.remove_thing_from_thing_group(+ thingGroupName=group, thingName=thing+ )+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ assert _executions_for(iot_client, things["inprogress"], job_id) == [+ (2, "QUEUED"), (1, "SUCCEEDED"),+ ]+ assert _executions_for(iot_client, things["succeeded"], job_id) == [+ (3, "QUEUED"), (2, "SUCCEEDED"), (1, "SUCCEEDED"),+ ]+ finally:+ _cleanup(iot_client, jobs=[job_id], things=list(things.values()), groups=[group])+++def test_iot_jobs_continuous_execution_finished_outside_group_runs_again(+ iot_client, iot_jobs_data+):+ """An IN_PROGRESS execution can still finish after its thing left the+ target group; rejoining afterwards queues execution 2 (measured+ eu-central-1 2026-10-05)."""+ group = _unique("jobs-group")+ thing = _unique("jobs-thing")+ job_id = _unique("job-cont")+ try:+ group_arn = iot_client.create_thing_group(thingGroupName=group)[+ "thingGroupArn"+ ]+ iot_client.create_job(+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,+ targetSelection="CONTINUOUS",+ )+ _create_thing(iot_client, thing)+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ _drive(iot_jobs_data, thing, job_id, "IN_PROGRESS")+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)++ state = iot_jobs_data.update_job_execution(+ thingName=thing, jobId=job_id, status="SUCCEEDED",+ includeJobExecutionState=True,+ )["executionState"]+ assert state["status"] == "SUCCEEDED"+ assert _executions_for(iot_client, thing, job_id) == [(1, "SUCCEEDED")]++ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ assert _executions_for(iot_client, thing, job_id) == [+ (2, "QUEUED"), (1, "SUCCEEDED"),+ ]+ finally:+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])+++def test_iot_jobs_continuous_join_event_queues_only_without_pending_execution(+ iot_client, iot_jobs_data+):+ """The job runs again when a thing JOINS one of its target groups while+ its newest execution is finished; membership alone does not re-run it+ (measured eu-central-1 2026-10-05). In both groups and finished, leaving+ one adds nothing and rejoining it queues #2; AWS also queued #2 right at+ the leave in 2 of 4 runs, and this pins the other answer. Finished in+ one group, joining the other queues #2. Adding a thing to a group it is+ already in is no join. IN_PROGRESS through a leave and rejoin stays the+ only execution, also after it finishes."""+ groups = [_unique("jobs-group"), _unique("jobs-group")]+ job_id = _unique("job-cont")+ cases = ["both", "second", "again", "busy"]+ things = {case: _unique(f"jobs-{case}") for case in cases}+ try:+ arns = [+ iot_client.create_thing_group(thingGroupName=g)["thingGroupArn"]+ for g in groups+ ]+ iot_client.create_job(+ jobId=job_id, targets=arns, document=_DOCUMENT,+ targetSelection="CONTINUOUS",+ )+ for case, thing in things.items():+ _create_thing(iot_client, thing)+ for g in groups if case in ("both", "busy") else groups[:1]:+ iot_client.add_thing_to_thing_group(thingGroupName=g, thingName=thing)+ for case in ("both", "second", "again"):+ _drive(iot_jobs_data, things[case], job_id, "SUCCEEDED")+ _drive(iot_jobs_data, things["busy"], job_id, "IN_PROGRESS")++ for case in ("both", "busy"):+ iot_client.remove_thing_from_thing_group(+ thingGroupName=groups[0], thingName=things[case]+ )+ iot_client.add_thing_to_thing_group(+ thingGroupName=groups[1], thingName=things["second"]+ )+ iot_client.add_thing_to_thing_group(+ thingGroupName=groups[0], thingName=things["again"]+ )+ assert _executions_for(iot_client, things["both"], job_id) == [(1, "SUCCEEDED")]+ assert _executions_for(iot_client, things["busy"], job_id) == [+ (1, "IN_PROGRESS")+ ]+ assert _executions_for(iot_client, things["second"], job_id) == [+ (2, "QUEUED"), (1, "SUCCEEDED"),+ ]+ assert _executions_for(iot_client, things["again"], job_id) == [+ (1, "SUCCEEDED")+ ]++ for case in ("both", "busy"):+ iot_client.add_thing_to_thing_group(+ thingGroupName=groups[0], thingName=things[case]+ )+ assert _executions_for(iot_client, things["both"], job_id) == [+ (2, "QUEUED"), (1, "SUCCEEDED"),+ ]+ assert _executions_for(iot_client, things["busy"], job_id) == [+ (1, "IN_PROGRESS")+ ]+ _drive(iot_jobs_data, things["busy"], job_id, "SUCCEEDED")+ assert _executions_for(iot_client, things["busy"], job_id) == [+ (1, "SUCCEEDED")+ ]+ finally:+ _cleanup(iot_client, jobs=[job_id], things=list(things.values()), groups=groups)++# ---------------------------------------------------------------------------# Listing + cancel/delete paths# ---------------------------------------------------------------------------@@ -1546,6 +1790,84 @@ def test_jobs_mqtt_update_include_flags_and_terminal_notify() < topics.index(f"{base}/notify-next")+def test_jobs_mqtt_leaving_continuous_job_group_notifies(iot_client, iot_data_client):+ """A thing leaving a CONTINUOUS job's target group loses its QUEUED+ execution (REMOVED): its pending set and front changed, so notify carries+ the emptied aggregate and notify-next goes bare."""+ group = _unique("jobs-mqtt-group")+ thing = _unique("jobs-mqtt-leave")+ job_id = _unique("job-cont")+ base = f"$aws/things/{thing}/jobs"+ try:+ group_arn = iot_client.create_thing_group(thingGroupName=group)[+ "thingGroupArn"+ ]+ iot_client.create_job(+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,+ targetSelection="CONTINUOUS",+ )+ _create_thing(iot_client, thing)+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)++ received = _collect_shadow_frames(+ f"{base}/#",+ lambda: iot_client.remove_thing_from_thing_group(+ thingGroupName=group, thingName=thing+ ),+ want=2,+ )+ frames = _frames_by_topic(received)+ assert frames[f"{base}/notify"]["jobs"] == {}+ assert set(frames[f"{base}/notify-next"]) == {"timestamp"}+ finally:+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])+++def test_jobs_mqtt_rejoining_continuous_job_group_notifies(+ iot_client, iot_data_client, iot_jobs_data+):+ """A thing that finished a CONTINUOUS job and rejoins its target group+ gets execution 2 QUEUED: notify lists it and notify-next carries it."""+ group = _unique("jobs-mqtt-group")+ thing = _unique("jobs-mqtt-rejoin")+ job_id = _unique("job-cont")+ base = f"$aws/things/{thing}/jobs"+ try:+ group_arn = iot_client.create_thing_group(thingGroupName=group)[+ "thingGroupArn"+ ]+ iot_client.create_job(+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,+ targetSelection="CONTINUOUS",+ )+ _create_thing(iot_client, thing)+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ _drive(iot_jobs_data, thing, job_id, "SUCCEEDED")+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)++ received = _collect_shadow_frames(+ f"{base}/#",+ lambda: iot_client.add_thing_to_thing_group(+ thingGroupName=group, thingName=thing+ ),+ want=2,+ )+ frames = _frames_by_topic(received)+ queued = frames[f"{base}/notify"]["jobs"]["QUEUED"]+ assert [(q["jobId"], q["executionNumber"]) for q in queued] == [(job_id, 2)]+ execution = frames[f"{base}/notify-next"]["execution"]+ assert (execution["executionNumber"], execution["status"]) == (2, "QUEUED")+ assert execution["jobDocument"] == _DOCUMENT_OBJECT+ # The members AWS sends (measured eu-central-1 2026-10-05): no+ # thingName, and no statusDetails while it is empty.+ assert set(execution) == {+ "jobId", "status", "queuedAt", "lastUpdatedAt", "versionNumber",+ "executionNumber", "jobDocument",+ }+ finally:+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])++def test_jobs_mqtt_version_mismatch_rejected(iot_client, iot_data_client):"""A stale expectedVersion answers update/rejected with the stringVersionMismatch code, the captured message wording, and the execution's