Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(iot): run a CONTINUOUS job again for a thing that rejoins its group (#2003)

iot-rocketc3f67beee4b8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 997c8e1e..90fe13ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Fixed
+- **IoT — a CONTINUOUS job runs again for a thing that rejoins its target group** — executions were kept one per thing and job, so a thing that finished the job, left the group and was added back never got another execution. Joining a target group now queues the next execution number when the thing's newest execution is finished, leaving the group moves a QUEUED execution to `REMOVED`, both publish `jobs/notify(-next)`, the `notify-next` execution leaves out `thingName` and an empty `statusDetails`, and `ListJobExecutionsForThing`, `DescribeJobExecution` with `executionNumber` and `jobProcessDetails` include the earlier executions, as AWS does. Contributed by @iot-rocket.
- **IoT — a `#` region no longer widens topic filters** — a WebSocket session whose credential names `#` as its region received every message published under that region, whatever its filter, because filters were matched with the account and region prefix in front of them. Contributed by @iot-rocket.
- **Cognito — app client secret hashes** — clients created with `GenerateSecret=true` require a valid `SecretHash` on signup, confirmation and password recovery, and `SECRET_HASH` on authentication and challenge responses. Missing or incorrect hashes return `NotAuthorizedException` before user changes, email delivery or Lambda triggers. Clients without a secret reject a supplied hash on self-service and initial authentication calls, while refresh and challenge responses ignore it, as on AWS. Refresh authentication verifies the token owner's username, or `sub` when the pool uses `UsernameAttributes`, and custom challenge sessions cannot be answered through a different client. The checks apply in both `AUTH` modes.
### Added
diff --git a/ministack/services/iot.py b/ministack/services/iot.py
index e1f99c24..e74dfcab 100644
--- a/ministack/services/iot.py
+++ b/ministack/services/iot.py
@@ -527,9 +527,9 @@ async def _route_request(
# ThingGroups — special add/remove paths must come BEFORE the
# generic ``/thing-groups/{name}`` handler.
if path == "/thing-groups/addThingToThingGroup" and method in ("PUT", "POST"):
- return _add_thing_to_group(_parse_body(body))
+ return await _add_thing_to_group(_parse_body(body))
if path == "/thing-groups/removeThingFromThingGroup" and method in ("PUT", "POST"):
- return _remove_thing_from_group(_parse_body(body))
+ return await _remove_thing_from_group(_parse_body(body))
if path == "/thing-groups" and method == "GET":
return _list_thing_groups(qp)
if path.startswith("/thing-groups/") and path.endswith("/things") and method == "GET":
@@ -1173,7 +1173,9 @@ def _delete_thing_group(name: str) -> tuple:
return json_response({})
-def _add_thing_to_group(payload: dict) -> tuple:
+async def _add_thing_to_group(payload: dict) -> tuple:
+ """Async because a CONTINUOUS job the thing joins publishes
+ jobs/notify(-next) through the broker."""
gname = payload.get("thingGroupName")
tname = payload.get("thingName")
if not gname or not tname:
@@ -1186,17 +1188,23 @@ def _add_thing_to_group(payload: dict) -> tuple:
thing = _things.get(tname)
if thing is None:
return _error_not_found("Thing", tname)
- if tname not in group.get("things", []):
+ joined = tname not in group.get("things", [])
+ if joined:
+ prev_next = jobs_first_pending_job_id(tname)
group.setdefault("things", []).append(tname)
_thing_groups[gname] = group
_membership_event("ADDED", group, thing)
if gname not in thing.get("thingGroupNames", []):
thing.setdefault("thingGroupNames", []).append(gname)
_things[tname] = thing
+ if joined and _jobs_thing_joined_group(tname, _thing_group_arn(gname)):
+ await jobs_notify_thing(get_account_id(), get_region(), tname, prev_next)
return json_response({})
-def _remove_thing_from_group(payload: dict) -> tuple:
+async def _remove_thing_from_group(payload: dict) -> tuple:
+ """Async because a QUEUED CONTINUOUS execution the removal retires
+ publishes jobs/notify(-next) through the broker."""
gname = payload.get("thingGroupName")
tname = payload.get("thingName")
group = _thing_groups.get(gname) if gname else None
@@ -1205,6 +1213,7 @@ def _remove_thing_from_group(payload: dict) -> tuple:
return _error_not_found("ThingGroup", gname or "")
if thing is None:
return _error_not_found("Thing", tname or "")
+ prev_next = jobs_first_pending_job_id(tname)
if tname in group.get("things", []):
group["things"].remove(tname)
_thing_groups[gname] = group
@@ -1212,6 +1221,8 @@ def _remove_thing_from_group(payload: dict) -> tuple:
if gname in thing.get("thingGroupNames", []):
thing["thingGroupNames"].remove(gname)
_things[tname] = thing
+ if _jobs_thing_left_group(tname):
+ await jobs_notify_thing(get_account_id(), get_region(), tname, prev_next)
return json_response({})
@@ -4043,13 +4054,15 @@ def _list_topic_rules(qp: dict) -> tuple:
#
# `versionNumber` gives optimistic concurrency: every transition bumps it, and
# a stale `expectedVersion` is rejected with a 409. `executionNumber` is NOT a
-# concurrency token here — an execution is created once at 1 and nothing
-# re-queues it, so it never changes (AWS increments it when a job execution is
-# retried, which MiniStack does not model).
+# concurrency token here. It counts a thing's executions of one job: 1 for the
+# first, one more each time the thing joins a CONTINUOUS job's target group
+# again after its last execution finished (AWS also increments it when a job
+# execution is retried, which MiniStack does not model). `_job_executions`
+# holds the newest execution per (thing, job); the older, terminal ones ride
+# along newest first under its internal `history` key.
#
-# TIMED_OUT and REMOVED are recognized as terminal (so a restored record in
-# either state behaves, and `jobProcessDetails` counts them) but nothing sets
-# them: there are no execution timeouts, and deleting a thing DELETES its
+# A thing leaving every target of a CONTINUOUS job moves its QUEUED execution
+# to REMOVED (see `_jobs_thing_left_group`). Deleting a thing DELETES its
# executions (see `_delete_thing`) rather than marking them REMOVED.
# Job ids are stricter than thing names, and identically so in both service
@@ -4170,21 +4183,97 @@ def _jobs_materialize_executions(job_id: str) -> None:
key = (thing, job_id)
if key in _job_executions:
continue
- _job_executions[key] = {
- "jobId": job_id,
- "thingName": thing,
- "status": "QUEUED",
- "statusDetails": {},
- "queuedAt": now,
- "startedAt": None,
- "lastUpdatedAt": now,
- "executionNumber": 1,
- "versionNumber": 1,
- }
+ _jobs_queue_execution(thing, job_id, now)
if job.get("targetSelection") == "SNAPSHOT":
job["snapshotted"] = True
+def _jobs_queue_execution(thing: str, job_id: str, now: int) -> None:
+ """Queue the thing's next execution of a job, at version 1.
+
+ The first one is number 1. A later one takes the next number and keeps
+ the finished one before it under `history`.
+ """
+ previous = _job_executions.get((thing, job_id))
+ execution = {
+ "jobId": job_id,
+ "thingName": thing,
+ "status": "QUEUED",
+ "statusDetails": {},
+ "queuedAt": now,
+ "startedAt": None,
+ "lastUpdatedAt": now,
+ "executionNumber": 1,
+ "versionNumber": 1,
+ }
+ if previous is not None:
+ execution["executionNumber"] = previous["executionNumber"] + 1
+ execution["history"] = [
+ {k: v for k, v in previous.items() if k != "history"},
+ *previous.get("history", []),
+ ]
+ _job_executions[(thing, job_id)] = execution
+
+
+def _jobs_thing_joined_group(thing: str, group_arn: str) -> bool:
+ """Run every CONTINUOUS job targeting a group for a thing that just
+ joined it.
+
+ The thing gets a new QUEUED execution unless its newest one is still
+ QUEUED or IN_PROGRESS. That covers a thing rejoining after its execution
+ finished, and a thing that finished the job joining another target group
+ of it (measured eu-central-1 2026-10-05). Returns whether anything was
+ queued, i.e. whether the thing's jobs/notify topics have news.
+ """
+ now = _jobs_now_ms()
+ queued = False
+ for job_id, job in _jobs.items():
+ if (
+ job.get("targetSelection") != "CONTINUOUS"
+ or job.get("status") == "CANCELED"
+ or group_arn not in (job.get("targets") or [])
+ ):
+ continue
+ previous = _job_executions.get((thing, job_id))
+ if previous is not None and previous["status"] not in _JOB_EXECUTION_TERMINAL:
+ continue
+ _jobs_queue_execution(thing, job_id, now)
+ queued = True
+ return queued
+
+
+def _jobs_thing_left_group(thing: str) -> bool:
+ """Retire a thing's QUEUED CONTINUOUS executions after it left a group.
+
+ Call after the membership change. For every CONTINUOUS job the thing no
+ longer resolves into, a QUEUED execution becomes REMOVED with its
+ `versionNumber` unchanged. IN_PROGRESS, SUCCEEDED and FAILED executions
+ keep their status (measured eu-central-1 2026-10-05). Returns whether a
+ pending execution went away, i.e. whether jobs/notify has news.
+ """
+ now = _jobs_now_ms()
+ removed = False
+ for job_id, job in _jobs.items():
+ execution = _job_executions.get((thing, job_id))
+ if (
+ execution is None
+ or execution["status"] != "QUEUED"
+ or job.get("targetSelection") != "CONTINUOUS"
+ or job.get("status") == "CANCELED"
+ or thing in _job_target_things(job.get("targets"))
+ ):
+ continue
+ execution["status"] = "REMOVED"
+ execution["lastUpdatedAt"] = now
+ removed = True
+ return removed
+
+
+def _jobs_with_history(execution: dict) -> list[dict]:
+ """A thing's executions of one job, newest first."""
+ return [execution, *execution.get("history", [])]
+
+
def _jobs_materialize_all() -> None:
for job_id in list(_jobs.keys()):
_jobs_materialize_executions(job_id)
@@ -4512,10 +4601,13 @@ def _describe_job(job_id: str) -> tuple:
if job is None:
return _error_not_found("Job", job_id)
_jobs_materialize_executions(job_id)
+ # Every execution counts, a rejoined thing's earlier ones included
+ # (measured eu-central-1 2026-10-05).
counts = {status: 0 for status in _JOB_EXECUTION_STATUSES}
- for execution in _job_executions.values():
- if execution["jobId"] == job_id:
- counts[execution["status"]] += 1
+ for latest in _job_executions.values():
+ if latest["jobId"] == job_id:
+ for execution in _jobs_with_history(latest):
+ counts[execution["status"]] += 1
job_doc = {
**_job_summary(job),
"targets": list(job.get("targets") or []),
@@ -4695,7 +4787,7 @@ async def _handle_thing_jobs(method: str, path: str, body: bytes, qp: dict) -> t
thing, tail[:-len("/cancel")], _parse_body(body), qp
)
if "/" not in tail and method == "GET":
- return _describe_job_execution(thing, tail)
+ return _describe_job_execution(thing, tail, qp)
return error_response_json(
"InvalidRequestException", f"Unsupported IoT path: {method} {path}", 400
)
@@ -4709,7 +4801,13 @@ def _list_job_executions_for_thing(thing: str, qp: dict) -> tuple:
# The wire shape nests the detail under `jobExecutionSummary`, with only
# the jobId beside it (JobExecutionSummaryForThing).
summaries = []
- for execution in _job_executions.values():
+ # A rejoined thing's executions of one job come newest first, as on AWS.
+ executions = [
+ execution
+ for latest in _job_executions.values()
+ for execution in _jobs_with_history(latest)
+ ]
+ for execution in executions:
if execution["thingName"] != thing:
continue
if wanted_status and execution["status"] != wanted_status:
@@ -4731,9 +4829,22 @@ def _list_job_executions_for_thing(thing: str, qp: dict) -> tuple:
return json_response({"executionSummaries": summaries})
-def _describe_job_execution(thing: str, job_id: str) -> tuple:
+def _describe_job_execution(thing: str, job_id: str, qp: dict) -> tuple:
+ """Without `executionNumber` the newest execution, with it that one."""
_jobs_materialize_executions(job_id)
execution = _job_executions.get((thing, job_id))
+ number = qp.get("executionNumber")
+ if execution is not None and number is not None:
+ try:
+ number = int(number)
+ except (TypeError, ValueError):
+ return error_response_json(
+ "InvalidRequestException", f"Invalid executionNumber: {number!r}", 400
+ )
+ execution = next(
+ (e for e in _jobs_with_history(execution) if e["executionNumber"] == number),
+ None,
+ )
if execution is None:
return _error_not_found("Job execution", f"{thing}/{job_id}")
view = {
@@ -5025,12 +5136,13 @@ async def jobs_notify_thing(
Payloads (all timestamps epoch seconds): ``notify`` carries
``{"timestamp", "jobs": {<STATUS>: [summaries]}}`` with empty status
lists omitted (``{}`` once nothing is pending); ``notify-next`` carries
- the full execution view — status and ``jobDocument`` as a JSON object
- included — or a bare ``{"timestamp"}`` when the queue emptied.
+ the execution view (``jobDocument`` as a JSON object, no ``thingName``)
+ or a bare ``{"timestamp"}`` when the queue emptied.
- Known boundaries: a CONTINUOUS job's execution materialized lazily on a
- *read* (a thing that joined the target group later) does not notify —
- the materialization sites are synchronous read paths — and DeleteThing's
+ Known boundaries: an execution materialized lazily on a *read* does not
+ notify — the materialization sites are synchronous read paths (a thing
+ joining a CONTINUOUS job's target group is queued eagerly by
+ AddThingToThingGroup, which does notify) — and DeleteThing's
execution sweep does not notify the deleted thing (that sweep is itself
a divergence: AWS keeps a deleted thing's executions, so it publishes
nothing there either)."""
@@ -5042,6 +5154,12 @@ async def jobs_notify_thing(
if nxt is not None
else None
)
+ if next_view is not None:
+ # AWS sends no thingName and no empty statusDetails here (measured
+ # eu-central-1 2026-10-05).
+ del next_view["thingName"]
+ if not next_view["statusDetails"]:
+ del next_view["statusDetails"]
ts = int(time.time())
jobs_by_status: dict = {}
for execution in pending:
diff --git a/ministack/services/iot_jobs_data.py b/ministack/services/iot_jobs_data.py
index a8decb4e..72f2edb7 100644
--- a/ministack/services/iot_jobs_data.py
+++ b/ministack/services/iot_jobs_data.py
@@ -199,7 +199,8 @@ def _describe_execution(thing: str, job_id: str, qp: dict) -> tuple:
async def _update_execution(thing: str, job_id: str, payload: dict) -> tuple:
- # Nothing re-queues here, so only execution number 1 exists.
+ # Only the newest execution takes updates; a rejoined thing's older ones
+ # are terminal.
execution_number = payload.get("executionNumber")
if execution_number is not None:
try:
diff --git a/tests/test_iot_jobs_data.py b/tests/test_iot_jobs_data.py
index 8f4f6156..84b7fa3d 100644
--- a/tests/test_iot_jobs_data.py
+++ b/tests/test_iot_jobs_data.py
@@ -728,6 +728,250 @@ def test_iot_jobs_continuous_late_thing_first_call_is_update(
_cleanup(iot_client, jobs=[job_id], things=[late], groups=[group])
+def _executions_for(iot_client, thing, job_id):
+ """(executionNumber, status) pairs ListJobExecutionsForThing lists."""
+ summaries = iot_client.list_job_executions_for_thing(
+ thingName=thing, jobId=job_id
+ )["executionSummaries"]
+ return [
+ (s["jobExecutionSummary"]["executionNumber"], s["jobExecutionSummary"]["status"])
+ for s in summaries
+ ]
+
+
+def _drive(iot_jobs_data, thing, job_id, status):
+ """Start the thing's pending execution, then report `status` unless it
+ is IN_PROGRESS."""
+ iot_jobs_data.start_next_pending_job_execution(thingName=thing)
+ if status != "IN_PROGRESS":
+ iot_jobs_data.update_job_execution(
+ thingName=thing, jobId=job_id, status=status
+ )
+
+
+def test_iot_jobs_continuous_thing_rejoining_group_runs_job_again(
+ iot_client, iot_jobs_data
+):
+ """A thing that leaves a CONTINUOUS job's target group and rejoins it
+ after its execution finished gets the next execution number, QUEUED.
+ Leaving moves a QUEUED execution to REMOVED; IN_PROGRESS, SUCCEEDED and
+ FAILED keep their status, and a thing rejoining while IN_PROGRESS gets
+ nothing new (measured eu-central-1 2026-10-05)."""
+ group = _unique("jobs-group")
+ job_id = _unique("job-cont")
+ cases = ["queued", "inprogress", "succeeded", "failed", "stays"]
+ things = {case: _unique(f"jobs-{case}") for case in cases}
+ try:
+ group_arn = iot_client.create_thing_group(thingGroupName=group)[
+ "thingGroupArn"
+ ]
+ iot_client.create_job(
+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,
+ targetSelection="CONTINUOUS",
+ )
+ for thing in things.values():
+ _create_thing(iot_client, thing)
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ _drive(iot_jobs_data, things["inprogress"], job_id, "IN_PROGRESS")
+ _drive(iot_jobs_data, things["succeeded"], job_id, "SUCCEEDED")
+ _drive(iot_jobs_data, things["failed"], job_id, "FAILED")
+ _drive(iot_jobs_data, things["stays"], job_id, "SUCCEEDED")
+
+ movers = [things[case] for case in cases if case != "stays"]
+ for thing in movers:
+ iot_client.remove_thing_from_thing_group(
+ thingGroupName=group, thingName=thing
+ )
+ removed = iot_client.describe_job_execution(
+ jobId=job_id, thingName=things["queued"]
+ )["execution"]
+ assert (removed["status"], removed["versionNumber"]) == ("REMOVED", 1)
+ pending = iot_jobs_data.get_pending_job_executions(thingName=things["queued"])
+ assert pending["queuedJobs"] == []
+ assert _executions_for(iot_client, things["inprogress"], job_id) == [
+ (1, "IN_PROGRESS")
+ ]
+ assert _executions_for(iot_client, things["failed"], job_id) == [(1, "FAILED")]
+
+ for thing in movers:
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ assert _executions_for(iot_client, things["queued"], job_id) == [
+ (2, "QUEUED"), (1, "REMOVED"),
+ ]
+ assert _executions_for(iot_client, things["succeeded"], job_id) == [
+ (2, "QUEUED"), (1, "SUCCEEDED"),
+ ]
+ assert _executions_for(iot_client, things["failed"], job_id) == [
+ (2, "QUEUED"), (1, "FAILED"),
+ ]
+ assert _executions_for(iot_client, things["inprogress"], job_id) == [
+ (1, "IN_PROGRESS")
+ ]
+ assert _executions_for(iot_client, things["stays"], job_id) == [
+ (1, "SUCCEEDED")
+ ]
+
+ # The device sees the new execution; the newest one answers a
+ # describe without executionNumber, a number picks an older one.
+ pending = iot_jobs_data.get_pending_job_executions(
+ thingName=things["succeeded"]
+ )
+ assert [(q["jobId"], q["executionNumber"]) for q in pending["queuedJobs"]] == [
+ (job_id, 2)
+ ]
+ newest = iot_client.describe_job_execution(
+ jobId=job_id, thingName=things["succeeded"]
+ )["execution"]
+ assert (newest["executionNumber"], newest["status"]) == (2, "QUEUED")
+ assert newest["versionNumber"] == 1
+ first = iot_client.describe_job_execution(
+ jobId=job_id, thingName=things["succeeded"], executionNumber=1
+ )["execution"]
+ assert (first["executionNumber"], first["status"]) == (1, "SUCCEEDED")
+ with pytest.raises(ClientError) as exc:
+ iot_client.describe_job_execution(
+ jobId=job_id, thingName=things["succeeded"], executionNumber=3
+ )
+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+
+ # Every execution counts, the earlier ones of a rejoined thing too.
+ details = iot_client.describe_job(jobId=job_id)["job"]["jobProcessDetails"]
+ assert details["numberOfQueuedThings"] == 3
+ assert details["numberOfInProgressThings"] == 1
+ assert details["numberOfRemovedThings"] == 1
+ assert details["numberOfSucceededThings"] == 2
+ assert details["numberOfFailedThings"] == 1
+
+ # The IN_PROGRESS execution that rejoined finishes as a member: no new
+ # execution follows. A second leave and rejoin runs the job again.
+ _drive(iot_jobs_data, things["inprogress"], job_id, "SUCCEEDED")
+ assert _executions_for(iot_client, things["inprogress"], job_id) == [
+ (1, "SUCCEEDED")
+ ]
+ _drive(iot_jobs_data, things["succeeded"], job_id, "SUCCEEDED")
+ for thing in (things["inprogress"], things["succeeded"]):
+ iot_client.remove_thing_from_thing_group(
+ thingGroupName=group, thingName=thing
+ )
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ assert _executions_for(iot_client, things["inprogress"], job_id) == [
+ (2, "QUEUED"), (1, "SUCCEEDED"),
+ ]
+ assert _executions_for(iot_client, things["succeeded"], job_id) == [
+ (3, "QUEUED"), (2, "SUCCEEDED"), (1, "SUCCEEDED"),
+ ]
+ finally:
+ _cleanup(iot_client, jobs=[job_id], things=list(things.values()), groups=[group])
+
+
+def test_iot_jobs_continuous_execution_finished_outside_group_runs_again(
+ iot_client, iot_jobs_data
+):
+ """An IN_PROGRESS execution can still finish after its thing left the
+ target group; rejoining afterwards queues execution 2 (measured
+ eu-central-1 2026-10-05)."""
+ group = _unique("jobs-group")
+ thing = _unique("jobs-thing")
+ job_id = _unique("job-cont")
+ try:
+ group_arn = iot_client.create_thing_group(thingGroupName=group)[
+ "thingGroupArn"
+ ]
+ iot_client.create_job(
+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,
+ targetSelection="CONTINUOUS",
+ )
+ _create_thing(iot_client, thing)
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ _drive(iot_jobs_data, thing, job_id, "IN_PROGRESS")
+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)
+
+ state = iot_jobs_data.update_job_execution(
+ thingName=thing, jobId=job_id, status="SUCCEEDED",
+ includeJobExecutionState=True,
+ )["executionState"]
+ assert state["status"] == "SUCCEEDED"
+ assert _executions_for(iot_client, thing, job_id) == [(1, "SUCCEEDED")]
+
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ assert _executions_for(iot_client, thing, job_id) == [
+ (2, "QUEUED"), (1, "SUCCEEDED"),
+ ]
+ finally:
+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])
+
+
+def test_iot_jobs_continuous_join_event_queues_only_without_pending_execution(
+ iot_client, iot_jobs_data
+):
+ """The job runs again when a thing JOINS one of its target groups while
+ its newest execution is finished; membership alone does not re-run it
+ (measured eu-central-1 2026-10-05). In both groups and finished, leaving
+ one adds nothing and rejoining it queues #2; AWS also queued #2 right at
+ the leave in 2 of 4 runs, and this pins the other answer. Finished in
+ one group, joining the other queues #2. Adding a thing to a group it is
+ already in is no join. IN_PROGRESS through a leave and rejoin stays the
+ only execution, also after it finishes."""
+ groups = [_unique("jobs-group"), _unique("jobs-group")]
+ job_id = _unique("job-cont")
+ cases = ["both", "second", "again", "busy"]
+ things = {case: _unique(f"jobs-{case}") for case in cases}
+ try:
+ arns = [
+ iot_client.create_thing_group(thingGroupName=g)["thingGroupArn"]
+ for g in groups
+ ]
+ iot_client.create_job(
+ jobId=job_id, targets=arns, document=_DOCUMENT,
+ targetSelection="CONTINUOUS",
+ )
+ for case, thing in things.items():
+ _create_thing(iot_client, thing)
+ for g in groups if case in ("both", "busy") else groups[:1]:
+ iot_client.add_thing_to_thing_group(thingGroupName=g, thingName=thing)
+ for case in ("both", "second", "again"):
+ _drive(iot_jobs_data, things[case], job_id, "SUCCEEDED")
+ _drive(iot_jobs_data, things["busy"], job_id, "IN_PROGRESS")
+
+ for case in ("both", "busy"):
+ iot_client.remove_thing_from_thing_group(
+ thingGroupName=groups[0], thingName=things[case]
+ )
+ iot_client.add_thing_to_thing_group(
+ thingGroupName=groups[1], thingName=things["second"]
+ )
+ iot_client.add_thing_to_thing_group(
+ thingGroupName=groups[0], thingName=things["again"]
+ )
+ assert _executions_for(iot_client, things["both"], job_id) == [(1, "SUCCEEDED")]
+ assert _executions_for(iot_client, things["busy"], job_id) == [
+ (1, "IN_PROGRESS")
+ ]
+ assert _executions_for(iot_client, things["second"], job_id) == [
+ (2, "QUEUED"), (1, "SUCCEEDED"),
+ ]
+ assert _executions_for(iot_client, things["again"], job_id) == [
+ (1, "SUCCEEDED")
+ ]
+
+ for case in ("both", "busy"):
+ iot_client.add_thing_to_thing_group(
+ thingGroupName=groups[0], thingName=things[case]
+ )
+ assert _executions_for(iot_client, things["both"], job_id) == [
+ (2, "QUEUED"), (1, "SUCCEEDED"),
+ ]
+ assert _executions_for(iot_client, things["busy"], job_id) == [
+ (1, "IN_PROGRESS")
+ ]
+ _drive(iot_jobs_data, things["busy"], job_id, "SUCCEEDED")
+ assert _executions_for(iot_client, things["busy"], job_id) == [
+ (1, "SUCCEEDED")
+ ]
+ finally:
+ _cleanup(iot_client, jobs=[job_id], things=list(things.values()), groups=groups)
+
+
# ---------------------------------------------------------------------------
# Listing + cancel/delete paths
# ---------------------------------------------------------------------------
@@ -1546,6 +1790,84 @@ def test_jobs_mqtt_update_include_flags_and_terminal_notify(
) < topics.index(f"{base}/notify-next")
+def test_jobs_mqtt_leaving_continuous_job_group_notifies(iot_client, iot_data_client):
+ """A thing leaving a CONTINUOUS job's target group loses its QUEUED
+ execution (REMOVED): its pending set and front changed, so notify carries
+ the emptied aggregate and notify-next goes bare."""
+ group = _unique("jobs-mqtt-group")
+ thing = _unique("jobs-mqtt-leave")
+ job_id = _unique("job-cont")
+ base = f"$aws/things/{thing}/jobs"
+ try:
+ group_arn = iot_client.create_thing_group(thingGroupName=group)[
+ "thingGroupArn"
+ ]
+ iot_client.create_job(
+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,
+ targetSelection="CONTINUOUS",
+ )
+ _create_thing(iot_client, thing)
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+
+ received = _collect_shadow_frames(
+ f"{base}/#",
+ lambda: iot_client.remove_thing_from_thing_group(
+ thingGroupName=group, thingName=thing
+ ),
+ want=2,
+ )
+ frames = _frames_by_topic(received)
+ assert frames[f"{base}/notify"]["jobs"] == {}
+ assert set(frames[f"{base}/notify-next"]) == {"timestamp"}
+ finally:
+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])
+
+
+def test_jobs_mqtt_rejoining_continuous_job_group_notifies(
+ iot_client, iot_data_client, iot_jobs_data
+):
+ """A thing that finished a CONTINUOUS job and rejoins its target group
+ gets execution 2 QUEUED: notify lists it and notify-next carries it."""
+ group = _unique("jobs-mqtt-group")
+ thing = _unique("jobs-mqtt-rejoin")
+ job_id = _unique("job-cont")
+ base = f"$aws/things/{thing}/jobs"
+ try:
+ group_arn = iot_client.create_thing_group(thingGroupName=group)[
+ "thingGroupArn"
+ ]
+ iot_client.create_job(
+ jobId=job_id, targets=[group_arn], document=_DOCUMENT,
+ targetSelection="CONTINUOUS",
+ )
+ _create_thing(iot_client, thing)
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ _drive(iot_jobs_data, thing, job_id, "SUCCEEDED")
+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)
+
+ received = _collect_shadow_frames(
+ f"{base}/#",
+ lambda: iot_client.add_thing_to_thing_group(
+ thingGroupName=group, thingName=thing
+ ),
+ want=2,
+ )
+ frames = _frames_by_topic(received)
+ queued = frames[f"{base}/notify"]["jobs"]["QUEUED"]
+ assert [(q["jobId"], q["executionNumber"]) for q in queued] == [(job_id, 2)]
+ execution = frames[f"{base}/notify-next"]["execution"]
+ assert (execution["executionNumber"], execution["status"]) == (2, "QUEUED")
+ assert execution["jobDocument"] == _DOCUMENT_OBJECT
+ # The members AWS sends (measured eu-central-1 2026-10-05): no
+ # thingName, and no statusDetails while it is empty.
+ assert set(execution) == {
+ "jobId", "status", "queuedAt", "lastUpdatedAt", "versionNumber",
+ "executionNumber", "jobDocument",
+ }
+ finally:
+ _cleanup(iot_client, jobs=[job_id], things=[thing], groups=[group])
+
+
def test_jobs_mqtt_version_mismatch_rejected(iot_client, iot_data_client):
"""A stale expectedVersion answers update/rejected with the string
VersionMismatch code, the captured message wording, and the execution's