Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(iot): refuse device reads of terminal job executions (#2192)

iot-rocket45bcc5bec005
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3b4c9e89..5c36a566 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **IoT Jobs — terminal job executions on the device plane** — a device's `DescribeJobExecution` (HTTP and the MQTT `get` topic) of a SUCCEEDED, FAILED, REJECTED, TIMED_OUT, CANCELED or REMOVED execution answered with the execution; it now answers `TerminalStateException` (410) and `TerminalStateReached` over MQTT, as AWS does. `UpdateJobExecution` and `CancelJobExecution` of a terminal execution now refuse with AWS's messages (over MQTT `InvalidStateTransition` with the `executionState`), a refused status such as QUEUED is checked first, and a device that reports SUCCEEDED or FAILED straight from QUEUED now stamps `startedAt`. Contributed by @iot-rocket.
- **IoT / CloudFormation — stack tags and provisioning templates** — stack-level tags never reached IoT resources; they now reach thing types, thing groups, policies, topic rules, CA certificates and provisioning templates without the `aws:cloudformation:` tags (job templates get none, as on AWS), and an update that only changes tags no longer adds a policy version. An `AWS::IoT::ProvisioningTemplate` `Enabled` of `"false"` from a parameter or the template enabled the template; it now disables it. `CreateProvisioningTemplate` and the CloudFormation type accepted any certificate resource; they now refuse one without a `Status` of `Active` or `Inactive` (any case) or whose `CertificateId` is not a reference to `AWS::IoT::Certificate::Id`, a second certificate resource and a resource without `Properties`, with AWS's messages. Contributed by @iot-rocket.
### Added
- **WAF v2 — regex pattern sets** — `CreateRegexPatternSet`, `GetRegexPatternSet`, `UpdateRegexPatternSet`, `DeleteRegexPatternSet` and `ListRegexPatternSets` now manage regional and CloudFront-scoped regex pattern sets, with their ARN (`regexpatternset`), tags and `LockToken` checks on update and delete (`WAFOptimisticLockException` for a stale token), as AWS does. Web ACL and rule group rules are stored as sent, so a `RegexPatternSetReferenceStatement` pointing at a pattern set ARN round-trips; the statement is not evaluated, as MiniStack does not inspect requests.
diff --git a/ministack/services/iot.py b/ministack/services/iot.py
index 45715e25..4539ad19 100644
--- a/ministack/services/iot.py
+++ b/ministack/services/iot.py
@@ -6489,15 +6489,21 @@ def _jobs_check_expected_version(
return None
-def _jobs_reject_if_terminal(execution: dict, verb: str) -> tuple | None:
+def _jobs_reject_if_terminal(execution: dict, message: str) -> tuple | None:
"""Terminal executions are frozen — no update, no cancel."""
+ if execution["status"] not in _JOB_EXECUTION_TERMINAL:
+ return None
+ return error_response_json("InvalidStateTransitionException", message, 409)
+
+
+def jobs_reject_terminal_describe(execution: dict) -> tuple | None:
+ """The device plane describes only QUEUED and IN_PROGRESS executions."""
if execution["status"] not in _JOB_EXECUTION_TERMINAL:
return None
return error_response_json(
- "InvalidStateTransitionException",
- f"Job execution is in terminal status {execution['status']} and "
- f"cannot be {verb}",
- 409,
+ "TerminalStateException",
+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED",
+ 410,
)
@@ -7713,7 +7719,9 @@ async def _cancel_job_execution(
)
if error:
return error
- error = _jobs_reject_if_terminal(execution, "canceled")
+ error = _jobs_reject_if_terminal(
+ execution, f"Job execution for thing {thing} within job {job_id} cannot be canceled."
+ )
if error:
return error
force = _qp_bool(qp, "force")
@@ -8068,9 +8076,6 @@ def jobs_update_execution(
error = _jobs_check_expected_version(
execution, expected_version, "InvalidStateTransitionException"
)
- if error:
- return None, error
- error = _jobs_reject_if_terminal(execution, "updated")
if error:
return None, error
if not status or status not in _JOB_EXECUTION_STATUSES:
@@ -8079,12 +8084,6 @@ def jobs_update_execution(
f"Invalid job execution status: {status!r}",
400,
)
- if status == "QUEUED":
- return None, error_response_json(
- "InvalidStateTransitionException",
- "A device cannot move a job execution back to QUEUED",
- 409,
- )
if status not in _DEVICE_SETTABLE_STATUSES:
# InvalidStateTransitionException, not InvalidRequestException
# (measured eu-north-1 2026-09-19).
@@ -8093,11 +8092,18 @@ def jobs_update_execution(
f"The status of job execution cannot be changed to be {status}",
409,
)
+ error = _jobs_reject_if_terminal(
+ execution,
+ f"Job {job_id} for {thing_name} has already finished with status {execution['status']}",
+ )
+ if error:
+ return None, error
now = _jobs_now_ms()
execution["status"] = status
if status_details:
execution["statusDetails"] = dict(status_details)
- if status == "IN_PROGRESS" and execution.get("startedAt") is None:
+ # The first report starts the execution, unless the device rejects the job.
+ if status != "REJECTED" and execution.get("startedAt") is None:
execution["startedAt"] = now
if status == "IN_PROGRESS" and step_timeout_minutes is not None:
# The device's own step timeout, reset every time it re-reports
@@ -9310,14 +9316,16 @@ def _jobs_error_tuple_to_rejected(
code = "ResourceNotFound"
elif status == 400:
code = "InvalidRequest"
+ elif status == 410:
+ code = "TerminalStateReached"
elif message.startswith("Expected version"):
code = "VersionMismatch"
- elif "terminal status" in message:
- code = "TerminalStateReached"
else:
code = "InvalidStateTransition"
execution_state = None
- if code in ("VersionMismatch", "InvalidStateTransition") and execution is not None:
+ # A refused status carries no executionState.
+ refused_status = message.startswith("The status of job execution cannot be changed")
+ if code in ("VersionMismatch", "InvalidStateTransition") and execution is not None and not refused_status:
execution_state = {
"status": execution["status"],
"versionNumber": execution["versionNumber"],
@@ -9440,6 +9448,10 @@ async def _handle_jobs_publish(
),
)
return
+ error = jobs_reject_terminal_describe(execution) if execution is not None else None
+ if error:
+ await _respond("rejected", _jobs_error_tuple_to_rejected(error, client_token))
+ return
# An idle thing's "$next" mirrors the HTTP plane's empty structure.
doc = {"timestamp": ts}
if execution is not None:
diff --git a/ministack/services/iot_jobs_data.py b/ministack/services/iot_jobs_data.py
index 4a769dd1..e3213dec 100644
--- a/ministack/services/iot_jobs_data.py
+++ b/ministack/services/iot_jobs_data.py
@@ -200,6 +200,9 @@ def _describe_execution(thing: str, job_id: str, qp: dict) -> tuple:
f"and job {job_id}",
404,
)
+ error = _iot_module.jobs_reject_terminal_describe(execution)
+ if error:
+ return error
include_document = str(qp.get("includeJobDocument", "true")).lower() != "false"
document = _iot_module.jobs_job_document(job_id) if include_document else None
return json_response({"execution": _execution_view(execution, document)})
diff --git a/tests/test_iot.py b/tests/test_iot.py
index c2c4ec30..71a3d5de 100644
--- a/tests/test_iot.py
+++ b/tests/test_iot.py
@@ -9493,7 +9493,7 @@ def test_iot_job_removed_timed_out_and_deleted_events():
await call("POST", "/things/ev-t4/jobs/ev-timeout",
{"status": "IN_PROGRESS", "statusDetails": {"d": "1"}}, device=True)
iot_module._job_executions[("ev-t4", "ev-timeout")]["startedAt"] -= 61_000
- await call("GET", "/things/ev-t4/jobs/ev-timeout", device=True)
+ await call("GET", "/things/ev-t4/jobs", device=True)
await call("DELETE", "/jobs/ev-timeout")
await call("DELETE", "/jobs/ev-cont", query={"force": "true"})
diff --git a/tests/test_iot_jobs_data.py b/tests/test_iot_jobs_data.py
index b986504b..f4c9392a 100644
--- a/tests/test_iot_jobs_data.py
+++ b/tests/test_iot_jobs_data.py
@@ -1320,6 +1320,31 @@ def test_iot_jobs_start_next_update_and_auto_complete(iot_client, iot_jobs_data)
_cleanup(iot_client, jobs=[job_id], things=[thing])
+def test_iot_jobs_first_report_stamps_started_at(iot_client, iot_jobs_data):
+ """QUEUED straight to SUCCEEDED or FAILED sets startedAt; REJECTED does not."""
+ thing = _unique("jobs-thing")
+ job_ids = {status: _unique("job") for status in ("SUCCEEDED", "FAILED", "REJECTED")}
+ try:
+ thing_arn = _create_thing(iot_client, thing)
+ for status, job_id in job_ids.items():
+ iot_client.create_job(jobId=job_id, targets=[thing_arn], document=_DOCUMENT)
+ iot_jobs_data.update_job_execution(jobId=job_id, thingName=thing, status=status)
+ listed = {
+ s["jobId"]: s["jobExecutionSummary"]
+ for s in iot_client.list_job_executions_for_thing(thingName=thing)["executionSummaries"]
+ }
+ for status, job_id in job_ids.items():
+ execution = iot_client.describe_job_execution(jobId=job_id, thingName=thing)["execution"]
+ if status == "REJECTED":
+ assert "startedAt" not in execution
+ assert "startedAt" not in listed[job_id]
+ else:
+ assert execution["startedAt"] == execution["lastUpdatedAt"]
+ assert listed[job_id]["startedAt"] == execution["startedAt"]
+ finally:
+ _cleanup(iot_client, jobs=list(job_ids.values()), things=[thing])
+
+
def test_iot_jobs_update_rejects_service_side_statuses(
iot_client, iot_jobs_data
):
@@ -1413,6 +1438,9 @@ def test_iot_jobs_device_cannot_move_execution_back_to_queued(
jobId=job_id, thingName=thing, status="QUEUED"
)
assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 409
+ assert ei.value.response["Error"]["Message"] == (
+ "The status of job execution cannot be changed to be QUEUED"
+ )
execution = iot_jobs_data.describe_job_execution(
jobId=job_id, thingName=thing
@@ -1423,6 +1451,114 @@ def test_iot_jobs_device_cannot_move_execution_back_to_queued(
_cleanup(iot_client, jobs=[job_id], things=[thing])
+def _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups):
+ """One execution per terminal status a test can reach at once: {status: (thing, jobId)}."""
+ executions = {}
+ for status in ("SUCCEEDED", "FAILED", "REJECTED", "CANCELED"):
+ thing, job_id = _unique("jobs-thing"), _unique("job")
+ things.append(thing)
+ jobs.append(job_id)
+ arn = _create_thing(iot_client, thing)
+ iot_client.create_job(jobId=job_id, targets=[arn], document=_DOCUMENT)
+ if status == "CANCELED":
+ iot_client.cancel_job_execution(jobId=job_id, thingName=thing)
+ else:
+ iot_jobs_data.update_job_execution(jobId=job_id, thingName=thing, status=status)
+ executions[status] = (thing, job_id)
+ group, thing, job_id = _unique("jobs-group"), _unique("jobs-thing"), _unique("job")
+ groups.append(group)
+ things.append(thing)
+ jobs.append(job_id)
+ group_arn = iot_client.create_thing_group(thingGroupName=group)["thingGroupArn"]
+ _create_thing(iot_client, thing)
+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)
+ iot_client.create_job(jobId=job_id, targets=[group_arn], document=_DOCUMENT,
+ targetSelection="CONTINUOUS")
+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)
+ executions["REMOVED"] = (thing, job_id)
+ for status, (thing, job_id) in executions.items():
+ execution = iot_client.describe_job_execution(jobId=job_id, thingName=thing)["execution"]
+ assert execution["status"] == status
+ return executions
+
+
+def test_iot_jobs_device_cannot_describe_a_terminal_execution(iot_client, iot_jobs_data):
+ """AWS answers the device 410 TerminalStateException for every terminal status."""
+ things, jobs, groups = [], [], []
+ try:
+ executions = _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups)
+ for status, (thing, job_id) in executions.items():
+ for extra in ({}, {"executionNumber": 1}, {"includeJobDocument": False}):
+ with pytest.raises(iot_jobs_data.exceptions.TerminalStateException) as ei:
+ iot_jobs_data.describe_job_execution(jobId=job_id, thingName=thing, **extra)
+ assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 410, status
+ assert ei.value.response["Error"]["Message"] == (
+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED"
+ ), status
+ finally:
+ _cleanup(iot_client, jobs=jobs, things=things, groups=groups)
+
+
+def test_iot_jobs_device_cannot_describe_a_timed_out_execution():
+ """The timeout flips the execution on the device's read, which then answers 410."""
+ from ministack.core.responses import request_scope
+ from ministack.services import iot as iot_module
+ from ministack.services import iot_jobs_data as jobs_data_module
+
+ iot_module.reset()
+ try:
+ with request_scope("123456789012", "us-east-1"):
+ started = iot_module._jobs_now_ms() - 120_000
+ iot_module._jobs["j1"] = {
+ "jobId": "j1", "targets": [], "targetSelection": "SNAPSHOT",
+ "status": "IN_PROGRESS", "document": "{}", "snapshotted": True,
+ "timeoutConfig": {"inProgressTimeoutInMinutes": 1},
+ }
+ iot_module._job_executions[("t1", "j1")] = {
+ "jobId": "j1", "thingName": "t1", "status": "IN_PROGRESS",
+ "statusDetails": {}, "queuedAt": started, "startedAt": started,
+ "lastUpdatedAt": started, "executionNumber": 1, "versionNumber": 2,
+ }
+ status, _, body = jobs_data_module._describe_execution("t1", "j1", {})
+ assert status == 410
+ assert json.loads(body)["__type"] == "TerminalStateException"
+ assert iot_module._job_executions[("t1", "j1")]["status"] == "TIMED_OUT"
+ finally:
+ iot_module.reset()
+
+
+def test_iot_jobs_terminal_execution_refuses_update_and_cancel(iot_client, iot_jobs_data):
+ """The refusals name the job, the thing and the final status; a refused status is checked first."""
+ things, jobs, groups = [], [], []
+ try:
+ executions = _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups)
+ for status, (thing, job_id) in executions.items():
+ for reported in ("IN_PROGRESS", "SUCCEEDED", "FAILED", "REJECTED"):
+ with pytest.raises(iot_jobs_data.exceptions.InvalidStateTransitionException) as ei:
+ iot_jobs_data.update_job_execution(
+ jobId=job_id, thingName=thing, status=reported
+ )
+ assert ei.value.response["Error"]["Message"] == (
+ f"Job {job_id} for {thing} has already finished with status {status}"
+ )
+ for reported in ("QUEUED", "CANCELED"):
+ with pytest.raises(iot_jobs_data.exceptions.InvalidStateTransitionException) as ei:
+ iot_jobs_data.update_job_execution(
+ jobId=job_id, thingName=thing, status=reported
+ )
+ assert ei.value.response["Error"]["Message"] == (
+ f"The status of job execution cannot be changed to be {reported}"
+ )
+ for force in (False, True):
+ with pytest.raises(iot_client.exceptions.InvalidStateTransitionException) as ei:
+ iot_client.cancel_job_execution(jobId=job_id, thingName=thing, force=force)
+ assert ei.value.response["Error"]["Message"] == (
+ f"Job execution for thing {thing} within job {job_id} cannot be canceled."
+ )
+ finally:
+ _cleanup(iot_client, jobs=jobs, things=things, groups=groups)
+
+
def test_iot_jobs_fleet_completes_only_when_every_execution_is_terminal(
iot_client, iot_jobs_data
):
@@ -3244,10 +3380,9 @@ def test_iot_jobs_status_details_are_nested_on_the_control_plane_only(
)["execution"]
assert control["status"] == "CANCELED"
assert control["statusDetails"] == {"detailsMap": cancel_details}
- device = iot_jobs_data.describe_job_execution(
- jobId=job_id, thingName=thing
- )["execution"]
- assert device["statusDetails"] == cancel_details
+ # The device plane does not describe a terminal execution.
+ with pytest.raises(iot_jobs_data.exceptions.TerminalStateException):
+ iot_jobs_data.describe_job_execution(jobId=job_id, thingName=thing)
finally:
_cleanup(iot_client, jobs=[job_id], things=[thing])
@@ -4117,7 +4252,7 @@ def test_jobs_mqtt_malformed_json_is_rejected_invalidjson(iot_client, iot_data_c
assert isinstance(doc["timestamp"], int)
-def test_jobs_mqtt_terminal_update_rejected_terminalstatereached(iot_client, iot_data_client):
+def test_jobs_mqtt_terminal_update_rejected_invalidstatetransition(iot_client, iot_data_client):
thing = _unique("qa-terminal")
job_id = _unique("job")
arn = _create_thing(iot_client, thing)
@@ -4135,7 +4270,23 @@ def test_jobs_mqtt_terminal_update_rejected_terminalstatereached(iot_client, iot
json.dumps({"status": "FAILED"}).encode()),
want=1,
)
- assert json.loads(received[0][1])["code"] == "TerminalStateReached"
+ doc = json.loads(received[0][1])
+ assert doc["code"] == "InvalidStateTransition"
+ assert doc["message"] == f"Job {job_id} for {thing} has already finished with status SUCCEEDED"
+ assert doc["executionState"] == {"status": "SUCCEEDED", "versionNumber": 2}
+ received = _collect_shadow_frames(
+ f"{base}/{job_id}/get/rejected",
+ lambda: _qa_publish(iot_data_client, f"{base}/{job_id}/get",
+ json.dumps({"clientToken": "g"}).encode()),
+ want=1,
+ )
+ doc = json.loads(received[0][1])
+ assert doc["code"] == "TerminalStateReached"
+ assert doc["message"] == (
+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED"
+ )
+ assert doc["clientToken"] == "g"
+ assert "executionState" not in doc
def test_jobs_mqtt_wrong_version_carries_execution_state(iot_client, iot_data_client):
@@ -4158,6 +4309,24 @@ def test_jobs_mqtt_wrong_version_carries_execution_state(iot_client, iot_data_cl
assert set(doc["executionState"]) == {"status", "versionNumber"}
+def test_jobs_mqtt_refused_status_carries_no_execution_state(iot_client, iot_data_client):
+ thing = _unique("qa-queued")
+ job_id = _unique("job")
+ arn = _create_thing(iot_client, thing)
+ iot_client.create_job(jobId=job_id, targets=[arn], document=_DOCUMENT)
+ base = f"$aws/things/{thing}/jobs"
+ received = _collect_shadow_frames(
+ f"{base}/{job_id}/update/rejected",
+ lambda: _qa_publish(iot_data_client, f"{base}/{job_id}/update",
+ json.dumps({"status": "QUEUED", "clientToken": "q"}).encode()),
+ want=1,
+ )
+ doc = json.loads(received[0][1])
+ assert doc["code"] == "InvalidStateTransition"
+ assert doc["message"] == "The status of job execution cannot be changed to be QUEUED"
+ assert "executionState" not in doc
+
+
def test_jobs_mqtt_job_named_get_does_not_break_thing_level_get(iot_client, iot_data_client):
thing = _unique("qa-collide")
arn = _create_thing(iot_client, thing)