Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(iot): refuse device reads of terminal job executions (#2192)
iot-rocket45bcc5bec005
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3b4c9e89..5c36a566 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **IoT Jobs — terminal job executions on the device plane** — a device's `DescribeJobExecution` (HTTP and the MQTT `get` topic) of a SUCCEEDED, FAILED, REJECTED, TIMED_OUT, CANCELED or REMOVED execution answered with the execution; it now answers `TerminalStateException` (410) and `TerminalStateReached` over MQTT, as AWS does. `UpdateJobExecution` and `CancelJobExecution` of a terminal execution now refuse with AWS's messages (over MQTT `InvalidStateTransition` with the `executionState`), a refused status such as QUEUED is checked first, and a device that reports SUCCEEDED or FAILED straight from QUEUED now stamps `startedAt`. Contributed by @iot-rocket.- **IoT / CloudFormation — stack tags and provisioning templates** — stack-level tags never reached IoT resources; they now reach thing types, thing groups, policies, topic rules, CA certificates and provisioning templates without the `aws:cloudformation:` tags (job templates get none, as on AWS), and an update that only changes tags no longer adds a policy version. An `AWS::IoT::ProvisioningTemplate` `Enabled` of `"false"` from a parameter or the template enabled the template; it now disables it. `CreateProvisioningTemplate` and the CloudFormation type accepted any certificate resource; they now refuse one without a `Status` of `Active` or `Inactive` (any case) or whose `CertificateId` is not a reference to `AWS::IoT::Certificate::Id`, a second certificate resource and a resource without `Properties`, with AWS's messages. Contributed by @iot-rocket.### Added- **WAF v2 — regex pattern sets** — `CreateRegexPatternSet`, `GetRegexPatternSet`, `UpdateRegexPatternSet`, `DeleteRegexPatternSet` and `ListRegexPatternSets` now manage regional and CloudFront-scoped regex pattern sets, with their ARN (`regexpatternset`), tags and `LockToken` checks on update and delete (`WAFOptimisticLockException` for a stale token), as AWS does. Web ACL and rule group rules are stored as sent, so a `RegexPatternSetReferenceStatement` pointing at a pattern set ARN round-trips; the statement is not evaluated, as MiniStack does not inspect requests.diff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex 45715e25..4539ad19 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -6489,15 +6489,21 @@ def _jobs_check_expected_version(return None-def _jobs_reject_if_terminal(execution: dict, verb: str) -> tuple | None:+def _jobs_reject_if_terminal(execution: dict, message: str) -> tuple | None:"""Terminal executions are frozen — no update, no cancel."""+ if execution["status"] not in _JOB_EXECUTION_TERMINAL:+ return None+ return error_response_json("InvalidStateTransitionException", message, 409)+++def jobs_reject_terminal_describe(execution: dict) -> tuple | None:+ """The device plane describes only QUEUED and IN_PROGRESS executions."""if execution["status"] not in _JOB_EXECUTION_TERMINAL:return Nonereturn error_response_json(- "InvalidStateTransitionException",- f"Job execution is in terminal status {execution['status']} and "- f"cannot be {verb}",- 409,+ "TerminalStateException",+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED",+ 410,)@@ -7713,7 +7719,9 @@ async def _cancel_job_execution()if error:return error- error = _jobs_reject_if_terminal(execution, "canceled")+ error = _jobs_reject_if_terminal(+ execution, f"Job execution for thing {thing} within job {job_id} cannot be canceled."+ )if error:return errorforce = _qp_bool(qp, "force")@@ -8068,9 +8076,6 @@ def jobs_update_execution(error = _jobs_check_expected_version(execution, expected_version, "InvalidStateTransitionException")- if error:- return None, error- error = _jobs_reject_if_terminal(execution, "updated")if error:return None, errorif not status or status not in _JOB_EXECUTION_STATUSES:@@ -8079,12 +8084,6 @@ def jobs_update_execution(f"Invalid job execution status: {status!r}",400,)- if status == "QUEUED":- return None, error_response_json(- "InvalidStateTransitionException",- "A device cannot move a job execution back to QUEUED",- 409,- )if status not in _DEVICE_SETTABLE_STATUSES:# InvalidStateTransitionException, not InvalidRequestException# (measured eu-north-1 2026-09-19).@@ -8093,11 +8092,18 @@ def jobs_update_execution(f"The status of job execution cannot be changed to be {status}",409,)+ error = _jobs_reject_if_terminal(+ execution,+ f"Job {job_id} for {thing_name} has already finished with status {execution['status']}",+ )+ if error:+ return None, errornow = _jobs_now_ms()execution["status"] = statusif status_details:execution["statusDetails"] = dict(status_details)- if status == "IN_PROGRESS" and execution.get("startedAt") is None:+ # The first report starts the execution, unless the device rejects the job.+ if status != "REJECTED" and execution.get("startedAt") is None:execution["startedAt"] = nowif status == "IN_PROGRESS" and step_timeout_minutes is not None:# The device's own step timeout, reset every time it re-reports@@ -9310,14 +9316,16 @@ def _jobs_error_tuple_to_rejected(code = "ResourceNotFound"elif status == 400:code = "InvalidRequest"+ elif status == 410:+ code = "TerminalStateReached"elif message.startswith("Expected version"):code = "VersionMismatch"- elif "terminal status" in message:- code = "TerminalStateReached"else:code = "InvalidStateTransition"execution_state = None- if code in ("VersionMismatch", "InvalidStateTransition") and execution is not None:+ # A refused status carries no executionState.+ refused_status = message.startswith("The status of job execution cannot be changed")+ if code in ("VersionMismatch", "InvalidStateTransition") and execution is not None and not refused_status:execution_state = {"status": execution["status"],"versionNumber": execution["versionNumber"],@@ -9440,6 +9448,10 @@ async def _handle_jobs_publish(),)return+ error = jobs_reject_terminal_describe(execution) if execution is not None else None+ if error:+ await _respond("rejected", _jobs_error_tuple_to_rejected(error, client_token))+ return# An idle thing's "$next" mirrors the HTTP plane's empty structure.doc = {"timestamp": ts}if execution is not None:diff --git a/ministack/services/iot_jobs_data.py b/ministack/services/iot_jobs_data.pyindex 4a769dd1..e3213dec 100644--- a/ministack/services/iot_jobs_data.py+++ b/ministack/services/iot_jobs_data.py@@ -200,6 +200,9 @@ def _describe_execution(thing: str, job_id: str, qp: dict) -> tuple:f"and job {job_id}",404,)+ error = _iot_module.jobs_reject_terminal_describe(execution)+ if error:+ return errorinclude_document = str(qp.get("includeJobDocument", "true")).lower() != "false"document = _iot_module.jobs_job_document(job_id) if include_document else Nonereturn json_response({"execution": _execution_view(execution, document)})diff --git a/tests/test_iot.py b/tests/test_iot.pyindex c2c4ec30..71a3d5de 100644--- a/tests/test_iot.py+++ b/tests/test_iot.py@@ -9493,7 +9493,7 @@ def test_iot_job_removed_timed_out_and_deleted_events():await call("POST", "/things/ev-t4/jobs/ev-timeout",{"status": "IN_PROGRESS", "statusDetails": {"d": "1"}}, device=True)iot_module._job_executions[("ev-t4", "ev-timeout")]["startedAt"] -= 61_000- await call("GET", "/things/ev-t4/jobs/ev-timeout", device=True)+ await call("GET", "/things/ev-t4/jobs", device=True)await call("DELETE", "/jobs/ev-timeout")await call("DELETE", "/jobs/ev-cont", query={"force": "true"})diff --git a/tests/test_iot_jobs_data.py b/tests/test_iot_jobs_data.pyindex b986504b..f4c9392a 100644--- a/tests/test_iot_jobs_data.py+++ b/tests/test_iot_jobs_data.py@@ -1320,6 +1320,31 @@ def test_iot_jobs_start_next_update_and_auto_complete(iot_client, iot_jobs_data)_cleanup(iot_client, jobs=[job_id], things=[thing])+def test_iot_jobs_first_report_stamps_started_at(iot_client, iot_jobs_data):+ """QUEUED straight to SUCCEEDED or FAILED sets startedAt; REJECTED does not."""+ thing = _unique("jobs-thing")+ job_ids = {status: _unique("job") for status in ("SUCCEEDED", "FAILED", "REJECTED")}+ try:+ thing_arn = _create_thing(iot_client, thing)+ for status, job_id in job_ids.items():+ iot_client.create_job(jobId=job_id, targets=[thing_arn], document=_DOCUMENT)+ iot_jobs_data.update_job_execution(jobId=job_id, thingName=thing, status=status)+ listed = {+ s["jobId"]: s["jobExecutionSummary"]+ for s in iot_client.list_job_executions_for_thing(thingName=thing)["executionSummaries"]+ }+ for status, job_id in job_ids.items():+ execution = iot_client.describe_job_execution(jobId=job_id, thingName=thing)["execution"]+ if status == "REJECTED":+ assert "startedAt" not in execution+ assert "startedAt" not in listed[job_id]+ else:+ assert execution["startedAt"] == execution["lastUpdatedAt"]+ assert listed[job_id]["startedAt"] == execution["startedAt"]+ finally:+ _cleanup(iot_client, jobs=list(job_ids.values()), things=[thing])++def test_iot_jobs_update_rejects_service_side_statuses(iot_client, iot_jobs_data):@@ -1413,6 +1438,9 @@ def test_iot_jobs_device_cannot_move_execution_back_to_queued(jobId=job_id, thingName=thing, status="QUEUED")assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 409+ assert ei.value.response["Error"]["Message"] == (+ "The status of job execution cannot be changed to be QUEUED"+ )execution = iot_jobs_data.describe_job_execution(jobId=job_id, thingName=thing@@ -1423,6 +1451,114 @@ def test_iot_jobs_device_cannot_move_execution_back_to_queued(_cleanup(iot_client, jobs=[job_id], things=[thing])+def _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups):+ """One execution per terminal status a test can reach at once: {status: (thing, jobId)}."""+ executions = {}+ for status in ("SUCCEEDED", "FAILED", "REJECTED", "CANCELED"):+ thing, job_id = _unique("jobs-thing"), _unique("job")+ things.append(thing)+ jobs.append(job_id)+ arn = _create_thing(iot_client, thing)+ iot_client.create_job(jobId=job_id, targets=[arn], document=_DOCUMENT)+ if status == "CANCELED":+ iot_client.cancel_job_execution(jobId=job_id, thingName=thing)+ else:+ iot_jobs_data.update_job_execution(jobId=job_id, thingName=thing, status=status)+ executions[status] = (thing, job_id)+ group, thing, job_id = _unique("jobs-group"), _unique("jobs-thing"), _unique("job")+ groups.append(group)+ things.append(thing)+ jobs.append(job_id)+ group_arn = iot_client.create_thing_group(thingGroupName=group)["thingGroupArn"]+ _create_thing(iot_client, thing)+ iot_client.add_thing_to_thing_group(thingGroupName=group, thingName=thing)+ iot_client.create_job(jobId=job_id, targets=[group_arn], document=_DOCUMENT,+ targetSelection="CONTINUOUS")+ iot_client.remove_thing_from_thing_group(thingGroupName=group, thingName=thing)+ executions["REMOVED"] = (thing, job_id)+ for status, (thing, job_id) in executions.items():+ execution = iot_client.describe_job_execution(jobId=job_id, thingName=thing)["execution"]+ assert execution["status"] == status+ return executions+++def test_iot_jobs_device_cannot_describe_a_terminal_execution(iot_client, iot_jobs_data):+ """AWS answers the device 410 TerminalStateException for every terminal status."""+ things, jobs, groups = [], [], []+ try:+ executions = _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups)+ for status, (thing, job_id) in executions.items():+ for extra in ({}, {"executionNumber": 1}, {"includeJobDocument": False}):+ with pytest.raises(iot_jobs_data.exceptions.TerminalStateException) as ei:+ iot_jobs_data.describe_job_execution(jobId=job_id, thingName=thing, **extra)+ assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 410, status+ assert ei.value.response["Error"]["Message"] == (+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED"+ ), status+ finally:+ _cleanup(iot_client, jobs=jobs, things=things, groups=groups)+++def test_iot_jobs_device_cannot_describe_a_timed_out_execution():+ """The timeout flips the execution on the device's read, which then answers 410."""+ from ministack.core.responses import request_scope+ from ministack.services import iot as iot_module+ from ministack.services import iot_jobs_data as jobs_data_module++ iot_module.reset()+ try:+ with request_scope("123456789012", "us-east-1"):+ started = iot_module._jobs_now_ms() - 120_000+ iot_module._jobs["j1"] = {+ "jobId": "j1", "targets": [], "targetSelection": "SNAPSHOT",+ "status": "IN_PROGRESS", "document": "{}", "snapshotted": True,+ "timeoutConfig": {"inProgressTimeoutInMinutes": 1},+ }+ iot_module._job_executions[("t1", "j1")] = {+ "jobId": "j1", "thingName": "t1", "status": "IN_PROGRESS",+ "statusDetails": {}, "queuedAt": started, "startedAt": started,+ "lastUpdatedAt": started, "executionNumber": 1, "versionNumber": 2,+ }+ status, _, body = jobs_data_module._describe_execution("t1", "j1", {})+ assert status == 410+ assert json.loads(body)["__type"] == "TerminalStateException"+ assert iot_module._job_executions[("t1", "j1")]["status"] == "TIMED_OUT"+ finally:+ iot_module.reset()+++def test_iot_jobs_terminal_execution_refuses_update_and_cancel(iot_client, iot_jobs_data):+ """The refusals name the job, the thing and the final status; a refused status is checked first."""+ things, jobs, groups = [], [], []+ try:+ executions = _terminal_executions(iot_client, iot_jobs_data, things, jobs, groups)+ for status, (thing, job_id) in executions.items():+ for reported in ("IN_PROGRESS", "SUCCEEDED", "FAILED", "REJECTED"):+ with pytest.raises(iot_jobs_data.exceptions.InvalidStateTransitionException) as ei:+ iot_jobs_data.update_job_execution(+ jobId=job_id, thingName=thing, status=reported+ )+ assert ei.value.response["Error"]["Message"] == (+ f"Job {job_id} for {thing} has already finished with status {status}"+ )+ for reported in ("QUEUED", "CANCELED"):+ with pytest.raises(iot_jobs_data.exceptions.InvalidStateTransitionException) as ei:+ iot_jobs_data.update_job_execution(+ jobId=job_id, thingName=thing, status=reported+ )+ assert ei.value.response["Error"]["Message"] == (+ f"The status of job execution cannot be changed to be {reported}"+ )+ for force in (False, True):+ with pytest.raises(iot_client.exceptions.InvalidStateTransitionException) as ei:+ iot_client.cancel_job_execution(jobId=job_id, thingName=thing, force=force)+ assert ei.value.response["Error"]["Message"] == (+ f"Job execution for thing {thing} within job {job_id} cannot be canceled."+ )+ finally:+ _cleanup(iot_client, jobs=jobs, things=things, groups=groups)++def test_iot_jobs_fleet_completes_only_when_every_execution_is_terminal(iot_client, iot_jobs_data):@@ -3244,10 +3380,9 @@ def test_iot_jobs_status_details_are_nested_on_the_control_plane_only()["execution"]assert control["status"] == "CANCELED"assert control["statusDetails"] == {"detailsMap": cancel_details}- device = iot_jobs_data.describe_job_execution(- jobId=job_id, thingName=thing- )["execution"]- assert device["statusDetails"] == cancel_details+ # The device plane does not describe a terminal execution.+ with pytest.raises(iot_jobs_data.exceptions.TerminalStateException):+ iot_jobs_data.describe_job_execution(jobId=job_id, thingName=thing)finally:_cleanup(iot_client, jobs=[job_id], things=[thing])@@ -4117,7 +4252,7 @@ def test_jobs_mqtt_malformed_json_is_rejected_invalidjson(iot_client, iot_data_cassert isinstance(doc["timestamp"], int)-def test_jobs_mqtt_terminal_update_rejected_terminalstatereached(iot_client, iot_data_client):+def test_jobs_mqtt_terminal_update_rejected_invalidstatetransition(iot_client, iot_data_client):thing = _unique("qa-terminal")job_id = _unique("job")arn = _create_thing(iot_client, thing)@@ -4135,7 +4270,23 @@ def test_jobs_mqtt_terminal_update_rejected_terminalstatereached(iot_client, iotjson.dumps({"status": "FAILED"}).encode()),want=1,)- assert json.loads(received[0][1])["code"] == "TerminalStateReached"+ doc = json.loads(received[0][1])+ assert doc["code"] == "InvalidStateTransition"+ assert doc["message"] == f"Job {job_id} for {thing} has already finished with status SUCCEEDED"+ assert doc["executionState"] == {"status": "SUCCEEDED", "versionNumber": 2}+ received = _collect_shadow_frames(+ f"{base}/{job_id}/get/rejected",+ lambda: _qa_publish(iot_data_client, f"{base}/{job_id}/get",+ json.dumps({"clientToken": "g"}).encode()),+ want=1,+ )+ doc = json.loads(received[0][1])+ assert doc["code"] == "TerminalStateReached"+ assert doc["message"] == (+ "Job Execution has reached terminal state. It is neither IN_PROGRESS nor QUEUED"+ )+ assert doc["clientToken"] == "g"+ assert "executionState" not in docdef test_jobs_mqtt_wrong_version_carries_execution_state(iot_client, iot_data_client):@@ -4158,6 +4309,24 @@ def test_jobs_mqtt_wrong_version_carries_execution_state(iot_client, iot_data_classert set(doc["executionState"]) == {"status", "versionNumber"}+def test_jobs_mqtt_refused_status_carries_no_execution_state(iot_client, iot_data_client):+ thing = _unique("qa-queued")+ job_id = _unique("job")+ arn = _create_thing(iot_client, thing)+ iot_client.create_job(jobId=job_id, targets=[arn], document=_DOCUMENT)+ base = f"$aws/things/{thing}/jobs"+ received = _collect_shadow_frames(+ f"{base}/{job_id}/update/rejected",+ lambda: _qa_publish(iot_data_client, f"{base}/{job_id}/update",+ json.dumps({"status": "QUEUED", "clientToken": "q"}).encode()),+ want=1,+ )+ doc = json.loads(received[0][1])+ assert doc["code"] == "InvalidStateTransition"+ assert doc["message"] == "The status of job execution cannot be changed to be QUEUED"+ assert "executionState" not in doc++def test_jobs_mqtt_job_named_get_does_not_break_thing_level_get(iot_client, iot_data_client):thing = _unique("qa-collide")arn = _create_thing(iot_client, thing)