Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(iot): publish shadow notifications for HTTP data-plane updates (#1531)
iot-rocket9ab4fb47ee98
diff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex 5b319a11..a4576823 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -4524,7 +4524,7 @@ async def broker_stop() -> None:_BASIC_INGEST_PREFIX = "$aws/rules/"-_SHADOW_TOPIC_PREFIX = "$aws/things/"+SHADOW_TOPIC_PREFIX = "$aws/things/"def _rules_for_account(account_id: str, region: str) -> list[dict]:@@ -4913,9 +4913,9 @@ def _parse_shadow_topic(topic: str) -> tuple[str, str, str | None] | None:Anything longer — the ``accepted`` / ``rejected`` / ``delta`` /``documents`` response suffixes — parses as ``None``, so the bridge's ownresponse publishes can never re-trigger it (no recursion)."""- if not topic.startswith(_SHADOW_TOPIC_PREFIX):+ if not topic.startswith(SHADOW_TOPIC_PREFIX):return None- rest = topic[len(_SHADOW_TOPIC_PREFIX):]+ rest = topic[len(SHADOW_TOPIC_PREFIX):]parts = rest.split("/")if len(parts) < 3 or parts[1] != "shadow":return None@@ -4942,15 +4942,18 @@ def _shadow_rejected(status: int, message: str, client_token: str | None = None)def _shadow_document(doc: dict) -> dict:- """Strip the computed ``delta`` sections out of a shadow snapshot.-- `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``) by- design, because that is what the GET response carries. The documents topic- is not a GET: on AWS the ``previous``/``current`` documents report only- ``desired`` and ``reported``, and the delta has its own topic. Publishing- the snapshot verbatim invented a section real devices never see.+ """Project a GET snapshot onto a ``documents`` member: ``state`` ++ ``metadata`` + ``version``, nothing else.++ `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``)+ and a top-level ``timestamp`` by design, because that is what the GET+ response carries. The documents topic is not a GET: on AWS the+ ``previous``/``current`` members report only ``desired`` and ``reported``+ (the delta has its own topic) and carry no ``timestamp`` of their own —+ the single timestamp sits on the envelope. Publishing the snapshot+ verbatim invented members real devices never see."""- stripped = dict(doc)+ stripped = {k: doc[k] for k in ("state", "metadata", "version") if k in doc}for section in ("state", "metadata"):block = stripped.get(section)if isinstance(block, dict) and "delta" in block:@@ -4958,6 +4961,62 @@ def _shadow_document(doc: dict) -> dict:return stripped+async def shadow_emit_update_responses(+ account_id: str,+ region: str,+ base: str,+ doc: dict,+ pre_status: int,+ pre_doc: dict,+ post_status: int,+ post_doc: dict,+ client_token: str | None,+) -> None:+ """Publish the reserved-topic responses for one ACCEPTED shadow update:+ ``update/accepted``, ``update/delta`` (when the post-update document+ carries one) and ``update/documents``.++ Shared by the MQTT bridge and the HTTP data plane: on AWS these are+ state-change notifications the Device Shadow service emits however the+ update arrived, which is what lets a topic rule on ``update/accepted``+ fire for an HTTP ``UpdateThingShadow``. The caller must hand in+ deep-copied pre/post snapshots (the store returns live dicts)."""++ async def _respond(suffix: str, payload_doc: dict) -> None:+ await broker_publish(+ account_id,+ region,+ f"{base}/{suffix}",+ json.dumps(payload_doc).encode("utf-8"),+ qos=1,+ )++ # The core update response already echoes clientToken when present.+ await _respond("update/accepted", doc)+ if post_status == 200:+ delta = (post_doc.get("state") or {}).get("delta")+ if delta:+ # The delta section of the GET document, with the metadata AWS+ # reports alongside it and the triggering request's clientToken.+ delta_doc = {+ "state": delta,+ "metadata": (post_doc.get("metadata") or {}).get("delta", {}),+ "version": post_doc["version"],+ "timestamp": post_doc["timestamp"],+ }+ if client_token is not None:+ delta_doc["clientToken"] = client_token+ await _respond("update/delta", delta_doc)+ documents_doc = {+ "previous": _shadow_document(pre_doc) if pre_status == 200 else None,+ "current": _shadow_document(post_doc) if post_status == 200 else None,+ "timestamp": _shadow_now(),+ }+ if client_token is not None:+ documents_doc["clientToken"] = client_token+ await _respond("update/documents", documents_doc)++async def _handle_shadow_publish(account_id: str,region: str,@@ -4972,7 +5031,7 @@ async def _handle_shadow_publish(# `name is not None` rather than a truthiness test: an empty named shadow# (`.../shadow/name//update`) must answer on the topic the requester is# listening on, not collapse onto the classic shadow's.- base = f"{_SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")+ base = f"{SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")shadow_name = name or ""async def _respond(suffix: str, doc: dict) -> None:@@ -5026,29 +5085,9 @@ async def _handle_shadow_publish(_shadow_rejected(status, doc.get("message", ""), client_token),)return- # The core update response already echoes clientToken when present.- await _respond("update/accepted", doc)- if post_status == 200:- delta = (post_doc.get("state") or {}).get("delta")- if delta:- # The delta section of the GET document, with the metadata AWS- # reports alongside it and the triggering request's clientToken.- delta_doc = {- "state": delta,- "metadata": (post_doc.get("metadata") or {}).get("delta", {}),- "version": post_doc["version"],- "timestamp": post_doc["timestamp"],- }- if client_token is not None:- delta_doc["clientToken"] = client_token- await _respond("update/delta", delta_doc)- await _respond(- "update/documents",- {- "previous": _shadow_document(pre_doc) if pre_status == 200 else None,- "current": _shadow_document(post_doc) if post_status == 200 else None,- "timestamp": _shadow_now(),- },+ await shadow_emit_update_responses(+ account_id, region, base, doc,+ pre_status, pre_doc, post_status, post_doc, client_token,)returndiff --git a/ministack/services/iot_data.py b/ministack/services/iot_data.pyindex b9894a4f..bf82249b 100644--- a/ministack/services/iot_data.py+++ b/ministack/services/iot_data.py@@ -14,6 +14,7 @@ which applies account-and-region-scoped topic prefixing transparently.from __future__ import annotations+import copyimport jsonimport loggingfrom urllib.parse import unquote@@ -86,7 +87,7 @@ async def handle_request(# Device Shadow: /things/{thingName}/shadow[?name={shadowName}]if path.startswith("/things/") and path.endswith("/shadow"):- return _shadow(method, path, body, qp)+ return await _shadow(method, path, body, qp)# Placeholders — return 501 so SDKs that probe these endpoints# get a clear "not implemented" rather than a generic 404.@@ -108,10 +109,23 @@ async def handle_request()-def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:- """Route ``GetThingShadow`` / ``UpdateThingShadow`` / ``DeleteThingShadow``."""+async def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:+ """Route ``GetThingShadow`` / ``UpdateThingShadow`` / ``DeleteThingShadow``.++ An accepted update or delete also publishes the reserved-topic+ notifications through the broker (``update/accepted``/``delta``/+ ``documents``, ``delete/accepted``): on AWS those are state-change+ messages the Device Shadow service emits however the change arrived, so a+ topic rule on them fires for HTTP callers too. A ``get`` publishes+ nothing — ``get/accepted`` is the MQTT request's response channel and an+ HTTP read is answered by its own response body. HTTP failures publish no+ ``rejected`` envelope either; the HTTP status already carries them."""thing = unquote(path[len("/things/"):-len("/shadow")])shadow_name = qp.get("name", "") or ""+ base = f"{_iot_module.SHADOW_TOPIC_PREFIX}{thing}/shadow" + (+ f"/name/{shadow_name}" if shadow_name else ""+ )+ account_id, region = get_account_id(), get_region()if method == "GET":status, doc = _iot_module.get_thing_shadow(thing, shadow_name)@@ -120,9 +134,30 @@ def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:request = json.loads(body) if body else {}except json.JSONDecodeError:return error_response_json("InvalidRequestException", "Invalid JSON", 400)+ # Snapshot around the mutation: the store hands back live dicts, and+ # the documents notification needs the before/after pair (same+ # discipline as the MQTT bridge).+ pre_status, pre_doc = _iot_module.get_thing_shadow(thing, shadow_name)+ pre_doc = copy.deepcopy(pre_doc)status, doc = _iot_module.update_thing_shadow(thing, shadow_name, request)+ if status == 200:+ post_status, post_doc = _iot_module.get_thing_shadow(thing, shadow_name)+ post_doc = copy.deepcopy(post_doc)+ await _iot_module.shadow_emit_update_responses(+ account_id, region, base, doc,+ pre_status, pre_doc, post_status, post_doc,+ request.get("clientToken") if isinstance(request, dict) else None,+ )elif method == "DELETE":status, doc = _iot_module.delete_thing_shadow(thing, shadow_name)+ if status == 200:+ await _iot_module.broker_publish(+ account_id,+ region,+ f"{base}/delete/accepted",+ json.dumps(doc).encode("utf-8"),+ qos=1,+ )else:return error_response_json("InvalidRequestException", f"Unsupported iot-data path: {method} {path}", 400diff --git a/tests/test_iot_data.py b/tests/test_iot_data.pyindex 811e024b..e9935005 100644--- a/tests/test_iot_data.py+++ b/tests/test_iot_data.py@@ -1688,6 +1688,140 @@ def test_shadow_update_over_mqtt_emits_accepted_delta_documents(iot_data_client)assert shadow["version"] == accepted["version"]+def test_shadow_update_over_http_emits_accepted_delta_documents(iot_data_client):+ """An HTTP `UpdateThingShadow` publishes the same state-change+ notifications as an MQTT update: accepted + delta + documents. On AWS the+ Device Shadow service emits these however the update arrived — it is what+ lets a topic rule on `update/accepted` fire for HTTP callers."""+ thing = _unique("shadow-http")+ base = f"$aws/things/{thing}/shadow"++ received = _collect_shadow_frames(+ f"{base}/update/+",+ lambda: iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps(+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-http"}+ ).encode(),+ ),+ want=3,+ )++ frames = {topic: json.loads(payload) for topic, payload in received}+ accepted = frames[f"{base}/update/accepted"]+ assert accepted["state"] == {"desired": {"led": "on"}}+ delta = frames[f"{base}/update/delta"]+ assert delta["state"] == {"led": "on"}+ assert delta["clientToken"] == "tok-http"+ docs = frames[f"{base}/update/documents"]+ # The envelope carries the triggering request's clientToken and the single+ # timestamp; previous/current are state + metadata + version only (a live+ # probe pinned the member keys — no inner timestamp).+ assert set(docs) == {"previous", "current", "timestamp", "clientToken"}+ assert docs["clientToken"] == "tok-http"+ assert docs["previous"] is None+ assert set(docs["current"]) == {"state", "metadata", "version"}+ assert docs["current"]["state"]["desired"] == {"led": "on"}+++def test_shadow_delete_over_http_emits_delete_accepted_but_get_nothing(iot_data_client):+ """An HTTP `DeleteThingShadow` publishes `delete/accepted` (a state+ change); an HTTP `GetThingShadow` publishes nothing — `get/accepted` is+ the MQTT request's response channel and the HTTP read has its own+ response body."""+ thing = _unique("shadow-http-del")+ base = f"$aws/things/{thing}/shadow"+ iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps({"state": {"reported": {"v": 1}}}).encode(),+ )++ received = _collect_shadow_frames(+ f"{base}/get/+",+ lambda: iot_data_client.get_thing_shadow(thingName=thing),+ want=1,+ timeout=2.0,+ )+ assert received == []++ received = _collect_shadow_frames(+ f"{base}/delete/+",+ lambda: iot_data_client.delete_thing_shadow(thingName=thing),+ want=1,+ )+ frames = {topic: json.loads(payload) for topic, payload in received}+ # AWS's delete/accepted body is abbreviated: version + timestamp only.+ assert set(frames[f"{base}/delete/accepted"]) == {"version", "timestamp"}+++def test_named_shadow_update_over_http_emits_on_named_topics(iot_data_client):+ """An HTTP update of a named shadow notifies on that shadow's own+ `/name/<n>/update/...` topics, not the classic shadow's."""+ thing = _unique("shadow-http-named")+ base = f"$aws/things/{thing}/shadow/name/cfg"++ received = _collect_shadow_frames(+ f"{base}/update/+",+ lambda: iot_data_client.update_thing_shadow(+ thingName=thing,+ shadowName="cfg",+ payload=json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),+ ),+ want=3,+ )++ frames = {topic: json.loads(payload) for topic, payload in received}+ accepted = frames[f"{base}/update/accepted"]+ assert accepted["state"] == {"desired": {"mode": "eco"}}+ docs = frames[f"{base}/update/documents"]+ # No clientToken in the request → none on the documents envelope.+ assert set(docs) == {"previous", "current", "timestamp"}+ assert docs["current"]["state"]["desired"] == {"mode": "eco"}+++def test_shadow_update_over_http_rejected_emits_no_frames(iot_data_client):+ """A failed HTTP update publishes nothing — no accepted trio and no+ `rejected` envelope, the HTTP 409 already carries the failure."""+ thing = _unique("shadow-http-conflict")+ base = f"$aws/things/{thing}/shadow"+ iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps({"state": {"reported": {"x": 1}}}).encode(),+ )++ def _stale_update():+ with pytest.raises(ClientError) as ei:+ iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps(+ {"state": {"reported": {"x": 2}}, "version": 99}+ ).encode(),+ )+ assert ei.value.response["Error"]["Code"] == "ConflictException"++ received = _collect_shadow_frames(+ f"{base}/update/+", _stale_update, want=1, timeout=2.0+ )+ assert received == []+++def test_shadow_delete_over_http_missing_emits_no_frames(iot_data_client):+ """An HTTP delete of a shadow that does not exist changes no state, so+ nothing is published — the HTTP 404 is the whole answer."""+ thing = _unique("shadow-http-del-miss")+ base = f"$aws/things/{thing}/shadow"++ def _delete_missing():+ with pytest.raises(ClientError) as ei:+ iot_data_client.delete_thing_shadow(thingName=thing)+ assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException"++ received = _collect_shadow_frames(+ f"{base}/delete/+", _delete_missing, want=1, timeout=2.0+ )+ assert received == []++def test_named_shadow_update_over_mqtt(iot_data_client):thing = _unique("shadow-named")base = f"$aws/things/{thing}/shadow/name/cfg"