Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(iot): device shadows over MQTT (#1395)
iot-rocket51a3c8c97cca
diff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex 13372a2d..61cf96c4 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -3281,6 +3281,7 @@ async def broker_stop() -> None:_BASIC_INGEST_PREFIX = "$aws/rules/"+_SHADOW_TOPIC_PREFIX = "$aws/things/"def _rules_for_account(account_id: str, region: str) -> list[dict]:@@ -3611,6 +3612,169 @@ async def _evaluate_topic_rules(await _run_rule_actions(account_id, region, rule, payload, topic, client_id)+def _parse_shadow_topic(topic: str) -> tuple[str, str, str | None] | None:+ """``$aws/things/<thing>/shadow[/name/<n>]/<verb>`` → (thing, verb, name).++ Anything longer — the ``accepted`` / ``rejected`` / ``delta`` /+ ``documents`` response suffixes — parses as ``None``, so the bridge's own+ response publishes can never re-trigger it (no recursion)."""+ if not topic.startswith(_SHADOW_TOPIC_PREFIX):+ return None+ rest = topic[len(_SHADOW_TOPIC_PREFIX):]+ parts = rest.split("/")+ if len(parts) < 3 or parts[1] != "shadow":+ return None+ thing = parts[0]+ tail = parts[2:]+ name = None+ if tail and tail[0] == "name" and len(tail) >= 2:+ name, tail = tail[1], tail[2:]+ if not tail:+ return None+ verb = tail[0]+ # Responses (accepted/rejected/delta/documents) are ours, never inputs.+ if len(tail) > 1 or verb not in ("update", "get", "delete"):+ return None+ return thing, verb, name+++def _shadow_rejected(status: int, message: str, client_token: str | None = None) -> dict:+ """AWS's shadow error response document."""+ doc = {"code": status, "message": message}+ if client_token is not None:+ doc["clientToken"] = client_token+ return doc+++def _shadow_document(doc: dict) -> dict:+ """Strip the computed ``delta`` sections out of a shadow snapshot.++ `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``) by+ design, because that is what the GET response carries. The documents topic+ is not a GET: on AWS the ``previous``/``current`` documents report only+ ``desired`` and ``reported``, and the delta has its own topic. Publishing+ the snapshot verbatim invented a section real devices never see.+ """+ stripped = dict(doc)+ for section in ("state", "metadata"):+ block = stripped.get(section)+ if isinstance(block, dict) and "delta" in block:+ stripped[section] = {k: v for k, v in block.items() if k != "delta"}+ return stripped+++async def _handle_shadow_publish(+ account_id: str,+ region: str,+ thing: str,+ verb: str,+ name: str | None,+ payload: bytes,+) -> None:+ """Shadow-over-MQTT bridge: feed a request-topic publish into the shadow+ store and publish the AWS reserved-topic responses back through the broker+ (``accepted`` / ``rejected``, plus ``delta`` and ``documents`` on update)."""+ # `name is not None` rather than a truthiness test: an empty named shadow+ # (`.../shadow/name//update`) must answer on the topic the requester is+ # listening on, not collapse onto the classic shadow's.+ base = f"{_SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")+ shadow_name = name or ""++ async def _respond(suffix: str, doc: dict) -> None:+ await broker_publish(+ account_id,+ region,+ f"{base}/{suffix}",+ json.dumps(doc).encode("utf-8"),+ qos=1,+ )++ try:+ request = json.loads(payload) if payload else {}+ except (ValueError, UnicodeDecodeError):+ # All three request verbs reject malformed JSON, as on AWS — a `get`+ # or `delete` payload is optional but must parse when present.+ await _respond(f"{verb}/rejected", _shadow_rejected(400, "invalid json"))+ return+ if not isinstance(request, dict):+ request = {}+ client_token = request.get("clientToken")++ if name is not None and not name:+ await _respond(+ f"{verb}/rejected",+ _shadow_rejected(400, "Invalid shadow name: must not be empty", client_token),+ )+ return++ if verb == "update":+ # The broker's WS path sets only the account contextvar, so pin both+ # around every shadow-store access.+ with request_scope(account_id, region):+ # get_thing_shadow hands back the stored state dicts themselves and+ # update_thing_shadow merges into them in place, so `previous` has+ # to be snapshotted before the update runs — otherwise it reports+ # the state *after* it, and from the second update on `previous`+ # and `current` are the same document. `current` is snapshotted for+ # the same reason: a concurrent update would otherwise edit it out+ # from under the publish below.+ pre_status, pre_doc = get_thing_shadow(thing, shadow_name)+ pre_doc = copy.deepcopy(pre_doc)+ status, doc = update_thing_shadow(thing, shadow_name, request)+ post_status, post_doc = (+ get_thing_shadow(thing, shadow_name) if status == 200 else (0, {})+ )+ post_doc = copy.deepcopy(post_doc)+ if status != 200:+ await _respond(+ "update/rejected",+ _shadow_rejected(status, doc.get("message", ""), client_token),+ )+ return+ # The core update response already echoes clientToken when present.+ await _respond("update/accepted", doc)+ if post_status == 200:+ delta = (post_doc.get("state") or {}).get("delta")+ if delta:+ # The delta section of the GET document, with the metadata AWS+ # reports alongside it and the triggering request's clientToken.+ delta_doc = {+ "state": delta,+ "metadata": (post_doc.get("metadata") or {}).get("delta", {}),+ "version": post_doc["version"],+ "timestamp": post_doc["timestamp"],+ }+ if client_token is not None:+ delta_doc["clientToken"] = client_token+ await _respond("update/delta", delta_doc)+ await _respond(+ "update/documents",+ {+ "previous": _shadow_document(pre_doc) if pre_status == 200 else None,+ "current": _shadow_document(post_doc) if post_status == 200 else None,+ "timestamp": _shadow_now(),+ },+ )+ return++ with request_scope(account_id, region):+ status, doc = (+ get_thing_shadow(thing, shadow_name)+ if verb == "get"+ else delete_thing_shadow(thing, shadow_name)+ )+ if status == 200:+ doc = copy.deepcopy(doc)+ if client_token is not None:+ doc["clientToken"] = client_token+ await _respond(f"{verb}/accepted", doc)+ else:+ await _respond(+ f"{verb}/rejected",+ _shadow_rejected(status, doc.get("message", ""), client_token),+ )++async def broker_publish(account_id: str,region: str,@@ -3681,6 +3845,27 @@ async def broker_publish(await _evaluate_topic_rules(account_id, region, topic, payload, client_id)+ # Shadow-over-MQTT bridge — AFTER delivery and rule evaluation, so rules+ # matching `$aws/things/+/shadow/update` fire on the request, and the+ # recursive `accepted`/`delta`/`documents` publishes then flow back+ # through this function to drive their own subscribers and rules. The+ # response suffixes parse as None, so the bridge never re-triggers itself.+ # NOTE known divergence: `_topic_matches` lets a bare `#` subscription+ # match `$aws/...` topics, unlike real AWS where `#` excludes the+ # reserved topic space.+ shadow = _parse_shadow_topic(topic)+ if shadow is not None:+ try:+ await _handle_shadow_publish(account_id, region, *shadow, payload)+ except Exception:+ # Guarded like subscriber delivery and rule dispatch above: a+ # payload the bridge chokes on (a deeply nested document raises+ # RecursionError out of json.loads, which the ValueError handler+ # inside does not catch) must not tear down the MQTT session.+ _broker_logger.warning(+ "IoT shadow bridge failed for %r", topic, exc_info=True+ )+async def broker_subscribe(account_id: str,diff --git a/tests/test_iot.py b/tests/test_iot.pyindex a1769003..87800504 100644--- a/tests/test_iot.py+++ b/tests/test_iot.py@@ -4356,3 +4356,606 @@ def test_ddb_attribute_value_mapping():assert _ddb_attribute_value({"a": 1}) == {"S": '{"a": 1}'}assert _ddb_attribute_value([1, 2]) == {"S": "[1, 2]"}assert _ddb_attribute_value("x") == {"S": "x"}+++# ----------------------------------------------------------------------+# Shadow-over-MQTT bridge (white-box).+# ----------------------------------------------------------------------+++ ("topic", "expected"),+ [+ # Request topics parse.+ ("$aws/things/dev1/shadow/get", ("dev1", "get", None)),+ ("$aws/things/dev1/shadow/update", ("dev1", "update", None)),+ ("$aws/things/dev1/shadow/delete", ("dev1", "delete", None)),+ ("$aws/things/dev1/shadow/name/cfg/get", ("dev1", "get", "cfg")),+ ("$aws/things/dev1/shadow/name/cfg/update", ("dev1", "update", "cfg")),+ ("$aws/things/dev1/shadow/name/cfg/delete", ("dev1", "delete", "cfg")),+ # An empty shadow name parses (so the bridge can reject it on the+ # requester's own reply topic) rather than collapsing onto classic.+ ("$aws/things/dev1/shadow/name//update", ("dev1", "update", "")),+ # Response suffixes never re-trigger the bridge.+ ("$aws/things/dev1/shadow/update/accepted", None),+ ("$aws/things/dev1/shadow/update/rejected", None),+ ("$aws/things/dev1/shadow/update/delta", None),+ ("$aws/things/dev1/shadow/update/documents", None),+ ("$aws/things/dev1/shadow/get/accepted", None),+ ("$aws/things/dev1/shadow/get/rejected", None),+ ("$aws/things/dev1/shadow/delete/accepted", None),+ ("$aws/things/dev1/shadow/name/cfg/update/accepted", None),+ ("$aws/things/dev1/shadow/name/cfg/update/delta", None),+ ("$aws/things/dev1/shadow/name/cfg/get/rejected", None),+ # Malformed / unrelated.+ ("$aws/things/dev1/shadow", None),+ ("$aws/things/dev1/shadow/name", None),+ ("$aws/things/dev1/shadow/name/cfg", None),+ ("$aws/things/dev1/shadow/frobnicate", None),+ ("$aws/things/dev1/notshadow/get", None),+ ("$aws/things", None),+ ("$aws/rules/rule1/some/topic", None),+ ("plain/topic", None),+ ("", None),+ ],+)+def test_parse_shadow_topic_truth_table(topic, expected):+ from ministack.services.iot import _parse_shadow_topic++ assert _parse_shadow_topic(topic) == expected+++def test_shadow_mqtt_update_publishes_accepted_delta_documents():+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(+ account_id, "$aws/things/dev1/shadow/update/+", _collect, granted_qos=1+ )+ await publish(+ account_id,+ "$aws/things/dev1/shadow/update",+ json.dumps(+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-1"}+ ).encode(),+ )++ try:+ asyncio.run(_run())+ accepted = frames["$aws/things/dev1/shadow/update/accepted"][0]+ assert accepted["state"] == {"desired": {"led": "on"}}+ assert accepted["version"] == 1+ assert accepted["clientToken"] == "tok-1"++ # desired != reported → a delta document, carrying the metadata AWS+ # reports for the delta and the triggering request's clientToken.+ delta = frames["$aws/things/dev1/shadow/update/delta"][0]+ assert delta["state"] == {"led": "on"}+ assert delta["version"] == 1+ assert "timestamp" in delta+ assert delta["clientToken"] == "tok-1"+ assert set(delta["metadata"]) == {"led"}+ assert delta["metadata"]["led"]["timestamp"] == accepted["metadata"]["desired"]["led"]["timestamp"]++ docs = frames["$aws/things/dev1/shadow/update/documents"][0]+ assert docs["previous"] is None+ assert docs["current"]["version"] == 1+ assert docs["current"]["state"]["desired"] == {"led": "on"}+ assert "timestamp" in docs++ # The MQTT update landed in the same store the HTTP data plane reads.+ with iot_module.request_scope(account_id, _TEST_REGION):+ status, doc = iot_module.get_thing_shadow("dev1", "")+ assert status == 200+ assert doc["state"]["desired"] == {"led": "on"}+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_update_reported_in_sync_emits_no_delta():+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(+ account_id, "$aws/things/dev2/shadow/update/+", _collect, granted_qos=1+ )+ await publish(+ account_id,+ "$aws/things/dev2/shadow/update",+ json.dumps(+ {"state": {"desired": {"led": "on"}, "reported": {"led": "on"}}}+ ).encode(),+ )++ try:+ asyncio.run(_run())+ assert len(frames["$aws/things/dev2/shadow/update/accepted"]) == 1+ assert "$aws/things/dev2/shadow/update/delta" not in frames+ assert len(frames["$aws/things/dev2/shadow/update/documents"]) == 1+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_update_rejections():+ """Invalid JSON → 400 (no token); version conflict → 409 with the token."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ rejected: list = []++ async def _run():+ async def _collect(topic, payload, qos):+ rejected.append(json.loads(payload))++ await subscribe(+ account_id, "$aws/things/dev3/shadow/update/rejected", _collect+ )+ await publish(+ account_id, "$aws/things/dev3/shadow/update", b"{not json"+ )+ await publish(+ account_id,+ "$aws/things/dev3/shadow/update",+ json.dumps({"state": {"desired": {"a": 1}}}).encode(),+ )+ await publish(+ account_id,+ "$aws/things/dev3/shadow/update",+ json.dumps(+ {"state": {"desired": {"a": 2}}, "version": 99, "clientToken": "t9"}+ ).encode(),+ )++ try:+ asyncio.run(_run())+ assert len(rejected) == 2+ assert rejected[0]["code"] == 400+ assert "clientToken" not in rejected[0]+ assert rejected[1]["code"] == 409+ assert rejected[1]["clientToken"] == "t9"+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_get_delete_roundtrip():+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ async def _run():+ for filt in (+ "$aws/things/dev4/shadow/get/+",+ "$aws/things/dev4/shadow/delete/+",+ ):+ await subscribe(account_id, filt, _collect)+ # get before any update → rejected 404+ await publish(account_id, "$aws/things/dev4/shadow/get", b"")+ await publish(+ account_id,+ "$aws/things/dev4/shadow/update",+ json.dumps({"state": {"reported": {"fw": "1.2"}}}).encode(),+ )+ # get with a clientToken → accepted echoes it+ await publish(+ account_id,+ "$aws/things/dev4/shadow/get",+ json.dumps({"clientToken": "get-1"}).encode(),+ )+ await publish(account_id, "$aws/things/dev4/shadow/delete", b"")+ # delete again → rejected 404+ await publish(account_id, "$aws/things/dev4/shadow/delete", b"")++ try:+ asyncio.run(_run())+ get_rejected = frames["$aws/things/dev4/shadow/get/rejected"]+ assert get_rejected[0]["code"] == 404+ get_accepted = frames["$aws/things/dev4/shadow/get/accepted"][0]+ assert get_accepted["state"]["reported"] == {"fw": "1.2"}+ assert get_accepted["clientToken"] == "get-1"+ del_accepted = frames["$aws/things/dev4/shadow/delete/accepted"][0]+ assert del_accepted["version"] == 1+ del_rejected = frames["$aws/things/dev4/shadow/delete/rejected"][0]+ assert del_rejected["code"] == 404+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_named_shadow_isolated_from_classic():+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(+ account_id, "$aws/things/dev5/shadow/name/cfg/update/+", _collect+ )+ await subscribe(account_id, "$aws/things/dev5/shadow/get/+", _collect)+ await publish(+ account_id,+ "$aws/things/dev5/shadow/name/cfg/update",+ json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),+ )+ # The classic shadow does not exist — only the named one was written.+ await publish(account_id, "$aws/things/dev5/shadow/get", b"")++ try:+ asyncio.run(_run())+ accepted = frames["$aws/things/dev5/shadow/name/cfg/update/accepted"][0]+ assert accepted["state"] == {"desired": {"mode": "eco"}}+ assert frames["$aws/things/dev5/shadow/get/rejected"][0]["code"] == 404+ with iot_module.request_scope(account_id, _TEST_REGION):+ status, doc = iot_module.get_thing_shadow("dev5", "cfg")+ assert status == 200+ assert doc["state"]["desired"] == {"mode": "eco"}+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_documents_previous_is_the_pre_update_state():+ """`documents.previous` must be the state before the update. The shadow+ store hands back its live dicts and updates merge into them in place, so an+ un-snapshotted `previous` reports the post-update state and the two+ documents come out identical from the second update on."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(+ account_id, "$aws/things/dev6/shadow/update/documents", _collect+ )+ for temp in (20, 21):+ await publish(+ account_id,+ "$aws/things/dev6/shadow/update",+ json.dumps({"state": {"reported": {"temp": temp}}}).encode(),+ )++ try:+ asyncio.run(_run())+ docs = frames["$aws/things/dev6/shadow/update/documents"]+ assert len(docs) == 2+ assert docs[0]["previous"] is None+ assert docs[0]["current"]["state"]["reported"] == {"temp": 20}+ second = docs[1]+ assert second["previous"]["state"]["reported"] == {"temp": 20}+ assert second["current"]["state"]["reported"] == {"temp": 21}+ assert second["previous"]["state"] != second["current"]["state"]+ assert second["previous"]["version"] == 1+ assert second["current"]["version"] == 2+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_documents_carry_no_delta_section():+ """A `documents` state reports `desired` and `reported` only.++ Both snapshots come from `get_thing_shadow`, which injects `state.delta`+ (and `metadata.delta`) because that is what a GET answers with. Publishing+ them verbatim put a section on the documents topic that real devices never+ receive — the delta has its own topic, published just above this one.+ """+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ for suffix in ("update/documents", "update/delta"):+ await subscribe(account_id, f"$aws/things/dev7/shadow/{suffix}", _collect)+ # A desired the reported state does not satisfy, so a delta exists on+ # both the pre-update and the post-update document.+ for reported in (10, 11):+ await publish(+ account_id,+ "$aws/things/dev7/shadow/update",+ json.dumps(+ {"state": {"desired": {"temp": 30}, "reported": {"temp": reported}}}+ ).encode(),+ )++ try:+ asyncio.run(_run())+ docs = frames["$aws/things/dev7/shadow/update/documents"]+ assert len(docs) == 2+ second = docs[1]+ for snapshot in ("previous", "current"):+ state = second[snapshot]["state"]+ assert set(state) == {"desired", "reported"}+ assert "delta" not in second[snapshot].get("metadata", {})+ assert second["previous"]["state"]["reported"] == {"temp": 10}+ assert second["current"]["state"]["reported"] == {"temp": 11}+ # The delta itself is still reported, on the topic that is meant for it.+ deltas = frames["$aws/things/dev7/shadow/update/delta"]+ assert deltas and deltas[-1]["state"] == {"temp": 30}+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_update_without_state_node_is_rejected():+ """The most common device-side mistake: valid JSON, no `state` node."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(account_id, "$aws/things/dev7/shadow/update/+", _collect)+ await publish(+ account_id,+ "$aws/things/dev7/shadow/update",+ json.dumps({"desired": {"led": "on"}, "clientToken": "tok-9"}).encode(),+ )++ try:+ asyncio.run(_run())+ rejected = frames["$aws/things/dev7/shadow/update/rejected"][0]+ assert rejected["code"] == 400+ assert "state" in rejected["message"]+ assert rejected["clientToken"] == "tok-9"+ assert "$aws/things/dev7/shadow/update/accepted" not in frames+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_empty_named_shadow_is_rejected_on_its_own_topic():+ """`.../shadow/name//update` names no shadow. Treating it as the classic+ shadow would write the wrong record and answer on a topic the requester is+ not subscribed to, so it is rejected where the requester is listening."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ await subscribe(account_id, "$aws/things/dev8/shadow/name//update/+", _collect)+ await subscribe(account_id, "$aws/things/dev8/shadow/update/+", _collect)+ await publish(+ account_id,+ "$aws/things/dev8/shadow/name//update",+ json.dumps({"state": {"desired": {"led": "on"}}, "clientToken": "t"}).encode(),+ )++ try:+ asyncio.run(_run())+ rejected = frames["$aws/things/dev8/shadow/name//update/rejected"][0]+ assert rejected["code"] == 400+ assert rejected["clientToken"] == "t"+ # Nothing landed on the classic shadow's topics or in its record.+ assert not any(t.startswith("$aws/things/dev8/shadow/update/") for t in frames)+ with iot_module.request_scope(account_id, _TEST_REGION):+ assert iot_module.get_thing_shadow("dev8", "")[0] == 404+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_named_shadow_full_lifecycle():+ """delta / documents / get / delete all answer on the named shadow's own+ topics — only update/accepted was covered for named shadows."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ frames: dict[str, list] = {}+ base = "$aws/things/dev9/shadow/name/cfg"++ async def _run():+ async def _collect(topic, payload, qos):+ frames.setdefault(topic, []).append(json.loads(payload))++ for filt in (f"{base}/update/+", f"{base}/get/+", f"{base}/delete/+"):+ await subscribe(account_id, filt, _collect, granted_qos=1)+ await publish(+ account_id,+ f"{base}/update",+ json.dumps(+ {"state": {"desired": {"mode": "eco"}}, "clientToken": "tok-n"}+ ).encode(),+ )+ await publish(account_id, f"{base}/get", b"")+ await publish(account_id, f"{base}/delete", b"")+ await publish(account_id, f"{base}/get", b"")++ try:+ asyncio.run(_run())+ delta = frames[f"{base}/update/delta"][0]+ assert delta["state"] == {"mode": "eco"}+ assert delta["clientToken"] == "tok-n"+ assert set(delta["metadata"]) == {"mode"}+ docs = frames[f"{base}/update/documents"][0]+ assert docs["previous"] is None+ assert docs["current"]["state"]["desired"] == {"mode": "eco"}+ assert frames[f"{base}/get/accepted"][0]["state"]["desired"] == {"mode": "eco"}+ assert frames[f"{base}/delete/accepted"][0]["version"] == 1+ assert frames[f"{base}/get/rejected"][0]["code"] == 404+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_bridge_failure_does_not_kill_the_publish(monkeypatch):+ """The bridge is the last side effect in broker_publish and the only one+ that used to run unguarded, so anything it raised propagated out and tore+ down the caller's MQTT session."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"++ async def _boom(*args, **kwargs):+ raise RecursionError("maximum recursion depth exceeded")++ monkeypatch.setattr(iot_module, "_handle_shadow_publish", _boom)+ delivered = []++ async def _run():+ async def _collect(topic, payload, qos):+ delivered.append(topic)++ await subscribe(account_id, "$aws/things/dev10/shadow/update", _collect)+ await publish(+ account_id,+ "$aws/things/dev10/shadow/update",+ json.dumps({"state": {"desired": {"led": "on"}}}).encode(),+ )++ try:+ asyncio.run(_run()) # no exception escapes+ # ...and the subscriber still got the request publish.+ assert delivered == ["$aws/things/dev10/shadow/update"]+ finally:+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_bridge_drives_topic_rules_on_request_and_accepted():+ """Rules fire on the request topic (before the bridge runs) AND on the+ bridge's own `update/accepted` publish (which recursively re-enters+ broker_publish and its now-async rule evaluation)."""+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ iot_module._topic_rules.clear()+ account_id = "123456789012"+ _put_rule(+ account_id,+ "shadow_req_rule",+ "SELECT * FROM '$aws/things/+/shadow/update'",+ [{"republish": {"topic": "seen/request", "qos": 0}}],+ )+ _put_rule(+ account_id,+ "shadow_acc_rule",+ "SELECT * FROM '$aws/things/+/shadow/update/accepted'",+ [{"republish": {"topic": "seen/accepted", "qos": 0}}],+ )+ seen: dict[str, list] = {}++ async def _run():+ async def _collect(topic, payload, qos):+ seen.setdefault(topic, []).append(json.loads(payload))++ await subscribe(account_id, "seen/#", _collect)+ await publish(+ account_id,+ "$aws/things/dev6/shadow/update",+ json.dumps({"state": {"desired": {"n": 1}}}).encode(),+ )++ try:+ asyncio.run(_run())+ assert len(seen.get("seen/request", [])) == 1+ assert seen["seen/request"][0] == {"state": {"desired": {"n": 1}}}+ accepted_events = seen.get("seen/accepted", [])+ assert len(accepted_events) == 1+ assert accepted_events[0]["version"] == 1+ assert accepted_events[0]["state"] == {"desired": {"n": 1}}+ finally:+ iot_module._topic_rules.clear()+ iot_module._shadows.clear()+ reset()+++def test_shadow_mqtt_get_and_delete_reject_invalid_json():+ """Malformed JSON rejects on every request verb, not only update.++ A `get`/`delete` payload is optional, but garbage must not be silently+ read as `{}` - as on AWS, the requester hears a 400 on its own+ `rejected` topic.+ """+ from ministack.services import iot as iot_module++ reset()+ iot_module._shadows.clear()+ account_id = "123456789012"+ rejected: list = []+ accepted: list = []++ async def _run():+ async def _collect_rejected(topic, payload, qos):+ rejected.append((topic, json.loads(payload)))++ async def _collect_accepted(topic, payload, qos):+ accepted.append(topic)++ for verb in ("get", "delete"):+ await subscribe(+ account_id, f"$aws/things/dev6/shadow/{verb}/rejected", _collect_rejected+ )+ await subscribe(+ account_id, f"$aws/things/dev6/shadow/{verb}/accepted", _collect_accepted+ )+ await publish(account_id, "$aws/things/dev6/shadow/get", b"{not json")+ await publish(account_id, "$aws/things/dev6/shadow/delete", b"\xff\xfe")++ try:+ asyncio.run(_run())+ assert [t.rsplit("/", 2)[-2] for t, _doc in rejected] == ["get", "delete"]+ assert all(doc["code"] == 400 for _t, doc in rejected)+ assert accepted == [], "malformed JSON must not reach the shadow store"+ finally:+ iot_module._shadows.clear()+ reset()diff --git a/tests/test_iot_data.py b/tests/test_iot_data.pyindex 6c32f21b..cf54a9a4 100644--- a/tests/test_iot_data.py+++ b/tests/test_iot_data.py@@ -1114,3 +1114,247 @@ def test_iot_rule_sns_action_publishes_to_topic(iot_client, iot_data_client, snsassert message == {"kind": "boom"}iot_client.delete_topic_rule(ruleName=rule)+++# ---------------------------------------------------------------------------+# Device Shadow over MQTT (reserved $aws/things/.../shadow/... topics)+# ---------------------------------------------------------------------------+++def _make_publish_frame(topic: str, payload: bytes) -> bytes:+ body = _enc_str(topic) + payload # QoS 0: no packet identifier+ return bytes([0x30]) + _enc_remaining(len(body)) + body+++def _default_ws_url() -> str:+ parsed = urlparse(ENDPOINT)+ ws_host = parsed.hostname or "localhost"+ ws_port = parsed.port or 4566+ return f"ws://prefix-ats.iot.us-east-1.{ws_host}:{ws_port}/mqtt"+++def _collect_shadow_frames(sub_filter, publish_fn, want, timeout=5.0):+ """Subscribe over WS, run ``publish_fn`` once ready, and return the+ collected ``(topic, payload)`` PUBLISH frames."""+ from conftest import patch_endpoint_dns++ ready = threading.Event()+ stop = threading.Event()+ received: list = []++ with patch_endpoint_dns():+ t = threading.Thread(+ target=lambda: asyncio.run(+ _ws_subscribe_and_collect(+ _default_ws_url(), sub_filter, ready, received, stop+ )+ ),+ daemon=True,+ )+ t.start()+ assert ready.wait(timeout=5), "WebSocket subscriber did not become ready"++ publish_fn()++ deadline = time.time() + timeout+ while time.time() < deadline and len(received) < want:+ time.sleep(0.05)+ stop.set()+ t.join(timeout=2)+ return received+++def test_shadow_update_over_mqtt_emits_accepted_delta_documents(iot_data_client):+ """HTTP `Publish` onto the shadow update topic drives the bridge: a WS+ subscriber sees accepted + delta + documents, and the HTTP data plane+ reads back the same stored state."""+ thing = _unique("shadow-thing")+ base = f"$aws/things/{thing}/shadow"++ received = _collect_shadow_frames(+ f"{base}/update/+",+ lambda: iot_data_client.publish(+ topic=f"{base}/update",+ payload=json.dumps(+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-a"}+ ).encode(),+ ),+ want=3,+ )++ frames = {topic: json.loads(payload) for topic, payload in received}+ accepted = frames[f"{base}/update/accepted"]+ assert accepted["state"] == {"desired": {"led": "on"}}+ assert accepted["clientToken"] == "tok-a"+ delta = frames[f"{base}/update/delta"]+ assert delta["state"] == {"led": "on"}+ assert delta["version"] == accepted["version"]+ # AWS includes the triggering request's clientToken on the delta, and the+ # metadata for the delta's attributes.+ assert delta["clientToken"] == "tok-a"+ assert set(delta["metadata"]) == {"led"}+ docs = frames[f"{base}/update/documents"]+ assert docs["previous"] is None+ assert docs["current"]["state"]["desired"] == {"led": "on"}++ shadow = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ assert shadow["state"]["desired"] == {"led": "on"}+ assert shadow["version"] == accepted["version"]+++def test_named_shadow_update_over_mqtt(iot_data_client):+ thing = _unique("shadow-named")+ base = f"$aws/things/{thing}/shadow/name/cfg"++ received = _collect_shadow_frames(+ f"{base}/update/+",+ lambda: iot_data_client.publish(+ topic=f"{base}/update",+ payload=json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),+ ),+ want=3,+ )++ frames = {topic: json.loads(payload) for topic, payload in received}+ accepted = frames[f"{base}/update/accepted"]+ assert accepted["state"] == {"desired": {"mode": "eco"}}++ named = _read_shadow(+ iot_data_client.get_thing_shadow(thingName=thing, shadowName="cfg")+ )+ assert named["state"]["desired"] == {"mode": "eco"}+ # Only the named shadow was written — the classic one must not exist.+ with pytest.raises(ClientError) as ei:+ iot_data_client.get_thing_shadow(thingName=thing)+ assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 404+++def test_shadow_get_missing_over_mqtt_rejected_404(iot_data_client):+ thing = _unique("shadow-miss")+ base = f"$aws/things/{thing}/shadow"++ received = _collect_shadow_frames(+ f"{base}/get/+",+ lambda: iot_data_client.publish(topic=f"{base}/get", payload=b""),+ want=1,+ )++ assert received, "expected a get/rejected frame"+ topic, payload = received[0]+ assert topic == f"{base}/get/rejected"+ doc = json.loads(payload)+ assert doc["code"] == 404+ assert thing in doc["message"]+++async def _ws_publish_shadow_and_collect(+ ws_url, sub_filter, pub_topic, pub_payload, received, want+):+ """One WS session: subscribe, publish an MQTT PUBLISH frame on the same+ connection, and collect the PUBLISH frames the broker sends back."""++ def _record(msg) -> int | None:+ buf = bytes(msg if isinstance(msg, (bytes, bytearray)) else msg.encode("latin-1"))+ parsed = _parse_packet(buf)+ if not parsed:+ return None+ ptype, _flags, body, _ = parsed+ if ptype == 3: # PUBLISH+ topic_len = struct.unpack_from("!H", body, 0)[0]+ received.append(+ (body[2:2 + topic_len].decode("utf-8"), body[2 + topic_len:])+ )+ return ptype++ async with websockets.connect(ws_url, subprotocols=["mqtt"]) as ws:+ await ws.send(_make_connect(_unique("shadow-dev")))+ await asyncio.wait_for(ws.recv(), timeout=2.0) # CONNACK+ await ws.send(_make_subscribe(packet_id=1, topic=sub_filter, qos=0))+ while True:+ msg = await asyncio.wait_for(ws.recv(), timeout=2.0)+ if _record(msg) == 9: # SUBACK+ break+ await ws.send(_make_publish_frame(pub_topic, pub_payload))+ end_at = time.time() + 5+ while time.time() < end_at and len(received) < want:+ try:+ msg = await asyncio.wait_for(ws.recv(), timeout=0.5)+ except asyncio.TimeoutError:+ continue+ _record(msg)+++def test_shadow_update_via_ws_publish_emits_accepted(iot_data_client):+ """The WS PUBLISH path (a device publishing over MQTT) triggers the+ bridge too — the same session receives its update/accepted response."""+ from conftest import patch_endpoint_dns++ thing = _unique("shadow-wspub")+ base = f"$aws/things/{thing}/shadow"+ received: list = []++ with patch_endpoint_dns():+ asyncio.run(+ _ws_publish_shadow_and_collect(+ _default_ws_url(),+ f"{base}/update/accepted",+ f"{base}/update",+ json.dumps({"state": {"reported": {"fw": "9"}}}).encode(),+ received,+ want=1,+ )+ )++ assert received, "expected an update/accepted frame over the same WS session"+ topic, payload = received[0]+ assert topic == f"{base}/update/accepted"+ doc = json.loads(payload)+ assert doc["state"] == {"reported": {"fw": "9"}}++ shadow = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ assert shadow["state"]["reported"] == {"fw": "9"}+++def test_shadow_accepted_drives_topic_rule_republish(iot_client, iot_data_client):+ """Rules-engine interplay: a rule on `$aws/things/+/shadow/update/accepted`+ fires on the bridge's own response publish (datasync-style pipelines)."""+ sink = _unique("shadow/sync")+ rule = _unique("shadowsync").replace("-", "_")+ thing = _unique("shadow-rule")+ iot_client.create_topic_rule(+ ruleName=rule,+ topicRulePayload={+ "sql": "SELECT * FROM '$aws/things/+/shadow/update/accepted'",+ "actions": [+ {+ "republish": {+ "topic": sink,+ "qos": 0,+ "roleArn": "arn:aws:iam::000000000000:role/rule",+ }+ }+ ],+ },+ )++ try:+ received = _collect_shadow_frames(+ sink,+ lambda: iot_data_client.publish(+ topic=f"$aws/things/{thing}/shadow/update",+ payload=json.dumps({"state": {"desired": {"sync": 1}}}).encode(),+ ),+ want=1,+ )++ assert received, "rule republish from update/accepted did not arrive"+ # The `+` thing wildcard means concurrently running shadow tests may+ # hit this rule too — find OUR update's projected event.+ events = [+ json.loads(payload) for topic, payload in received if topic == sink+ ]+ matching = [e for e in events if e.get("state") == {"desired": {"sync": 1}}]+ assert matching, f"expected our accepted-doc projection, got {events}"+ assert matching[0]["version"] >= 1+ finally:+ iot_client.delete_topic_rule(ruleName=rule)