Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(iot): device shadows over MQTT (#1395)

iot-rocket51a3c8c97cca
diff --git a/ministack/services/iot.py b/ministack/services/iot.py
index 13372a2d..61cf96c4 100644
--- a/ministack/services/iot.py
+++ b/ministack/services/iot.py
@@ -3281,6 +3281,7 @@ async def broker_stop() -> None:
_BASIC_INGEST_PREFIX = "$aws/rules/"
+_SHADOW_TOPIC_PREFIX = "$aws/things/"
def _rules_for_account(account_id: str, region: str) -> list[dict]:
@@ -3611,6 +3612,169 @@ async def _evaluate_topic_rules(
await _run_rule_actions(account_id, region, rule, payload, topic, client_id)
+def _parse_shadow_topic(topic: str) -> tuple[str, str, str | None] | None:
+ """``$aws/things/<thing>/shadow[/name/<n>]/<verb>`` → (thing, verb, name).
+
+ Anything longer — the ``accepted`` / ``rejected`` / ``delta`` /
+ ``documents`` response suffixes — parses as ``None``, so the bridge's own
+ response publishes can never re-trigger it (no recursion)."""
+ if not topic.startswith(_SHADOW_TOPIC_PREFIX):
+ return None
+ rest = topic[len(_SHADOW_TOPIC_PREFIX):]
+ parts = rest.split("/")
+ if len(parts) < 3 or parts[1] != "shadow":
+ return None
+ thing = parts[0]
+ tail = parts[2:]
+ name = None
+ if tail and tail[0] == "name" and len(tail) >= 2:
+ name, tail = tail[1], tail[2:]
+ if not tail:
+ return None
+ verb = tail[0]
+ # Responses (accepted/rejected/delta/documents) are ours, never inputs.
+ if len(tail) > 1 or verb not in ("update", "get", "delete"):
+ return None
+ return thing, verb, name
+
+
+def _shadow_rejected(status: int, message: str, client_token: str | None = None) -> dict:
+ """AWS's shadow error response document."""
+ doc = {"code": status, "message": message}
+ if client_token is not None:
+ doc["clientToken"] = client_token
+ return doc
+
+
+def _shadow_document(doc: dict) -> dict:
+ """Strip the computed ``delta`` sections out of a shadow snapshot.
+
+ `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``) by
+ design, because that is what the GET response carries. The documents topic
+ is not a GET: on AWS the ``previous``/``current`` documents report only
+ ``desired`` and ``reported``, and the delta has its own topic. Publishing
+ the snapshot verbatim invented a section real devices never see.
+ """
+ stripped = dict(doc)
+ for section in ("state", "metadata"):
+ block = stripped.get(section)
+ if isinstance(block, dict) and "delta" in block:
+ stripped[section] = {k: v for k, v in block.items() if k != "delta"}
+ return stripped
+
+
+async def _handle_shadow_publish(
+ account_id: str,
+ region: str,
+ thing: str,
+ verb: str,
+ name: str | None,
+ payload: bytes,
+) -> None:
+ """Shadow-over-MQTT bridge: feed a request-topic publish into the shadow
+ store and publish the AWS reserved-topic responses back through the broker
+ (``accepted`` / ``rejected``, plus ``delta`` and ``documents`` on update)."""
+ # `name is not None` rather than a truthiness test: an empty named shadow
+ # (`.../shadow/name//update`) must answer on the topic the requester is
+ # listening on, not collapse onto the classic shadow's.
+ base = f"{_SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")
+ shadow_name = name or ""
+
+ async def _respond(suffix: str, doc: dict) -> None:
+ await broker_publish(
+ account_id,
+ region,
+ f"{base}/{suffix}",
+ json.dumps(doc).encode("utf-8"),
+ qos=1,
+ )
+
+ try:
+ request = json.loads(payload) if payload else {}
+ except (ValueError, UnicodeDecodeError):
+ # All three request verbs reject malformed JSON, as on AWS — a `get`
+ # or `delete` payload is optional but must parse when present.
+ await _respond(f"{verb}/rejected", _shadow_rejected(400, "invalid json"))
+ return
+ if not isinstance(request, dict):
+ request = {}
+ client_token = request.get("clientToken")
+
+ if name is not None and not name:
+ await _respond(
+ f"{verb}/rejected",
+ _shadow_rejected(400, "Invalid shadow name: must not be empty", client_token),
+ )
+ return
+
+ if verb == "update":
+ # The broker's WS path sets only the account contextvar, so pin both
+ # around every shadow-store access.
+ with request_scope(account_id, region):
+ # get_thing_shadow hands back the stored state dicts themselves and
+ # update_thing_shadow merges into them in place, so `previous` has
+ # to be snapshotted before the update runs — otherwise it reports
+ # the state *after* it, and from the second update on `previous`
+ # and `current` are the same document. `current` is snapshotted for
+ # the same reason: a concurrent update would otherwise edit it out
+ # from under the publish below.
+ pre_status, pre_doc = get_thing_shadow(thing, shadow_name)
+ pre_doc = copy.deepcopy(pre_doc)
+ status, doc = update_thing_shadow(thing, shadow_name, request)
+ post_status, post_doc = (
+ get_thing_shadow(thing, shadow_name) if status == 200 else (0, {})
+ )
+ post_doc = copy.deepcopy(post_doc)
+ if status != 200:
+ await _respond(
+ "update/rejected",
+ _shadow_rejected(status, doc.get("message", ""), client_token),
+ )
+ return
+ # The core update response already echoes clientToken when present.
+ await _respond("update/accepted", doc)
+ if post_status == 200:
+ delta = (post_doc.get("state") or {}).get("delta")
+ if delta:
+ # The delta section of the GET document, with the metadata AWS
+ # reports alongside it and the triggering request's clientToken.
+ delta_doc = {
+ "state": delta,
+ "metadata": (post_doc.get("metadata") or {}).get("delta", {}),
+ "version": post_doc["version"],
+ "timestamp": post_doc["timestamp"],
+ }
+ if client_token is not None:
+ delta_doc["clientToken"] = client_token
+ await _respond("update/delta", delta_doc)
+ await _respond(
+ "update/documents",
+ {
+ "previous": _shadow_document(pre_doc) if pre_status == 200 else None,
+ "current": _shadow_document(post_doc) if post_status == 200 else None,
+ "timestamp": _shadow_now(),
+ },
+ )
+ return
+
+ with request_scope(account_id, region):
+ status, doc = (
+ get_thing_shadow(thing, shadow_name)
+ if verb == "get"
+ else delete_thing_shadow(thing, shadow_name)
+ )
+ if status == 200:
+ doc = copy.deepcopy(doc)
+ if client_token is not None:
+ doc["clientToken"] = client_token
+ await _respond(f"{verb}/accepted", doc)
+ else:
+ await _respond(
+ f"{verb}/rejected",
+ _shadow_rejected(status, doc.get("message", ""), client_token),
+ )
+
+
async def broker_publish(
account_id: str,
region: str,
@@ -3681,6 +3845,27 @@ async def broker_publish(
await _evaluate_topic_rules(account_id, region, topic, payload, client_id)
+ # Shadow-over-MQTT bridge — AFTER delivery and rule evaluation, so rules
+ # matching `$aws/things/+/shadow/update` fire on the request, and the
+ # recursive `accepted`/`delta`/`documents` publishes then flow back
+ # through this function to drive their own subscribers and rules. The
+ # response suffixes parse as None, so the bridge never re-triggers itself.
+ # NOTE known divergence: `_topic_matches` lets a bare `#` subscription
+ # match `$aws/...` topics, unlike real AWS where `#` excludes the
+ # reserved topic space.
+ shadow = _parse_shadow_topic(topic)
+ if shadow is not None:
+ try:
+ await _handle_shadow_publish(account_id, region, *shadow, payload)
+ except Exception:
+ # Guarded like subscriber delivery and rule dispatch above: a
+ # payload the bridge chokes on (a deeply nested document raises
+ # RecursionError out of json.loads, which the ValueError handler
+ # inside does not catch) must not tear down the MQTT session.
+ _broker_logger.warning(
+ "IoT shadow bridge failed for %r", topic, exc_info=True
+ )
+
async def broker_subscribe(
account_id: str,
diff --git a/tests/test_iot.py b/tests/test_iot.py
index a1769003..87800504 100644
--- a/tests/test_iot.py
+++ b/tests/test_iot.py
@@ -4356,3 +4356,606 @@ def test_ddb_attribute_value_mapping():
assert _ddb_attribute_value({"a": 1}) == {"S": '{"a": 1}'}
assert _ddb_attribute_value([1, 2]) == {"S": "[1, 2]"}
assert _ddb_attribute_value("x") == {"S": "x"}
+
+
+# ----------------------------------------------------------------------
+# Shadow-over-MQTT bridge (white-box).
+# ----------------------------------------------------------------------
+
+
+ ("topic", "expected"),
+ [
+ # Request topics parse.
+ ("$aws/things/dev1/shadow/get", ("dev1", "get", None)),
+ ("$aws/things/dev1/shadow/update", ("dev1", "update", None)),
+ ("$aws/things/dev1/shadow/delete", ("dev1", "delete", None)),
+ ("$aws/things/dev1/shadow/name/cfg/get", ("dev1", "get", "cfg")),
+ ("$aws/things/dev1/shadow/name/cfg/update", ("dev1", "update", "cfg")),
+ ("$aws/things/dev1/shadow/name/cfg/delete", ("dev1", "delete", "cfg")),
+ # An empty shadow name parses (so the bridge can reject it on the
+ # requester's own reply topic) rather than collapsing onto classic.
+ ("$aws/things/dev1/shadow/name//update", ("dev1", "update", "")),
+ # Response suffixes never re-trigger the bridge.
+ ("$aws/things/dev1/shadow/update/accepted", None),
+ ("$aws/things/dev1/shadow/update/rejected", None),
+ ("$aws/things/dev1/shadow/update/delta", None),
+ ("$aws/things/dev1/shadow/update/documents", None),
+ ("$aws/things/dev1/shadow/get/accepted", None),
+ ("$aws/things/dev1/shadow/get/rejected", None),
+ ("$aws/things/dev1/shadow/delete/accepted", None),
+ ("$aws/things/dev1/shadow/name/cfg/update/accepted", None),
+ ("$aws/things/dev1/shadow/name/cfg/update/delta", None),
+ ("$aws/things/dev1/shadow/name/cfg/get/rejected", None),
+ # Malformed / unrelated.
+ ("$aws/things/dev1/shadow", None),
+ ("$aws/things/dev1/shadow/name", None),
+ ("$aws/things/dev1/shadow/name/cfg", None),
+ ("$aws/things/dev1/shadow/frobnicate", None),
+ ("$aws/things/dev1/notshadow/get", None),
+ ("$aws/things", None),
+ ("$aws/rules/rule1/some/topic", None),
+ ("plain/topic", None),
+ ("", None),
+ ],
+)
+def test_parse_shadow_topic_truth_table(topic, expected):
+ from ministack.services.iot import _parse_shadow_topic
+
+ assert _parse_shadow_topic(topic) == expected
+
+
+def test_shadow_mqtt_update_publishes_accepted_delta_documents():
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(
+ account_id, "$aws/things/dev1/shadow/update/+", _collect, granted_qos=1
+ )
+ await publish(
+ account_id,
+ "$aws/things/dev1/shadow/update",
+ json.dumps(
+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-1"}
+ ).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ accepted = frames["$aws/things/dev1/shadow/update/accepted"][0]
+ assert accepted["state"] == {"desired": {"led": "on"}}
+ assert accepted["version"] == 1
+ assert accepted["clientToken"] == "tok-1"
+
+ # desired != reported → a delta document, carrying the metadata AWS
+ # reports for the delta and the triggering request's clientToken.
+ delta = frames["$aws/things/dev1/shadow/update/delta"][0]
+ assert delta["state"] == {"led": "on"}
+ assert delta["version"] == 1
+ assert "timestamp" in delta
+ assert delta["clientToken"] == "tok-1"
+ assert set(delta["metadata"]) == {"led"}
+ assert delta["metadata"]["led"]["timestamp"] == accepted["metadata"]["desired"]["led"]["timestamp"]
+
+ docs = frames["$aws/things/dev1/shadow/update/documents"][0]
+ assert docs["previous"] is None
+ assert docs["current"]["version"] == 1
+ assert docs["current"]["state"]["desired"] == {"led": "on"}
+ assert "timestamp" in docs
+
+ # The MQTT update landed in the same store the HTTP data plane reads.
+ with iot_module.request_scope(account_id, _TEST_REGION):
+ status, doc = iot_module.get_thing_shadow("dev1", "")
+ assert status == 200
+ assert doc["state"]["desired"] == {"led": "on"}
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_update_reported_in_sync_emits_no_delta():
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(
+ account_id, "$aws/things/dev2/shadow/update/+", _collect, granted_qos=1
+ )
+ await publish(
+ account_id,
+ "$aws/things/dev2/shadow/update",
+ json.dumps(
+ {"state": {"desired": {"led": "on"}, "reported": {"led": "on"}}}
+ ).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ assert len(frames["$aws/things/dev2/shadow/update/accepted"]) == 1
+ assert "$aws/things/dev2/shadow/update/delta" not in frames
+ assert len(frames["$aws/things/dev2/shadow/update/documents"]) == 1
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_update_rejections():
+ """Invalid JSON → 400 (no token); version conflict → 409 with the token."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ rejected: list = []
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ rejected.append(json.loads(payload))
+
+ await subscribe(
+ account_id, "$aws/things/dev3/shadow/update/rejected", _collect
+ )
+ await publish(
+ account_id, "$aws/things/dev3/shadow/update", b"{not json"
+ )
+ await publish(
+ account_id,
+ "$aws/things/dev3/shadow/update",
+ json.dumps({"state": {"desired": {"a": 1}}}).encode(),
+ )
+ await publish(
+ account_id,
+ "$aws/things/dev3/shadow/update",
+ json.dumps(
+ {"state": {"desired": {"a": 2}}, "version": 99, "clientToken": "t9"}
+ ).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ assert len(rejected) == 2
+ assert rejected[0]["code"] == 400
+ assert "clientToken" not in rejected[0]
+ assert rejected[1]["code"] == 409
+ assert rejected[1]["clientToken"] == "t9"
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_get_delete_roundtrip():
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ async def _run():
+ for filt in (
+ "$aws/things/dev4/shadow/get/+",
+ "$aws/things/dev4/shadow/delete/+",
+ ):
+ await subscribe(account_id, filt, _collect)
+ # get before any update → rejected 404
+ await publish(account_id, "$aws/things/dev4/shadow/get", b"")
+ await publish(
+ account_id,
+ "$aws/things/dev4/shadow/update",
+ json.dumps({"state": {"reported": {"fw": "1.2"}}}).encode(),
+ )
+ # get with a clientToken → accepted echoes it
+ await publish(
+ account_id,
+ "$aws/things/dev4/shadow/get",
+ json.dumps({"clientToken": "get-1"}).encode(),
+ )
+ await publish(account_id, "$aws/things/dev4/shadow/delete", b"")
+ # delete again → rejected 404
+ await publish(account_id, "$aws/things/dev4/shadow/delete", b"")
+
+ try:
+ asyncio.run(_run())
+ get_rejected = frames["$aws/things/dev4/shadow/get/rejected"]
+ assert get_rejected[0]["code"] == 404
+ get_accepted = frames["$aws/things/dev4/shadow/get/accepted"][0]
+ assert get_accepted["state"]["reported"] == {"fw": "1.2"}
+ assert get_accepted["clientToken"] == "get-1"
+ del_accepted = frames["$aws/things/dev4/shadow/delete/accepted"][0]
+ assert del_accepted["version"] == 1
+ del_rejected = frames["$aws/things/dev4/shadow/delete/rejected"][0]
+ assert del_rejected["code"] == 404
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_named_shadow_isolated_from_classic():
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(
+ account_id, "$aws/things/dev5/shadow/name/cfg/update/+", _collect
+ )
+ await subscribe(account_id, "$aws/things/dev5/shadow/get/+", _collect)
+ await publish(
+ account_id,
+ "$aws/things/dev5/shadow/name/cfg/update",
+ json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),
+ )
+ # The classic shadow does not exist — only the named one was written.
+ await publish(account_id, "$aws/things/dev5/shadow/get", b"")
+
+ try:
+ asyncio.run(_run())
+ accepted = frames["$aws/things/dev5/shadow/name/cfg/update/accepted"][0]
+ assert accepted["state"] == {"desired": {"mode": "eco"}}
+ assert frames["$aws/things/dev5/shadow/get/rejected"][0]["code"] == 404
+ with iot_module.request_scope(account_id, _TEST_REGION):
+ status, doc = iot_module.get_thing_shadow("dev5", "cfg")
+ assert status == 200
+ assert doc["state"]["desired"] == {"mode": "eco"}
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_documents_previous_is_the_pre_update_state():
+ """`documents.previous` must be the state before the update. The shadow
+ store hands back its live dicts and updates merge into them in place, so an
+ un-snapshotted `previous` reports the post-update state and the two
+ documents come out identical from the second update on."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(
+ account_id, "$aws/things/dev6/shadow/update/documents", _collect
+ )
+ for temp in (20, 21):
+ await publish(
+ account_id,
+ "$aws/things/dev6/shadow/update",
+ json.dumps({"state": {"reported": {"temp": temp}}}).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ docs = frames["$aws/things/dev6/shadow/update/documents"]
+ assert len(docs) == 2
+ assert docs[0]["previous"] is None
+ assert docs[0]["current"]["state"]["reported"] == {"temp": 20}
+ second = docs[1]
+ assert second["previous"]["state"]["reported"] == {"temp": 20}
+ assert second["current"]["state"]["reported"] == {"temp": 21}
+ assert second["previous"]["state"] != second["current"]["state"]
+ assert second["previous"]["version"] == 1
+ assert second["current"]["version"] == 2
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_documents_carry_no_delta_section():
+ """A `documents` state reports `desired` and `reported` only.
+
+ Both snapshots come from `get_thing_shadow`, which injects `state.delta`
+ (and `metadata.delta`) because that is what a GET answers with. Publishing
+ them verbatim put a section on the documents topic that real devices never
+ receive — the delta has its own topic, published just above this one.
+ """
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ for suffix in ("update/documents", "update/delta"):
+ await subscribe(account_id, f"$aws/things/dev7/shadow/{suffix}", _collect)
+ # A desired the reported state does not satisfy, so a delta exists on
+ # both the pre-update and the post-update document.
+ for reported in (10, 11):
+ await publish(
+ account_id,
+ "$aws/things/dev7/shadow/update",
+ json.dumps(
+ {"state": {"desired": {"temp": 30}, "reported": {"temp": reported}}}
+ ).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ docs = frames["$aws/things/dev7/shadow/update/documents"]
+ assert len(docs) == 2
+ second = docs[1]
+ for snapshot in ("previous", "current"):
+ state = second[snapshot]["state"]
+ assert set(state) == {"desired", "reported"}
+ assert "delta" not in second[snapshot].get("metadata", {})
+ assert second["previous"]["state"]["reported"] == {"temp": 10}
+ assert second["current"]["state"]["reported"] == {"temp": 11}
+ # The delta itself is still reported, on the topic that is meant for it.
+ deltas = frames["$aws/things/dev7/shadow/update/delta"]
+ assert deltas and deltas[-1]["state"] == {"temp": 30}
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_update_without_state_node_is_rejected():
+ """The most common device-side mistake: valid JSON, no `state` node."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(account_id, "$aws/things/dev7/shadow/update/+", _collect)
+ await publish(
+ account_id,
+ "$aws/things/dev7/shadow/update",
+ json.dumps({"desired": {"led": "on"}, "clientToken": "tok-9"}).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ rejected = frames["$aws/things/dev7/shadow/update/rejected"][0]
+ assert rejected["code"] == 400
+ assert "state" in rejected["message"]
+ assert rejected["clientToken"] == "tok-9"
+ assert "$aws/things/dev7/shadow/update/accepted" not in frames
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_empty_named_shadow_is_rejected_on_its_own_topic():
+ """`.../shadow/name//update` names no shadow. Treating it as the classic
+ shadow would write the wrong record and answer on a topic the requester is
+ not subscribed to, so it is rejected where the requester is listening."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(account_id, "$aws/things/dev8/shadow/name//update/+", _collect)
+ await subscribe(account_id, "$aws/things/dev8/shadow/update/+", _collect)
+ await publish(
+ account_id,
+ "$aws/things/dev8/shadow/name//update",
+ json.dumps({"state": {"desired": {"led": "on"}}, "clientToken": "t"}).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ rejected = frames["$aws/things/dev8/shadow/name//update/rejected"][0]
+ assert rejected["code"] == 400
+ assert rejected["clientToken"] == "t"
+ # Nothing landed on the classic shadow's topics or in its record.
+ assert not any(t.startswith("$aws/things/dev8/shadow/update/") for t in frames)
+ with iot_module.request_scope(account_id, _TEST_REGION):
+ assert iot_module.get_thing_shadow("dev8", "")[0] == 404
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_named_shadow_full_lifecycle():
+ """delta / documents / get / delete all answer on the named shadow's own
+ topics — only update/accepted was covered for named shadows."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ frames: dict[str, list] = {}
+ base = "$aws/things/dev9/shadow/name/cfg"
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ frames.setdefault(topic, []).append(json.loads(payload))
+
+ for filt in (f"{base}/update/+", f"{base}/get/+", f"{base}/delete/+"):
+ await subscribe(account_id, filt, _collect, granted_qos=1)
+ await publish(
+ account_id,
+ f"{base}/update",
+ json.dumps(
+ {"state": {"desired": {"mode": "eco"}}, "clientToken": "tok-n"}
+ ).encode(),
+ )
+ await publish(account_id, f"{base}/get", b"")
+ await publish(account_id, f"{base}/delete", b"")
+ await publish(account_id, f"{base}/get", b"")
+
+ try:
+ asyncio.run(_run())
+ delta = frames[f"{base}/update/delta"][0]
+ assert delta["state"] == {"mode": "eco"}
+ assert delta["clientToken"] == "tok-n"
+ assert set(delta["metadata"]) == {"mode"}
+ docs = frames[f"{base}/update/documents"][0]
+ assert docs["previous"] is None
+ assert docs["current"]["state"]["desired"] == {"mode": "eco"}
+ assert frames[f"{base}/get/accepted"][0]["state"]["desired"] == {"mode": "eco"}
+ assert frames[f"{base}/delete/accepted"][0]["version"] == 1
+ assert frames[f"{base}/get/rejected"][0]["code"] == 404
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_bridge_failure_does_not_kill_the_publish(monkeypatch):
+ """The bridge is the last side effect in broker_publish and the only one
+ that used to run unguarded, so anything it raised propagated out and tore
+ down the caller's MQTT session."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+
+ async def _boom(*args, **kwargs):
+ raise RecursionError("maximum recursion depth exceeded")
+
+ monkeypatch.setattr(iot_module, "_handle_shadow_publish", _boom)
+ delivered = []
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ delivered.append(topic)
+
+ await subscribe(account_id, "$aws/things/dev10/shadow/update", _collect)
+ await publish(
+ account_id,
+ "$aws/things/dev10/shadow/update",
+ json.dumps({"state": {"desired": {"led": "on"}}}).encode(),
+ )
+
+ try:
+ asyncio.run(_run()) # no exception escapes
+ # ...and the subscriber still got the request publish.
+ assert delivered == ["$aws/things/dev10/shadow/update"]
+ finally:
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_bridge_drives_topic_rules_on_request_and_accepted():
+ """Rules fire on the request topic (before the bridge runs) AND on the
+ bridge's own `update/accepted` publish (which recursively re-enters
+ broker_publish and its now-async rule evaluation)."""
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ iot_module._topic_rules.clear()
+ account_id = "123456789012"
+ _put_rule(
+ account_id,
+ "shadow_req_rule",
+ "SELECT * FROM '$aws/things/+/shadow/update'",
+ [{"republish": {"topic": "seen/request", "qos": 0}}],
+ )
+ _put_rule(
+ account_id,
+ "shadow_acc_rule",
+ "SELECT * FROM '$aws/things/+/shadow/update/accepted'",
+ [{"republish": {"topic": "seen/accepted", "qos": 0}}],
+ )
+ seen: dict[str, list] = {}
+
+ async def _run():
+ async def _collect(topic, payload, qos):
+ seen.setdefault(topic, []).append(json.loads(payload))
+
+ await subscribe(account_id, "seen/#", _collect)
+ await publish(
+ account_id,
+ "$aws/things/dev6/shadow/update",
+ json.dumps({"state": {"desired": {"n": 1}}}).encode(),
+ )
+
+ try:
+ asyncio.run(_run())
+ assert len(seen.get("seen/request", [])) == 1
+ assert seen["seen/request"][0] == {"state": {"desired": {"n": 1}}}
+ accepted_events = seen.get("seen/accepted", [])
+ assert len(accepted_events) == 1
+ assert accepted_events[0]["version"] == 1
+ assert accepted_events[0]["state"] == {"desired": {"n": 1}}
+ finally:
+ iot_module._topic_rules.clear()
+ iot_module._shadows.clear()
+ reset()
+
+
+def test_shadow_mqtt_get_and_delete_reject_invalid_json():
+ """Malformed JSON rejects on every request verb, not only update.
+
+ A `get`/`delete` payload is optional, but garbage must not be silently
+ read as `{}` - as on AWS, the requester hears a 400 on its own
+ `rejected` topic.
+ """
+ from ministack.services import iot as iot_module
+
+ reset()
+ iot_module._shadows.clear()
+ account_id = "123456789012"
+ rejected: list = []
+ accepted: list = []
+
+ async def _run():
+ async def _collect_rejected(topic, payload, qos):
+ rejected.append((topic, json.loads(payload)))
+
+ async def _collect_accepted(topic, payload, qos):
+ accepted.append(topic)
+
+ for verb in ("get", "delete"):
+ await subscribe(
+ account_id, f"$aws/things/dev6/shadow/{verb}/rejected", _collect_rejected
+ )
+ await subscribe(
+ account_id, f"$aws/things/dev6/shadow/{verb}/accepted", _collect_accepted
+ )
+ await publish(account_id, "$aws/things/dev6/shadow/get", b"{not json")
+ await publish(account_id, "$aws/things/dev6/shadow/delete", b"\xff\xfe")
+
+ try:
+ asyncio.run(_run())
+ assert [t.rsplit("/", 2)[-2] for t, _doc in rejected] == ["get", "delete"]
+ assert all(doc["code"] == 400 for _t, doc in rejected)
+ assert accepted == [], "malformed JSON must not reach the shadow store"
+ finally:
+ iot_module._shadows.clear()
+ reset()
diff --git a/tests/test_iot_data.py b/tests/test_iot_data.py
index 6c32f21b..cf54a9a4 100644
--- a/tests/test_iot_data.py
+++ b/tests/test_iot_data.py
@@ -1114,3 +1114,247 @@ def test_iot_rule_sns_action_publishes_to_topic(iot_client, iot_data_client, sns
assert message == {"kind": "boom"}
iot_client.delete_topic_rule(ruleName=rule)
+
+
+# ---------------------------------------------------------------------------
+# Device Shadow over MQTT (reserved $aws/things/.../shadow/... topics)
+# ---------------------------------------------------------------------------
+
+
+def _make_publish_frame(topic: str, payload: bytes) -> bytes:
+ body = _enc_str(topic) + payload # QoS 0: no packet identifier
+ return bytes([0x30]) + _enc_remaining(len(body)) + body
+
+
+def _default_ws_url() -> str:
+ parsed = urlparse(ENDPOINT)
+ ws_host = parsed.hostname or "localhost"
+ ws_port = parsed.port or 4566
+ return f"ws://prefix-ats.iot.us-east-1.{ws_host}:{ws_port}/mqtt"
+
+
+def _collect_shadow_frames(sub_filter, publish_fn, want, timeout=5.0):
+ """Subscribe over WS, run ``publish_fn`` once ready, and return the
+ collected ``(topic, payload)`` PUBLISH frames."""
+ from conftest import patch_endpoint_dns
+
+ ready = threading.Event()
+ stop = threading.Event()
+ received: list = []
+
+ with patch_endpoint_dns():
+ t = threading.Thread(
+ target=lambda: asyncio.run(
+ _ws_subscribe_and_collect(
+ _default_ws_url(), sub_filter, ready, received, stop
+ )
+ ),
+ daemon=True,
+ )
+ t.start()
+ assert ready.wait(timeout=5), "WebSocket subscriber did not become ready"
+
+ publish_fn()
+
+ deadline = time.time() + timeout
+ while time.time() < deadline and len(received) < want:
+ time.sleep(0.05)
+ stop.set()
+ t.join(timeout=2)
+ return received
+
+
+def test_shadow_update_over_mqtt_emits_accepted_delta_documents(iot_data_client):
+ """HTTP `Publish` onto the shadow update topic drives the bridge: a WS
+ subscriber sees accepted + delta + documents, and the HTTP data plane
+ reads back the same stored state."""
+ thing = _unique("shadow-thing")
+ base = f"$aws/things/{thing}/shadow"
+
+ received = _collect_shadow_frames(
+ f"{base}/update/+",
+ lambda: iot_data_client.publish(
+ topic=f"{base}/update",
+ payload=json.dumps(
+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-a"}
+ ).encode(),
+ ),
+ want=3,
+ )
+
+ frames = {topic: json.loads(payload) for topic, payload in received}
+ accepted = frames[f"{base}/update/accepted"]
+ assert accepted["state"] == {"desired": {"led": "on"}}
+ assert accepted["clientToken"] == "tok-a"
+ delta = frames[f"{base}/update/delta"]
+ assert delta["state"] == {"led": "on"}
+ assert delta["version"] == accepted["version"]
+ # AWS includes the triggering request's clientToken on the delta, and the
+ # metadata for the delta's attributes.
+ assert delta["clientToken"] == "tok-a"
+ assert set(delta["metadata"]) == {"led"}
+ docs = frames[f"{base}/update/documents"]
+ assert docs["previous"] is None
+ assert docs["current"]["state"]["desired"] == {"led": "on"}
+
+ shadow = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))
+ assert shadow["state"]["desired"] == {"led": "on"}
+ assert shadow["version"] == accepted["version"]
+
+
+def test_named_shadow_update_over_mqtt(iot_data_client):
+ thing = _unique("shadow-named")
+ base = f"$aws/things/{thing}/shadow/name/cfg"
+
+ received = _collect_shadow_frames(
+ f"{base}/update/+",
+ lambda: iot_data_client.publish(
+ topic=f"{base}/update",
+ payload=json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),
+ ),
+ want=3,
+ )
+
+ frames = {topic: json.loads(payload) for topic, payload in received}
+ accepted = frames[f"{base}/update/accepted"]
+ assert accepted["state"] == {"desired": {"mode": "eco"}}
+
+ named = _read_shadow(
+ iot_data_client.get_thing_shadow(thingName=thing, shadowName="cfg")
+ )
+ assert named["state"]["desired"] == {"mode": "eco"}
+ # Only the named shadow was written — the classic one must not exist.
+ with pytest.raises(ClientError) as ei:
+ iot_data_client.get_thing_shadow(thingName=thing)
+ assert ei.value.response["ResponseMetadata"]["HTTPStatusCode"] == 404
+
+
+def test_shadow_get_missing_over_mqtt_rejected_404(iot_data_client):
+ thing = _unique("shadow-miss")
+ base = f"$aws/things/{thing}/shadow"
+
+ received = _collect_shadow_frames(
+ f"{base}/get/+",
+ lambda: iot_data_client.publish(topic=f"{base}/get", payload=b""),
+ want=1,
+ )
+
+ assert received, "expected a get/rejected frame"
+ topic, payload = received[0]
+ assert topic == f"{base}/get/rejected"
+ doc = json.loads(payload)
+ assert doc["code"] == 404
+ assert thing in doc["message"]
+
+
+async def _ws_publish_shadow_and_collect(
+ ws_url, sub_filter, pub_topic, pub_payload, received, want
+):
+ """One WS session: subscribe, publish an MQTT PUBLISH frame on the same
+ connection, and collect the PUBLISH frames the broker sends back."""
+
+ def _record(msg) -> int | None:
+ buf = bytes(msg if isinstance(msg, (bytes, bytearray)) else msg.encode("latin-1"))
+ parsed = _parse_packet(buf)
+ if not parsed:
+ return None
+ ptype, _flags, body, _ = parsed
+ if ptype == 3: # PUBLISH
+ topic_len = struct.unpack_from("!H", body, 0)[0]
+ received.append(
+ (body[2:2 + topic_len].decode("utf-8"), body[2 + topic_len:])
+ )
+ return ptype
+
+ async with websockets.connect(ws_url, subprotocols=["mqtt"]) as ws:
+ await ws.send(_make_connect(_unique("shadow-dev")))
+ await asyncio.wait_for(ws.recv(), timeout=2.0) # CONNACK
+ await ws.send(_make_subscribe(packet_id=1, topic=sub_filter, qos=0))
+ while True:
+ msg = await asyncio.wait_for(ws.recv(), timeout=2.0)
+ if _record(msg) == 9: # SUBACK
+ break
+ await ws.send(_make_publish_frame(pub_topic, pub_payload))
+ end_at = time.time() + 5
+ while time.time() < end_at and len(received) < want:
+ try:
+ msg = await asyncio.wait_for(ws.recv(), timeout=0.5)
+ except asyncio.TimeoutError:
+ continue
+ _record(msg)
+
+
+def test_shadow_update_via_ws_publish_emits_accepted(iot_data_client):
+ """The WS PUBLISH path (a device publishing over MQTT) triggers the
+ bridge too — the same session receives its update/accepted response."""
+ from conftest import patch_endpoint_dns
+
+ thing = _unique("shadow-wspub")
+ base = f"$aws/things/{thing}/shadow"
+ received: list = []
+
+ with patch_endpoint_dns():
+ asyncio.run(
+ _ws_publish_shadow_and_collect(
+ _default_ws_url(),
+ f"{base}/update/accepted",
+ f"{base}/update",
+ json.dumps({"state": {"reported": {"fw": "9"}}}).encode(),
+ received,
+ want=1,
+ )
+ )
+
+ assert received, "expected an update/accepted frame over the same WS session"
+ topic, payload = received[0]
+ assert topic == f"{base}/update/accepted"
+ doc = json.loads(payload)
+ assert doc["state"] == {"reported": {"fw": "9"}}
+
+ shadow = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))
+ assert shadow["state"]["reported"] == {"fw": "9"}
+
+
+def test_shadow_accepted_drives_topic_rule_republish(iot_client, iot_data_client):
+ """Rules-engine interplay: a rule on `$aws/things/+/shadow/update/accepted`
+ fires on the bridge's own response publish (datasync-style pipelines)."""
+ sink = _unique("shadow/sync")
+ rule = _unique("shadowsync").replace("-", "_")
+ thing = _unique("shadow-rule")
+ iot_client.create_topic_rule(
+ ruleName=rule,
+ topicRulePayload={
+ "sql": "SELECT * FROM '$aws/things/+/shadow/update/accepted'",
+ "actions": [
+ {
+ "republish": {
+ "topic": sink,
+ "qos": 0,
+ "roleArn": "arn:aws:iam::000000000000:role/rule",
+ }
+ }
+ ],
+ },
+ )
+
+ try:
+ received = _collect_shadow_frames(
+ sink,
+ lambda: iot_data_client.publish(
+ topic=f"$aws/things/{thing}/shadow/update",
+ payload=json.dumps({"state": {"desired": {"sync": 1}}}).encode(),
+ ),
+ want=1,
+ )
+
+ assert received, "rule republish from update/accepted did not arrive"
+ # The `+` thing wildcard means concurrently running shadow tests may
+ # hit this rule too — find OUR update's projected event.
+ events = [
+ json.loads(payload) for topic, payload in received if topic == sink
+ ]
+ matching = [e for e in events if e.get("state") == {"desired": {"sync": 1}}]
+ assert matching, f"expected our accepted-doc projection, got {events}"
+ assert matching[0]["version"] >= 1
+ finally:
+ iot_client.delete_topic_rule(ruleName=rule)