Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(iot): publish shadow notifications for HTTP data-plane updates (#1531)

iot-rocket9ab4fb47ee98
diff --git a/ministack/services/iot.py b/ministack/services/iot.py
index 5b319a11..a4576823 100644
--- a/ministack/services/iot.py
+++ b/ministack/services/iot.py
@@ -4524,7 +4524,7 @@ async def broker_stop() -> None:
_BASIC_INGEST_PREFIX = "$aws/rules/"
-_SHADOW_TOPIC_PREFIX = "$aws/things/"
+SHADOW_TOPIC_PREFIX = "$aws/things/"
def _rules_for_account(account_id: str, region: str) -> list[dict]:
@@ -4913,9 +4913,9 @@ def _parse_shadow_topic(topic: str) -> tuple[str, str, str | None] | None:
Anything longer — the ``accepted`` / ``rejected`` / ``delta`` /
``documents`` response suffixes — parses as ``None``, so the bridge's own
response publishes can never re-trigger it (no recursion)."""
- if not topic.startswith(_SHADOW_TOPIC_PREFIX):
+ if not topic.startswith(SHADOW_TOPIC_PREFIX):
return None
- rest = topic[len(_SHADOW_TOPIC_PREFIX):]
+ rest = topic[len(SHADOW_TOPIC_PREFIX):]
parts = rest.split("/")
if len(parts) < 3 or parts[1] != "shadow":
return None
@@ -4942,15 +4942,18 @@ def _shadow_rejected(status: int, message: str, client_token: str | None = None)
def _shadow_document(doc: dict) -> dict:
- """Strip the computed ``delta`` sections out of a shadow snapshot.
-
- `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``) by
- design, because that is what the GET response carries. The documents topic
- is not a GET: on AWS the ``previous``/``current`` documents report only
- ``desired`` and ``reported``, and the delta has its own topic. Publishing
- the snapshot verbatim invented a section real devices never see.
+ """Project a GET snapshot onto a ``documents`` member: ``state`` +
+ ``metadata`` + ``version``, nothing else.
+
+ `get_thing_shadow` injects ``state.delta`` (and its ``metadata.delta``)
+ and a top-level ``timestamp`` by design, because that is what the GET
+ response carries. The documents topic is not a GET: on AWS the
+ ``previous``/``current`` members report only ``desired`` and ``reported``
+ (the delta has its own topic) and carry no ``timestamp`` of their own —
+ the single timestamp sits on the envelope. Publishing the snapshot
+ verbatim invented members real devices never see.
"""
- stripped = dict(doc)
+ stripped = {k: doc[k] for k in ("state", "metadata", "version") if k in doc}
for section in ("state", "metadata"):
block = stripped.get(section)
if isinstance(block, dict) and "delta" in block:
@@ -4958,6 +4961,62 @@ def _shadow_document(doc: dict) -> dict:
return stripped
+async def shadow_emit_update_responses(
+ account_id: str,
+ region: str,
+ base: str,
+ doc: dict,
+ pre_status: int,
+ pre_doc: dict,
+ post_status: int,
+ post_doc: dict,
+ client_token: str | None,
+) -> None:
+ """Publish the reserved-topic responses for one ACCEPTED shadow update:
+ ``update/accepted``, ``update/delta`` (when the post-update document
+ carries one) and ``update/documents``.
+
+ Shared by the MQTT bridge and the HTTP data plane: on AWS these are
+ state-change notifications the Device Shadow service emits however the
+ update arrived, which is what lets a topic rule on ``update/accepted``
+ fire for an HTTP ``UpdateThingShadow``. The caller must hand in
+ deep-copied pre/post snapshots (the store returns live dicts)."""
+
+ async def _respond(suffix: str, payload_doc: dict) -> None:
+ await broker_publish(
+ account_id,
+ region,
+ f"{base}/{suffix}",
+ json.dumps(payload_doc).encode("utf-8"),
+ qos=1,
+ )
+
+ # The core update response already echoes clientToken when present.
+ await _respond("update/accepted", doc)
+ if post_status == 200:
+ delta = (post_doc.get("state") or {}).get("delta")
+ if delta:
+ # The delta section of the GET document, with the metadata AWS
+ # reports alongside it and the triggering request's clientToken.
+ delta_doc = {
+ "state": delta,
+ "metadata": (post_doc.get("metadata") or {}).get("delta", {}),
+ "version": post_doc["version"],
+ "timestamp": post_doc["timestamp"],
+ }
+ if client_token is not None:
+ delta_doc["clientToken"] = client_token
+ await _respond("update/delta", delta_doc)
+ documents_doc = {
+ "previous": _shadow_document(pre_doc) if pre_status == 200 else None,
+ "current": _shadow_document(post_doc) if post_status == 200 else None,
+ "timestamp": _shadow_now(),
+ }
+ if client_token is not None:
+ documents_doc["clientToken"] = client_token
+ await _respond("update/documents", documents_doc)
+
+
async def _handle_shadow_publish(
account_id: str,
region: str,
@@ -4972,7 +5031,7 @@ async def _handle_shadow_publish(
# `name is not None` rather than a truthiness test: an empty named shadow
# (`.../shadow/name//update`) must answer on the topic the requester is
# listening on, not collapse onto the classic shadow's.
- base = f"{_SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")
+ base = f"{SHADOW_TOPIC_PREFIX}{thing}/shadow" + (f"/name/{name}" if name is not None else "")
shadow_name = name or ""
async def _respond(suffix: str, doc: dict) -> None:
@@ -5026,29 +5085,9 @@ async def _handle_shadow_publish(
_shadow_rejected(status, doc.get("message", ""), client_token),
)
return
- # The core update response already echoes clientToken when present.
- await _respond("update/accepted", doc)
- if post_status == 200:
- delta = (post_doc.get("state") or {}).get("delta")
- if delta:
- # The delta section of the GET document, with the metadata AWS
- # reports alongside it and the triggering request's clientToken.
- delta_doc = {
- "state": delta,
- "metadata": (post_doc.get("metadata") or {}).get("delta", {}),
- "version": post_doc["version"],
- "timestamp": post_doc["timestamp"],
- }
- if client_token is not None:
- delta_doc["clientToken"] = client_token
- await _respond("update/delta", delta_doc)
- await _respond(
- "update/documents",
- {
- "previous": _shadow_document(pre_doc) if pre_status == 200 else None,
- "current": _shadow_document(post_doc) if post_status == 200 else None,
- "timestamp": _shadow_now(),
- },
+ await shadow_emit_update_responses(
+ account_id, region, base, doc,
+ pre_status, pre_doc, post_status, post_doc, client_token,
)
return
diff --git a/ministack/services/iot_data.py b/ministack/services/iot_data.py
index b9894a4f..bf82249b 100644
--- a/ministack/services/iot_data.py
+++ b/ministack/services/iot_data.py
@@ -14,6 +14,7 @@ which applies account-and-region-scoped topic prefixing transparently.
from __future__ import annotations
+import copy
import json
import logging
from urllib.parse import unquote
@@ -86,7 +87,7 @@ async def handle_request(
# Device Shadow: /things/{thingName}/shadow[?name={shadowName}]
if path.startswith("/things/") and path.endswith("/shadow"):
- return _shadow(method, path, body, qp)
+ return await _shadow(method, path, body, qp)
# Placeholders — return 501 so SDKs that probe these endpoints
# get a clear "not implemented" rather than a generic 404.
@@ -108,10 +109,23 @@ async def handle_request(
)
-def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:
- """Route ``GetThingShadow`` / ``UpdateThingShadow`` / ``DeleteThingShadow``."""
+async def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:
+ """Route ``GetThingShadow`` / ``UpdateThingShadow`` / ``DeleteThingShadow``.
+
+ An accepted update or delete also publishes the reserved-topic
+ notifications through the broker (``update/accepted``/``delta``/
+ ``documents``, ``delete/accepted``): on AWS those are state-change
+ messages the Device Shadow service emits however the change arrived, so a
+ topic rule on them fires for HTTP callers too. A ``get`` publishes
+ nothing — ``get/accepted`` is the MQTT request's response channel and an
+ HTTP read is answered by its own response body. HTTP failures publish no
+ ``rejected`` envelope either; the HTTP status already carries them."""
thing = unquote(path[len("/things/"):-len("/shadow")])
shadow_name = qp.get("name", "") or ""
+ base = f"{_iot_module.SHADOW_TOPIC_PREFIX}{thing}/shadow" + (
+ f"/name/{shadow_name}" if shadow_name else ""
+ )
+ account_id, region = get_account_id(), get_region()
if method == "GET":
status, doc = _iot_module.get_thing_shadow(thing, shadow_name)
@@ -120,9 +134,30 @@ def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:
request = json.loads(body) if body else {}
except json.JSONDecodeError:
return error_response_json("InvalidRequestException", "Invalid JSON", 400)
+ # Snapshot around the mutation: the store hands back live dicts, and
+ # the documents notification needs the before/after pair (same
+ # discipline as the MQTT bridge).
+ pre_status, pre_doc = _iot_module.get_thing_shadow(thing, shadow_name)
+ pre_doc = copy.deepcopy(pre_doc)
status, doc = _iot_module.update_thing_shadow(thing, shadow_name, request)
+ if status == 200:
+ post_status, post_doc = _iot_module.get_thing_shadow(thing, shadow_name)
+ post_doc = copy.deepcopy(post_doc)
+ await _iot_module.shadow_emit_update_responses(
+ account_id, region, base, doc,
+ pre_status, pre_doc, post_status, post_doc,
+ request.get("clientToken") if isinstance(request, dict) else None,
+ )
elif method == "DELETE":
status, doc = _iot_module.delete_thing_shadow(thing, shadow_name)
+ if status == 200:
+ await _iot_module.broker_publish(
+ account_id,
+ region,
+ f"{base}/delete/accepted",
+ json.dumps(doc).encode("utf-8"),
+ qos=1,
+ )
else:
return error_response_json(
"InvalidRequestException", f"Unsupported iot-data path: {method} {path}", 400
diff --git a/tests/test_iot_data.py b/tests/test_iot_data.py
index 811e024b..e9935005 100644
--- a/tests/test_iot_data.py
+++ b/tests/test_iot_data.py
@@ -1688,6 +1688,140 @@ def test_shadow_update_over_mqtt_emits_accepted_delta_documents(iot_data_client)
assert shadow["version"] == accepted["version"]
+def test_shadow_update_over_http_emits_accepted_delta_documents(iot_data_client):
+ """An HTTP `UpdateThingShadow` publishes the same state-change
+ notifications as an MQTT update: accepted + delta + documents. On AWS the
+ Device Shadow service emits these however the update arrived — it is what
+ lets a topic rule on `update/accepted` fire for HTTP callers."""
+ thing = _unique("shadow-http")
+ base = f"$aws/things/{thing}/shadow"
+
+ received = _collect_shadow_frames(
+ f"{base}/update/+",
+ lambda: iot_data_client.update_thing_shadow(
+ thingName=thing,
+ payload=json.dumps(
+ {"state": {"desired": {"led": "on"}}, "clientToken": "tok-http"}
+ ).encode(),
+ ),
+ want=3,
+ )
+
+ frames = {topic: json.loads(payload) for topic, payload in received}
+ accepted = frames[f"{base}/update/accepted"]
+ assert accepted["state"] == {"desired": {"led": "on"}}
+ delta = frames[f"{base}/update/delta"]
+ assert delta["state"] == {"led": "on"}
+ assert delta["clientToken"] == "tok-http"
+ docs = frames[f"{base}/update/documents"]
+ # The envelope carries the triggering request's clientToken and the single
+ # timestamp; previous/current are state + metadata + version only (a live
+ # probe pinned the member keys — no inner timestamp).
+ assert set(docs) == {"previous", "current", "timestamp", "clientToken"}
+ assert docs["clientToken"] == "tok-http"
+ assert docs["previous"] is None
+ assert set(docs["current"]) == {"state", "metadata", "version"}
+ assert docs["current"]["state"]["desired"] == {"led": "on"}
+
+
+def test_shadow_delete_over_http_emits_delete_accepted_but_get_nothing(iot_data_client):
+ """An HTTP `DeleteThingShadow` publishes `delete/accepted` (a state
+ change); an HTTP `GetThingShadow` publishes nothing — `get/accepted` is
+ the MQTT request's response channel and the HTTP read has its own
+ response body."""
+ thing = _unique("shadow-http-del")
+ base = f"$aws/things/{thing}/shadow"
+ iot_data_client.update_thing_shadow(
+ thingName=thing,
+ payload=json.dumps({"state": {"reported": {"v": 1}}}).encode(),
+ )
+
+ received = _collect_shadow_frames(
+ f"{base}/get/+",
+ lambda: iot_data_client.get_thing_shadow(thingName=thing),
+ want=1,
+ timeout=2.0,
+ )
+ assert received == []
+
+ received = _collect_shadow_frames(
+ f"{base}/delete/+",
+ lambda: iot_data_client.delete_thing_shadow(thingName=thing),
+ want=1,
+ )
+ frames = {topic: json.loads(payload) for topic, payload in received}
+ # AWS's delete/accepted body is abbreviated: version + timestamp only.
+ assert set(frames[f"{base}/delete/accepted"]) == {"version", "timestamp"}
+
+
+def test_named_shadow_update_over_http_emits_on_named_topics(iot_data_client):
+ """An HTTP update of a named shadow notifies on that shadow's own
+ `/name/<n>/update/...` topics, not the classic shadow's."""
+ thing = _unique("shadow-http-named")
+ base = f"$aws/things/{thing}/shadow/name/cfg"
+
+ received = _collect_shadow_frames(
+ f"{base}/update/+",
+ lambda: iot_data_client.update_thing_shadow(
+ thingName=thing,
+ shadowName="cfg",
+ payload=json.dumps({"state": {"desired": {"mode": "eco"}}}).encode(),
+ ),
+ want=3,
+ )
+
+ frames = {topic: json.loads(payload) for topic, payload in received}
+ accepted = frames[f"{base}/update/accepted"]
+ assert accepted["state"] == {"desired": {"mode": "eco"}}
+ docs = frames[f"{base}/update/documents"]
+ # No clientToken in the request → none on the documents envelope.
+ assert set(docs) == {"previous", "current", "timestamp"}
+ assert docs["current"]["state"]["desired"] == {"mode": "eco"}
+
+
+def test_shadow_update_over_http_rejected_emits_no_frames(iot_data_client):
+ """A failed HTTP update publishes nothing — no accepted trio and no
+ `rejected` envelope, the HTTP 409 already carries the failure."""
+ thing = _unique("shadow-http-conflict")
+ base = f"$aws/things/{thing}/shadow"
+ iot_data_client.update_thing_shadow(
+ thingName=thing,
+ payload=json.dumps({"state": {"reported": {"x": 1}}}).encode(),
+ )
+
+ def _stale_update():
+ with pytest.raises(ClientError) as ei:
+ iot_data_client.update_thing_shadow(
+ thingName=thing,
+ payload=json.dumps(
+ {"state": {"reported": {"x": 2}}, "version": 99}
+ ).encode(),
+ )
+ assert ei.value.response["Error"]["Code"] == "ConflictException"
+
+ received = _collect_shadow_frames(
+ f"{base}/update/+", _stale_update, want=1, timeout=2.0
+ )
+ assert received == []
+
+
+def test_shadow_delete_over_http_missing_emits_no_frames(iot_data_client):
+ """An HTTP delete of a shadow that does not exist changes no state, so
+ nothing is published — the HTTP 404 is the whole answer."""
+ thing = _unique("shadow-http-del-miss")
+ base = f"$aws/things/{thing}/shadow"
+
+ def _delete_missing():
+ with pytest.raises(ClientError) as ei:
+ iot_data_client.delete_thing_shadow(thingName=thing)
+ assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException"
+
+ received = _collect_shadow_frames(
+ f"{base}/delete/+", _delete_missing, want=1, timeout=2.0
+ )
+ assert received == []
+
+
def test_named_shadow_update_over_mqtt(iot_data_client):
thing = _unique("shadow-named")
base = f"$aws/things/{thing}/shadow/name/cfg"