Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(iot): support device shadow operations on iot-data (#1081)
maximoosemined942abb6497f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fa80f472..fbaeadf1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]+### Added+- **IoT — device shadow operations** — `GetThingShadow`, `UpdateThingShadow`, and `DeleteThingShadow` on the `iot-data` endpoint (classic and named shadows via `?name=`) previously returned `InvalidRequestException: Unsupported iot-data path: .../shadow`. Shadows now store `desired`/`reported` state: updates deep-merge into the stored document (a `null` value removes the field), bump the version, and stamp per-field `metadata` timestamps; the `/accepted` response echoes only the sections sent, while `GetThingShadow` returns the full document including the computed `state.delta` (desired fields that differ from reported). A stale `version` in an update is rejected with a `409`, and reading or deleting a nonexistent shadow returns `ResourceNotFoundException`. Shadows are account-scoped and persisted with the rest of the IoT state.### Fixed- **Lambda — Node warm-worker invoke no longer fails on fd 1 writes** — `fs.writeSync` / `fs.write` on stdout bypassed the existing `process.stdout.write` redirect to stderr (#373), so sync loggers (e.g. pino with `{ sync: true }`) could emit on the JSON-line protocol channel and surface false `Runtime.HandlerError` / `Unknown error` responses even when the handler succeeded. The Node worker bootstrap now redirects fd 1 through `fs.writeSync` / `fs.write`, and the warm-worker stdout reader accepts only protocol lines whose `status` is `ok` or `error`. Reported by @kofuk (#1093).- **Cognito — `ListUsers` `status` filter matched against the wrong field** — `Filter='status = "Enabled"'`/`"Disabled"` was compared against `UserStatus` (the confirmation-state enum: `CONFIRMED`, `FORCE_CHANGE_PASSWORD`, `UNCONFIRMED`, etc.), which never equals the literal string `"Enabled"`/`"Disabled"`. As a result, filtering by `status` always returned an empty list, regardless of how many users existed or their actual enabled/disabled state. `status` now correctly reflects the account's `Enabled` boolean toggled by `AdminEnableUser`/`AdminDisableUser`, matching real AWS Cognito's `ListUsers` Filter semantics. Contributed by @jey-mfv.diff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex 0f47832b..fd75f09a 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -76,6 +76,7 @@ _thing_groups: AccountScopedDict = AccountScopedDict()_certificates: AccountScopedDict = AccountScopedDict() # certificateId -> Certificate dict_policies: AccountScopedDict = AccountScopedDict() # policyName -> Policy dict_topic_rules: AccountScopedDict = AccountScopedDict() # ruleName -> TopicRule dict+_shadows: AccountScopedDict = AccountScopedDict() # (thingName, shadowName) -> shadow dict# Local CA state — lazily generated on first use, persisted across restarts.import threading@@ -157,6 +158,7 @@ def get_state() -> dict:"certificates": copy.deepcopy(_certificates),"policies": copy.deepcopy(_policies),"topic_rules": copy.deepcopy(_topic_rules),+ "shadows": copy.deepcopy(_shadows),"ca": {"ca_cert_pem": _ca_cert_pem, "ca_key_pem": _ca_key_pem}if _ca_cert_pem and _ca_key_pemelse {},@@ -174,6 +176,7 @@ def restore_state(data: dict | None) -> None:_certificates.update(data.get("certificates", {}))_policies.update(data.get("policies", {}))_topic_rules.update(data.get("topic_rules", {}))+ _shadows.update(data.get("shadows", {}))ca_data = data.get("ca")if ca_data:cert = ca_data.get("ca_cert_pem")@@ -194,6 +197,7 @@ def reset() -> None:_certificates.clear()_policies.clear()_topic_rules.clear()+ _shadows.clear()with _CA_LOCK:_ca_cert_pem = None_ca_key_pem = None@@ -1454,6 +1458,127 @@ def lookup_certificate_by_id(cert_id: str) -> dict | None:return _certificates.get(cert_id)+# ---------------------------------------------------------------------------+# Device Shadow (consumed by iot_data.py)+# ---------------------------------------------------------------------------+++def _shadow_now() -> int:+ return int(time.time())+++def _deep_merge(base: dict, patch: dict) -> dict:+ """Merge ``patch`` into ``base`` in place. A ``null`` value removes the key."""+ for k, v in patch.items():+ if v is None:+ base.pop(k, None)+ elif isinstance(v, dict) and isinstance(base.get(k), dict):+ _deep_merge(base[k], v)+ else:+ base[k] = v+ return base+++def _build_metadata(state: dict, ts: int):+ """Mirror a state subtree, replacing each leaf with ``{"timestamp": ts}``."""+ if isinstance(state, dict):+ return {k: _build_metadata(v, ts) for k, v in state.items()}+ return {"timestamp": ts}+++def _compute_delta(desired: dict, reported: dict):+ """Fields in ``desired`` that differ from ``reported`` (recursing into dicts)."""+ delta = {}+ for k, dv in desired.items():+ rv = reported.get(k)+ if isinstance(dv, dict) and isinstance(rv, dict):+ sub = _compute_delta(dv, rv)+ if sub:+ delta[k] = sub+ elif dv != rv:+ delta[k] = dv+ return delta+++def _shadow_error(status: int, message: str) -> tuple:+ return status, {"message": message}+++def update_thing_shadow(thing_name: str, shadow_name: str, request: dict) -> tuple:+ """Merge ``request`` into the shadow. Returns (status, response_doc)."""+ if not isinstance(request, dict) or not isinstance(request.get("state"), dict):+ return _shadow_error(400, "Missing required node: state")++ req_state = request["state"]+ key = (thing_name, shadow_name)+ rec = _shadows.get(key) or {"state": {"desired": {}, "reported": {}}, "version": 0}++ expected = request.get("version")+ if expected is not None and expected != rec["version"]:+ return _shadow_error(409, "Version conflict")++ ts = _shadow_now()+ for section in ("desired", "reported"):+ if section in req_state:+ patch = req_state[section]+ if patch is None:+ rec["state"][section] = {}+ elif isinstance(patch, dict):+ _deep_merge(rec["state"].setdefault(section, {}), patch)++ rec["version"] += 1+ rec["metadata"] = {+ "desired": _build_metadata(rec["state"].get("desired", {}), ts),+ "reported": _build_metadata(rec["state"].get("reported", {}), ts),+ }+ _shadows[key] = rec++ # The /accepted response echoes only the sections present in the request.+ resp_state = {s: req_state[s] for s in ("desired", "reported") if s in req_state and req_state[s] is not None}+ resp_meta = {s: _build_metadata(req_state[s], ts) for s in resp_state}+ doc = {"state": resp_state, "metadata": resp_meta, "version": rec["version"], "timestamp": ts}+ if request.get("clientToken") is not None:+ doc["clientToken"] = request["clientToken"]+ return 200, doc+++def get_thing_shadow(thing_name: str, shadow_name: str) -> tuple:+ """Return (status, full_shadow_doc) or a 404 error doc if none exists."""+ rec = _shadows.get((thing_name, shadow_name))+ if rec is None:+ label = f"{thing_name}" if not shadow_name else f"{thing_name}/{shadow_name}"+ return _shadow_error(404, f"No shadow exists with name: {label}")++ desired = rec["state"].get("desired", {})+ reported = rec["state"].get("reported", {})+ state = {}+ if desired:+ state["desired"] = desired+ if reported:+ state["reported"] = reported+ delta = _compute_delta(desired, reported)+ if delta:+ state["delta"] = delta+ doc = {+ "state": state,+ "metadata": rec.get("metadata", {"desired": {}, "reported": {}}),+ "version": rec["version"],+ "timestamp": _shadow_now(),+ }+ return 200, doc+++def delete_thing_shadow(thing_name: str, shadow_name: str) -> tuple:+ """Delete a shadow. Returns (status, doc). 404 if it does not exist."""+ key = (thing_name, shadow_name)+ rec = _shadows.get(key)+ if rec is None:+ label = f"{thing_name}" if not shadow_name else f"{thing_name}/{shadow_name}"+ return _shadow_error(404, f"No shadow exists with name: {label}")+ _shadows.pop(key, None)+ return 200, {"version": rec["version"], "timestamp": _shadow_now()}++# ===========================================================================# MQTT Broker — embedded MQTT 3.1.1 broker logic over WebSocket# ===========================================================================diff --git a/ministack/services/iot_data.py b/ministack/services/iot_data.pyindex af6bce95..f97c3d64 100644--- a/ministack/services/iot_data.py+++ b/ministack/services/iot_data.py@@ -83,6 +83,10 @@ async def handle_request(if method == "POST" and path.startswith("/topics/"):return await _publish(path[len("/topics/"):], body, qp)+ # Device Shadow: /things/{thingName}/shadow[?name={shadowName}]+ if path.startswith("/things/") and path.endswith("/shadow"):+ return _shadow(method, path, body, qp)+# Placeholders — return 501 so SDKs that probe these endpoints# get a clear "not implemented" rather than a generic 404.if path == "/retainedMessage":@@ -103,6 +107,35 @@ async def handle_request()+def _shadow(method: str, path: str, body: bytes, qp: dict) -> tuple:+ """Route ``GetThingShadow`` / ``UpdateThingShadow`` / ``DeleteThingShadow``."""+ thing = unquote(path[len("/things/"):-len("/shadow")])+ shadow_name = qp.get("name", "") or ""++ if method == "GET":+ status, doc = _iot_module.get_thing_shadow(thing, shadow_name)+ elif method == "POST":+ try:+ request = json.loads(body) if body else {}+ except json.JSONDecodeError:+ return error_response_json("InvalidRequestException", "Invalid JSON", 400)+ status, doc = _iot_module.update_thing_shadow(thing, shadow_name, request)+ elif method == "DELETE":+ status, doc = _iot_module.delete_thing_shadow(thing, shadow_name)+ else:+ return error_response_json(+ "InvalidRequestException", f"Unsupported iot-data path: {method} {path}", 400+ )++ if status == 404:+ return error_response_json("ResourceNotFoundException", doc["message"], 404)+ if status == 409:+ return error_response_json("ConflictException", doc["message"], 409)+ if status >= 400:+ return error_response_json("InvalidRequestException", doc["message"], status)+ return json_response(doc)++async def _publish(raw_topic: str, body: bytes, qp: dict) -> tuple:"""``POST /topics/{topic}`` — forward body to the broker.diff --git a/tests/test_iot_data.py b/tests/test_iot_data.pyindex 02f4d48e..5ec40294 100644--- a/tests/test_iot_data.py+++ b/tests/test_iot_data.py@@ -363,6 +363,107 @@ def test_iot_ws_same_account_publish_delivers(iot_data_client):# ---------------------------------------------------------------------------+# Device Shadow (GetThingShadow / UpdateThingShadow / DeleteThingShadow)+# ---------------------------------------------------------------------------+++def _read_shadow(resp) -> dict:+ return json.loads(resp["payload"].read())+++def test_get_thing_shadow_missing_raises_not_found(iot_data_client):+ with pytest.raises(ClientError) as ei:+ iot_data_client.get_thing_shadow(thingName=_unique("nothing"))+ assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException"+++def test_update_thing_shadow_reported_and_read_back(iot_data_client):+ thing = _unique("dev")+ resp = iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps({"state": {"reported": {"temp": 22, "missedReadings": 3}}}).encode(),+ )+ accepted = _read_shadow(resp)+ # The /accepted response echoes only the reported section it received.+ assert accepted["state"] == {"reported": {"temp": 22, "missedReadings": 3}}+ assert accepted["version"] == 1+ assert "reported" in accepted["metadata"]++ got = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ assert got["state"]["reported"] == {"temp": 22, "missedReadings": 3}+ assert got["version"] == 1+++def test_update_thing_shadow_merges_and_computes_delta(iot_data_client):+ thing = _unique("dev")+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"reported": {"temp": 22}}}).encode()+ )+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"desired": {"temp": 25}}}).encode()+ )+ got = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ assert got["state"]["desired"] == {"temp": 25}+ assert got["state"]["reported"] == {"temp": 22}+ # delta = desired fields differing from reported.+ assert got["state"]["delta"] == {"temp": 25}+ assert got["version"] == 2+++def test_update_thing_shadow_null_removes_field(iot_data_client):+ thing = _unique("dev")+ iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps({"state": {"reported": {"a": 1, "b": 2}}}).encode(),+ )+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"reported": {"b": None}}}).encode()+ )+ got = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ assert got["state"]["reported"] == {"a": 1}+++def test_named_shadow_is_isolated_from_classic(iot_data_client):+ thing = _unique("dev")+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"reported": {"classic": True}}}).encode()+ )+ iot_data_client.update_thing_shadow(+ thingName=thing, shadowName="cfg",+ payload=json.dumps({"state": {"reported": {"named": True}}}).encode(),+ )+ classic = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing))+ named = _read_shadow(iot_data_client.get_thing_shadow(thingName=thing, shadowName="cfg"))+ assert classic["state"]["reported"] == {"classic": True}+ assert named["state"]["reported"] == {"named": True}+++def test_delete_thing_shadow(iot_data_client):+ thing = _unique("dev")+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"reported": {"x": 1}}}).encode()+ )+ iot_data_client.delete_thing_shadow(thingName=thing)+ with pytest.raises(ClientError) as ei:+ iot_data_client.get_thing_shadow(thingName=thing)+ assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException"+++def test_update_thing_shadow_version_conflict(iot_data_client):+ thing = _unique("dev")+ iot_data_client.update_thing_shadow(+ thingName=thing, payload=json.dumps({"state": {"reported": {"x": 1}}}).encode()+ )+ # Stale version is rejected.+ with pytest.raises(ClientError) as ei:+ iot_data_client.update_thing_shadow(+ thingName=thing,+ payload=json.dumps({"state": {"reported": {"x": 2}}, "version": 99}).encode(),+ )+ assert ei.value.response["Error"]["Code"] == "ConflictException"+++# ---------------------------------------------------------------------------# Topic-rule routing (publish → rule → Lambda)# ---------------------------------------------------------------------------