Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
IoT — binary topic-rule payloads are no longer corrupted (#1253)
maximoosemined64d2cbfb2f9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 08f2d7c4..528735bd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **IoT — binary topic-rule payloads are no longer corrupted** — the rules engine built every rule event by decoding the payload as UTF-8 with `errors="replace"`, so a payload that is not valid UTF-8 reached the action with each non-ASCII byte replaced by `U+FFFD`. `encode(<expr>, 'base64')` is now supported and encodes the payload as published, so `SELECT encode(*, 'base64') AS data FROM 'telemetry'` delivers `{"data": "<base64>"}` that decodes back to the published bytes — the documented way to reach a Lambda action, which does not accept binary input. No lossy decode remains on the publish → rule → invoke path; a payload that is not valid UTF-8 and whose SELECT projects no attributes dispatches no action rather than dispatching corrupted text.- **IoT — topic-rule SQL is now evaluated** — the rules engine routed a publish to its actions but ignored the rule's `SELECT` clause, so a rule declaring `SELECT deviceId AS id FROM 'sensors/+/telemetry'` delivered the whole message instead of the projection. The SELECT clause is now parsed and projected — `*`, attribute paths, `AS` aliases, `topic()` / `topic(n)`, `timestamp()`, and literals, with unaliased items named as AWS names them and missing attributes omitted — for both delivery paths, Basic Ingest (where `topic()` reports the topic after the rule prefix) and a publish matching the `FROM` filter. A JSON payload under `SELECT *` still arrives as the parsed object.## [1.4.9] — 2026-08-01diff --git a/ministack/services/iot.py b/ministack/services/iot.pyindex bc6595d8..8c2a2483 100644--- a/ministack/services/iot.py+++ b/ministack/services/iot.py@@ -1444,7 +1444,29 @@ def _resolve_attribute(message, path: str):return value-def _eval_select_function(name: str, args: list[str], topic: str, message):+def _encode_base64(value, payload: bytes):+ if value is _MISSING:+ return _MISSING+ if isinstance(value, (bytes, bytearray)):+ raw = bytes(value)+ elif isinstance(value, str):+ raw = value.encode("utf-8")+ else:+ raw = json.dumps(value, separators=(",", ":")).encode("utf-8")+ return base64.b64encode(raw).decode("ascii")+++def _eval_select_function(name: str, args: list[str], topic: str, payload: bytes, message):+ if name == "encode":+ if len(args) != 2:+ return _MISSING+ source, encoding = args[0].strip(), args[1].strip().strip("'").lower()+ if encoding != "base64":+ return _MISSING+ # `encode(*, 'base64')` encodes the payload as published — the bytes+ # never round-trip through a text decode.+ value = payload if source == "*" else _eval_select_expr(source, topic, payload, message)+ return _encode_base64(value, payload)if name == "topic":if not args:return topic@@ -1461,7 +1483,7 @@ def _eval_select_function(name: str, args: list[str], topic: str, message):return _MISSING-def _eval_select_expr(expr: str, topic: str, message):+def _eval_select_expr(expr: str, topic: str, payload: bytes, message):expr = expr.strip()if expr == "*":return message@@ -1473,6 +1495,7 @@ def _eval_select_expr(expr: str, topic: str, message):m.group("name").lower(),_split_select_items(m.group("args")),topic,+ payload,message,)if _SELECT_ATTR_RE.match(expr):@@ -1923,11 +1946,45 @@ def _rules_for_account(account_id: str, region: str) -> list[dict]:def _rule_message(payload: bytes):- """Decode a publish payload into the message the SELECT clause reads."""+ """Decode a publish payload into the message the SELECT clause reads.++ Returns ``_MISSING`` for a payload that is not valid UTF-8. Such a payload+ has no attributes to project, but its bytes stay intact for+ ``encode(*, 'base64')``; the decode is never lossy.+ """try:- return json.loads(payload.decode("utf-8"))- except (UnicodeDecodeError, json.JSONDecodeError):- return (payload or b"").decode("utf-8", "replace")+ text = (payload or b"").decode("utf-8")+ except UnicodeDecodeError:+ return _MISSING+ try:+ return json.loads(text)+ except json.JSONDecodeError:+ return text+++def _rule_event(sql: str, topic: str, payload: bytes):+ """Project a publish payload through a rule's SELECT clause."""+ payload = payload or b""+ message = _rule_message(payload)+ items = _split_select_items(_rule_select_clause(sql)) or ["*"]++ if len(items) == 1:+ expr, alias = _split_select_alias(items[0])+ if expr == "*" and alias is None:+ return message++ event: dict = {}+ for item in items:+ expr, alias = _split_select_alias(item)+ value = _eval_select_expr(expr, topic, payload, message)+ if value is _MISSING:+ continue+ if expr == "*" and alias is None:+ if isinstance(value, dict):+ event.update(value)+ continue+ event[alias or _select_default_key(expr)] = value+ return eventdef _rule_event(sql: str, topic: str, payload: bytes):@@ -1980,6 +2037,13 @@ def _run_rule_actions(if not rule or rule.get("ruleDisabled"):returnevent = _rule_event(rule.get("sql", ""), topic, payload)+ if event is _MISSING:+ _broker_logger.warning(+ "IoT rule %s: payload is not valid UTF-8 and its SELECT clause "+ "projects no attributes — no action dispatched",+ rule.get("ruleName"),+ )+ returnfor action in rule.get("actions", []) or []:lam = action.get("lambda")if lam and lam.get("functionArn"):diff --git a/tests/test_iot.py b/tests/test_iot.pyindex cda69acc..9bb7c7e2 100644--- a/tests/test_iot.py+++ b/tests/test_iot.py@@ -5,6 +5,7 @@ Local CA), Policies, and DescribeEndpoint. The data plane (broker / WS /iot-data Publish) is covered separately in ``test_iot_data.py``."""+import base64import jsonimport timeimport uuid@@ -2370,6 +2371,11 @@ def test_puback_for_first_of_multiple_in_flight():# Rule SQL SELECT projection (white-box tests for _rule_event).# ----------------------------------------------------------------------+# Every byte value: not valid UTF-8, so it survives the rule path only if the+# payload is never text-decoded.+_BINARY_PAYLOAD = bytes(range(256))++def test_rule_event_select_star_returns_parsed_json():from ministack.services.iot import _rule_event@@ -2377,6 +2383,18 @@ def test_rule_event_select_star_returns_parsed_json():assert event == {"temp": 22}+def test_rule_event_encode_base64_round_trips_binary_payload():+ from ministack.services.iot import _rule_event++ event = _rule_event(+ "SELECT encode(*, 'base64') AS data FROM 'telemetry'",+ "telemetry",+ _BINARY_PAYLOAD,+ )+ assert list(event) == ["data"]+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD++def test_rule_event_projects_attributes_with_aliases():from ministack.services.iot import _rule_event@@ -2398,6 +2416,12 @@ def test_rule_event_omits_missing_attributes():assert event == {"deviceId": "d1"}+def test_rule_event_select_star_skips_non_utf8_payload():+ from ministack.services.iot import _MISSING, _rule_event++ assert _rule_event("SELECT * FROM 'telemetry'", "telemetry", _BINARY_PAYLOAD) is _MISSING++def test_rule_event_aliased_star_nests_the_message():from ministack.services.iot import _rule_eventdiff --git a/tests/test_iot_data.py b/tests/test_iot_data.pyindex a0fdcfd9..63285209 100644--- a/tests/test_iot_data.py+++ b/tests/test_iot_data.py@@ -11,6 +11,7 @@ bridge layer.from __future__ import annotations+import base64import io as _ioimport jsonimport os@@ -724,6 +725,56 @@ def test_iot_disabled_rule_does_not_fire(iot_client, iot_data_client, lam, sqs):iot_client.delete_topic_rule(ruleName=rule)+# Every byte value: not valid UTF-8, so it survives the rule path only if the+# payload is never text-decoded.+_BINARY_PAYLOAD = bytes(range(256))+++def test_iot_rule_encode_base64_projection_basic_ingest(+ iot_client, iot_data_client, lam, sqs+):+ sink = sqs.create_queue(QueueName=_unique("encode-sink"))["QueueUrl"]+ fn_arn = _make_sink_lambda(lam, sink)+ rule = _unique("encode").replace("-", "_")+ iot_client.create_topic_rule(+ ruleName=rule,+ topicRulePayload={+ "sql": "SELECT encode(*, 'base64') AS data FROM 'telemetry'",+ "awsIotSqlVersion": "2016-03-23",+ "actions": [{"lambda": {"functionArn": fn_arn}}],+ },+ )++ iot_data_client.publish(topic=f"$aws/rules/{rule}", payload=_BINARY_PAYLOAD)+ event = _poll_sink(sqs, sink)+ assert list(event) == ["data"]+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD++ iot_client.delete_topic_rule(ruleName=rule)+++def test_iot_rule_encode_base64_projection_topic_filter(+ iot_client, iot_data_client, lam, sqs+):+ sink = sqs.create_queue(QueueName=_unique("encode-filter-sink"))["QueueUrl"]+ fn_arn = _make_sink_lambda(lam, sink)+ rule = _unique("encfilter").replace("-", "_")+ iot_client.create_topic_rule(+ ruleName=rule,+ topicRulePayload={+ "sql": "SELECT encode(*, 'base64') AS data FROM 'telemetry'",+ "actions": [{"lambda": {"functionArn": fn_arn}}],+ },+ )++ iot_data_client.publish(topic="telemetry", payload=_BINARY_PAYLOAD)+ event = _poll_sink(sqs, sink)+ assert list(event) == ["data"]+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD++ iot_client.delete_topic_rule(ruleName=rule)++def test_iot_rule_attribute_projection(iot_client, iot_data_client, lam, sqs):sink = sqs.create_queue(QueueName=_unique("project-sink"))["QueueUrl"]fn_arn = _make_sink_lambda(lam, sink)