Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

IoT — binary topic-rule payloads are no longer corrupted (#1253)

maximoosemined64d2cbfb2f9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08f2d7c4..528735bd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **IoT — binary topic-rule payloads are no longer corrupted** — the rules engine built every rule event by decoding the payload as UTF-8 with `errors="replace"`, so a payload that is not valid UTF-8 reached the action with each non-ASCII byte replaced by `U+FFFD`. `encode(<expr>, 'base64')` is now supported and encodes the payload as published, so `SELECT encode(*, 'base64') AS data FROM 'telemetry'` delivers `{"data": "<base64>"}` that decodes back to the published bytes — the documented way to reach a Lambda action, which does not accept binary input. No lossy decode remains on the publish → rule → invoke path; a payload that is not valid UTF-8 and whose SELECT projects no attributes dispatches no action rather than dispatching corrupted text.
- **IoT — topic-rule SQL is now evaluated** — the rules engine routed a publish to its actions but ignored the rule's `SELECT` clause, so a rule declaring `SELECT deviceId AS id FROM 'sensors/+/telemetry'` delivered the whole message instead of the projection. The SELECT clause is now parsed and projected — `*`, attribute paths, `AS` aliases, `topic()` / `topic(n)`, `timestamp()`, and literals, with unaliased items named as AWS names them and missing attributes omitted — for both delivery paths, Basic Ingest (where `topic()` reports the topic after the rule prefix) and a publish matching the `FROM` filter. A JSON payload under `SELECT *` still arrives as the parsed object.
## [1.4.9] — 2026-08-01
diff --git a/ministack/services/iot.py b/ministack/services/iot.py
index bc6595d8..8c2a2483 100644
--- a/ministack/services/iot.py
+++ b/ministack/services/iot.py
@@ -1444,7 +1444,29 @@ def _resolve_attribute(message, path: str):
return value
-def _eval_select_function(name: str, args: list[str], topic: str, message):
+def _encode_base64(value, payload: bytes):
+ if value is _MISSING:
+ return _MISSING
+ if isinstance(value, (bytes, bytearray)):
+ raw = bytes(value)
+ elif isinstance(value, str):
+ raw = value.encode("utf-8")
+ else:
+ raw = json.dumps(value, separators=(",", ":")).encode("utf-8")
+ return base64.b64encode(raw).decode("ascii")
+
+
+def _eval_select_function(name: str, args: list[str], topic: str, payload: bytes, message):
+ if name == "encode":
+ if len(args) != 2:
+ return _MISSING
+ source, encoding = args[0].strip(), args[1].strip().strip("'").lower()
+ if encoding != "base64":
+ return _MISSING
+ # `encode(*, 'base64')` encodes the payload as published — the bytes
+ # never round-trip through a text decode.
+ value = payload if source == "*" else _eval_select_expr(source, topic, payload, message)
+ return _encode_base64(value, payload)
if name == "topic":
if not args:
return topic
@@ -1461,7 +1483,7 @@ def _eval_select_function(name: str, args: list[str], topic: str, message):
return _MISSING
-def _eval_select_expr(expr: str, topic: str, message):
+def _eval_select_expr(expr: str, topic: str, payload: bytes, message):
expr = expr.strip()
if expr == "*":
return message
@@ -1473,6 +1495,7 @@ def _eval_select_expr(expr: str, topic: str, message):
m.group("name").lower(),
_split_select_items(m.group("args")),
topic,
+ payload,
message,
)
if _SELECT_ATTR_RE.match(expr):
@@ -1923,11 +1946,45 @@ def _rules_for_account(account_id: str, region: str) -> list[dict]:
def _rule_message(payload: bytes):
- """Decode a publish payload into the message the SELECT clause reads."""
+ """Decode a publish payload into the message the SELECT clause reads.
+
+ Returns ``_MISSING`` for a payload that is not valid UTF-8. Such a payload
+ has no attributes to project, but its bytes stay intact for
+ ``encode(*, 'base64')``; the decode is never lossy.
+ """
try:
- return json.loads(payload.decode("utf-8"))
- except (UnicodeDecodeError, json.JSONDecodeError):
- return (payload or b"").decode("utf-8", "replace")
+ text = (payload or b"").decode("utf-8")
+ except UnicodeDecodeError:
+ return _MISSING
+ try:
+ return json.loads(text)
+ except json.JSONDecodeError:
+ return text
+
+
+def _rule_event(sql: str, topic: str, payload: bytes):
+ """Project a publish payload through a rule's SELECT clause."""
+ payload = payload or b""
+ message = _rule_message(payload)
+ items = _split_select_items(_rule_select_clause(sql)) or ["*"]
+
+ if len(items) == 1:
+ expr, alias = _split_select_alias(items[0])
+ if expr == "*" and alias is None:
+ return message
+
+ event: dict = {}
+ for item in items:
+ expr, alias = _split_select_alias(item)
+ value = _eval_select_expr(expr, topic, payload, message)
+ if value is _MISSING:
+ continue
+ if expr == "*" and alias is None:
+ if isinstance(value, dict):
+ event.update(value)
+ continue
+ event[alias or _select_default_key(expr)] = value
+ return event
def _rule_event(sql: str, topic: str, payload: bytes):
@@ -1980,6 +2037,13 @@ def _run_rule_actions(
if not rule or rule.get("ruleDisabled"):
return
event = _rule_event(rule.get("sql", ""), topic, payload)
+ if event is _MISSING:
+ _broker_logger.warning(
+ "IoT rule %s: payload is not valid UTF-8 and its SELECT clause "
+ "projects no attributes — no action dispatched",
+ rule.get("ruleName"),
+ )
+ return
for action in rule.get("actions", []) or []:
lam = action.get("lambda")
if lam and lam.get("functionArn"):
diff --git a/tests/test_iot.py b/tests/test_iot.py
index cda69acc..9bb7c7e2 100644
--- a/tests/test_iot.py
+++ b/tests/test_iot.py
@@ -5,6 +5,7 @@ Local CA), Policies, and DescribeEndpoint. The data plane (broker / WS /
iot-data Publish) is covered separately in ``test_iot_data.py``.
"""
+import base64
import json
import time
import uuid
@@ -2370,6 +2371,11 @@ def test_puback_for_first_of_multiple_in_flight():
# Rule SQL SELECT projection (white-box tests for _rule_event).
# ----------------------------------------------------------------------
+# Every byte value: not valid UTF-8, so it survives the rule path only if the
+# payload is never text-decoded.
+_BINARY_PAYLOAD = bytes(range(256))
+
+
def test_rule_event_select_star_returns_parsed_json():
from ministack.services.iot import _rule_event
@@ -2377,6 +2383,18 @@ def test_rule_event_select_star_returns_parsed_json():
assert event == {"temp": 22}
+def test_rule_event_encode_base64_round_trips_binary_payload():
+ from ministack.services.iot import _rule_event
+
+ event = _rule_event(
+ "SELECT encode(*, 'base64') AS data FROM 'telemetry'",
+ "telemetry",
+ _BINARY_PAYLOAD,
+ )
+ assert list(event) == ["data"]
+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD
+
+
def test_rule_event_projects_attributes_with_aliases():
from ministack.services.iot import _rule_event
@@ -2398,6 +2416,12 @@ def test_rule_event_omits_missing_attributes():
assert event == {"deviceId": "d1"}
+def test_rule_event_select_star_skips_non_utf8_payload():
+ from ministack.services.iot import _MISSING, _rule_event
+
+ assert _rule_event("SELECT * FROM 'telemetry'", "telemetry", _BINARY_PAYLOAD) is _MISSING
+
+
def test_rule_event_aliased_star_nests_the_message():
from ministack.services.iot import _rule_event
diff --git a/tests/test_iot_data.py b/tests/test_iot_data.py
index a0fdcfd9..63285209 100644
--- a/tests/test_iot_data.py
+++ b/tests/test_iot_data.py
@@ -11,6 +11,7 @@ bridge layer.
from __future__ import annotations
+import base64
import io as _io
import json
import os
@@ -724,6 +725,56 @@ def test_iot_disabled_rule_does_not_fire(iot_client, iot_data_client, lam, sqs):
iot_client.delete_topic_rule(ruleName=rule)
+# Every byte value: not valid UTF-8, so it survives the rule path only if the
+# payload is never text-decoded.
+_BINARY_PAYLOAD = bytes(range(256))
+
+
+def test_iot_rule_encode_base64_projection_basic_ingest(
+ iot_client, iot_data_client, lam, sqs
+):
+ sink = sqs.create_queue(QueueName=_unique("encode-sink"))["QueueUrl"]
+ fn_arn = _make_sink_lambda(lam, sink)
+ rule = _unique("encode").replace("-", "_")
+ iot_client.create_topic_rule(
+ ruleName=rule,
+ topicRulePayload={
+ "sql": "SELECT encode(*, 'base64') AS data FROM 'telemetry'",
+ "awsIotSqlVersion": "2016-03-23",
+ "actions": [{"lambda": {"functionArn": fn_arn}}],
+ },
+ )
+
+ iot_data_client.publish(topic=f"$aws/rules/{rule}", payload=_BINARY_PAYLOAD)
+ event = _poll_sink(sqs, sink)
+ assert list(event) == ["data"]
+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD
+
+ iot_client.delete_topic_rule(ruleName=rule)
+
+
+def test_iot_rule_encode_base64_projection_topic_filter(
+ iot_client, iot_data_client, lam, sqs
+):
+ sink = sqs.create_queue(QueueName=_unique("encode-filter-sink"))["QueueUrl"]
+ fn_arn = _make_sink_lambda(lam, sink)
+ rule = _unique("encfilter").replace("-", "_")
+ iot_client.create_topic_rule(
+ ruleName=rule,
+ topicRulePayload={
+ "sql": "SELECT encode(*, 'base64') AS data FROM 'telemetry'",
+ "actions": [{"lambda": {"functionArn": fn_arn}}],
+ },
+ )
+
+ iot_data_client.publish(topic="telemetry", payload=_BINARY_PAYLOAD)
+ event = _poll_sink(sqs, sink)
+ assert list(event) == ["data"]
+ assert base64.b64decode(event["data"]) == _BINARY_PAYLOAD
+
+ iot_client.delete_topic_rule(ruleName=rule)
+
+
def test_iot_rule_attribute_projection(iot_client, iot_data_client, lam, sqs):
sink = sqs.create_queue(QueueName=_unique("project-sink"))["QueueUrl"]
fn_arn = _make_sink_lambda(lam, sink)