Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(lambda): pace per-ESM invoke retries instead of throttling the whole poll loop (#1090)

Adam Woodsfaa9892c8da5
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index f69e347f..abdc8b2b 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -5478,6 +5478,12 @@ _kinesis_positions = AccountRegionScopedDict()
# Per-ESM DynamoDB stream tracking: esm_uuid -> {shard_id: position}
_dynamodb_stream_positions = AccountRegionScopedDict()
_dynamodb_stream_positions_lock = threading.Lock()
+# Per-ESM invoke-failure cooldown: esm_uuid -> monotonic time before which the
+# poller skips this ESM. Keeps a broken ESM's retries paced even while other
+# ESMs are busy and keep _poll_loop from sleeping between passes. Not
+# persisted — a warm boot simply starts with no cooldowns in effect.
+_esm_backoff_until = AccountRegionScopedDict()
+_ESM_BACKOFF_SECONDS = 1.0
def _init_stream_position(esm_id, source_arn, starting):
@@ -5513,19 +5519,24 @@ def _ensure_poller():
def _poll_loop():
"""Background thread: polls SQS/Kinesis/DynamoDB for active ESMs and invokes Lambda."""
while True:
+ processed = False
try:
- _poll_sqs()
+ processed = _poll_sqs() or processed
except Exception as e:
logger.error("ESM SQS poller error: %s", e)
try:
- _poll_kinesis()
+ processed = _poll_kinesis() or processed
except Exception as e:
logger.error("ESM Kinesis poller error: %s", e)
try:
- _poll_dynamodb_streams()
+ processed = _poll_dynamodb_streams() or processed
except Exception as e:
logger.error("ESM DynamoDB streams poller error: %s", e)
- time.sleep(1 if _esms.has_any() else 5)
+ # A pass that found work likely left more behind it - loop again
+ # immediately rather than waiting out the idle cadence below, so
+ # throughput isn't throttled to batch_size-per-tick.
+ if not processed:
+ time.sleep(1 if _esms.has_any() else 5)
def _iter_all_esms():
@@ -5552,8 +5563,12 @@ def _sqs_message_attributes_to_camel_case(attrs: dict) -> dict:
def _poll_sqs():
+ """Returns True if any ESM advanced past a batch this pass (successfully
+ invoked, or filtered out entirely)."""
from ministack.services import sqs as _sqs
+ processed_any = False
+
for acct_id, region, esm in _iter_all_esms():
account_token = _request_account_id.set(acct_id)
region_token = _request_region.set(region)
@@ -5586,6 +5601,10 @@ def _poll_sqs():
if not queue or queue.get("attributes", {}).get("QueueArn") != source_arn:
continue
+ esm_id = esm["UUID"]
+ if _esm_backoff_until.get(esm_id, 0) > time.time():
+ continue
+
batch_size = esm.get("BatchSize", 10)
now = time.time()
@@ -5623,6 +5642,7 @@ def _poll_sqs():
# All records filtered out — treat the batch as processed.
for msg in batch:
queue["messages"].remove(msg)
+ processed_any = True
continue
event = {"Records": records}
@@ -5637,7 +5657,12 @@ def _poll_sqs():
"ESM: Lambda %s failed processing SQS batch from %s (errorType=%s errorMessage=%s)\n%s",
func_name, queue_name, err_type, err_msg, result.get("log", ""),
)
+ # Failed messages stay invisible for their visibility timeout
+ # rather than advancing, so don't report this as processed.
+ _esm_backoff_until[esm_id] = time.time() + _ESM_BACKOFF_SECONDS
else:
+ processed_any = True
+ _esm_backoff_until.pop(esm_id, None)
# Check for ReportBatchItemFailures — partial batch response
failed_ids = set()
if "ReportBatchItemFailures" in esm.get("FunctionResponseTypes", []):
@@ -5678,10 +5703,16 @@ def _poll_sqs():
_request_account_id.reset(account_token)
_request_region.reset(region_token)
+ return processed_any
+
def _poll_kinesis():
+ """Returns True if any shard advanced past a batch this pass (successfully
+ invoked, or filtered out entirely)."""
from ministack.services import kinesis as _kin
+ processed_any = False
+
for acct_id, region, esm in _iter_all_esms():
account_token = _request_account_id.set(acct_id)
region_token = _request_region.set(region)
@@ -5714,6 +5745,9 @@ def _poll_kinesis():
continue
esm_id = esm["UUID"]
+ if _esm_backoff_until.get(esm_id, 0) > time.time():
+ continue
+
if esm_id not in _kinesis_positions:
starting = esm.get("StartingPosition", "LATEST")
_kinesis_positions[esm_id] = {}
@@ -5774,6 +5808,7 @@ def _poll_kinesis():
records = _apply_filter_criteria(records, esm)
if not records:
_kinesis_positions[esm_id][shard_id] = pos + len(raw_records)
+ processed_any = True
continue
event = {"Records": records}
@@ -5788,7 +5823,13 @@ def _poll_kinesis():
"ESM: Lambda %s failed processing Kinesis batch from %s/%s (errorType=%s errorMessage=%s)\n%s",
func_name, stream_name, shard_id, err_type, err_msg, result.get("log", ""),
)
+ # Position doesn't advance on failure, so the next pass
+ # would refetch this exact batch — don't report it as
+ # processed.
+ _esm_backoff_until[esm_id] = time.time() + _ESM_BACKOFF_SECONDS
else:
+ processed_any = True
+ _esm_backoff_until.pop(esm_id, None)
positions[shard_id] = pos + len(raw_records)
esm["LastProcessingResult"] = f"OK - {len(raw_records)} records"
log_output = result.get("log", "")
@@ -5802,13 +5843,19 @@ def _poll_kinesis():
_request_account_id.reset(account_token)
_request_region.reset(region_token)
+ return processed_any
+
def _poll_dynamodb_streams():
+ """Returns True if any table advanced past a batch this pass (successfully
+ invoked, or filtered out entirely)."""
from ministack.services import dynamodb as _ddb
stream_records = getattr(_ddb, "_stream_records", None)
if stream_records is None:
- return
+ return False
+
+ processed_any = False
for acct_id, region, esm in _iter_all_esms():
account_token = _request_account_id.set(acct_id)
@@ -5846,6 +5893,9 @@ def _poll_dynamodb_streams():
continue
esm_id = esm["UUID"]
+ if _esm_backoff_until.get(esm_id, 0) > time.time():
+ continue
+
with _dynamodb_stream_positions_lock:
if esm_id not in _dynamodb_stream_positions:
starting = esm.get("StartingPosition", "LATEST")
@@ -5866,6 +5916,7 @@ def _poll_dynamodb_streams():
# All records filtered — advance position so we don't re-evaluate.
with _dynamodb_stream_positions_lock:
_dynamodb_stream_positions[esm_id] = pos + raw_len
+ processed_any = True
continue
event = {"Records": batch}
@@ -5880,7 +5931,13 @@ def _poll_dynamodb_streams():
"ESM: Lambda %s failed processing DynamoDB stream batch from %s (errorType=%s errorMessage=%s)\n%s",
func_name, table_name, err_type, err_msg, result.get("log", ""),
)
+ # Position doesn't advance on failure, so the next pass
+ # would refetch this exact batch — don't report it as
+ # processed.
+ _esm_backoff_until[esm_id] = time.time() + _ESM_BACKOFF_SECONDS
else:
+ processed_any = True
+ _esm_backoff_until.pop(esm_id, None)
with _dynamodb_stream_positions_lock:
_dynamodb_stream_positions[esm_id] = pos + raw_len
esm["LastProcessingResult"] = f"OK - {len(batch)} records"
@@ -5895,6 +5952,8 @@ def _poll_dynamodb_streams():
_request_account_id.reset(account_token)
_request_region.reset(region_token)
+ return processed_any
+
# ---------------------------------------------------------------------------
# Function URL Config
@@ -5997,6 +6056,7 @@ def reset():
_function_urls.clear()
_kinesis_positions.clear()
_dynamodb_stream_positions.clear()
+ _esm_backoff_until.clear()
_pool_clear_all()
lambda_runtime.reset()
with _provided_code_lock:
diff --git a/tests/test_lambda.py b/tests/test_lambda.py
index 8878e4f7..9acb38f1 100644
--- a/tests/test_lambda.py
+++ b/tests/test_lambda.py
@@ -4559,6 +4559,551 @@ def test_lambda_kinesis_poller_does_not_tail_match_invalid_event_source_arn(
set_request_region(original_region)
+def esm_poll_state(tmp_path, monkeypatch):
+ """Snapshot Lambda/SQS/Kinesis/DynamoDB state via each module's own
+ get_state()/restore_state() (the same pair ``lambda_svc_isolated`` and
+ the real persistence path use), hand back the emptied modules for a
+ test to populate, then restore on teardown.
+
+ Like ``lambda_svc_isolated``, redirects ``CODE_BLOB_DIR`` to ``tmp_path``
+ before calling ``lsvc.get_state()`` — otherwise get_state()'s blob
+ externalization/orphan-pruning would touch the real on-disk
+ CODE_BLOB_DIR/STATE_DIR as a side effect of an unrelated ESM-poller test.
+
+ dynamodb's ``_stream_records`` isn't part of its get_state()/
+ restore_state() contract (stream backlogs aren't persisted), so it's
+ snapshotted/restored by hand alongside the rest.
+ """
+ from ministack.services import dynamodb as _ddb
+ from ministack.services import kinesis as _kin
+ from ministack.services import sqs as _sqs
+
+ monkeypatch.setattr(lsvc, "CODE_BLOB_DIR", str(tmp_path / "lambda-blobs"))
+ lambda_state = lsvc.get_state()
+ sqs_state = _sqs.get_state()
+ kinesis_state = _kin.get_state()
+ dynamodb_state = _ddb.get_state()
+ stream_records = dict(_ddb._stream_records._data)
+
+ def _clear_all():
+ lsvc._functions._data.clear()
+ lsvc._esms._data.clear()
+ lsvc._kinesis_positions._data.clear()
+ lsvc._dynamodb_stream_positions._data.clear()
+ lsvc._esm_backoff_until._data.clear()
+ _sqs._queues._data.clear()
+ _kin._streams._data.clear()
+ _ddb._tables._data.clear()
+ _ddb._stream_records._data.clear()
+
+ _clear_all()
+ try:
+ yield lsvc, _sqs, _kin, _ddb
+ finally:
+ _clear_all()
+ lsvc.restore_state(lambda_state)
+ _sqs.restore_state(sqs_state)
+ _kin.restore_state(kinesis_state)
+ _ddb.restore_state(dynamodb_state)
+ _ddb._stream_records._data.update(stream_records)
+
+
+def test_poll_sqs_returns_true_when_batch_processed(esm_poll_state, monkeypatch):
+ """_poll_loop uses this return value to skip its idle sleep and keep
+ draining a burst immediately instead of throttling to one batch/tick."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ queue_name = "esm-drain-signal"
+ queue_url = f"http://localhost:4566/000000000000/{queue_name}"
+ _sqs._queues[queue_url] = {
+ "name": queue_name,
+ "messages": [{
+ "id": "msg-1",
+ "body": "payload",
+ "md5_body": "",
+ "receipt_handle": "rh-1",
+ "sent_at": time.time(),
+ "visible_at": 0,
+ "receive_count": 0,
+ "first_receive_at": None,
+ "message_attributes": {},
+ }],
+ "attributes": {"QueueArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}"},
+ "is_fifo": False,
+ "dedup_cache": {},
+ "fifo_seq": 0,
+ }
+ _lsvc._functions["esm-drain-signal-fn"] = {
+ "config": {
+ "FunctionName": "esm-drain-signal-fn",
+ "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:esm-drain-signal-fn",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _lsvc._esms["esm-drain-signal"] = {
+ "UUID": "esm-drain-signal",
+ "EventSourceArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}",
+ "FunctionName": "esm-drain-signal-fn",
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ }
+ monkeypatch.setattr(_lsvc, "_execute_function", lambda _func, _event: {"error": False, "body": {}})
+
+ assert _lsvc._poll_sqs() is True
+
+
+def test_poll_kinesis_returns_true_when_batch_processed(esm_poll_state, monkeypatch):
+ """_poll_loop uses this return value to skip its idle sleep and keep
+ draining a burst immediately instead of throttling to one batch/tick."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ stream_name = "esm-kinesis-drain-signal"
+ stream_arn = f"arn:aws:kinesis:us-east-1:000000000000:stream/{stream_name}"
+ function_name = "esm-kinesis-drain-signal-fn"
+
+ _lsvc._functions[function_name] = {
+ "config": {
+ "FunctionName": function_name,
+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{function_name}",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _kin._streams[stream_name] = _kinesis_stream_record(stream_name, stream_arn)
+ _lsvc._esms["esm-kinesis-drain-signal"] = {
+ "UUID": "esm-kinesis-drain-signal",
+ "EventSourceArn": stream_arn,
+ "FunctionName": function_name,
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ monkeypatch.setattr(_lsvc, "_execute_function", lambda _func, _event: {"error": False, "body": {}})
+
+ assert _lsvc._poll_kinesis() is True
+
+
+def test_poll_dynamodb_streams_returns_true_when_batch_processed(esm_poll_state, monkeypatch):
+ """_poll_loop uses this return value to skip its idle sleep and keep
+ draining a burst immediately instead of throttling to one batch/tick."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ table_name = "esm-ddb-drain-signal"
+ stream_arn = f"arn:aws:dynamodb:us-east-1:000000000000:table/{table_name}/stream/2024-01-01T00:00:00.000"
+ function_name = "esm-ddb-drain-signal-fn"
+
+ _lsvc._functions[function_name] = {
+ "config": {
+ "FunctionName": function_name,
+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{function_name}",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _ddb._tables[table_name] = {"LatestStreamArn": stream_arn}
+ _ddb._stream_records[table_name] = [{
+ "eventID": "1",
+ "eventName": "INSERT",
+ "eventSource": "aws:dynamodb",
+ "dynamodb": {
+ "Keys": {},
+ "SequenceNumber": "1",
+ "SizeBytes": 1,
+ "StreamViewType": "NEW_AND_OLD_IMAGES",
+ },
+ "eventSourceARN": stream_arn,
+ }]
+ _lsvc._esms["esm-ddb-drain-signal"] = {
+ "UUID": "esm-ddb-drain-signal",
+ "EventSourceArn": stream_arn,
+ "FunctionName": function_name,
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ monkeypatch.setattr(_lsvc, "_execute_function", lambda _func, _event: {"error": False, "body": {}})
+
+ assert _lsvc._poll_dynamodb_streams() is True
+
+
+def test_poll_sqs_returns_false_when_invoke_fails(esm_poll_state, monkeypatch):
+ """A failed invoke leaves the message undeleted (just invisible for its
+ visibility timeout) rather than advancing — _poll_loop must not skip its
+ idle sleep for a pass that made no real progress."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ queue_name = "esm-drain-signal-failure"
+ queue_url = f"http://localhost:4566/000000000000/{queue_name}"
+ _sqs._queues[queue_url] = {
+ "name": queue_name,
+ "messages": [{
+ "id": "msg-1",
+ "body": "payload",
+ "md5_body": "",
+ "receipt_handle": "rh-1",
+ "sent_at": time.time(),
+ "visible_at": 0,
+ "receive_count": 0,
+ "first_receive_at": None,
+ "message_attributes": {},
+ }],
+ "attributes": {"QueueArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}"},
+ "is_fifo": False,
+ "dedup_cache": {},
+ "fifo_seq": 0,
+ }
+ _lsvc._functions["esm-drain-signal-failure-fn"] = {
+ "config": {
+ "FunctionName": "esm-drain-signal-failure-fn",
+ "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:esm-drain-signal-failure-fn",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _lsvc._esms["esm-drain-signal-failure"] = {
+ "UUID": "esm-drain-signal-failure",
+ "EventSourceArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}",
+ "FunctionName": "esm-drain-signal-failure-fn",
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ }
+ monkeypatch.setattr(
+ _lsvc, "_execute_function",
+ lambda _func, _event: {"error": True, "body": {"errorType": "Error", "errorMessage": "boom"}},
+ )
+
+ assert _lsvc._poll_sqs() is False
+ assert len(_sqs._queues[queue_url]["messages"]) == 1
+
+
+def test_poll_sqs_backs_off_failing_esm_without_starving_other_esms(esm_poll_state, monkeypatch):
+ """A broken ESM must not be retried at full loop speed just because some
+ other healthy ESM keeps _poll_loop from sleeping — each ESM paces its own
+ retries independently via a per-ESM backoff, not a single loop-wide flag."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ def make_queue(name):
+ queue_url = f"http://localhost:4566/000000000000/{name}"
+ _sqs._queues[queue_url] = {
+ "name": name,
+ "messages": [{
+ "id": "msg-1",
+ "body": "payload",
+ "md5_body": "",
+ "receipt_handle": "rh-1",
+ "sent_at": time.time(),
+ "visible_at": 0,
+ "receive_count": 0,
+ "first_receive_at": None,
+ "message_attributes": {},
+ }],
+ "attributes": {"QueueArn": f"arn:aws:sqs:us-east-1:000000000000:{name}"},
+ "is_fifo": False,
+ "dedup_cache": {},
+ "fifo_seq": 0,
+ }
+ return queue_url
+
+ healthy_queue_url = make_queue("esm-healthy")
+ broken_queue_url = make_queue("esm-broken")
+
+ _lsvc._functions["esm-healthy-fn"] = {
+ "config": {
+ "FunctionName": "esm-healthy-fn",
+ "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:esm-healthy-fn",
+ },
+ "versions": {}, "aliases": {},
+ }
+ _lsvc._functions["esm-broken-fn"] = {
+ "config": {
+ "FunctionName": "esm-broken-fn",
+ "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:esm-broken-fn",
+ },
+ "versions": {}, "aliases": {},
+ }
+ _lsvc._esms["esm-healthy"] = {
+ "UUID": "esm-healthy",
+ "EventSourceArn": "arn:aws:sqs:us-east-1:000000000000:esm-healthy",
+ "FunctionName": "esm-healthy-fn",
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ }
+ _lsvc._esms["esm-broken"] = {
+ "UUID": "esm-broken",
+ "EventSourceArn": "arn:aws:sqs:us-east-1:000000000000:esm-broken",
+ "FunctionName": "esm-broken-fn",
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ }
+
+ invoke_calls = []
+
+ def fake_execute(func, _event):
+ func_name = func["config"]["FunctionName"]
+ invoke_calls.append(func_name)
+ if func_name == "esm-broken-fn":
+ return {"error": True, "body": {"errorType": "Error", "errorMessage": "boom"}}
+ # Refill the healthy queue so every pass keeps finding work, mirroring
+ # the sustained-traffic scenario that starves _poll_loop's idle sleep.
+ _sqs._queues[healthy_queue_url]["messages"].append({
+ "id": f"msg-{len(invoke_calls)}",
+ "body": "payload",
+ "md5_body": "",
+ "receipt_handle": f"rh-{len(invoke_calls)}",
+ "sent_at": time.time(),
+ "visible_at": 0,
+ "receive_count": 0,
+ "first_receive_at": None,
+ "message_attributes": {},
+ })
+ return {"error": False, "body": {}}
+
+ monkeypatch.setattr(_lsvc, "_execute_function", fake_execute)
+
+ assert _lsvc._poll_sqs() is True
+ assert invoke_calls.count("esm-broken-fn") == 1
+
+ # Second pass: the healthy ESM keeps reporting True (so _poll_loop would
+ # never sleep), but the broken ESM must still be skipped — it's within
+ # its own backoff window regardless of what the healthy ESM is doing.
+ assert _lsvc._poll_sqs() is True
+ assert invoke_calls.count("esm-broken-fn") == 1
+ assert len(_sqs._queues[broken_queue_url]["messages"]) == 1
+
+
+def test_poll_sqs_retries_esm_after_backoff_expires(esm_poll_state, monkeypatch):
+ """Once the cooldown elapses, a previously-failing ESM is retried again —
+ the backoff paces retries, it doesn't disable the ESM."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ queue_name = "esm-drain-signal-recovers"
+ queue_url = f"http://localhost:4566/000000000000/{queue_name}"
+ _sqs._queues[queue_url] = {
+ "name": queue_name,
+ "messages": [{
+ "id": "msg-1",
+ "body": "payload",
+ "md5_body": "",
+ "receipt_handle": "rh-1",
+ "sent_at": time.time(),
+ "visible_at": 0,
+ "receive_count": 0,
+ "first_receive_at": None,
+ "message_attributes": {},
+ }],
+ # VisibilityTimeout=0 so the message is immediately re-receivable —
+ # isolates the assertions below to the backoff mechanism itself
+ # rather than real-world SQS visibility timing.
+ "attributes": {
+ "QueueArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}",
+ "VisibilityTimeout": "0",
+ },
+ "is_fifo": False,
+ "dedup_cache": {},
+ "fifo_seq": 0,
+ }
+ _lsvc._functions["esm-drain-signal-recovers-fn"] = {
+ "config": {
+ "FunctionName": "esm-drain-signal-recovers-fn",
+ "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:esm-drain-signal-recovers-fn",
+ },
+ "versions": {}, "aliases": {},
+ }
+ _lsvc._esms["esm-drain-signal-recovers"] = {
+ "UUID": "esm-drain-signal-recovers",
+ "EventSourceArn": f"arn:aws:sqs:us-east-1:000000000000:{queue_name}",
+ "FunctionName": "esm-drain-signal-recovers-fn",
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ }
+ invoke_calls = []
+ monkeypatch.setattr(
+ _lsvc, "_execute_function",
+ lambda _func, _event: (invoke_calls.append(1), {"error": True, "body": {"errorType": "Error", "errorMessage": "boom"}})[1],
+ )
+
+ fake_now = [1_000_000.0]
+ monkeypatch.setattr(_lsvc.time, "time", lambda: fake_now[0])
+
+ assert _lsvc._poll_sqs() is False
+ assert len(invoke_calls) == 1
+
+ # Still within the backoff window — skipped before it would even receive.
+ fake_now[0] += _lsvc._ESM_BACKOFF_SECONDS / 2
+ assert _lsvc._poll_sqs() is False
+ assert len(invoke_calls) == 1
+
+ # Backoff has elapsed — the ESM is retried (and fails again).
+ fake_now[0] += _lsvc._ESM_BACKOFF_SECONDS
+ assert _lsvc._poll_sqs() is False
+ assert len(invoke_calls) == 2
+
+
+def test_poll_kinesis_returns_false_when_invoke_fails(esm_poll_state, monkeypatch):
+ """A failed invoke doesn't advance the shard position, so the next pass
+ would refetch the same batch — _poll_loop must not skip its idle sleep
+ for a pass that made no real progress, or it spins retrying forever."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ stream_name = "esm-kinesis-drain-signal-failure"
+ stream_arn = f"arn:aws:kinesis:us-east-1:000000000000:stream/{stream_name}"
+ function_name = "esm-kinesis-drain-signal-failure-fn"
+
+ _lsvc._functions[function_name] = {
+ "config": {
+ "FunctionName": function_name,
+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{function_name}",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _kin._streams[stream_name] = _kinesis_stream_record(stream_name, stream_arn)
+ _lsvc._esms["esm-kinesis-drain-signal-failure"] = {
+ "UUID": "esm-kinesis-drain-signal-failure",
+ "EventSourceArn": stream_arn,
+ "FunctionName": function_name,
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ monkeypatch.setattr(
+ _lsvc, "_execute_function",
+ lambda _func, _event: {"error": True, "body": {"errorType": "Error", "errorMessage": "boom"}},
+ )
+
+ assert _lsvc._poll_kinesis() is False
+ # Position didn't advance, so a second pass sees the exact same batch.
+ assert _lsvc._poll_kinesis() is False
+
+
+def test_poll_dynamodb_streams_returns_false_when_invoke_fails(esm_poll_state, monkeypatch):
+ """A failed invoke doesn't advance the stream position, so the next pass
+ would refetch the same batch — _poll_loop must not skip its idle sleep
+ for a pass that made no real progress, or it spins retrying forever."""
+ _lsvc, _sqs, _kin, _ddb = esm_poll_state
+
+ table_name = "esm-ddb-drain-signal-failure"
+ stream_arn = f"arn:aws:dynamodb:us-east-1:000000000000:table/{table_name}/stream/2024-01-01T00:00:00.000"
+ function_name = "esm-ddb-drain-signal-failure-fn"
+
+ _lsvc._functions[function_name] = {
+ "config": {
+ "FunctionName": function_name,
+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{function_name}",
+ },
+ "versions": {},
+ "aliases": {},
+ }
+ _ddb._tables[table_name] = {"LatestStreamArn": stream_arn}
+ _ddb._stream_records[table_name] = [{
+ "eventID": "1",
+ "eventName": "INSERT",
+ "eventSource": "aws:dynamodb",
+ "dynamodb": {
+ "Keys": {},
+ "SequenceNumber": "1",
+ "SizeBytes": 1,
+ "StreamViewType": "NEW_AND_OLD_IMAGES",
+ },
+ "eventSourceARN": stream_arn,
+ }]
+ _lsvc._esms["esm-ddb-drain-signal-failure"] = {
+ "UUID": "esm-ddb-drain-signal-failure",
+ "EventSourceArn": stream_arn,
+ "FunctionName": function_name,
+ "State": "Enabled",
+ "Enabled": True,
+ "BatchSize": 10,
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ monkeypatch.setattr(
+ _lsvc, "_execute_function",
+ lambda _func, _event: {"error": True, "body": {"errorType": "Error", "errorMessage": "boom"}},
+ )
+
+ assert _lsvc._poll_dynamodb_streams() is False
+ # Position didn't advance, so a second pass sees the exact same batch.
+ assert _lsvc._poll_dynamodb_streams() is False
+
+
+def test_pollers_return_false_when_idle(esm_poll_state):
+ """No enabled ESMs -> every poller's per-pass loop body never runs, so
+ each reports nothing processed."""
+ lsvc, _sqs, _kin, _ddb = esm_poll_state
+ assert lsvc._poll_sqs() is False
+ assert lsvc._poll_kinesis() is False
+ assert lsvc._poll_dynamodb_streams() is False
+
+
+def test_poll_dynamodb_streams_returns_false_when_stream_records_unavailable(monkeypatch):
+ from ministack.services import dynamodb as _ddb
+
+ monkeypatch.delattr(_ddb, "_stream_records")
+
+ assert lsvc._poll_dynamodb_streams() is False
+
+
+class _StopPollLoop(BaseException):
+ """Sentinel used to break out of _poll_loop's `while True` after the
+ iteration under test — raised from a spot _poll_loop doesn't wrap in a
+ bare ``except Exception``, so it isn't swallowed and logged away like a
+ real poller error would be."""
+
+
+def test_poll_loop_skips_sleep_when_a_poller_processed_work(monkeypatch):
+ calls = {"sqs": 0}
+
+ def fake_poll_sqs():
+ calls["sqs"] += 1
+ if calls["sqs"] > 1:
+ raise _StopPollLoop()
+ return True
+
+ sleep_calls = []
+ monkeypatch.setattr(lsvc, "_poll_sqs", fake_poll_sqs)
+ monkeypatch.setattr(lsvc, "_poll_kinesis", lambda: False)
+ monkeypatch.setattr(lsvc, "_poll_dynamodb_streams", lambda: False)
+ monkeypatch.setattr(lsvc.time, "sleep", lambda secs: sleep_calls.append(secs))
+
+ with pytest.raises(_StopPollLoop):
+ lsvc._poll_loop()
+
+ # First pass processed a batch, so it must loop again immediately
+ # instead of sleeping; the second pass is what raises _StopPollLoop.
+ assert sleep_calls == []
+ assert calls["sqs"] == 2
+
+
+def test_poll_loop_sleeps_when_no_poller_processed_work(esm_poll_state, monkeypatch):
+ sleep_calls = []
+
+ def fake_sleep(secs):
+ sleep_calls.append(secs)
+ raise _StopPollLoop()
+
+ # esm_poll_state clears _esms, so has_any() -> False and an idle pass
+ # sleeps 5s (not 1s).
+ monkeypatch.setattr(lsvc, "_poll_sqs", lambda: False)
+ monkeypatch.setattr(lsvc, "_poll_kinesis", lambda: False)
+ monkeypatch.setattr(lsvc, "_poll_dynamodb_streams", lambda: False)
+ monkeypatch.setattr(lsvc.time, "sleep", fake_sleep)
+
+ with pytest.raises(_StopPollLoop):
+ lsvc._poll_loop()
+
+ assert sleep_calls == [5]
+
+
def test_lambda_create_esm_rejects_unresolved_function_arn():
original_account = get_account_id()
original_region = get_region()