Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
S3: event notification records match S3's (#1832)
Nathan66c8bbbb31a4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 77fcdc5f..569f4289 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed- **DynamoDB — `DescribeTable` accepts a table ARN** — `TableName` given as the table's ARN answered `ResourceNotFoundException`, where AWS accepts either form. It now resolves through the same ARN normalization the other table operations use, and an ARN from another account or region still answers `ResourceNotFoundException`. This unblocks OpenSearch Data Prepper's DynamoDB source, which always describes the table by ARN.+- **S3 — event notification records match S3's** — the object block carries the key URL-encoded (`red flower.jpg` → `red+flower.jpg`), the `versionId` the write made or removed, and a `sequencer` that grows with every create and delete (it was always `"0"`); records are `eventVersion` `2.6`. A delete that leaves a delete marker is `ObjectRemoved:DeleteMarkerCreated` (it was `ObjectRemoved:Delete`, so a subscription to permanent deletes received markers), and `DeleteObjects` sends an event for each object it removes (it sent none). Checked against the S3 User Guide's "Event message structure" and "Event notification types and destinations" pages; not validated against a real AWS account.- **STS — `GetCallerIdentity` resolves IAM-user callers** — keys created with `CreateAccessKey` returned the `root` ARN; they now return the user's ARN and ID in both XML and JSON protocols. Under `AUTH=true`, unknown and inactive keys are rejected instead of reported as root.## [1.5.17] — 2026-09-25diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex d78002b1..52aeb4da 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -29,6 +29,7 @@ import contextvarsimport copyimport datetime as _dtimport hashlib+import itertoolsimport jsonimport loggingimport os@@ -42,6 +43,7 @@ import timeimport zlibfrom urllib.parse import parse_qs as _parse_qsfrom urllib.parse import quote as url_quote+from urllib.parse import quote_plus as _url_quote_plusfrom urllib.parse import unquote as url_unquotefrom urllib.parse import urlparse as _urlparsefrom xml.etree.ElementTree import Element, ParseError, SubElement, tostring@@ -3369,6 +3371,33 @@ def _s3_event_to_eventbridge(event_name: str) -> tuple[str, str | None]:return detail_type, reason+# The notification record's `sequencer`: a hexadecimal string that grows with+# every create and delete, so "the event notification with the greater+# `sequencer` hexadecimal value is the event that occurred later" for one key+# (S3 User Guide, "Event message structure"). Taken when the write happens,+# not when the background thread builds the record, so it orders the writes.+# Seeded from the clock so values keep growing across a restart.+_event_sequence = itertools.count(time.time_ns() // 1000)+_EVENT_SEQUENCED_FAMILIES = ("s3:ObjectCreated:", "s3:ObjectRemoved:")+++def _next_event_sequencer(event_name: str) -> str | None:+ """A sequencer for an object create or delete event; None for the others,+ which the record carries no sequencer for ("only used with PUT and DELETE+ requests")."""+ if not event_name.startswith(_EVENT_SEQUENCED_FAMILIES):+ return None+ return f"{next(_event_sequence):018X}"+++def _event_object_key(key: str) -> str:+ """The key as a notification record carries it: "The object key name value+ is URL encoded. For example, `red flower.jpg` becomes `red+flower.jpg`"+ (S3 User Guide, "Event message structure"); slashes are left as they are.+ EventBridge details carry the key as stored."""+ return _url_quote_plus(key, safe="/")++def _fire_s3_event(bucket_name: str,key: str,@@ -3377,8 +3406,18 @@ def _fire_s3_event(etag: str = "",deletion_type: str | None = None,restore_event_data: dict | None = None,+ version_id: str | None = None,+ sequencer: str | None = None,+ object_extra: dict | None = None,+ record_extra: dict | None = None,) -> None:- """Build and deliver an S3 event notification. Best-effort — errors are logged."""+ """Build and deliver an S3 event notification. Best-effort — errors are logged.++ `version_id` is the version the write made or removed ("null or not present+ if the bucket isn't versioning-enabled"), `sequencer` the one taken when it+ happened, `object_extra` fields the record's `s3.object` gains for this+ event (`hasObjectAnnotation` on a copy), and `record_extra` fields the record+ itself gains (`objectAnnotation` on an annotation event)."""try:configs = _parse_notification_config(bucket_name)raw_xml = _bucket_notifications.get(bucket_name, "")@@ -3391,11 +3430,19 @@ def _fire_s3_event(event_time = now_iso()request_id = new_uuid()clean_etag = etag.strip('"')+ event_object = {"key": _event_object_key(key), "size": size, "eTag": clean_etag}+ if version_id:+ event_object["versionId"] = version_id+ if sequencer:+ event_object["sequencer"] = sequencer+ if object_extra:+ event_object.update(object_extra)event_payload = {"Records": [{- "eventVersion": "2.1",+ # One unified version for every event type since 2.4.+ "eventVersion": "2.6","eventSource": "aws:s3","awsRegion": bucket_region,"eventTime": event_time,@@ -3414,16 +3461,13 @@ def _fire_s3_event("ownerIdentity": {"principalId": "EXAMPLE"},"arn": f"arn:aws:s3:::{bucket_name}",},- "object": {- "key": key,- "size": size,- "eTag": clean_etag,- "sequencer": "0",- },+ "object": event_object,},}],}+ if record_extra:+ event_payload["Records"][0].update(record_extra)if restore_event_data:# glacierEventData appears only on s3:ObjectRestore:Completed.event_payload["Records"][0]["glacierEventData"] = {@@ -3461,7 +3505,7 @@ def _fire_s3_event("version": "0","event-version": "1.0","bucket": {"name": bucket_name},- "object": {"key": key, "size": size, "etag": clean_etag, "sequencer": "0"},+ "object": {"key": key, "size": size, "etag": clean_etag, "sequencer": sequencer or "0"},"request-id": request_id,"requester": get_account_id(),"source-ip-address": "127.0.0.1",@@ -3590,10 +3634,14 @@ def _fire_s3_event_async(size: int = 0,etag: str = "",deletion_type: str | None = None,+ version_id: str | None = None,+ object_extra: dict | None = None,+ record_extra: dict | None = None,) -> None:"""Fire S3 event notification in a background thread (non-blocking)."""if bucket_name not in _bucket_notifications:return+ sequencer = _next_event_sequencer(event_name)# threading.Thread does not copy contextvars, so without this snapshot the# worker runs under the default account (000000000000): the account-scoped# _bucket_notifications lookup comes back empty and the event is silently@@ -3604,6 +3652,12 @@ def _fire_s3_event_async(t = threading.Thread(target=ctx.run,args=(_fire_s3_event, bucket_name, key, event_name, size, etag, deletion_type),+ kwargs={+ "version_id": version_id,+ "sequencer": sequencer,+ "object_extra": object_extra,+ "record_extra": record_extra,+ },daemon=True,)t.start()@@ -3696,14 +3750,16 @@ def _put_object(bucket_name: str, key: str, body: bytes, headers: dict):if len(pending_tags) > 10:return _error("BadRequest", "Object tags cannot be greater than 10", 400)- _fire_s3_event_async(bucket_name, key, "s3:ObjectCreated:Put", size=obj["size"], etag=obj["etag"])-resp_headers = {"ETag": obj["etag"], "Content-Length": "0"}resp_headers.update(sse_headers)_maybe_replicate(bucket_name, key, obj, body)version_id = _record_object_version(bucket_name, key, prior_obj, obj, body)if version_id:resp_headers["x-amz-version-id"] = version_id+ # After the version is cut, so the record names it.+ _fire_s3_event_async(+ bucket_name, key, "s3:ObjectCreated:Put", size=obj["size"], etag=obj["etag"], version_id=version_id+ )if pending_tags is not None:_object_tags[(bucket_name, key, obj.get("version_id"))] = pending_tags@@ -3941,10 +3997,9 @@ def _post_object(bucket_name: str, body: bytes, headers: dict):if len(parsed) <= 10:pending_tags = parsed- _fire_s3_event_async(bucket_name, key, "s3:ObjectCreated:Post", size=obj["size"], etag=etag)-_maybe_replicate(bucket_name, key, obj, file_value)version_id = _record_object_version(bucket_name, key, prior_obj, obj, file_value)+ _fire_s3_event_async(bucket_name, key, "s3:ObjectCreated:Post", size=obj["size"], etag=etag, version_id=version_id)if pending_tags is not None:_object_tags[(bucket_name, key, version_id)] = pending_tags@@ -4697,7 +4752,7 @@ def _delete_object(bucket_name: str, key: str, headers: dict | None = None, querif was_delete_marker:resp_headers["x-amz-delete-marker"] = "true"if _found:- _fire_s3_event_async(bucket_name, key, "s3:ObjectRemoved:Delete")+ _fire_s3_event_async(bucket_name, key, "s3:ObjectRemoved:Delete", version_id=version_id)return 204, resp_headers, b""versioning = _bucket_versioning.get(bucket_name, "")@@ -4707,7 +4762,15 @@ def _delete_object(bucket_name: str, key: str, headers: dict | None = None, querexisted = key in bucket["objects"]bucket["objects"].pop(key, None)if existed:- _fire_s3_event_async(bucket_name, key, "s3:ObjectRemoved:Delete", deletion_type="Delete Marker Created")+ # "s3:ObjectRemoved:DeleteMarkerCreated" is its own event type:+ # s3:ObjectRemoved:Delete is for an object or version that is gone.+ _fire_s3_event_async(+ bucket_name,+ key,+ "s3:ObjectRemoved:DeleteMarkerCreated",+ deletion_type="Delete Marker Created",+ version_id=delete_marker_id,+ )return 204, {"x-amz-delete-marker": "true", "x-amz-version-id": delete_marker_id}, b""existed = key in bucket["objects"]@@ -4993,14 +5056,6 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):else:_object_legal_hold.pop((bucket_name, dest_key), None)- _fire_s3_event_async(- bucket_name,- dest_key,- "s3:ObjectCreated:Copy",- size=dest_obj["size"],- etag=new_etag,- )-resp_headers = {"Content-Type": "application/xml"}resp_headers.update(dest_sse)if copy_src_vid:@@ -5011,6 +5066,14 @@ def _copy_object(bucket_name: str, dest_key: str, headers: dict):version_id = _record_object_version(bucket_name, dest_key, dest_prior_obj, dest_obj, src_body)if version_id:resp_headers["x-amz-version-id"] = version_id+ _fire_s3_event_async(+ bucket_name,+ dest_key,+ "s3:ObjectCreated:Copy",+ size=dest_obj["size"],+ etag=new_etag,+ version_id=version_id,+ )dest_version_id = dest_obj.get("version_id")if pending_dest_tags is not None:@@ -5955,15 +6018,31 @@ def _delete_objects(bucket_name: str, body: bytes, headers: dict = None):# S3 reports the delete as successful even if the version was absent._found, was_marker = _delete_object_version(bucket, bucket_name, k, version_id)deleted.append({"key": k, "version_id": version_id, "was_marker": was_marker})+ # "notification when an object or a batch of objects is removed":+ # each entry sends what the single-object DELETE would.+ if _found:+ _fire_s3_event_async(bucket_name, k, "s3:ObjectRemoved:Delete", version_id=version_id)elif _bucket_versioning.get(bucket_name) in ("Enabled", "Suspended"):# No VersionId on a versioned bucket: create a delete marker —# even for a key that never existed — exactly as the single-object# DELETE does, and report it on the Deleted entry.+ existed = k in bucket["objects"]marker_id = _record_delete_marker(bucket_name, k, bucket["objects"].get(k))bucket["objects"].pop(k, None)deleted.append({"key": k, "version_id": "", "was_marker": False, "marker_created": marker_id})+ if existed:+ _fire_s3_event_async(+ bucket_name,+ k,+ "s3:ObjectRemoved:DeleteMarkerCreated",+ deletion_type="Delete Marker Created",+ version_id=marker_id,+ )else:# No VersionId → plain delete of the current object.+ existed = k in bucket["objects"]+ if existed:+ _fire_s3_event_async(bucket_name, k, "s3:ObjectRemoved:Delete")bucket["objects"].pop(k, None)_object_tags.pop((bucket_name, k, None), None)_object_retention.pop((bucket_name, k), None)@@ -6388,21 +6467,21 @@ def _complete_multipart_upload(del _multipart_uploads[upload_id]+ resp_headers = {"Content-Type": "application/xml"}+ resp_headers.update(_stored_sse_headers(obj))+ _maybe_replicate(bucket_name, key, obj, combined)+ version_id = _record_object_version(bucket_name, key, prior_obj, obj, combined)+ if version_id:+ resp_headers["x-amz-version-id"] = version_id_fire_s3_event_async(bucket_name,key,"s3:ObjectCreated:CompleteMultipartUpload",size=obj["size"],etag=final_etag,+ version_id=version_id,)- resp_headers = {"Content-Type": "application/xml"}- resp_headers.update(_stored_sse_headers(obj))- _maybe_replicate(bucket_name, key, obj, combined)- version_id = _record_object_version(bucket_name, key, prior_obj, obj, combined)- if version_id:- resp_headers["x-amz-version-id"] = version_id-# Persist only after the versioning block: the .meta.json sidecar must# carry the version_id assigned above (#1058).if S3_PERSIST:diff --git a/tests/test_s3_event_records.py b/tests/test_s3_event_records.pynew file mode 100644index 00000000..16faa583--- /dev/null+++ b/tests/test_s3_event_records.py@@ -0,0 +1,170 @@+"""The S3 notification record's object block, per the S3 User Guide's+"Event message structure" and "Event notification types and destinations"+pages: a URL-encoded key, the version the write made or removed, a sequencer+that grows with each create and delete, eventVersion 2.6, a delete marker+reported as ObjectRemoved:DeleteMarkerCreated, and one event per object a+DeleteObjects removes. Evidence: official AWS documentation, not a run against+a real AWS account."""++import json+import time+import uuid++import pytest+++def _queue(sqs, name):+ url = sqs.create_queue(QueueName=name)["QueueUrl"]+ arn = sqs.get_queue_attributes(QueueUrl=url, AttributeNames=["QueueArn"])["Attributes"]["QueueArn"]+ return url, arn+++def _records(sqs, url, expected, timeout=10):+ """Every S3 record delivered to the queue, waiting until `expected` have arrived."""+ records = []+ deadline = time.time() + timeout+ while len(records) < expected and time.time() < deadline:+ resp = sqs.receive_message(QueueUrl=url, MaxNumberOfMessages=10, WaitTimeSeconds=1)+ for m in resp.get("Messages", []):+ body = json.loads(m["Body"])+ records.extend(body.get("Records", []))+ sqs.delete_message(QueueUrl=url, ReceiptHandle=m["ReceiptHandle"])+ return records+++def notified(s3, sqs):+ """A bucket whose object events go to a fresh queue: (bucket, queue url, configure)."""+ suffix = uuid.uuid4().hex[:8]+ bucket = f"evt-records-{suffix}"+ s3.create_bucket(Bucket=bucket)+ url, arn = _queue(sqs, f"evt-records-{suffix}")++ def configure(*events, versioned=False):+ if versioned:+ s3.put_bucket_versioning(Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})+ s3.put_bucket_notification_configuration(+ Bucket=bucket,+ NotificationConfiguration={"QueueConfigurations": [{"QueueArn": arn, "Events": list(events)}]},+ )+ # The configuration's own s3:TestEvent carries no Records; drain it.+ _records(sqs, url, 0, timeout=1)++ return bucket, url, configure+++def test_record_key_is_url_encoded(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:*")+ s3.put_object(Bucket=bucket, Key="photos/red flower+(1).jpg", Body=b"x")+ [record] = _records(sqs, url, 1)+ # "`red flower.jpg` becomes `red+flower.jpg`"; slashes stay, a literal + is escaped.+ assert record["s3"]["object"]["key"] == "photos/red+flower%2B%281%29.jpg"+ assert record["eventVersion"] == "2.6"+++def test_record_carries_the_version_the_put_made(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:*", versioned=True)+ first = s3.put_object(Bucket=bucket, Key="doc.txt", Body=b"one")["VersionId"]+ second = s3.put_object(Bucket=bucket, Key="doc.txt", Body=b"two")["VersionId"]+ records = _records(sqs, url, 2)+ assert sorted(r["s3"]["object"]["versionId"] for r in records) == sorted([first, second])+++def test_record_has_no_version_in_an_unversioned_bucket(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:*")+ s3.put_object(Bucket=bucket, Key="plain.txt", Body=b"x")+ [record] = _records(sqs, url, 1)+ assert "versionId" not in record["s3"]["object"]+++def test_sequencer_grows_with_each_write(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:*", "s3:ObjectRemoved:*", versioned=True)+ v1 = s3.put_object(Bucket=bucket, Key="seq.txt", Body=b"1")["VersionId"]+ v2 = s3.put_object(Bucket=bucket, Key="seq.txt", Body=b"2")["VersionId"]+ marker = s3.delete_object(Bucket=bucket, Key="seq.txt")["VersionId"]+ by_version = {r["s3"]["object"]["versionId"]: r["s3"]["object"]["sequencer"] for r in _records(sqs, url, 3)}+ # Compared as S3 says to: left-pad the shorter with zeros, then compare.+ width = max(len(v) for v in by_version.values())+ ordered = [by_version[v].rjust(width, "0") for v in (v1, v2, marker)]+ assert ordered == sorted(ordered) and len(set(ordered)) == 3+ assert all(int(s, 16) >= 0 for s in ordered)+++def test_delete_without_version_reports_the_marker(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectRemoved:*", versioned=True)+ s3.put_object(Bucket=bucket, Key="gone.txt", Body=b"x")+ marker = s3.delete_object(Bucket=bucket, Key="gone.txt")["VersionId"]+ [record] = _records(sqs, url, 1)+ assert record["eventName"] == "ObjectRemoved:DeleteMarkerCreated"+ assert record["s3"]["object"]["versionId"] == marker+++def test_marker_is_not_an_object_removed_delete(s3, sqs, notified):+ bucket, url, configure = notified+ # Subscribed to permanent deletes only: a marker is its own event type.+ configure("s3:ObjectRemoved:Delete", versioned=True)+ version = s3.put_object(Bucket=bucket, Key="kept.txt", Body=b"x")["VersionId"]+ s3.delete_object(Bucket=bucket, Key="kept.txt")+ s3.delete_object(Bucket=bucket, Key="kept.txt", VersionId=version)+ [record] = _records(sqs, url, 1)+ assert record["eventName"] == "ObjectRemoved:Delete"+ assert record["s3"]["object"]["versionId"] == version+ assert _records(sqs, url, 1, timeout=2) == []+++def test_copy_record_names_the_new_version(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:Copy", versioned=True)+ s3.put_object(Bucket=bucket, Key="src.txt", Body=b"x")+ copied = s3.copy_object(Bucket=bucket, Key="dst.txt", CopySource={"Bucket": bucket, "Key": "src.txt"})+ [record] = _records(sqs, url, 1)+ assert record["eventName"] == "ObjectCreated:Copy"+ assert record["s3"]["object"]["versionId"] == copied["VersionId"]+++def test_multipart_record_names_the_new_version(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectCreated:CompleteMultipartUpload", versioned=True)+ upload = s3.create_multipart_upload(Bucket=bucket, Key="big.bin")["UploadId"]+ part = s3.upload_part(Bucket=bucket, Key="big.bin", UploadId=upload, PartNumber=1, Body=b"x" * 16)+ done = s3.complete_multipart_upload(+ Bucket=bucket,+ Key="big.bin",+ UploadId=upload,+ MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": part["ETag"]}]},+ )+ [record] = _records(sqs, url, 1)+ assert record["s3"]["object"]["versionId"] == done["VersionId"]+++def test_delete_objects_sends_an_event_per_object(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectRemoved:*", versioned=True)+ for k in ("a.txt", "b.txt"):+ s3.put_object(Bucket=bucket, Key=k, Body=b"x")+ version = s3.put_object(Bucket=bucket, Key="c.txt", Body=b"x")["VersionId"]+ s3.delete_objects(+ Bucket=bucket,+ Delete={"Objects": [{"Key": "a.txt"}, {"Key": "b.txt"}, {"Key": "c.txt", "VersionId": version}]},+ )+ records = _records(sqs, url, 3)+ got = sorted((r["s3"]["object"]["key"], r["eventName"]) for r in records)+ assert got == [+ ("a.txt", "ObjectRemoved:DeleteMarkerCreated"),+ ("b.txt", "ObjectRemoved:DeleteMarkerCreated"),+ ("c.txt", "ObjectRemoved:Delete"),+ ]+++def test_delete_objects_in_an_unversioned_bucket(s3, sqs, notified):+ bucket, url, configure = notified+ configure("s3:ObjectRemoved:*")+ s3.put_object(Bucket=bucket, Key="u.txt", Body=b"x")+ s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": "u.txt"}, {"Key": "never-there.txt"}]})+ [record] = _records(sqs, url, 1)+ assert (record["s3"]["object"]["key"], record["eventName"]) == ("u.txt", "ObjectRemoved:Delete")