Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3): answer the conditional-read headers on GetObject and HeadObject (#1390)
Andrew Gaul29cf0232d14a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7d4d7658..279a3c95 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed- **SES v2 — error responses carry `x-amzn-errortype`** — restJson1 resolves the error shape from that header, so without it SDKs surfaced a bare HTTP status instead of the modelled exception: boto3 reported `An error occurred (404)` rather than `NotFoundException`, and typed handling never matched on any SESv2 operation. The 1.3.24 sweep that added the header centrally in `error_response_json` reached `ses` but not `ses_v2`, which builds its error bodies itself; it is now set on all of them.+- **S3 — GetObject and HeadObject answer the conditional-read headers** — `If-Match`, `If-None-Match`, `If-Modified-Since` and `If-Unmodified-Since` were honoured on PutObject, CopyObject and DeleteObject but ignored on the reads, so every conditional read returned `200` and the whole body: a cache revalidating an unchanged object refetched it in full, and a read guarded against a concurrent overwrite never noticed one. Reads now answer `412 PreconditionFailed` or `304 Not Modified`, with the precedence RFC 9110 13.2.2 and the AWS GetObject reference define — an entity tag decides, and its date counterpart applies only in its absence — and a `412` preempts a `Range` rather than slicing a representation the caller rejected. The `304` keeps the validators (`ETag`, `Last-Modified`) and drops the headers describing a payload it cannot carry, `x-amz-checksum-*` included, which boto3 asks for by default and would otherwise try to validate against the empty body. Reported by @gaul.## [1.4.17] — 2026-08-14diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex ec947a6c..923058be 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -512,6 +512,95 @@ def _check_put_preconditions(headers: dict, existing_obj: dict | None):return None+def _etag_condition_matches(field_value: str, etag: str) -> bool:+ """Whether an `If-Match`/`If-None-Match` field value selects this object.++ The value is a comma-separated list of entity tags, or `*` for "any+ representation" — which is always satisfied here, since the caller has+ already resolved the object. A weak tag compares equal to its strong form:+ S3 only ever mints strong ETags, and If-None-Match uses the weak comparison+ anyway (RFC 9110 13.1.2). `etag` is expected unquoted.+ """+ if field_value == "*":+ return True+ for candidate in field_value.split(","):+ candidate = candidate.strip()+ if candidate[:2].upper() == "W/":+ candidate = candidate[2:]+ if candidate.strip('"') == etag:+ return True+ return False+++def _not_modified(resp_headers: dict):+ """A 304 for a read whose caller already holds the current representation.++ Keeps the validators the 200 would have carried — the ETag above all, which+ is how the client confirms which representation it just revalidated — and+ drops the headers that describe a payload, since a 304 ends at the header+ section and can never carry one (RFC 9110 15.4.5).+ """+ payload_headers = {+ "content-length", "content-type", "content-encoding", "accept-ranges",+ }++ def describes_payload(name: str) -> bool:+ lowered = name.lower()+ # x-amz-checksum-* is a checksum of bytes this response does not carry.+ # boto3 asks for them by default and, seeing one, wraps the body in a+ # validating stream it then cannot parse — the same hazard that keeps+ # them off a 206 partial response.+ return lowered in payload_headers or lowered.startswith("x-amz-checksum-")++ return (+ 304,+ {k: v for k, v in resp_headers.items() if not describes_payload(k)},+ b"",+ )+++def _check_read_preconditions(headers: dict, obj: dict, resp_headers: dict):+ """Evaluate the four conditional-read headers on GetObject/HeadObject.++ Returns a finished response tuple — 412 PreconditionFailed when the request+ asked for a representation this object no longer is, or 304 Not Modified+ when the caller's copy is still current — otherwise None, and the read+ proceeds normally. Answering conditional reads is what lets caches, `curl+ -z`, and every "download only if it changed" client avoid refetching a body+ they already have.++ Precedence follows RFC 9110 13.2.2 and the AWS GetObject reference: the+ entity-tag header wins over its date counterpart, so If-Unmodified-Since is+ consulted only without If-Match, and If-Modified-Since only without+ If-None-Match.+ """+ _precond_msg = "At least one of the pre-conditions you specified did not hold"+ etag = (obj.get("etag") or "").strip('"')+ mtime = _object_mtime_dt(obj)++ if_match = (headers.get("if-match") or "").strip()+ if if_match:+ if not _etag_condition_matches(if_match, etag):+ return _error("PreconditionFailed", _precond_msg, 412)+ else:+ unmod = _parse_http_date(headers.get("if-unmodified-since", ""))+ # "Unmodified since" fails when the object was written after that time.+ if unmod and mtime and mtime > unmod:+ return _error("PreconditionFailed", _precond_msg, 412)++ if_none_match = (headers.get("if-none-match") or "").strip()+ if if_none_match:+ if _etag_condition_matches(if_none_match, etag):+ return _not_modified(resp_headers)+ else:+ mod = _parse_http_date(headers.get("if-modified-since", ""))+ # Not modified since that time means the caller's copy is still current.+ if mod and mtime and mtime <= mod:+ return _not_modified(resp_headers)++ return None++def _find_xml_tag(parent, tag_name, ns=S3_NS):el = parent.find("{%s}%s" % (ns, tag_name))if el is None:@@ -2881,6 +2970,9 @@ def _get_object(bucket_name: str, key: str, headers: dict, query_params: dict =resp_headers = _object_response_headers(vobj, bucket_name, key, include_checksums=include_checksums)resp_headers.update(_object_tagging_count_header(bucket_name, key, version_id))+ precondition = _check_read_preconditions(headers, vobj, resp_headers)+ if precondition is not None:+ return preconditionbody = v.get("data")if body is None:body = _read_body(bucket_name, key, bucket["objects"].get(key, {}))@@ -2907,6 +2999,13 @@ def _get_object(bucket_name: str, key: str, headers: dict, query_params: dict =include_checksums=include_checksums)resp_headers.update(_object_tagging_count_header(bucket_name, key, obj.get("version_id")))+ # Evaluated before the range and partNumber slicing below: a failed+ # precondition preempts the read entirely, and a 304 describes the whole+ # representation the caller already has, not the slice they asked for.+ precondition = _check_read_preconditions(headers, obj, resp_headers)+ if precondition is not None:+ return precondition+body = _read_body(bucket_name, key, obj)# partNumber: return a single part of a completed multipart object as 206@@ -3106,6 +3205,9 @@ def _head_object(bucket_name: str, key: str, headers: dict | None = None,include_checksums=include_checksums)if version_id:resp_headers["x-amz-version-id"] = version_id+ precondition = _check_read_preconditions(headers, obj, resp_headers)+ if precondition is not None:+ return preconditionreturn 200, resp_headers, b""diff --git a/tests/test_s3.py b/tests/test_s3.pyindex a42c23a8..78c588bc 100644--- a/tests/test_s3.py+++ b/tests/test_s3.py@@ -4365,6 +4365,151 @@ def test_s3_copy_object_source_date_preconditions(s3):assert e3.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412+def _conditional_read_bucket(s3):+ """A bucket holding one object, with its ETag and Last-Modified."""+ bkt = f"cond-read-{_uuid_mod.uuid4().hex[:8]}"+ s3.create_bucket(Bucket=bkt)+ etag = s3.put_object(Bucket=bkt, Key="obj", Body=b"payload")["ETag"]+ mtime = s3.head_object(Bucket=bkt, Key="obj")["LastModified"]+ return bkt, etag, mtime+++def test_s3_get_object_etag_preconditions(s3):+ """GetObject answers If-Match and If-None-Match instead of always returning+ the body: a stale If-Match is 412, and a current If-None-Match is 304."""+ bkt, etag, _ = _conditional_read_bucket(s3)++ # The condition holds -> the read proceeds as normal.+ assert s3.get_object(Bucket=bkt, Key="obj", IfMatch=etag)["Body"].read() == b"payload"+ assert s3.get_object(Bucket=bkt, Key="obj", IfMatch="*")["Body"].read() == b"payload"+ assert s3.get_object(Bucket=bkt, Key="obj", IfNoneMatch='"ABCORZ"')["Body"].read() == b"payload"++ # If-Match against some other representation -> 412, nothing transferred.+ with pytest.raises(ClientError) as e:+ s3.get_object(Bucket=bkt, Key="obj", IfMatch='"ABCORZ"')+ assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412+ assert e.value.response["Error"]["Code"] == "PreconditionFailed"++ # If-None-Match naming the current representation -> the caller's copy is+ # current, so 304 and no body. The ETag confirms which one they revalidated.+ for value in (etag, "*", f'"nomatch", {etag}', f"W/{etag}"):+ with pytest.raises(ClientError) as e2:+ s3.get_object(Bucket=bkt, Key="obj", IfNoneMatch=value)+ resp = e2.value.response+ assert resp["ResponseMetadata"]["HTTPStatusCode"] == 304, value+ assert resp["ResponseMetadata"]["HTTPHeaders"]["etag"] == etag, value+++def test_s3_get_object_date_preconditions(s3):+ """GetObject answers If-Modified-Since and If-Unmodified-Since: unchanged+ since the caller's timestamp is 304, changed after it is 412."""+ from datetime import timedelta++ bkt, etag, mtime = _conditional_read_bucket(s3)+ before = mtime - timedelta(days=1)+ after = mtime + timedelta(seconds=1)++ # Modified since a day ago (true) and unmodified since a second from now+ # (true) both leave the read alone.+ assert s3.get_object(Bucket=bkt, Key="obj", IfModifiedSince=before)["Body"].read() == b"payload"+ assert s3.get_object(Bucket=bkt, Key="obj", IfUnmodifiedSince=after)["Body"].read() == b"payload"++ # Not modified since a second from now -> 304.+ with pytest.raises(ClientError) as e:+ s3.get_object(Bucket=bkt, Key="obj", IfModifiedSince=after)+ assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 304+ assert e.value.response["ResponseMetadata"]["HTTPHeaders"]["etag"] == etag++ # Modified since a day ago -> the If-Unmodified-Since condition fails -> 412.+ with pytest.raises(ClientError) as e2:+ s3.get_object(Bucket=bkt, Key="obj", IfUnmodifiedSince=before)+ assert e2.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412+ assert e2.value.response["Error"]["Code"] == "PreconditionFailed"+++def test_s3_read_precondition_precedence(s3):+ """An entity tag beats its date counterpart, per RFC 9110 13.2.2 and the AWS+ GetObject reference: the date header applies only when the tag is absent."""+ from datetime import timedelta++ bkt, etag, mtime = _conditional_read_bucket(s3)+ before = mtime - timedelta(days=1)+ after = mtime + timedelta(seconds=1)++ # If-Match holds, so the failing If-Unmodified-Since is not consulted -> 200.+ resp = s3.get_object(Bucket=bkt, Key="obj", IfMatch=etag, IfUnmodifiedSince=before)+ assert resp["Body"].read() == b"payload"++ # If-None-Match matches, so the satisfied If-Modified-Since is not+ # consulted -> 304 rather than the body.+ with pytest.raises(ClientError) as e:+ s3.get_object(Bucket=bkt, Key="obj", IfNoneMatch=etag, IfModifiedSince=before)+ assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 304++ # The tag is consulted even when it does not decide the read: a present+ # If-None-Match that misses suppresses If-Modified-Since outright, so the+ # body comes back although that date alone would have meant 304.+ resp = s3.get_object(Bucket=bkt, Key="obj", IfNoneMatch='"ABCORZ"', IfModifiedSince=after)+ assert resp["Body"].read() == b"payload"+++def test_s3_head_object_preconditions(s3):+ """HeadObject evaluates the same four conditions as GetObject."""+ from datetime import timedelta++ bkt, etag, mtime = _conditional_read_bucket(s3)++ assert s3.head_object(Bucket=bkt, Key="obj", IfMatch=etag)["ETag"] == etag++ with pytest.raises(ClientError) as e:+ s3.head_object(Bucket=bkt, Key="obj", IfMatch='"ABCORZ"')+ assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412++ with pytest.raises(ClientError) as e2:+ s3.head_object(Bucket=bkt, Key="obj", IfNoneMatch=etag)+ assert e2.value.response["ResponseMetadata"]["HTTPStatusCode"] == 304++ with pytest.raises(ClientError) as e3:+ s3.head_object(Bucket=bkt, Key="obj", IfUnmodifiedSince=mtime - timedelta(days=1))+ assert e3.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412+++def test_s3_not_modified_carries_no_payload(s3):+ """Checked on the wire, not through boto3: a 304 ends at the header section+ and can never carry a body (RFC 9110 15.4.5), so it must not describe one+ either. A stray body would desynchronise the next request on the connection."""+ import http.client++ bkt, etag, _ = _conditional_read_bucket(s3)+ parsed = urlparse(ENDPOINT)+ conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 4566, timeout=10)+ # Checksum mode is on, as boto3 turns it on by default, so the checksum+ # headers below are ones the 200 really would have carried.+ conn.request("GET", f"/{bkt}/obj",+ headers={"If-None-Match": etag, "x-amz-checksum-mode": "ENABLED"})+ resp = conn.getresponse()+ body = resp.read()++ assert resp.status == 304+ assert body == b""+ assert resp.getheader("ETag") == etag+ assert resp.getheader("Last-Modified")+ for absent in ("Content-Type", "Content-Encoding", "Accept-Ranges"):+ assert resp.getheader(absent) is None, absent+ assert (resp.getheader("Content-Length") or "0") == "0"+ # A checksum of bytes that were not sent belongs to no one: boto3 asks for+ # these by default and chokes on the empty body if it gets one back.+ assert not [h for h, _ in resp.getheaders() if h.lower().startswith("x-amz-checksum-")]++ # The connection is still usable, which is the point of getting the framing+ # right: a second request on it reads its own response.+ conn.request("GET", f"/{bkt}/obj")+ second = conn.getresponse()+ assert second.status == 200+ assert second.read() == b"payload"+ conn.close()++def test_s3_canned_acl_expands_to_group_grants(s3):"""A canned ACL (at PutObject or PutObjectAcl) expands to the group grants itimplies: public-read -> AllUsers READ; authenticated-read -> AuthenticatedUsers