Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
refactor(sigv4): share presigning primitives (#1634)
Ian Oberstdd9484cfa4ed
diff --git a/ministack/core/sigv4.py b/ministack/core/sigv4.pynew file mode 100644index 00000000..5423468b--- /dev/null+++ b/ministack/core/sigv4.py@@ -0,0 +1,134 @@+"""Shared AWS Signature Version 4 primitives.++This module owns only deterministic signing mechanics. Services remain+responsible for validating their own credential scope, resolving secrets, and+mapping verification failures to service-specific responses.+"""++import datetime as dt+import hashlib+import hmac+from collections.abc import Mapping+from typing import Any+from urllib.parse import quote++UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"+++def uri_encode(value: str, encode_slash: bool = True) -> str:+ """Encode a SigV4 URI component using the RFC 3986 unreserved set."""+ safe = "-_.~" + ("" if encode_slash else "/")+ return quote(value, safe=safe)+++def build_canonical_query_string(+ query_params: Mapping[str, Any],+ *,+ exclude: tuple[str, ...] = ("x-amz-signature",),+) -> str:+ """Encode and sort query parameters for a SigV4 canonical request."""+ excluded = {name.lower() for name in exclude}+ pairs = []+ for name, values in query_params.items():+ if name.lower() in excluded:+ continue+ value_list = values if isinstance(values, list) else [values]+ for value in value_list:+ pairs.append((uri_encode(name), uri_encode(value)))+ pairs.sort()+ return "&".join(f"{key}={value}" for key, value in pairs)+++def build_canonical_headers(headers: Mapping[str, Any], signed_headers: str) -> str:+ """Normalize the headers named by a SigV4 signed-headers value."""+ canonical = ""+ for name in (name for name in signed_headers.split(";") if name):+ raw = headers.get(name, headers.get(name.lower(), ""))+ canonical += f"{name.lower()}:{' '.join(str(raw).split())}\n"+ return canonical+++def build_canonical_request(+ method: str,+ path: str,+ headers: Mapping[str, Any],+ query_params: Mapping[str, Any],+ signed_headers: str,+ *,+ payload_hash: str = UNSIGNED_PAYLOAD,+ exclude_query_params: tuple[str, ...] = ("x-amz-signature",),+) -> str:+ """Build the canonical request hashed by SigV4."""+ return "\n".join(+ [+ method,+ uri_encode(path, encode_slash=False),+ build_canonical_query_string(query_params, exclude=exclude_query_params),+ build_canonical_headers(headers, signed_headers),+ signed_headers,+ payload_hash,+ ]+ )+++def build_string_to_sign(+ amz_date: str,+ date_stamp: str,+ region: str,+ service: str,+ canonical_request: str,+) -> str:+ """Build the SigV4 string to sign for a canonical request."""+ return "\n".join(+ [+ "AWS4-HMAC-SHA256",+ amz_date,+ f"{date_stamp}/{region}/{service}/aws4_request",+ hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),+ ]+ )+++def derive_signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:+ """Derive the SigV4 signing key for one date, region, and service."""++ def _sign(key: bytes, value: str) -> bytes:+ return hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest()++ date_key = _sign(("AWS4" + secret).encode("utf-8"), date_stamp)+ region_key = _sign(date_key, region)+ service_key = _sign(region_key, service)+ return _sign(service_key, "aws4_request")+++def calculate_signature(+ secret: str,+ date_stamp: str,+ region: str,+ service: str,+ string_to_sign: str,+) -> str:+ """Calculate the lowercase hexadecimal SigV4 signature."""+ signing_key = derive_signing_key(secret, date_stamp, region, service)+ return hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()+++def presigned_request_is_expired(+ amz_date: str,+ expires: str | int,+ *,+ now: dt.datetime | None = None,+) -> bool:+ """Return whether a presigned request is past its exclusive expiry time.++ Invalid timestamps and durations raise ``ValueError`` so each service can+ apply its own validation or compatibility behavior.+ """+ signed_at = dt.datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)+ current_time = now if now is not None else dt.datetime.now(dt.timezone.utc)+ return current_time > signed_at + dt.timedelta(seconds=int(expires))+++def signatures_match(computed: str, provided: str) -> bool:+ """Compare signatures without leaking a mismatch position through timing."""+ return hmac.compare_digest(computed, provided)diff --git a/ministack/services/s3.py b/ministack/services/s3.pyindex bb85fc19..fc3d363d 100644--- a/ministack/services/s3.py+++ b/ministack/services/s3.py@@ -29,7 +29,6 @@ import contextvarsimport copyimport datetime as _dtimport hashlib-import hmacimport jsonimport loggingimport os@@ -61,6 +60,13 @@ from ministack.core.responses import (set_request_account_id,set_request_region,)+from ministack.core.sigv4 import (+ build_canonical_request,+ build_string_to_sign,+ calculate_signature,+ presigned_request_is_expired,+ signatures_match,+)logger = logging.getLogger("s3")@@ -1492,27 +1498,6 @@ def _object_response_headers(obj: dict, bucket_name: str = "", key: str = "", in# ----------------------------------------------------------------------------_SIGV4_UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"---def _uri_encode(value: str, encode_slash: bool = True) -> str:- """RFC3986 encoding per the SigV4 spec: unreserved chars (A-Za-z0-9-_.~)- stay literal, everything else is percent-encoded. ``/`` is preserved in- the canonical URI (path separators) and encoded everywhere else."""- safe = "-_.~" + ("" if encode_slash else "/")- return url_quote(value, safe=safe)---def _sigv4_signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:- def _h(key, msg):- return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()-- k_date = _h(("AWS4" + secret).encode("utf-8"), date_stamp)- k_region = _h(k_date, region)- k_service = _h(k_region, service)- return _h(k_service, "aws4_request")--def _resolve_presign_secret(access_key_id):"""The secret a presigned URL was signed with.@@ -1574,55 +1559,30 @@ def _verify_presigned_sigv4(method, path, headers, query_params):expires = _qp(query_params, "X-Amz-Expires", "") or _qp(query_params, "x-amz-expires", "")if expires:try:- signed_at = _dt.datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=_dt.timezone.utc)- if _dt.datetime.now(_dt.timezone.utc) > signed_at + _dt.timedelta(seconds=int(expires)):+ if presigned_request_is_expired(amz_date, expires):return _error("AccessDenied", "Request has expired", 403, path)except (ValueError, TypeError):pass- # Canonical query string: every query param except X-Amz-Signature,- # RFC3986-encoded, sorted by encoded key then value.- pairs = []- for name, values in query_params.items():- if name.lower() == "x-amz-signature":- continue- vlist = values if isinstance(values, list) else [values]- for v in vlist:- pairs.append((_uri_encode(name), _uri_encode(v)))- pairs.sort()- canonical_qs = "&".join(f"{k}={v}" for k, v in pairs)-- # Canonical headers: the signed headers, lowercased names, trimmed values.- canonical_headers = ""- for hname in (h for h in signed_headers.split(";") if h):- raw = headers.get(hname, headers.get(hname.lower(), ""))- canonical_headers += f"{hname.lower()}:{' '.join(str(raw).split())}\n"-- canonical_request = "\n".join(- [- method,- _uri_encode(path, encode_slash=False),- canonical_qs,- canonical_headers,- signed_headers,- _SIGV4_UNSIGNED_PAYLOAD,- ]+ canonical_request = build_canonical_request(+ method,+ path,+ headers,+ query_params,+ signed_headers,)-- string_to_sign = "\n".join(- [- "AWS4-HMAC-SHA256",- amz_date,- f"{date_stamp}/{region}/{service}/aws4_request",- hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),- ]+ string_to_sign = build_string_to_sign(+ amz_date,+ date_stamp,+ region,+ service,+ canonical_request,)secret = _resolve_presign_secret(_akid)- signing_key = _sigv4_signing_key(secret, date_stamp, region, service)- computed = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()+ computed = calculate_signature(secret, date_stamp, region, service, string_to_sign)- if not hmac.compare_digest(computed, signature):+ if not signatures_match(computed, signature):return _bad_signature()return Nonediff --git a/tests/test_sigv4.py b/tests/test_sigv4.pynew file mode 100644index 00000000..308a9e0f--- /dev/null+++ b/tests/test_sigv4.py@@ -0,0 +1,157 @@+import datetime as dt++import pytest++from ministack.core.sigv4 import (+ build_canonical_headers,+ build_canonical_query_string,+ build_canonical_request,+ build_string_to_sign,+ calculate_signature,+ derive_signing_key,+ presigned_request_is_expired,+ signatures_match,+ uri_encode,+)+++ "value, encode_slash, expected",+ [+ ("abcXYZ012-_.~", True, "abcXYZ012-_.~"),+ ("space + percent%", True, "space%20%2B%20percent%25"),+ ("folder/object", True, "folder%2Fobject"),+ ("folder/object", False, "folder/object"),+ ("caf\N{LATIN SMALL LETTER E WITH ACUTE}", True, "caf%C3%A9"),+ ],+)+def test_uri_encode_uses_sigv4_rules(value, encode_slash, expected):+ assert uri_encode(value, encode_slash=encode_slash) == expected+++def test_build_canonical_query_string_encodes_sorts_and_excludes_signature():+ query_params = {+ "space key": ["b/a", "a+b"],+ "Action": "connect",+ "X-Amz-Signature": "not-part-of-the-canonical-query",+ }++ assert build_canonical_query_string(query_params) == (+ "Action=connect&space%20key=a%2Bb&space%20key=b%2Fa"+ )+++def test_build_canonical_query_string_accepts_custom_exclusions():+ query_params = {"Action": "connect", "DBUser": "alice", "Nonce": "123"}++ assert build_canonical_query_string(query_params, exclude=("nonce",)) == "Action=connect&DBUser=alice"+++def test_build_canonical_headers_normalizes_names_and_values():+ headers = {+ "host": "examplebucket.s3.amazonaws.com",+ "x-custom": " spaced\t value ",+ }++ assert build_canonical_headers(headers, "host;x-custom") == (+ "host:examplebucket.s3.amazonaws.com\n"+ "x-custom:spaced value\n"+ )+++def test_build_canonical_request_matches_fixed_vector():+ query_params = {+ "X-Amz-Algorithm": "AWS4-HMAC-SHA256",+ "X-Amz-Credential": "AKIDEXAMPLE/20130524/us-east-1/s3/aws4_request",+ "X-Amz-Date": "20130524T000000Z",+ "X-Amz-Expires": "86400",+ "X-Amz-SignedHeaders": "host",+ "X-Amz-Signature": "excluded",+ }++ assert build_canonical_request(+ "GET",+ "/test.txt",+ {"host": "examplebucket.s3.amazonaws.com"},+ query_params,+ "host",+ ) == (+ "GET\n"+ "/test.txt\n"+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&"+ "X-Amz-Credential=AKIDEXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&"+ "X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host\n"+ "host:examplebucket.s3.amazonaws.com\n"+ "\n"+ "host\n"+ "UNSIGNED-PAYLOAD"+ )+++def test_signing_helpers_match_botocore_fixed_vector():+ canonical_request = (+ "GET\n"+ "/test.txt\n"+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&"+ "X-Amz-Credential=AKIDEXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&"+ "X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host\n"+ "host:examplebucket.s3.amazonaws.com\n"+ "\n"+ "host\n"+ "UNSIGNED-PAYLOAD"+ )+ string_to_sign = build_string_to_sign(+ "20130524T000000Z",+ "20130524",+ "us-east-1",+ "s3",+ canonical_request,+ )++ assert string_to_sign == (+ "AWS4-HMAC-SHA256\n"+ "20130524T000000Z\n"+ "20130524/us-east-1/s3/aws4_request\n"+ "fe76c9a452b5c779479d88b7efe53bc3935d1a56dd76e83e930f401e91272d73"+ )+ assert derive_signing_key(+ "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",+ "20130524",+ "us-east-1",+ "s3",+ ).hex() == "f117494eff5d09da21cbf7f0339559ea04fc9582d31299cb992be70a6b27c97a"+ assert calculate_signature(+ "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",+ "20130524",+ "us-east-1",+ "s3",+ string_to_sign,+ ) == "ca6159ff16837c055653a722d9f10b6a529b7c62c84174a2859958324bc78766"+++def test_presigned_request_expiry_is_exclusive_at_boundary():+ expires_at = dt.datetime(2026, 9, 4, 12, 15, tzinfo=dt.timezone.utc)++ assert not presigned_request_is_expired("20260904T120000Z", "900", now=expires_at)+ assert presigned_request_is_expired(+ "20260904T120000Z",+ "900",+ now=expires_at + dt.timedelta(microseconds=1),+ )+++ "amz_date, expires",+ [+ ("not-a-date", "900"),+ ("20260904T120000Z", "not-a-duration"),+ ],+)+def test_presigned_request_expiry_rejects_malformed_values(amz_date, expires):+ with pytest.raises(ValueError):+ presigned_request_is_expired(amz_date, expires)+++def test_signatures_match_accepts_only_equal_values():+ assert signatures_match("abc123", "abc123")+ assert not signatures_match("abc123", "abc124")