Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

refactor(sigv4): share presigning primitives (#1634)

Ian Oberstdd9484cfa4ed
diff --git a/ministack/core/sigv4.py b/ministack/core/sigv4.py
new file mode 100644
index 00000000..5423468b
--- /dev/null
+++ b/ministack/core/sigv4.py
@@ -0,0 +1,134 @@
+"""Shared AWS Signature Version 4 primitives.
+
+This module owns only deterministic signing mechanics. Services remain
+responsible for validating their own credential scope, resolving secrets, and
+mapping verification failures to service-specific responses.
+"""
+
+import datetime as dt
+import hashlib
+import hmac
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import quote
+
+UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"
+
+
+def uri_encode(value: str, encode_slash: bool = True) -> str:
+ """Encode a SigV4 URI component using the RFC 3986 unreserved set."""
+ safe = "-_.~" + ("" if encode_slash else "/")
+ return quote(value, safe=safe)
+
+
+def build_canonical_query_string(
+ query_params: Mapping[str, Any],
+ *,
+ exclude: tuple[str, ...] = ("x-amz-signature",),
+) -> str:
+ """Encode and sort query parameters for a SigV4 canonical request."""
+ excluded = {name.lower() for name in exclude}
+ pairs = []
+ for name, values in query_params.items():
+ if name.lower() in excluded:
+ continue
+ value_list = values if isinstance(values, list) else [values]
+ for value in value_list:
+ pairs.append((uri_encode(name), uri_encode(value)))
+ pairs.sort()
+ return "&".join(f"{key}={value}" for key, value in pairs)
+
+
+def build_canonical_headers(headers: Mapping[str, Any], signed_headers: str) -> str:
+ """Normalize the headers named by a SigV4 signed-headers value."""
+ canonical = ""
+ for name in (name for name in signed_headers.split(";") if name):
+ raw = headers.get(name, headers.get(name.lower(), ""))
+ canonical += f"{name.lower()}:{' '.join(str(raw).split())}\n"
+ return canonical
+
+
+def build_canonical_request(
+ method: str,
+ path: str,
+ headers: Mapping[str, Any],
+ query_params: Mapping[str, Any],
+ signed_headers: str,
+ *,
+ payload_hash: str = UNSIGNED_PAYLOAD,
+ exclude_query_params: tuple[str, ...] = ("x-amz-signature",),
+) -> str:
+ """Build the canonical request hashed by SigV4."""
+ return "\n".join(
+ [
+ method,
+ uri_encode(path, encode_slash=False),
+ build_canonical_query_string(query_params, exclude=exclude_query_params),
+ build_canonical_headers(headers, signed_headers),
+ signed_headers,
+ payload_hash,
+ ]
+ )
+
+
+def build_string_to_sign(
+ amz_date: str,
+ date_stamp: str,
+ region: str,
+ service: str,
+ canonical_request: str,
+) -> str:
+ """Build the SigV4 string to sign for a canonical request."""
+ return "\n".join(
+ [
+ "AWS4-HMAC-SHA256",
+ amz_date,
+ f"{date_stamp}/{region}/{service}/aws4_request",
+ hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
+ ]
+ )
+
+
+def derive_signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:
+ """Derive the SigV4 signing key for one date, region, and service."""
+
+ def _sign(key: bytes, value: str) -> bytes:
+ return hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest()
+
+ date_key = _sign(("AWS4" + secret).encode("utf-8"), date_stamp)
+ region_key = _sign(date_key, region)
+ service_key = _sign(region_key, service)
+ return _sign(service_key, "aws4_request")
+
+
+def calculate_signature(
+ secret: str,
+ date_stamp: str,
+ region: str,
+ service: str,
+ string_to_sign: str,
+) -> str:
+ """Calculate the lowercase hexadecimal SigV4 signature."""
+ signing_key = derive_signing_key(secret, date_stamp, region, service)
+ return hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
+
+
+def presigned_request_is_expired(
+ amz_date: str,
+ expires: str | int,
+ *,
+ now: dt.datetime | None = None,
+) -> bool:
+ """Return whether a presigned request is past its exclusive expiry time.
+
+ Invalid timestamps and durations raise ``ValueError`` so each service can
+ apply its own validation or compatibility behavior.
+ """
+ signed_at = dt.datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)
+ current_time = now if now is not None else dt.datetime.now(dt.timezone.utc)
+ return current_time > signed_at + dt.timedelta(seconds=int(expires))
+
+
+def signatures_match(computed: str, provided: str) -> bool:
+ """Compare signatures without leaking a mismatch position through timing."""
+ return hmac.compare_digest(computed, provided)
diff --git a/ministack/services/s3.py b/ministack/services/s3.py
index bb85fc19..fc3d363d 100644
--- a/ministack/services/s3.py
+++ b/ministack/services/s3.py
@@ -29,7 +29,6 @@ import contextvars
import copy
import datetime as _dt
import hashlib
-import hmac
import json
import logging
import os
@@ -61,6 +60,13 @@ from ministack.core.responses import (
set_request_account_id,
set_request_region,
)
+from ministack.core.sigv4 import (
+ build_canonical_request,
+ build_string_to_sign,
+ calculate_signature,
+ presigned_request_is_expired,
+ signatures_match,
+)
logger = logging.getLogger("s3")
@@ -1492,27 +1498,6 @@ def _object_response_headers(obj: dict, bucket_name: str = "", key: str = "", in
# ---------------------------------------------------------------------------
-_SIGV4_UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"
-
-
-def _uri_encode(value: str, encode_slash: bool = True) -> str:
- """RFC3986 encoding per the SigV4 spec: unreserved chars (A-Za-z0-9-_.~)
- stay literal, everything else is percent-encoded. ``/`` is preserved in
- the canonical URI (path separators) and encoded everywhere else."""
- safe = "-_.~" + ("" if encode_slash else "/")
- return url_quote(value, safe=safe)
-
-
-def _sigv4_signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:
- def _h(key, msg):
- return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
-
- k_date = _h(("AWS4" + secret).encode("utf-8"), date_stamp)
- k_region = _h(k_date, region)
- k_service = _h(k_region, service)
- return _h(k_service, "aws4_request")
-
-
def _resolve_presign_secret(access_key_id):
"""The secret a presigned URL was signed with.
@@ -1574,55 +1559,30 @@ def _verify_presigned_sigv4(method, path, headers, query_params):
expires = _qp(query_params, "X-Amz-Expires", "") or _qp(query_params, "x-amz-expires", "")
if expires:
try:
- signed_at = _dt.datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=_dt.timezone.utc)
- if _dt.datetime.now(_dt.timezone.utc) > signed_at + _dt.timedelta(seconds=int(expires)):
+ if presigned_request_is_expired(amz_date, expires):
return _error("AccessDenied", "Request has expired", 403, path)
except (ValueError, TypeError):
pass
- # Canonical query string: every query param except X-Amz-Signature,
- # RFC3986-encoded, sorted by encoded key then value.
- pairs = []
- for name, values in query_params.items():
- if name.lower() == "x-amz-signature":
- continue
- vlist = values if isinstance(values, list) else [values]
- for v in vlist:
- pairs.append((_uri_encode(name), _uri_encode(v)))
- pairs.sort()
- canonical_qs = "&".join(f"{k}={v}" for k, v in pairs)
-
- # Canonical headers: the signed headers, lowercased names, trimmed values.
- canonical_headers = ""
- for hname in (h for h in signed_headers.split(";") if h):
- raw = headers.get(hname, headers.get(hname.lower(), ""))
- canonical_headers += f"{hname.lower()}:{' '.join(str(raw).split())}\n"
-
- canonical_request = "\n".join(
- [
- method,
- _uri_encode(path, encode_slash=False),
- canonical_qs,
- canonical_headers,
- signed_headers,
- _SIGV4_UNSIGNED_PAYLOAD,
- ]
+ canonical_request = build_canonical_request(
+ method,
+ path,
+ headers,
+ query_params,
+ signed_headers,
)
-
- string_to_sign = "\n".join(
- [
- "AWS4-HMAC-SHA256",
- amz_date,
- f"{date_stamp}/{region}/{service}/aws4_request",
- hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
- ]
+ string_to_sign = build_string_to_sign(
+ amz_date,
+ date_stamp,
+ region,
+ service,
+ canonical_request,
)
secret = _resolve_presign_secret(_akid)
- signing_key = _sigv4_signing_key(secret, date_stamp, region, service)
- computed = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
+ computed = calculate_signature(secret, date_stamp, region, service, string_to_sign)
- if not hmac.compare_digest(computed, signature):
+ if not signatures_match(computed, signature):
return _bad_signature()
return None
diff --git a/tests/test_sigv4.py b/tests/test_sigv4.py
new file mode 100644
index 00000000..308a9e0f
--- /dev/null
+++ b/tests/test_sigv4.py
@@ -0,0 +1,157 @@
+import datetime as dt
+
+import pytest
+
+from ministack.core.sigv4 import (
+ build_canonical_headers,
+ build_canonical_query_string,
+ build_canonical_request,
+ build_string_to_sign,
+ calculate_signature,
+ derive_signing_key,
+ presigned_request_is_expired,
+ signatures_match,
+ uri_encode,
+)
+
+
+ "value, encode_slash, expected",
+ [
+ ("abcXYZ012-_.~", True, "abcXYZ012-_.~"),
+ ("space + percent%", True, "space%20%2B%20percent%25"),
+ ("folder/object", True, "folder%2Fobject"),
+ ("folder/object", False, "folder/object"),
+ ("caf\N{LATIN SMALL LETTER E WITH ACUTE}", True, "caf%C3%A9"),
+ ],
+)
+def test_uri_encode_uses_sigv4_rules(value, encode_slash, expected):
+ assert uri_encode(value, encode_slash=encode_slash) == expected
+
+
+def test_build_canonical_query_string_encodes_sorts_and_excludes_signature():
+ query_params = {
+ "space key": ["b/a", "a+b"],
+ "Action": "connect",
+ "X-Amz-Signature": "not-part-of-the-canonical-query",
+ }
+
+ assert build_canonical_query_string(query_params) == (
+ "Action=connect&space%20key=a%2Bb&space%20key=b%2Fa"
+ )
+
+
+def test_build_canonical_query_string_accepts_custom_exclusions():
+ query_params = {"Action": "connect", "DBUser": "alice", "Nonce": "123"}
+
+ assert build_canonical_query_string(query_params, exclude=("nonce",)) == "Action=connect&DBUser=alice"
+
+
+def test_build_canonical_headers_normalizes_names_and_values():
+ headers = {
+ "host": "examplebucket.s3.amazonaws.com",
+ "x-custom": " spaced\t value ",
+ }
+
+ assert build_canonical_headers(headers, "host;x-custom") == (
+ "host:examplebucket.s3.amazonaws.com\n"
+ "x-custom:spaced value\n"
+ )
+
+
+def test_build_canonical_request_matches_fixed_vector():
+ query_params = {
+ "X-Amz-Algorithm": "AWS4-HMAC-SHA256",
+ "X-Amz-Credential": "AKIDEXAMPLE/20130524/us-east-1/s3/aws4_request",
+ "X-Amz-Date": "20130524T000000Z",
+ "X-Amz-Expires": "86400",
+ "X-Amz-SignedHeaders": "host",
+ "X-Amz-Signature": "excluded",
+ }
+
+ assert build_canonical_request(
+ "GET",
+ "/test.txt",
+ {"host": "examplebucket.s3.amazonaws.com"},
+ query_params,
+ "host",
+ ) == (
+ "GET\n"
+ "/test.txt\n"
+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&"
+ "X-Amz-Credential=AKIDEXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&"
+ "X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host\n"
+ "host:examplebucket.s3.amazonaws.com\n"
+ "\n"
+ "host\n"
+ "UNSIGNED-PAYLOAD"
+ )
+
+
+def test_signing_helpers_match_botocore_fixed_vector():
+ canonical_request = (
+ "GET\n"
+ "/test.txt\n"
+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&"
+ "X-Amz-Credential=AKIDEXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&"
+ "X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host\n"
+ "host:examplebucket.s3.amazonaws.com\n"
+ "\n"
+ "host\n"
+ "UNSIGNED-PAYLOAD"
+ )
+ string_to_sign = build_string_to_sign(
+ "20130524T000000Z",
+ "20130524",
+ "us-east-1",
+ "s3",
+ canonical_request,
+ )
+
+ assert string_to_sign == (
+ "AWS4-HMAC-SHA256\n"
+ "20130524T000000Z\n"
+ "20130524/us-east-1/s3/aws4_request\n"
+ "fe76c9a452b5c779479d88b7efe53bc3935d1a56dd76e83e930f401e91272d73"
+ )
+ assert derive_signing_key(
+ "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
+ "20130524",
+ "us-east-1",
+ "s3",
+ ).hex() == "f117494eff5d09da21cbf7f0339559ea04fc9582d31299cb992be70a6b27c97a"
+ assert calculate_signature(
+ "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
+ "20130524",
+ "us-east-1",
+ "s3",
+ string_to_sign,
+ ) == "ca6159ff16837c055653a722d9f10b6a529b7c62c84174a2859958324bc78766"
+
+
+def test_presigned_request_expiry_is_exclusive_at_boundary():
+ expires_at = dt.datetime(2026, 9, 4, 12, 15, tzinfo=dt.timezone.utc)
+
+ assert not presigned_request_is_expired("20260904T120000Z", "900", now=expires_at)
+ assert presigned_request_is_expired(
+ "20260904T120000Z",
+ "900",
+ now=expires_at + dt.timedelta(microseconds=1),
+ )
+
+
+ "amz_date, expires",
+ [
+ ("not-a-date", "900"),
+ ("20260904T120000Z", "not-a-duration"),
+ ],
+)
+def test_presigned_request_expiry_rejects_malformed_values(amz_date, expires):
+ with pytest.raises(ValueError):
+ presigned_request_is_expired(amz_date, expires)
+
+
+def test_signatures_match_accepts_only_equal_values():
+ assert signatures_match("abc123", "abc123")
+ assert not signatures_match("abc123", "abc124")