Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(cloudtrail): add CloudTrail service with in-memory audit log (#547)
Akhil Adiga7bb1857b15c0
diff --git a/ministack/app.py b/ministack/app.pyindex b448e012..2f174f48 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -249,6 +249,7 @@ SERVICE_REGISTRY = {"waf": {"module": "waf_v1"},"waf-regional": {"module": "waf_v1"},"wafv2": {"module": "waf"},+ "cloudtrail": {"module": "cloudtrail"},}SERVICE_HANDLERS = {@@ -282,6 +283,7 @@ _state_map = {"scheduler": "scheduler", "autoscaling": "autoscaling","eks": "eks", "backup": "backup", "pipes": "pipes","cloudfront_keyvaluestore": "cloudfront_keyvaluestore",+ "cloudtrail": "cloudtrail",}SERVICE_NAME_ALIASES = {@@ -722,6 +724,7 @@ async def _handle_admin_config_request(path: str, method: str, body: bytes):"stepfunctions._sfn_mock_config","stepfunctions._SFN_WAIT_SCALE","lambda_svc.LAMBDA_EXECUTOR",+ "cloudtrail._recording_enabled",}try:config = json.loads(body) if body else {}@@ -749,6 +752,8 @@ async def _handle_admin_config_request(path: str, method: str, body: bytes):logger.warning("/_ministack/config: invalid SFN_WAIT_SCALE=%r", value)continuevalue = float_value+ elif key == "cloudtrail._recording_enabled":+ value = str(value).lower() in ("1", "true", "yes")setattr(mod, var_name, value)applied[key] = valueexcept (ImportError, AttributeError) as e:@@ -1098,6 +1103,186 @@ async def _handle_special_data_plane_request(return None+# ---------------------------------------------------------------------------+# CloudTrail event recording helpers+# ---------------------------------------------------------------------------++_S3_PATH_EVENTS = {+ ("GET", 0): "ListBuckets",+ ("PUT", 1): "CreateBucket",+ ("DELETE", 1): "DeleteBucket",+ ("HEAD", 1): "HeadBucket",+ ("GET", 1): "ListObjects",+ ("PUT", 2): "PutObject",+ ("GET", 2): "GetObject",+ ("DELETE", 2): "DeleteObject",+ ("HEAD", 2): "HeadObject",+ ("POST", 2): "CreateMultipartUpload",+}+++def _ct_event_name(service: str, method: str, path: str, headers: dict, query_params: dict) -> str:+ target = headers.get("x-amz-target", "")+ if target and "." in target:+ return target.rsplit(".", 1)[-1]++ action = query_params.get("Action", "")+ if isinstance(action, list):+ action = action[0] if action else ""+ if action:+ return action++ if service == "s3":+ parts = [p for p in path.split("/") if p]+ depth = min(len(parts), 2)+ return _S3_PATH_EVENTS.get((method, depth), f"{method}.s3")++ if service == "lambda":+ parts = [p for p in path.split("/") if p]+ if "functions" in parts:+ fi = parts.index("functions")+ rest = parts[fi + 1 :]+ if not rest:+ return "CreateFunction" if method == "POST" else "ListFunctions"+ sub = rest[1] if len(rest) > 1 else None+ _sub_map = {+ "invocations": "Invoke",+ "code": "UpdateFunctionCode",+ "configuration": "UpdateFunctionConfiguration",+ "aliases": "CreateAlias" if method == "POST" else "ListAliases",+ "versions": "PublishVersion" if method == "POST" else "ListVersionsByFunction",+ }+ if sub in _sub_map:+ return _sub_map[sub]+ return {"GET": "GetFunction", "DELETE": "DeleteFunction", "PUT": "UpdateFunctionCode"}.get(+ method, f"{method}.lambda"+ )++ return f"{method}.{service}"+++def _ct_resources(service: str, method: str, path: str, body: bytes) -> list:+ if service == "s3":+ parts = [p for p in path.split("/") if p]+ if not parts:+ return []+ resources = [{"ResourceName": parts[0], "ResourceType": "AWS::S3::Bucket"}]+ if len(parts) >= 2:+ resources.append(+ {"ResourceName": "/".join(parts[1:]), "ResourceType": "AWS::S3::Object"}+ )+ return resources++ if service in ("dynamodb", "lambda", "sqs", "sns", "kinesis"):+ try:+ parsed = json.loads(body) if body else {}+ except Exception:+ parsed = {}++ if service == "dynamodb":+ table = parsed.get("TableName", "")+ if table:+ return [{"ResourceName": table, "ResourceType": "AWS::DynamoDB::Table"}]++ if service == "lambda":+ fn = parsed.get("FunctionName", "")+ if not fn:+ parts = [p for p in path.split("/") if p]+ if "functions" in parts:+ fi = parts.index("functions")+ rest = parts[fi + 1 :]+ fn = rest[0] if rest else ""+ if fn:+ return [{"ResourceName": fn, "ResourceType": "AWS::Lambda::Function"}]++ if service == "sqs":+ parts = [p for p in path.split("/") if p]+ if len(parts) >= 2:+ return [{"ResourceName": parts[-1], "ResourceType": "AWS::SQS::Queue"}]++ if service == "sns":+ topic = parsed.get("TopicArn", "")+ if topic:+ return [{"ResourceName": topic, "ResourceType": "AWS::SNS::Topic"}]++ if service == "kinesis":+ stream = parsed.get("StreamName", "")+ if stream:+ return [{"ResourceName": stream, "ResourceType": "AWS::Kinesis::Stream"}]++ return []+++def _ct_request_params(headers: dict, body: bytes, query_params: dict) -> dict:+ ct = headers.get("content-type", "")+ if "json" in ct:+ try:+ return json.loads(body) if body else {}+ except Exception:+ return {}+ if "form" in ct:+ try:+ from urllib.parse import parse_qs as _pqs+ raw = {k: v[0] if len(v) == 1 else v for k, v in _pqs(body.decode("utf-8", errors="replace")).items()}+ return raw+ except Exception:+ return {}+ return {}+++def _maybe_record_cloudtrail(+ service: str,+ method: str,+ path: str,+ headers: dict,+ body: bytes,+ query_params: dict,+ request_id: str,+ region: str,+):+ """Best-effort CloudTrail event recording.++ Zero hot-path cost when CLOUDTRAIL_RECORDING is not set: the cloudtrail+ module is never loaded and the dict lookup short-circuits immediately.+ When CLOUDTRAIL_RECORDING=1 is set, the module is loaded on the first+ request so recording begins from the very first API call, not just after+ someone has explicitly called a CloudTrail endpoint.+ """+ if service == "cloudtrail" or path.startswith("/_"):+ return+ ct_mod = _loaded_modules.get("cloudtrail")+ if ct_mod is None:+ # Only pay the import cost if CLOUDTRAIL_RECORDING is explicitly on.+ # This keeps the default-off hot path to a single O(1) dict lookup.+ if os.environ.get("CLOUDTRAIL_RECORDING", "0") != "1":+ return+ ct_mod = _get_module("cloudtrail")+ if isinstance(ct_mod, _ErrorModule):+ return+ if not getattr(ct_mod, "_recording_enabled", False):+ return+ try:+ event_name = _ct_event_name(service, method, path, headers, query_params)+ resources = _ct_resources(service, method, path, body)+ access_key_id = extract_access_key_id(headers) or "test"+ user_agent = headers.get("user-agent", "")+ request_params = _ct_request_params(headers, body, query_params)+ ct_mod.record_event(+ service=service,+ event_name=event_name,+ username=access_key_id,+ access_key_id=access_key_id,+ resources=resources,+ region=region,+ request_id=request_id,+ user_agent=user_agent,+ request_params=request_params,+ method=method,+ )+ except Exception:+ pass++# ---------------------------------------------------------------------------# Tier 4 — Generic service dispatch# ---------------------------------------------------------------------------@@ -1143,6 +1328,8 @@ async def _dispatch_service_request(json.dumps({"__type": "InternalError", "message": str(e)}).encode(),)+ _maybe_record_cloudtrail(service, method, path, headers, body, query_params, request_id, region)+resp_headers.update({"Access-Control-Allow-Origin": "*",diff --git a/ministack/core/router.py b/ministack/core/router.pyindex f771c015..cedef118 100644--- a/ministack/core/router.py+++ b/ministack/core/router.py@@ -308,6 +308,11 @@ SERVICE_PATTERNS = {"path_prefixes": ["/backup-vaults", "/backup/plans", "/backup-jobs", "/untag"],"credential_scope": "backup",},+ "cloudtrail": {+ "target_prefixes": ["com.amazonaws.cloudtrail.v20131101.AmazonCloudTrailService"],+ "host_patterns": [r"cloudtrail\."],+ "credential_scope": "cloudtrail",+ },}@@ -385,6 +390,7 @@ def detect_service(method: str, path: str, headers: dict, query_params: dict) ->"scheduler": "scheduler","eks": "eks","tagging": "tagging",+ "cloudtrail": "cloudtrail",}if svc_name in scope_map:return scope_map[svc_name]diff --git a/ministack/services/cloudtrail.py b/ministack/services/cloudtrail.pynew file mode 100644index 00000000..b28dfd6b--- /dev/null+++ b/ministack/services/cloudtrail.py@@ -0,0 +1,442 @@+"""+AWS CloudTrail Service Emulator.+In-memory audit log — records all API calls and exposes LookupEvents for test assertions.++Recording is off by default. Enable with CLOUDTRAIL_RECORDING=1 or via+POST /_ministack/config {"cloudtrail._recording_enabled": "true"}.+Cap the per-account ring buffer with CLOUDTRAIL_MAX_EVENTS (default 10000).++Supported operations:+ Audit log: LookupEvents+ Control plane: CreateTrail, DeleteTrail, GetTrail, DescribeTrails,+ GetTrailStatus, StartLogging, StopLogging,+ PutEventSelectors, GetEventSelectors,+ AddTags, ListTags, RemoveTags+"""++import collections+import json+import logging+import os+import time++from ministack.core.persistence import load_state+from ministack.core.responses import AccountScopedDict, get_account_id, get_region, new_uuid++logger = logging.getLogger("cloudtrail")++_recording_enabled: bool = os.environ.get("CLOUDTRAIL_RECORDING", "0") == "1"+_MAX_EVENTS: int = int(os.environ.get("CLOUDTRAIL_MAX_EVENTS", "10000"))++_events = AccountScopedDict() # "events" -> deque[event_dict]+_trails = AccountScopedDict() # trail_name -> trail_record+_event_selectors = AccountScopedDict() # trail_name -> list[EventSelector]+_trail_tags = AccountScopedDict() # trail_arn -> {tag_key: tag_value}++_SCRUB_KEYS = frozenset(+ {+ "secretaccesskey",+ "password",+ "authtoken",+ "signature",+ "authorization",+ "x-amz-security-token",+ "credentials",+ "secretstring",+ "secretbinary",+ }+)+++def reset():+ _events.clear()+ _trails.clear()+ _event_selectors.clear()+ _trail_tags.clear()+++def get_state():+ # Trail config persists; events are ephemeral (timestamps meaningless after restart).+ return {+ "trails": {k: v for k, v in _trails.items()},+ "event_selectors": {k: v for k, v in _event_selectors.items()},+ "trail_tags": {k: v for k, v in _trail_tags.items()},+ }+++def restore_state(data):+ if not isinstance(data, dict):+ return+ for k, v in data.get("trails", {}).items():+ _trails[k] = v+ for k, v in data.get("event_selectors", {}).items():+ _event_selectors[k] = v+ for k, v in data.get("trail_tags", {}).items():+ _trail_tags[k] = v+++def load_persisted_state(data):+ restore_state(data)+++def _scrub(params: dict) -> dict:+ if not isinstance(params, dict):+ return {}+ return {+ k: "***REDACTED***" if k.lower() in _SCRUB_KEYS else v+ for k, v in params.items()+ }+++def _trail_arn(name: str) -> str:+ return f"arn:aws:cloudtrail:{get_region()}:{get_account_id()}:trail/{name}"+++def _get_event_queue() -> collections.deque:+ q = _events.get("events")+ if q is None:+ q = collections.deque(maxlen=_MAX_EVENTS)+ _events["events"] = q+ return q+++def record_event(+ service: str,+ event_name: str,+ username: str,+ access_key_id: str,+ resources: list,+ region: str,+ request_id: str,+ user_agent: str,+ request_params: dict,+ method: str,+):+ """Append an API call as a CloudTrail event. Called from the ASGI dispatch loop.+ Only runs when _recording_enabled is True."""+ account_id = get_account_id()+ event_id = new_uuid()+ ts = time.time()+ readonly = _is_readonly(method, event_name)++ event_source = f"{service}.amazonaws.com"++ ct_event = {+ "eventVersion": "1.08",+ "userIdentity": {+ "type": "IAMUser",+ "principalId": access_key_id or "AIDATEST",+ "arn": f"arn:aws:iam::{account_id}:user/{username or 'test'}",+ "accountId": account_id,+ "accessKeyId": access_key_id or "",+ "userName": username or "test",+ },+ "eventTime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(ts)),+ "eventSource": event_source,+ "eventName": event_name,+ "awsRegion": region,+ "sourceIPAddress": "127.0.0.1",+ "userAgent": user_agent or "boto3",+ "requestParameters": _scrub(request_params),+ "responseElements": None,+ "requestID": request_id,+ "eventID": event_id,+ "eventType": "AwsApiCall",+ "readOnly": readonly,+ "recipientAccountId": account_id,+ }++ event_record = {+ "EventId": event_id,+ "EventName": event_name,+ "EventSource": event_source,+ "EventTime": ts,+ "Username": username or "test",+ "AccessKeyId": access_key_id or "",+ "ReadOnly": readonly,+ "Resources": list(resources),+ "CloudTrailEvent": json.dumps(ct_event),+ }++ _get_event_queue().append(event_record)+++def _is_readonly(method: str, event_name: str) -> str:+ if method in ("GET", "HEAD"):+ return "true"+ for prefix in ("Get", "List", "Describe", "Head"):+ if event_name.startswith(prefix):+ return "true"+ return "false"+++def _ok(body: dict):+ return 200, {"Content-Type": "application/x-amz-json-1.1"}, json.dumps(body).encode()+++def _err(code: str, msg: str, status: int = 400):+ return (+ status,+ {"Content-Type": "application/x-amz-json-1.1"},+ json.dumps({"__type": code, "message": msg}).encode(),+ )+++def _lookup_events(body: dict):+ event_queue = _get_event_queue()++ attrs = {a["AttributeKey"]: a["AttributeValue"] for a in body.get("LookupAttributes", [])}+ start_time = body.get("StartTime")+ end_time = body.get("EndTime")+ try:+ max_results = min(int(body.get("MaxResults", 50)), 50)+ except (TypeError, ValueError):+ max_results = 50++ filtered = []+ for ev in reversed(list(event_queue)): # newest first+ t = ev["EventTime"]+ if start_time is not None and t < float(start_time):+ continue+ if end_time is not None and t > float(end_time):+ continue+ if "EventName" in attrs and ev["EventName"] != attrs["EventName"]:+ continue+ if "Username" in attrs and ev["Username"] != attrs["Username"]:+ continue+ if "AccessKeyId" in attrs and ev.get("AccessKeyId", "") != attrs["AccessKeyId"]:+ continue+ if "ReadOnly" in attrs and ev.get("ReadOnly", "false") != attrs["ReadOnly"]:+ continue+ if "EventId" in attrs and ev["EventId"] != attrs["EventId"]:+ continue+ if "ResourceName" in attrs:+ rn = attrs["ResourceName"]+ if not any(r.get("ResourceName") == rn for r in ev.get("Resources", [])):+ continue+ if "ResourceType" in attrs:+ rt = attrs["ResourceType"]+ if not any(r.get("ResourceType") == rt for r in ev.get("Resources", [])):+ continue+ if "EventSource" in attrs and ev.get("EventSource", "") != attrs["EventSource"]:+ continue+ filtered.append(ev)+ if len(filtered) >= max_results:+ break++ return _ok({"Events": filtered})+++def _resolve_trail_name(name: str) -> str:+ if name.startswith("arn:"):+ return name.rsplit("/", 1)[-1]+ return name+++def _create_trail(body: dict):+ name = body.get("Name", "").strip()+ if not name:+ return _err("InvalidTrailNameException", "Trail name is required.")+ if _trails.get(name) is not None:+ return _err("TrailAlreadyExistsException", f"Trail {name!r} already exists.")+ arn = _trail_arn(name)+ trail = {+ "Name": name,+ "S3BucketName": body.get("S3BucketName", ""),+ "S3KeyPrefix": body.get("S3KeyPrefix", ""),+ "SnsTopicName": body.get("SnsTopicName", ""),+ "IncludeGlobalServiceEvents": body.get("IncludeGlobalServiceEvents", True),+ "IsMultiRegionTrail": body.get("IsMultiRegionTrail", False),+ "LogFileValidationEnabled": body.get("EnableLogFileValidation", False),+ "HomeRegion": get_region(),+ "TrailARN": arn,+ "HasCustomEventSelectors": False,+ "HasInsightSelectors": False,+ "IsOrganizationTrail": body.get("IsOrganizationTrail", False),+ "IsLogging": True,+ }+ _trails[name] = trail+ return _ok(+ {+ "Name": name,+ "S3BucketName": trail["S3BucketName"],+ "S3KeyPrefix": trail["S3KeyPrefix"],+ "IncludeGlobalServiceEvents": trail["IncludeGlobalServiceEvents"],+ "IsMultiRegionTrail": trail["IsMultiRegionTrail"],+ "TrailARN": arn,+ "LogFileValidationEnabled": trail["LogFileValidationEnabled"],+ "IsOrganizationTrail": trail["IsOrganizationTrail"],+ }+ )+++def _delete_trail(body: dict):+ raw = body.get("Name", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ if _trails.get(name) is None:+ return _err("TrailNotFoundException", f"Unknown trail: {name!r}", 404)+ del _trails[name]+ _event_selectors.pop(name, None)+ return _ok({})+++def _get_trail(body: dict):+ raw = body.get("Name", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ trail = _trails.get(name)+ if trail is None:+ return _err("TrailNotFoundException", f"Unknown trail: {name!r}", 404)+ return _ok({"Trail": trail})+++def _describe_trails(body: dict):+ trail_names = body.get("trailNameList", [])+ all_trails = [v for _, v in _trails.items()]+ if trail_names:+ resolved = {_resolve_trail_name(n) for n in trail_names}+ all_trails = [t for t in all_trails if t["Name"] in resolved]+ return _ok({"trailList": all_trails})+++def _get_trail_status(body: dict):+ raw = body.get("Name", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ if _trails.get(name) is None:+ return _err("TrailNotFoundException", f"Unknown trail: {name!r}", 404)+ now = time.time()+ return _ok(+ {+ "IsLogging": True,+ "LatestDeliveryTime": now,+ "StartLoggingTime": now - 3600,+ "StopLoggingTime": None,+ "LatestDeliveryError": "",+ "LatestNotificationError": "",+ }+ )+++def _start_logging(body: dict):+ raw = body.get("Name", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ if _trails.get(name) is None:+ return _err("TrailNotFoundException", f"Unknown trail: {name!r}", 404)+ return _ok({})+++def _stop_logging(body: dict):+ raw = body.get("Name", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ if _trails.get(name) is None:+ return _err("TrailNotFoundException", f"Unknown trail: {name!r}", 404)+ return _ok({})+++def _put_event_selectors(body: dict):+ raw = body.get("TrailName", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ selectors = body.get("EventSelectors", [])+ _event_selectors[name] = selectors+ return _ok({"TrailARN": _trail_arn(name), "EventSelectors": selectors})+++def _get_event_selectors(body: dict):+ raw = body.get("TrailName", "").strip()+ if not raw:+ return _err("InvalidTrailNameException", "Trail name is required.")+ name = _resolve_trail_name(raw)+ selectors = _event_selectors.get(name) or []+ return _ok(+ {+ "TrailARN": _trail_arn(name),+ "EventSelectors": selectors,+ "AdvancedEventSelectors": [],+ }+ )+++def _add_tags(body: dict):+ arn = body.get("ResourceId", "").strip()+ if not arn:+ return _err("CloudTrailARNInvalidException", "ResourceId (trail ARN) is required.")+ existing = _trail_tags.get(arn) or {}+ for tag in body.get("TagsList", []):+ existing[tag["Key"]] = tag["Value"]+ _trail_tags[arn] = existing+ return _ok({})+++def _list_tags(body: dict):+ arns = body.get("ResourceIdList", [])+ result = [+ {+ "ResourceId": arn,+ "TagsList": [{"Key": k, "Value": v} for k, v in (_trail_tags.get(arn) or {}).items()],+ }+ for arn in arns+ ]+ return _ok({"ResourceTagList": result})+++def _remove_tags(body: dict):+ arn = body.get("ResourceId", "").strip()+ if not arn:+ return _err("CloudTrailARNInvalidException", "ResourceId (trail ARN) is required.")+ existing = _trail_tags.get(arn) or {}+ for tag in body.get("TagsList", []):+ existing.pop(tag.get("Key", ""), None)+ _trail_tags[arn] = existing+ return _ok({})+++_DISPATCH = {+ "LookupEvents": _lookup_events,+ "CreateTrail": _create_trail,+ "DeleteTrail": _delete_trail,+ "GetTrail": _get_trail,+ "DescribeTrails": _describe_trails,+ "GetTrailStatus": _get_trail_status,+ "StartLogging": _start_logging,+ "StopLogging": _stop_logging,+ "PutEventSelectors": _put_event_selectors,+ "GetEventSelectors": _get_event_selectors,+ "AddTags": _add_tags,+ "ListTags": _list_tags,+ "RemoveTags": _remove_tags,+}+++async def handle_request(method, path, headers, body_bytes, query_params):+ target = headers.get("x-amz-target", "")+ action = target.rsplit(".", 1)[-1] if "." in target else target++ try:+ body = json.loads(body_bytes) if body_bytes else {}+ except json.JSONDecodeError:+ body = {}++ handler = _DISPATCH.get(action)+ if handler is None:+ logger.warning("cloudtrail: unknown action %r", action)+ return _err("InvalidParameterException", f"Unknown CloudTrail action: {action!r}")+ return handler(body)+++try:+ _restored = load_state("cloudtrail")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted cloudtrail state; continuing fresh")diff --git a/tests/bench_recording.py b/tests/bench_recording.pynew file mode 100644index 00000000..4e5e6b05--- /dev/null+++ b/tests/bench_recording.py@@ -0,0 +1,191 @@+"""+Benchmark: CloudTrail recording hook overhead.++Measures req/s and per-request latency across three service protocols+(S3 REST, DynamoDB JSON-target, SQS query-string) with recording off+and on, so the delta can be included in the PR description.++Usage:+ # Start ministack first, then run:+ python tests/bench_recording.py # recording off+ CLOUDTRAIL_RECORDING=1 python tests/bench_recording.py # recording on++ # Or compare both in one run (server must already have recording toggled):+ python tests/bench_recording.py --compare++Options:+ --endpoint Ministack URL (default: http://localhost:4566)+ --n Requests per scenario (default: 300)+ --compare Run off then on by toggling via /_ministack/config (no restart needed)+"""++import argparse+import statistics+import time+import uuid++import boto3+import requests++ENDPOINT = "http://localhost:4566"+REGION = "us-east-1"+++def _client(service, endpoint=ENDPOINT):+ return boto3.client(+ service,+ endpoint_url=endpoint,+ aws_access_key_id="test",+ aws_secret_access_key="test",+ region_name=REGION,+ )+++def _uid():+ return uuid.uuid4().hex[:8]+++def _set_recording(enabled: bool, endpoint=ENDPOINT):+ resp = requests.post(+ f"{endpoint}/_ministack/config",+ json={"cloudtrail._recording_enabled": "true" if enabled else "false"},+ )+ if resp.status_code != 200:+ raise RuntimeError(f"Failed to toggle recording: {resp.text}")+++def _reset(endpoint=ENDPOINT):+ requests.post(f"{endpoint}/_ministack/reset")+++def _percentile(data, pct):+ return statistics.quantiles(data, n=100)[pct - 1]+++def bench_s3(n, endpoint=ENDPOINT):+ """GET on a known bucket — exercises the S3 REST path."""+ s3 = _client("s3", endpoint)+ bucket = f"bench-{_uid()}"+ s3.create_bucket(Bucket=bucket)++ latencies = []+ for _ in range(n):+ t0 = time.perf_counter()+ s3.head_bucket(Bucket=bucket)+ latencies.append((time.perf_counter() - t0) * 1000)++ s3.delete_bucket(Bucket=bucket)+ return latencies+++def bench_dynamodb(n, endpoint=ENDPOINT):+ """GetItem on a known table — exercises the DynamoDB JSON-target path."""+ ddb = _client("dynamodb", endpoint)+ table = f"bench-{_uid()}"+ ddb.create_table(+ TableName=table,+ KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],+ AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],+ BillingMode="PAY_PER_REQUEST",+ )++ latencies = []+ for _ in range(n):+ t0 = time.perf_counter()+ ddb.get_item(TableName=table, Key={"pk": {"S": "bench-key"}})+ latencies.append((time.perf_counter() - t0) * 1000)++ ddb.delete_table(TableName=table)+ return latencies+++def bench_sqs(n, endpoint=ENDPOINT):+ """ReceiveMessage on an empty queue — exercises the SQS query-string path."""+ sqs = _client("sqs", endpoint)+ url = sqs.create_queue(QueueName=f"bench-{_uid()}")["QueueUrl"]++ latencies = []+ for _ in range(n):+ t0 = time.perf_counter()+ sqs.receive_message(QueueUrl=url, MaxNumberOfMessages=1)+ latencies.append((time.perf_counter() - t0) * 1000)++ sqs.delete_queue(QueueUrl=url)+ return latencies+++def _report(label, latencies):+ n = len(latencies)+ total_s = sum(latencies) / 1000+ req_s = n / total_s+ p50 = statistics.median(latencies)+ p95 = _percentile(latencies, 95)+ p99 = _percentile(latencies, 99)+ print(+ f" {label:<10} {req_s:>7.0f} req/s "+ f"p50={p50:.1f}ms p95={p95:.1f}ms p99={p99:.1f}ms"+ )+ return req_s+++def _run_all(label, n, endpoint):+ print(f"\n{'─' * 60}")+ print(f" {label}")+ print(f"{'─' * 60}")+ warmup = max(10, n // 10)+ print(f" Warming up ({warmup} req per scenario)...")+ bench_s3(warmup, endpoint)+ bench_dynamodb(warmup, endpoint)+ bench_sqs(warmup, endpoint)+ print(f" Measuring ({n} req per scenario)...")+ results = {+ "s3": _report("S3 HeadBucket", bench_s3(n, endpoint)),+ "dynamodb": _report("DDB GetItem", bench_dynamodb(n, endpoint)),+ "sqs": _report("SQS Recv", bench_sqs(n, endpoint)),+ }+ return results+++def _delta_row(scenario, off, on):+ delta = (on - off) / off * 100+ sign = "+" if delta > 0 else ""+ flag = " ⚠️ overhead >2%" if delta < -2 else ""+ print(f" {scenario:<12} off={off:>7.0f} on={on:>7.0f} delta={sign}{delta:.1f}%{flag}")+++def main():+ parser = argparse.ArgumentParser(description="Bench CloudTrail recording overhead")+ parser.add_argument("--endpoint", default=ENDPOINT)+ parser.add_argument("--n", type=int, default=300, help="Requests per scenario")+ parser.add_argument("--compare", action="store_true", help="Run off then on in one pass")+ args = parser.parse_args()++ if args.compare:+ print("Running with recording OFF...")+ _set_recording(False, args.endpoint)+ off = _run_all("CLOUDTRAIL_RECORDING=0 (off)", args.n, args.endpoint)++ _reset(args.endpoint)++ print("\nRunning with recording ON...")+ _set_recording(True, args.endpoint)+ on = _run_all("CLOUDTRAIL_RECORDING=1 (on)", args.n, args.endpoint)++ _set_recording(False, args.endpoint)++ print(f"\n{'─' * 60}")+ print(" Delta summary (req/s, higher is better)")+ print(f"{'─' * 60}")+ for scenario in ("s3", "dynamodb", "sqs"):+ labels = {"s3": "S3", "dynamodb": "DynamoDB", "sqs": "SQS"}+ _delta_row(labels[scenario], off[scenario], on[scenario])+ print()+ else:+ recording = __import__("os").environ.get("CLOUDTRAIL_RECORDING", "0")+ label = f"CLOUDTRAIL_RECORDING={recording}"+ _run_all(label, args.n, args.endpoint)+ print()+++if __name__ == "__main__":+ main()diff --git a/tests/test_cloudtrail.py b/tests/test_cloudtrail.pynew file mode 100644index 00000000..31853abe--- /dev/null+++ b/tests/test_cloudtrail.py@@ -0,0 +1,471 @@+"""+CloudTrail integration tests.++Recording must be enabled for event-recording tests. The module-scoped+`enable_recording` fixture toggles it on via /_ministack/config before any+test runs and resets state after the module completes, so these tests are+safe to run alongside the rest of the suite.++Tests are split into two sections:+ - Control plane: trail CRUD, stubs (always available)+ - Event recording: LookupEvents and filter variants (require recording enabled)+"""++import time++import boto3+import pytest+import requests+from botocore.exceptions import ClientError++ENDPOINT = "http://localhost:4566"+REGION = "us-east-1"+++def _client(service):+ return boto3.client(+ service,+ endpoint_url=ENDPOINT,+ aws_access_key_id="test",+ aws_secret_access_key="test",+ region_name=REGION,+ )+++def _uid():+ import uuid++ return uuid.uuid4().hex[:8]++[email protected](scope="module")+def ct():+ return _client("cloudtrail")++[email protected](scope="module")+def s3():+ return _client("s3")++[email protected](scope="module")+def ddb():+ return _client("dynamodb")++[email protected](scope="module", autouse=True)+def enable_recording():+ """Enable CloudTrail recording for this test module via the runtime config endpoint."""+ resp = requests.post(+ f"{ENDPOINT}/_ministack/config",+ json={"cloudtrail._recording_enabled": "true"},+ )+ assert resp.status_code == 200, f"Failed to enable recording: {resp.text}"+ yield+ # Disable recording after module; reset clears events+ requests.post(+ f"{ENDPOINT}/_ministack/config",+ json={"cloudtrail._recording_enabled": "false"},+ )+++# ---------------------------------------------------------------------------+# Control plane — trail CRUD+# ---------------------------------------------------------------------------+++def test_create_trail(ct):+ name = f"trail-{_uid()}"+ resp = ct.create_trail(Name=name, S3BucketName="my-logs")+ assert resp["Name"] == name+ assert "TrailARN" in resp+ assert "cloudtrail" in resp["TrailARN"]+ assert f"/{name}" in resp["TrailARN"]+++def test_create_trail_duplicate(ct):+ name = f"trail-dup-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ with pytest.raises(ClientError) as exc:+ ct.create_trail(Name=name, S3BucketName="bucket")+ assert "TrailAlreadyExistsException" in str(exc.value)+++def test_get_trail(ct):+ name = f"trail-get-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ resp = ct.get_trail(Name=name)+ assert resp["Trail"]["Name"] == name+ assert "TrailARN" in resp["Trail"]+++def test_get_trail_not_found(ct):+ with pytest.raises(ClientError) as exc:+ ct.get_trail(Name=f"nonexistent-{_uid()}")+ assert "TrailNotFoundException" in str(exc.value)+++def test_delete_trail(ct):+ name = f"trail-del-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ ct.delete_trail(Name=name)+ resp = ct.describe_trails(trailNameList=[name])+ assert not any(t["Name"] == name for t in resp["trailList"])+++def test_delete_trail_not_found(ct):+ with pytest.raises(ClientError) as exc:+ ct.delete_trail(Name=f"nonexistent-{_uid()}")+ assert "TrailNotFoundException" in str(exc.value)+++def test_describe_trails_all(ct):+ name = f"trail-desc-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ resp = ct.describe_trails()+ assert any(t["Name"] == name for t in resp["trailList"])+++def test_describe_trails_by_name(ct):+ name = f"trail-byname-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ resp = ct.describe_trails(trailNameList=[name])+ assert len(resp["trailList"]) == 1+ assert resp["trailList"][0]["Name"] == name+++def test_describe_trails_name_not_found(ct):+ resp = ct.describe_trails(trailNameList=[f"nonexistent-{_uid()}"])+ assert resp["trailList"] == []+++def test_get_trail_status(ct):+ name = f"trail-status-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ resp = ct.get_trail_status(Name=name)+ assert resp["IsLogging"] is True+++def test_get_trail_status_not_found(ct):+ with pytest.raises(ClientError):+ ct.get_trail_status(Name=f"nonexistent-{_uid()}")+++def test_start_stop_logging(ct):+ name = f"trail-log-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ ct.start_logging(Name=name)+ ct.stop_logging(Name=name)+++def test_start_logging_not_found(ct):+ with pytest.raises(ClientError) as exc:+ ct.start_logging(Name=f"nonexistent-{_uid()}")+ assert "TrailNotFoundException" in str(exc.value)+++def test_stop_logging_not_found(ct):+ with pytest.raises(ClientError) as exc:+ ct.stop_logging(Name=f"nonexistent-{_uid()}")+ assert "TrailNotFoundException" in str(exc.value)+++def test_put_get_event_selectors(ct):+ name = f"trail-sel-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ selectors = [{"ReadWriteType": "All", "IncludeManagementEvents": True, "DataResources": []}]+ put_resp = ct.put_event_selectors(TrailName=name, EventSelectors=selectors)+ assert "TrailARN" in put_resp+ assert put_resp["EventSelectors"] == selectors++ get_resp = ct.get_event_selectors(TrailName=name)+ assert get_resp["EventSelectors"] == selectors+ assert get_resp["AdvancedEventSelectors"] == []+++def test_get_event_selectors_empty(ct):+ name = f"trail-nosel-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ resp = ct.get_event_selectors(TrailName=name)+ assert resp["EventSelectors"] == []+++def test_add_list_remove_tags(ct):+ name = f"trail-tags-{_uid()}"+ ct.create_trail(Name=name, S3BucketName="bucket")+ arn = ct.get_trail(Name=name)["Trail"]["TrailARN"]++ ct.add_tags(ResourceId=arn, TagsList=[{"Key": "env", "Value": "test"}, {"Key": "team", "Value": "ops"}])+ list_resp = ct.list_tags(ResourceIdList=[arn])+ tags = {t["Key"]: t["Value"] for item in list_resp["ResourceTagList"] for t in item["TagsList"]}+ assert tags["env"] == "test"+ assert tags["team"] == "ops"++ ct.remove_tags(ResourceId=arn, TagsList=[{"Key": "env", "Value": "test"}])+ list_resp2 = ct.list_tags(ResourceIdList=[arn])+ tags2 = {t["Key"]: t["Value"] for item in list_resp2["ResourceTagList"] for t in item["TagsList"]}+ assert "env" not in tags2+ assert tags2["team"] == "ops"+++# ---------------------------------------------------------------------------+# Event recording — LookupEvents and filters+# ---------------------------------------------------------------------------+++def test_lookup_all_events_s3(ct, s3):+ bucket = f"ct-all-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events()+ assert any(e["EventName"] == "CreateBucket" for e in resp["Events"])+++def test_lookup_filter_event_name(ct, s3):+ bucket = f"ct-ename-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": "CreateBucket"}]+ )+ assert len(resp["Events"]) > 0+ assert all(e["EventName"] == "CreateBucket" for e in resp["Events"])+++def test_lookup_filter_resource_name(ct, s3):+ bucket = f"ct-rname-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": bucket}]+ )+ assert len(resp["Events"]) > 0+ for ev in resp["Events"]:+ assert any(r.get("ResourceName") == bucket for r in ev.get("Resources", []))+++def test_lookup_filter_resource_type(ct, s3):+ bucket = f"ct-rtype-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceType", "AttributeValue": "AWS::S3::Bucket"}]+ )+ assert len(resp["Events"]) > 0+ for ev in resp["Events"]:+ assert any(r.get("ResourceType") == "AWS::S3::Bucket" for r in ev.get("Resources", []))+++def test_lookup_filter_username(ct, s3):+ bucket = f"ct-user-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "Username", "AttributeValue": "test"}]+ )+ assert len(resp["Events"]) > 0+ assert all(e["Username"] == "test" for e in resp["Events"])+++def test_lookup_filter_access_key_id(ct, s3):+ bucket = f"ct-akid-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "AccessKeyId", "AttributeValue": "test"}]+ )+ assert len(resp["Events"]) > 0+ assert all(e.get("AccessKeyId") == "test" for e in resp["Events"])+++def test_lookup_filter_readonly_false(ct, s3):+ bucket = f"ct-rw-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ReadOnly", "AttributeValue": "false"}]+ )+ assert len(resp["Events"]) > 0+ assert all(e.get("ReadOnly") == "false" for e in resp["Events"])+++def test_lookup_filter_event_source(ct, s3):+ bucket = f"ct-src-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventSource", "AttributeValue": "s3.amazonaws.com"}]+ )+ assert len(resp["Events"]) > 0+ for ev in resp["Events"]:+ assert ev.get("EventSource") == "s3.amazonaws.com"+++def test_lookup_filter_event_id(ct, s3):+ bucket = f"ct-eid-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ all_resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": bucket}]+ )+ assert len(all_resp["Events"]) >= 1+ target_id = all_resp["Events"][0]["EventId"]++ id_resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventId", "AttributeValue": target_id}]+ )+ assert len(id_resp["Events"]) == 1+ assert id_resp["Events"][0]["EventId"] == target_id+++def test_lookup_no_match(ct):+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": f"NoSuchAction{_uid()}"}]+ )+ assert resp["Events"] == []+++def test_lookup_time_range_match(ct, s3):+ bucket = f"ct-time-{_uid()}"+ from datetime import datetime, timezone, timedelta++ before = datetime.now(timezone.utc) - timedelta(seconds=2)+ s3.create_bucket(Bucket=bucket)+ after = datetime.now(timezone.utc) + timedelta(seconds=2)++ resp = ct.lookup_events(+ StartTime=before,+ EndTime=after,+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": bucket}],+ )+ assert len(resp["Events"]) >= 1+++def test_lookup_time_range_future_empty(ct):+ from datetime import datetime, timezone, timedelta++ future_start = datetime.now(timezone.utc) + timedelta(hours=1)+ future_end = datetime.now(timezone.utc) + timedelta(hours=2)+ resp = ct.lookup_events(StartTime=future_start, EndTime=future_end)+ assert resp["Events"] == []+++def test_lookup_max_results(ct, s3):+ for _ in range(6):+ s3.create_bucket(Bucket=f"ct-maxr-{_uid()}")+ resp = ct.lookup_events(MaxResults=3)+ assert len(resp["Events"]) <= 3+++def test_lookup_newest_first(ct, s3):+ b1 = f"ct-ord1-{_uid()}"+ b2 = f"ct-ord2-{_uid()}"+ s3.create_bucket(Bucket=b1)+ s3.create_bucket(Bucket=b2)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": "CreateBucket"}],+ MaxResults=10,+ )+ events = resp["Events"]+ assert len(events) >= 2+ # EventTime should be non-increasing (newest first)+ ts = [e["EventTime"].timestamp() if hasattr(e["EventTime"], "timestamp") else float(e["EventTime"]) for e in events]+ assert ts == sorted(ts, reverse=True)+++def test_dynamodb_create_table_recorded(ct, ddb):+ table = f"ct-tbl-{_uid()}"+ ddb.create_table(+ TableName=table,+ KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],+ AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],+ BillingMode="PAY_PER_REQUEST",+ )+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": "CreateTable"}]+ )+ assert any(e["EventName"] == "CreateTable" for e in resp["Events"])+++def test_dynamodb_resource_captured(ct, ddb):+ table = f"ct-res-{_uid()}"+ ddb.create_table(+ TableName=table,+ KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],+ AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],+ BillingMode="PAY_PER_REQUEST",+ )+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": table}]+ )+ assert len(resp["Events"]) >= 1+ ev = resp["Events"][0]+ assert any(r["ResourceName"] == table for r in ev["Resources"])+ assert any(r["ResourceType"] == "AWS::DynamoDB::Table" for r in ev["Resources"])+++def test_event_record_full_structure(ct, s3):+ import json as _json++ bucket = f"ct-struct-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": bucket}]+ )+ assert len(resp["Events"]) >= 1+ ev = resp["Events"][0]++ # Top-level required fields+ for field in ("EventId", "EventName", "EventSource", "EventTime", "Username", "CloudTrailEvent"):+ assert field in ev, f"Missing field: {field}"++ # Full CloudTrailEvent JSON shape+ ct_ev = _json.loads(ev["CloudTrailEvent"])+ for field in (+ "eventVersion",+ "userIdentity",+ "eventTime",+ "eventSource",+ "eventName",+ "awsRegion",+ "sourceIPAddress",+ "userAgent",+ "requestParameters",+ "responseElements",+ "requestID",+ "eventID",+ "eventType",+ "recipientAccountId",+ ):+ assert field in ct_ev, f"Missing CloudTrailEvent field: {field}"++ assert ct_ev["eventType"] == "AwsApiCall"+ assert ct_ev["eventSource"].endswith(".amazonaws.com")+ assert ct_ev["userIdentity"]["type"] == "IAMUser"+ # readOnly is stored as a string ("true"/"false") in both the event record and CloudTrailEvent+ assert ev.get("ReadOnly") in ("true", "false")+ assert ct_ev.get("readOnly") in ("true", "false")+++def test_cloudtrail_calls_not_self_recorded(ct):+ """CloudTrail management calls (DescribeTrails) must not appear in LookupEvents."""+ before = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": "DescribeTrails"}]+ )["Events"]+ ct.describe_trails()+ after = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "EventName", "AttributeValue": "DescribeTrails"}]+ )["Events"]+ assert len(after) == len(before)+++def test_recording_disabled_no_new_events(ct, s3):+ """Disabling recording stops new events from being appended."""+ requests.post(+ f"{ENDPOINT}/_ministack/config",+ json={"cloudtrail._recording_enabled": "false"},+ )+ try:+ bucket = f"ct-off-{_uid()}"+ s3.create_bucket(Bucket=bucket)+ resp = ct.lookup_events(+ LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": bucket}]+ )+ assert resp["Events"] == []+ finally:+ # Re-enable so subsequent tests in this module still work+ requests.post(+ f"{ENDPOINT}/_ministack/config",+ json={"cloudtrail._recording_enabled": "true"},+ )