Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add insights and lake query tables to aws cloudtrail (#116973)

Tom Owers2cbd9feef164
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 82008cf78a39..945bf237d676 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1110,14 +1110,14 @@ Note: Compared every Describe and List operation in the official botocore servic
## AWS CloudTrail — gaps
-Today (4): `event_data_stores`, `events`, `insight_events`, `trails`
+Today (7): `event_data_stores`, `events`, `insight_events`, `insights_data`, `insights_metric_data`, `lake_queries`, `trails`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/cloudtrail/2013-11-01/service-2.json>
-- [ ] `CloudTrail Lake event query results (StartQuery and GetQueryResults, POST /)` — exposes stored data events and older activity unavailable through the 90-day management event lookup (high)
-- [ ] `Insights metric data (ListInsightsMetricData, POST /)` — measures API call and error rates over time around detected anomalies (high)
-- [ ] `Data-event Insights (ListInsightsData, POST /)` — adds anomalies generated from data events, beyond the management Insights available through LookupEvents (high)
-- [ ] `Lake query history (ListQueries and DescribeQuery, POST /)` — tracks query status, runtime, and data scanned for analysis of Lake usage and cost (medium)
+- ~~`CloudTrail Lake event query results (StartQuery and GetQueryResults, POST /)`~~ — skipped: `StartQuery` runs a SQL scan that AWS bills to the customer by data scanned, so a scheduled sync would add a charge on every run, and it needs a SQL statement per event store rather than a list to page
+- [x] `Insights metric data (ListInsightsMetricData, POST /)` — measures API call and error rates over time around detected anomalies (high). Added as `insights_metric_data`: one hourly series for each activity that raised an Insights event in the past 90 days.
+- [x] `Data-event Insights (ListInsightsData, POST /)` — adds anomalies generated from data events, beyond the management Insights available through LookupEvents (high). Added as `insights_data` (fan-out over trails with Insights selectors).
+- [x] `Lake query history (ListQueries and DescribeQuery, POST /)` — tracks query status, runtime, and data scanned for analysis of Lake usage and cost (medium). Added as `lake_queries` (fan-out over event stores, one `DescribeQuery` per query).
Note: Parsed all read operations and response shapes in the official botocore service model. Existing insight_events use LookupEvents; data-event Insights and metric series use separate operations. Lake exports need query submission, polling, and result pagination; Lake closed to new customers on May 31, 2026. Excluded trail health, selectors, delivery configuration, keys, and import plumbing.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/aws_cloudtrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/aws_cloudtrail.py
index 69206ca15dfd..36d80f63a911 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/aws_cloudtrail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/aws_cloudtrail.py
@@ -6,6 +6,7 @@ from collections.abc import Iterator
from typing import Any
import requests
+import structlog
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
@@ -17,7 +18,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cloudt
ACCESS_DENIED_CODES,
AWS_CLOUDTRAIL_ENDPOINTS,
CREDENTIAL_ERRORS,
+ INSIGHTS_METRIC_PERIOD_SECONDS,
LOOKBACK_SECONDS,
+ SKIPPED_PARENT_CODES,
TARGET_PREFIXES,
THROTTLE_CODES,
CloudTrailEndpoint,
@@ -27,6 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+logger = structlog.get_logger(__name__)
+
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
TRANSPORT_RETRY = BoundedRetry(
total=3,
@@ -35,6 +40,9 @@ TRANSPORT_RETRY = BoundedRetry(
allowed_methods=frozenset(["POST"]),
raise_on_status=False,
)
+# Minimum seconds between requests. LookupEvents and ListInsightsData allow two requests per second per account and
+# region. ListInsightsMetricData allows one.
+SPACED_OPERATIONS = {"LookupEvents": 0.5, "ListInsightsData": 0.5, "ListInsightsMetricData": 1.0}
@frozen
@@ -43,6 +51,8 @@ class AwsCloudTrailResumeConfig:
start_time: float | None = None
end_time: float | None = None
finished: bool = False
+ # The fan-out parent of `next_token`. With no token, every row of this parent is already read.
+ parent: str | None = None
class AwsCloudTrailError(Exception):
@@ -96,7 +106,7 @@ class AwsCloudTrailClient:
retry=TRANSPORT_RETRY,
redact_values=tuple(value for value in (access_key_id, secret_access_key, session_token) if value),
)
- self._last_lookup: float | None = None
+ self._last_request: dict[str, float] = {}
@retry(
retry=retry_if_exception_type(AwsCloudTrailThrottled),
@@ -105,10 +115,11 @@ class AwsCloudTrailClient:
reraise=True,
)
def request(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
- if operation == "LookupEvents":
- if self._last_lookup is not None:
- time.sleep(max(0, 0.5 - (time.monotonic() - self._last_lookup)))
- self._last_lookup = time.monotonic()
+ if operation in SPACED_OPERATIONS:
+ last = self._last_request.get(operation)
+ if last is not None:
+ time.sleep(max(0, SPACED_OPERATIONS[operation] - (time.monotonic() - last)))
+ self._last_request[operation] = time.monotonic()
body = json.dumps(payload).encode("utf-8")
request = AWSRequest(
method="POST",
@@ -155,6 +166,19 @@ def request_payload(endpoint: CloudTrailEndpoint, page_size: int | None = None)
return payload
+def _epoch(value: dt.datetime | str | int | float) -> float:
+ if isinstance(value, str):
+ try:
+ return float(value)
+ except ValueError:
+ value = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if isinstance(value, dt.datetime):
+ if value.tzinfo is None:
+ value = value.replace(tzinfo=dt.UTC)
+ return value.timestamp()
+ return float(value)
+
+
def get_rows(
client: AwsCloudTrailClient,
endpoint: CloudTrailEndpoint,
@@ -169,13 +193,7 @@ def get_rows(
if endpoint.is_events and end_time is None:
end_time = dt.datetime.now(dt.UTC).timestamp()
if last_value is not None:
- if isinstance(last_value, str):
- last_value = dt.datetime.fromisoformat(last_value.replace("Z", "+00:00"))
- if isinstance(last_value, dt.datetime):
- if last_value.tzinfo is None:
- last_value = last_value.replace(tzinfo=dt.UTC)
- last_value = last_value.timestamp()
- start_time = max(float(last_value), end_time - LOOKBACK_SECONDS)
+ start_time = max(_epoch(last_value), end_time - LOOKBACK_SECONDS)
payload = request_payload(endpoint)
if endpoint.is_events:
payload["EndTime"] = end_time
@@ -213,6 +231,246 @@ def get_rows(
client.close()
+@frozen
+class FanOutParent:
+ key: str
+ payload: dict[str, Any]
+ columns: dict[str, Any]
+
+
+def _pages(
+ client: AwsCloudTrailClient,
+ operation: str,
+ payload: dict[str, Any],
+ manager: ResumableSourceManager[AwsCloudTrailResumeConfig],
+) -> Iterator[dict[str, Any]]:
+ next_token: str | None = None
+ while True:
+ params = {**payload, "NextToken": next_token} if next_token else dict(payload)
+ body = client.request(operation, params)
+ manager.safe_point()
+ yield body
+ next_token = body.get("NextToken")
+ if next_token and next_token == params.get("NextToken"):
+ raise ValueError("AWS CloudTrail returned a repeated page token.")
+ if not next_token:
+ return
+
+
+def _skip_parent(error: AwsCloudTrailError, operation: str, parent: str) -> None:
+ if error.code not in SKIPPED_PARENT_CODES:
+ raise error
+ logger.warning(
+ "AWS CloudTrail skipped a parent it cannot read", operation=operation, parent=parent, code=error.code
+ )
+
+
+def _insights_trails(
+ client: AwsCloudTrailClient, manager: ResumableSourceManager[AwsCloudTrailResumeConfig]
+) -> list[str]:
+ body = client.request("DescribeTrails", {"includeShadowTrails": True})
+ manager.safe_point()
+ return sorted(
+ {
+ trail["TrailARN"]
+ for trail in body.get("trailList", [])
+ if trail.get("HasInsightSelectors") and trail.get("TrailARN")
+ }
+ )
+
+
+@frozen
+class InsightSeries:
+ trail_arn: str
+ event_source: str
+ event_name: str
+ insight_type: str
+ error_code: str
+
+
+def _series(event: dict[str, Any], trail_arn: str) -> InsightSeries | None:
+ try:
+ details = json.loads(event.get("CloudTrailEvent") or "{}").get("insightDetails") or {}
+ except (ValueError, AttributeError):
+ return None
+ event_source = details.get("eventSource") or event.get("EventSource")
+ event_name = details.get("eventName") or event.get("EventName")
+ insight_type = details.get("insightType")
+ if not (event_source and event_name and insight_type):
+ return None
+ return InsightSeries(
+ trail_arn=trail_arn,
+ event_source=event_source,
+ event_name=event_name,
+ insight_type=insight_type,
+ error_code=details.get("errorCode") or "",
+ )
+
+
+def insight_series_parents(
+ client: AwsCloudTrailClient, manager: ResumableSourceManager[AwsCloudTrailResumeConfig]
+) -> list[FanOutParent]:
+ # Metrics exist only for activity that triggered an Insights event, so the events name every series.
+ series: set[InsightSeries] = set()
+ for body in _pages(client, "LookupEvents", {"MaxResults": 50, "EventCategory": "insight"}, manager):
+ series.update(key for event in body.get("Events", []) if (key := _series(event, "")))
+ for trail_arn in _insights_trails(client, manager):
+ listing = {"InsightSource": trail_arn, "DataType": "InsightsEvents", "MaxResults": 50}
+ try:
+ for body in _pages(client, "ListInsightsData", listing, manager):
+ series.update(key for event in body.get("Events", []) if (key := _series(event, trail_arn)))
+ except AwsCloudTrailError as error:
+ _skip_parent(error, "ListInsightsData", trail_arn)
+ parents: list[FanOutParent] = []
+ for item in series:
+ payload: dict[str, Any] = {
+ "EventSource": item.event_source,
+ "EventName": item.event_name,
+ "InsightType": item.insight_type,
+ "Period": INSIGHTS_METRIC_PERIOD_SECONDS,
+ }
+ if item.error_code:
+ payload["ErrorCode"] = item.error_code
+ if item.trail_arn:
+ payload["TrailName"] = item.trail_arn
+ columns = {
+ "trail_arn": item.trail_arn,
+ "event_source": item.event_source,
+ "event_name": item.event_name,
+ "insight_type": item.insight_type,
+ "error_code": item.error_code,
+ }
+ parents.append(FanOutParent(key=json.dumps(list(columns.values())), payload=payload, columns=columns))
+ # A resumed run skips parents by comparing keys, so the walk follows key order.
+ return sorted(parents, key=lambda parent: parent.key)
+
+
+def fan_out_parents(
+ client: AwsCloudTrailClient,
+ endpoint: CloudTrailEndpoint,
+ manager: ResumableSourceManager[AwsCloudTrailResumeConfig],
+) -> list[FanOutParent]:
+ if endpoint.parent == "trails":
+ return [
+ FanOutParent(
+ key=arn, payload={"InsightSource": arn, "DataType": "InsightsEvents"}, columns={"trail_arn": arn}
+ )
+ for arn in _insights_trails(client, manager)
+ ]
+ if endpoint.parent == "event_data_stores":
+ stores = {
+ store["EventDataStoreArn"]
+ for body in _pages(client, "ListEventDataStores", {"MaxResults": 100}, manager)
+ for store in body.get("EventDataStores", [])
+ if store.get("EventDataStoreArn")
+ }
+ return [
+ FanOutParent(key=arn, payload={"EventDataStore": arn}, columns={"event_data_store_arn": arn})
+ for arn in sorted(stores)
+ ]
+ if endpoint.parent == "insight_series":
+ return insight_series_parents(client, manager)
+ raise ValueError(f"Unknown AWS CloudTrail parent: {endpoint.parent}")
+
+
+def _page_rows(
+ body: dict[str, Any], endpoint: CloudTrailEndpoint, parent: FanOutParent, region: str
+) -> list[dict[str, Any]]:
+ if endpoint.operation == "ListInsightsMetricData":
+ return [
+ {
+ "region": region,
+ **parent.columns,
+ "timestamp": dt.datetime.fromtimestamp(timestamp, dt.UTC),
+ "value": value,
+ }
+ for timestamp, value in zip(body.get("Timestamps") or [], body.get("Values") or [], strict=False)
+ ]
+ return [{**normalize_row(item, endpoint, region), **parent.columns} for item in body.get(endpoint.result_key) or []]
+
+
+def _describe_query(client: AwsCloudTrailClient, endpoint: CloudTrailEndpoint, row: dict[str, Any]) -> dict[str, Any]:
+ try:
+ details = client.request(
+ "DescribeQuery", {"EventDataStore": row["event_data_store_arn"], "QueryId": row["query_id"]}
+ )
+ except AwsCloudTrailError as error:
+ if error.code != "QueryIdNotFoundException":
+ raise
+ return row
+ return {**row, **normalize_row(details, endpoint, client.region)}
+
+
+def get_fan_out_rows(
+ client: AwsCloudTrailClient,
+ endpoint: CloudTrailEndpoint,
+ manager: ResumableSourceManager[AwsCloudTrailResumeConfig],
+ last_value: dt.datetime | str | int | float | None,
+) -> Iterator[list[dict[str, Any]]]:
+ try:
+ state = manager.load_state() or AwsCloudTrailResumeConfig()
+ if state.finished:
+ return
+ start_time, end_time = state.start_time, state.end_time
+ if endpoint.retention_seconds is not None and end_time is None:
+ end_time = dt.datetime.now(dt.UTC).timestamp()
+ start_time = end_time - endpoint.retention_seconds
+ if last_value is not None:
+ start_time = max(start_time, _epoch(last_value) - endpoint.refresh_seconds)
+ window = {} if end_time is None else {"StartTime": start_time, "EndTime": end_time}
+ parents = fan_out_parents(client, endpoint, manager)
+ for index, parent in enumerate(parents):
+ if state.parent is not None and (
+ parent.key < state.parent or (parent.key == state.parent and not state.next_token)
+ ):
+ continue
+ resumed_token = state.next_token if parent.key == state.parent else None
+ next_token = resumed_token
+ try:
+ while True:
+ params = {**request_payload(endpoint), **window, **parent.payload}
+ if next_token:
+ params["NextToken"] = next_token
+ try:
+ body = client.request(endpoint.operation, params)
+ except AwsCloudTrailError as error:
+ if error.code == "InvalidNextTokenException" and resumed_token and next_token == resumed_token:
+ next_token = resumed_token = None
+ continue
+ raise
+ next_token = body.get("NextToken")
+ if next_token and next_token == params.get("NextToken"):
+ raise ValueError("AWS CloudTrail returned a repeated page token.")
+ checkpoint = AwsCloudTrailResumeConfig(
+ next_token=next_token,
+ start_time=start_time,
+ end_time=end_time,
+ finished=not next_token and index == len(parents) - 1,
+ parent=parent.key,
+ )
+ rows = _page_rows(body, endpoint, parent, client.region)
+ if endpoint.operation != "ListQueries":
+ manager.save_state(checkpoint)
+ if rows:
+ yield rows
+ else:
+ # One describe call per row, so each row is handed over as soon as it is read. The cursor
+ # staged for the earlier rows still points at this page, so a hand-off reads it again.
+ for row in rows[:-1]:
+ yield [_describe_query(client, endpoint, row)]
+ last = [_describe_query(client, endpoint, rows[-1])] if rows else []
+ manager.save_state(checkpoint)
+ if last:
+ yield last
+ manager.safe_point()
+ if not next_token:
+ break
+ except AwsCloudTrailError as error:
+ _skip_parent(error, endpoint.operation, parent.key)
+ finally:
+ client.close()
+
+
def validate_credentials(
access_key_id: str,
secret_access_key: str,
@@ -224,6 +482,9 @@ def validate_credentials(
endpoint = AWS_CLOUDTRAIL_ENDPOINTS.get(schema_name or "trails")
if endpoint is None:
return False, f"Unknown AWS CloudTrail table: {schema_name}"
+ permissions = endpoint.permissions or (endpoint.operation,)
+ if endpoint.probe is not None:
+ endpoint = AWS_CLOUDTRAIL_ENDPOINTS[endpoint.probe]
try:
client = AwsCloudTrailClient(access_key_id, secret_access_key, session_token, region, api_version)
except ValueError as error:
@@ -234,7 +495,10 @@ def validate_credentials(
if error.code in ACCESS_DENIED_CODES:
if schema_name is None:
return True, None
- return False, f"Grant the cloudtrail:{endpoint.operation} IAM permission to read this table."
+ if len(permissions) == 1:
+ return False, f"Grant the cloudtrail:{permissions[0]} IAM permission to read this table."
+ granted = ", ".join(f"cloudtrail:{permission}" for permission in permissions)
+ return False, f"Grant the {granted} IAM permissions to read this table."
return False, CREDENTIAL_ERRORS.get(error.code, "Could not read from AWS CloudTrail. Try again.")
except (requests.RequestException, ValueError):
return False, "Could not reach the AWS CloudTrail API. Check the selected region and try again."
@@ -256,18 +520,20 @@ def aws_cloudtrail_source(
endpoint = AWS_CLOUDTRAIL_ENDPOINTS.get(endpoint_name)
if endpoint is None:
raise ValueError(f"Unknown AWS CloudTrail table: {endpoint_name}")
+ rows = get_rows if endpoint.parent is None else get_fan_out_rows
return SourceResponse(
name=endpoint_name,
- items=lambda: get_rows(
+ items=lambda: rows(
AwsCloudTrailClient(access_key_id, secret_access_key, session_token, region, api_version),
endpoint,
manager,
last_value,
),
primary_keys=list(endpoint.primary_key),
- sort_mode="desc" if endpoint.is_events else None,
- partition_keys=["event_time"] if endpoint.is_events else None,
- partition_mode="datetime" if endpoint.is_events else None,
- partition_format="month" if endpoint.is_events else None,
+ # Event lookups return newest first, and fan-out tables keep an order only within one parent.
+ sort_mode="desc" if endpoint.incremental_field else None,
+ partition_keys=[endpoint.partition_key] if endpoint.partition_key else None,
+ partition_mode="datetime" if endpoint.partition_key else None,
+ partition_format="month" if endpoint.partition_key else None,
on_complete=manager.clear_state,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/canonical_descriptions.py
index 449390304d21..f78b39a8dd72 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/canonical_descriptions.py
@@ -60,4 +60,41 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"region": "The AWS region selected for this import.",
},
},
+ "insights_data": {
+ "description": "Insights events generated on trails that log data events, within the past 90 days.",
+ "docs_url": "https://docs.aws.amazon.com/awscloudtrail/latest/APIReference/API_ListInsightsData.html",
+ "columns": {**_EVENT_COLUMNS, "trail_arn": "The ARN of the trail that generated the Insights event."},
+ },
+ "insights_metric_data": {
+ "description": "Hourly API call or error rate for each activity that triggered an Insights event in the past 90 days.",
+ "docs_url": "https://docs.aws.amazon.com/awscloudtrail/latest/APIReference/API_ListInsightsMetricData.html",
+ "columns": {
+ "trail_arn": "The ARN of the trail for data event Insights. Empty for management event Insights.",
+ "event_source": "The AWS service that received the requests, such as iam.amazonaws.com.",
+ "event_name": "The AWS API action on which unusual activity was recorded.",
+ "insight_type": "The Insights type: ApiCallRateInsight or ApiErrorRateInsight.",
+ "error_code": "The error measured by an ApiErrorRateInsight metric. Empty for ApiCallRateInsight.",
+ "timestamp": "The start of the hour that the data point covers, in UTC.",
+ "value": "The API call count or error count in the hour.",
+ "region": "The AWS region selected for this import.",
+ },
+ },
+ "lake_queries": {
+ "description": "CloudTrail Lake queries run in the past 7 days, with status, run time, and data scanned.",
+ "docs_url": "https://docs.aws.amazon.com/awscloudtrail/latest/APIReference/API_DescribeQuery.html",
+ "columns": {
+ "event_data_store_arn": "The ARN of the event store that the query ran on.",
+ "query_id": "The query ID.",
+ "query_status": "The query status: QUEUED, RUNNING, FINISHED, FAILED, CANCELLED, or TIMED_OUT.",
+ "creation_time": "The time when the query was created.",
+ "query_string": "The SQL code of the query.",
+ "query_statistics": "Events matched, events scanned, bytes scanned, run time in milliseconds, and creation time.",
+ "error_message": "The error message if the query failed.",
+ "delivery_s3_uri": "The S3 URI where CloudTrail delivered the query results, if applicable.",
+ "delivery_status": "The delivery status of the query results.",
+ "prompt": "The natural language prompt used to generate the query, if any.",
+ "event_data_store_owner_account_id": "The account ID of the event store owner.",
+ "region": "The AWS region selected for this import.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/settings.py
index 8eeecf6b02fb..a9639bb989a1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/settings.py
@@ -6,6 +6,11 @@ from products.warehouse_sources.backend.types import IncrementalField
CLOUDTRAIL_API_VERSION = "2013-11-01"
TARGET_PREFIXES = {CLOUDTRAIL_API_VERSION: "com.amazonaws.cloudtrail.v20131101.CloudTrail_20131101"}
LOOKBACK_SECONDS = 90 * 24 * 60 * 60
+LAKE_QUERY_HISTORY_SECONDS = 7 * 24 * 60 * 60
+# Lake queries still running at sync time are read again on the next sync so their final status lands.
+LAKE_QUERY_REFRESH_SECONDS = 24 * 60 * 60
+# Hourly points are the only granularity CloudTrail keeps for the full 90 days of Insights metrics.
+INSIGHTS_METRIC_PERIOD_SECONDS = 3600
@frozen
@@ -16,6 +21,16 @@ class CloudTrailEndpoint:
page_size: int | None = None
event_category: str | None = None
timestamps: tuple[str, ...] = ()
+ incremental_field: str | None = None
+ partition_key: str | None = None
+ # Fan-out endpoints read one parent listing per run and call `operation` once per parent.
+ parent: str | None = None
+ # How far back the API keeps rows. Fan-out endpoints send this window on every request.
+ retention_seconds: int | None = None
+ refresh_seconds: int = 0
+ # The table to probe when validating, for endpoints that need a parent before they can be called.
+ probe: str | None = None
+ permissions: tuple[str, ...] = ()
@property
def is_events(self) -> bool:
@@ -29,6 +44,8 @@ AWS_CLOUDTRAIL_ENDPOINTS: dict[str, CloudTrailEndpoint] = {
primary_key=("region", "event_id"),
page_size=50,
timestamps=("event_time",),
+ incremental_field="event_time",
+ partition_key="event_time",
),
"insight_events": CloudTrailEndpoint(
operation="LookupEvents",
@@ -37,6 +54,8 @@ AWS_CLOUDTRAIL_ENDPOINTS: dict[str, CloudTrailEndpoint] = {
page_size=50,
event_category="insight",
timestamps=("event_time",),
+ incremental_field="event_time",
+ partition_key="event_time",
),
"trails": CloudTrailEndpoint(
operation="DescribeTrails",
@@ -50,20 +69,75 @@ AWS_CLOUDTRAIL_ENDPOINTS: dict[str, CloudTrailEndpoint] = {
page_size=100,
timestamps=("created_timestamp", "updated_timestamp"),
),
+ "insights_data": CloudTrailEndpoint(
+ operation="ListInsightsData",
+ result_key="Events",
+ primary_key=("region", "trail_arn", "event_id"),
+ page_size=50,
+ timestamps=("event_time",),
+ incremental_field="event_time",
+ partition_key="event_time",
+ parent="trails",
+ retention_seconds=LOOKBACK_SECONDS,
+ probe="trails",
+ permissions=("DescribeTrails", "ListInsightsData"),
+ ),
+ "insights_metric_data": CloudTrailEndpoint(
+ operation="ListInsightsMetricData",
+ result_key="Values",
+ primary_key=("region", "trail_arn", "event_source", "event_name", "insight_type", "error_code", "timestamp"),
+ page_size=21600,
+ partition_key="timestamp",
+ parent="insight_series",
+ probe="insight_events",
+ permissions=("LookupEvents", "DescribeTrails", "ListInsightsData"),
+ ),
+ "lake_queries": CloudTrailEndpoint(
+ operation="ListQueries",
+ result_key="Queries",
+ primary_key=("event_data_store_arn", "query_id"),
+ page_size=1000,
+ timestamps=("creation_time",),
+ incremental_field="creation_time",
+ partition_key="creation_time",
+ parent="event_data_stores",
+ retention_seconds=LAKE_QUERY_HISTORY_SECONDS,
+ refresh_seconds=LAKE_QUERY_REFRESH_SECONDS,
+ probe="event_data_stores",
+ permissions=("ListEventDataStores", "ListQueries", "DescribeQuery"),
+ ),
}
ENDPOINTS = tuple(AWS_CLOUDTRAIL_ENDPOINTS)
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
- name: [incremental_field("event_time")] for name, endpoint in AWS_CLOUDTRAIL_ENDPOINTS.items() if endpoint.is_events
+ name: [incremental_field(endpoint.incremental_field)]
+ for name, endpoint in AWS_CLOUDTRAIL_ENDPOINTS.items()
+ if endpoint.incremental_field is not None
}
ENDPOINT_DESCRIPTIONS = {
"events": "Management events from the past 90 days in the selected AWS region. Data events are excluded.",
"insight_events": "Insights events from the past 90 days for trails with Insights enabled in the selected region.",
"trails": "Trail settings in the selected region, including shadow trails.",
"event_data_stores": "CloudTrail Lake event store settings in the selected region. Stored events are excluded.",
+ "insights_data": "Insights events from the past 90 days for trails with Insights enabled on data events.",
+ "insights_metric_data": "Hourly API call and error rates from the past 90 days for each activity that triggered an Insights event.",
+ "lake_queries": "CloudTrail Lake queries from the past 7 days, with their status, run time, and data scanned.",
}
ACCESS_DENIED_CODES = frozenset({"AccessDenied", "AccessDeniedException"})
THROTTLE_CODES = frozenset({"Throttling", "ThrottlingException", "TooManyRequestsException", "RequestLimitExceeded"})
+# A parent that cannot serve the child call (an organization trail seen from a member account, a trail
+# without data event Insights, a stopped event store) is skipped rather than failing the whole table.
+SKIPPED_PARENT_CODES = frozenset(
+ {
+ "InvalidParameterException",
+ "OperationNotPermittedException",
+ "UnsupportedOperationException",
+ "TrailNotFoundException",
+ "InvalidTrailNameException",
+ "EventDataStoreNotFoundException",
+ "InactiveEventDataStoreException",
+ }
+)
CREDENTIAL_ERRORS = {
"UnrecognizedClientException": "AWS rejected the access key. Check the access key ID, secret access key, and session token.",
"InvalidClientTokenId": "AWS rejected the access key. Check the access key ID and secret access key.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/source.py
index 3d9622c3ec68..b270fc252b1a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/source.py
@@ -70,7 +70,7 @@ class AwsCloudTrailSource(ResumableSource[AwsCloudTrailSourceConfig, AwsCloudTra
}
errors.update(
{
- f"AWS CloudTrail request failed: {code}": "Grant cloudtrail:LookupEvents, cloudtrail:DescribeTrails, or cloudtrail:ListEventDataStores for the tables you select."
+ f"AWS CloudTrail request failed: {code}": "Grant the CloudTrail permissions listed in the source setup for the tables you select."
for code in ACCESS_DENIED_CODES
}
)
@@ -139,11 +139,14 @@ class AwsCloudTrailSource(ResumableSource[AwsCloudTrailSourceConfig, AwsCloudTra
caption="""Sync CloudTrail events and settings from one AWS region.
Grant `cloudtrail:LookupEvents`, `cloudtrail:DescribeTrails`, and `cloudtrail:ListEventDataStores` for the tables you select.
+Insights tables also need `cloudtrail:ListInsightsData`.
+The Lake queries table also needs `cloudtrail:ListQueries` and `cloudtrail:DescribeQuery`.
Enter an access key ID and secret access key. Add a session token for temporary credentials.
Event imports cover the past 90 days. They include management events and trail Insights events, but exclude data events.
Insights must be enabled on a trail to return Insights events.
The event stores table imports CloudTrail Lake settings, not stored events.
+The Lake queries table imports query history from the past 7 days, not query results.
CloudTrail Lake is available only to existing customers after May 31, 2026.""",
iconPath="/static/services/aws_cloudtrail.png",
docsUrl="https://posthog.com/docs/cdp/sources/aws-cloudtrail",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/tests/test_aws_cloudtrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/tests/test_aws_cloudtrail.py
index 11a75c5cacd7..d29c2e5d97f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/tests/test_aws_cloudtrail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cloudtrail/tests/test_aws_cloudtrail.py
@@ -1,6 +1,6 @@
import json
import datetime as dt
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
from typing import Any, cast
import pytest
@@ -56,6 +56,41 @@ def request_payloads(requests_log: list[RecordedRequest]) -> list[dict[str, Any]
return [cast(dict[str, Any], request.json()) for request in requests_log]
+def by_operation(answers: dict[str, list[ScriptedResponse]]) -> Callable[[RecordedRequest], ScriptedResponse]:
+ queues = {operation: list(responses) for operation, responses in answers.items()}
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ return queues[operation_of(request)].pop(0)
+
+ return respond
+
+
+def operation_of(request: RecordedRequest) -> str:
+ return request.headers["x-amz-target"].rsplit(".", 1)[-1]
+
+
+def calls(requests_log: list[RecordedRequest], operation: str) -> list[dict[str, Any]]:
+ return [cast(dict[str, Any], request.json()) for request in requests_log if operation_of(request) == operation]
+
+
+def insight(source: str, name: str, insight_type: str, error_code: str | None = None) -> dict[str, Any]:
+ details = {"eventSource": source, "eventName": name, "insightType": insight_type}
+ if error_code:
+ details["errorCode"] = error_code
+ return {"EventId": f"{name}-{insight_type}", "CloudTrailEvent": json.dumps({"insightDetails": details})}
+
+
+TRAILS = ScriptedResponse(
+ json={
+ "trailList": [
+ {"TrailARN": "arn:trail/b", "HasInsightSelectors": True},
+ {"TrailARN": "arn:trail/a", "HasInsightSelectors": True},
+ {"TrailARN": "arn:trail/plain", "HasInsightSelectors": False},
+ ]
+ }
+)
+
+
def response(body: object, status: int = 200, headers: dict[str, str] | None = None) -> requests.Response:
result = requests.Response()
result.status_code = status
@@ -337,10 +372,45 @@ def test_lookup_requests_are_spaced(session: Mock) -> None:
assert sleep.call_args.args[0] == pytest.approx(0.4)
+ "value,expected",
+ [
+ ("1712345678", 1712345678.0),
+ ("2024-04-05T19:34:38Z", 1712345678.0),
+ (1712345678, 1712345678.0),
+ (dt.datetime(2024, 4, 5, 19, 34, 38, tzinfo=dt.UTC), 1712345678.0),
+ ],
+)
+def test_epoch_accepts_numeric_strings_and_timestamps(value: Any, expected: float) -> None:
+ assert transport._epoch(value) == expected
+
+
+def test_metric_requests_are_spaced_to_one_per_second(session: Mock) -> None:
+ with (
+ patch.object(transport.time, "monotonic", return_value=10.0) as monotonic,
+ patch.object(transport.time, "sleep") as sleep,
+ ):
+ api = client()
+ api.request("ListInsightsMetricData", {})
+ monotonic.return_value = 10.1
+ api.request("ListInsightsMetricData", {})
+ sleep.assert_called_once()
+ assert sleep.call_args.args[0] == pytest.approx(0.9)
+
+
@pytest.mark.parametrize("code", ["AccessDenied", "AccessDeniedException"])
@pytest.mark.parametrize(
"schema_name,valid",
- [(None, True), ("events", False), ("insight_events", False), ("trails", False), ("event_data_stores", False)],
+ [
+ (None, True),
+ ("events", False),
+ ("insight_events", False),
+ ("trails", False),
+ ("event_data_stores", False),
+ ("insights_data", False),
+ ("insights_metric_data", False),
+ ("lake_queries", False),
+ ],
)
def test_access_denial_only_accepts_source_creation(
session: Mock, code: str, schema_name: str | None, valid: bool
@@ -350,7 +420,10 @@ def test_access_denial_only_accepts_source_creation(
assert ok is valid
if schema_name is not None:
assert reason is not None
- assert f"cloudtrail:{AWS_CLOUDTRAIL_ENDPOINTS[schema_name].operation}" in reason
+ endpoint = AWS_CLOUDTRAIL_ENDPOINTS[schema_name]
+ for permission in endpoint.permissions or (endpoint.operation,):
+ assert f"cloudtrail:{permission}" in reason
+ assert "cloudtrail:ListInsightsMetricData" not in reason
else:
assert reason is None
assert session.post.call_count == 1
@@ -388,6 +461,9 @@ def test_credentials_errors_are_actionable_and_terminal(code: str, message: str)
("events", "LookupEvents", {"MaxResults": 1}),
("insight_events", "LookupEvents", {"MaxResults": 1, "EventCategory": "insight"}),
("event_data_stores", "ListEventDataStores", {"MaxResults": 1}),
+ ("insights_data", "DescribeTrails", {"includeShadowTrails": True}),
+ ("insights_metric_data", "LookupEvents", {"MaxResults": 1, "EventCategory": "insight"}),
+ ("lake_queries", "ListEventDataStores", {"MaxResults": 1}),
],
)
def test_validation_uses_one_cheap_probe(schema_name: str | None, target: str, params: dict[str, Any]) -> None:
@@ -423,3 +499,196 @@ def test_connection_error_has_useful_message(session: Mock) -> None:
assert not valid
assert reason is not None and "region" in reason
session.close.assert_called_once()
+
+
+@time_machine.travel(NOW, tick=False)
+def test_insights_data_walks_each_insights_trail_and_skips_unreadable_ones() -> None:
+ last_value = NOW - dt.timedelta(days=1)
+ result = driver().run(
+ "insights_data",
+ by_operation(
+ {
+ "DescribeTrails": [TRAILS],
+ "ListInsightsData": [
+ ScriptedResponse(
+ json={"Events": [{"EventId": "one", "EventTime": NOW.timestamp()}], "NextToken": "p2"}
+ ),
+ ScriptedResponse(json={"Events": [{"EventId": "two", "EventTime": NOW.timestamp()}]}),
+ ScriptedResponse(status=400, json={"__type": "OperationNotPermittedException"}),
+ ],
+ }
+ ),
+ incremental_field="event_time",
+ db_incremental_field_last_value=last_value,
+ )
+ assert result.raised is None
+ window = {"StartTime": last_value.timestamp(), "EndTime": NOW.timestamp()}
+ base = {"MaxResults": 50, "DataType": "InsightsEvents", **window}
+ assert calls(result.requests, "DescribeTrails") == [{"includeShadowTrails": True}]
+ assert calls(result.requests, "ListInsightsData") == [
+ {**base, "InsightSource": "arn:trail/a"},
+ {**base, "InsightSource": "arn:trail/a", "NextToken": "p2"},
+ {**base, "InsightSource": "arn:trail/b"},
+ ]
+ assert [(row["trail_arn"], row["event_id"], row["event_time"]) for row in result.rows] == [
+ ("arn:trail/a", "one", NOW),
+ ("arn:trail/a", "two", NOW),
+ ]
+ assert result.response is not None and result.response.sort_mode == "desc"
+
+
+ "state,expected",
+ [
+ (
+ AwsCloudTrailResumeConfig(parent="arn:trail/a", next_token="p2", start_time=100, end_time=200),
+ [("arn:trail/a", "p2"), ("arn:trail/b", None)],
+ ),
+ (AwsCloudTrailResumeConfig(parent="arn:trail/a", start_time=100, end_time=200), [("arn:trail/b", None)]),
+ ],
+)
+def test_fan_out_resumes_from_saved_parent(
+ state: AwsCloudTrailResumeConfig, expected: list[tuple[str, str | None]]
+) -> None:
+ result = driver().run(
+ "insights_data",
+ by_operation(
+ {
+ "DescribeTrails": [TRAILS],
+ "ListInsightsData": [ScriptedResponse(json={"Events": [{"EventId": "x"}]})] * len(expected),
+ }
+ ),
+ resume_state=state,
+ )
+ assert result.raised is None
+ sent = calls(result.requests, "ListInsightsData")
+ assert [(call["InsightSource"], call.get("NextToken")) for call in sent] == expected
+ assert all(call["StartTime"] == 100 and call["EndTime"] == 200 for call in sent)
+ assert result.saved_states[-1] == AwsCloudTrailResumeConfig(
+ parent="arn:trail/b", start_time=100, end_time=200, finished=True
+ )
+
+
+def test_insights_metric_data_reads_one_series_per_insight() -> None:
+ result = driver().run(
+ "insights_metric_data",
+ by_operation(
+ {
+ "LookupEvents": [
+ ScriptedResponse(
+ json={
+ "Events": [
+ insight("iam.amazonaws.com", "CreateUser", "ApiCallRateInsight"),
+ insight("iam.amazonaws.com", "CreateUser", "ApiCallRateInsight"),
+ insight("iam.amazonaws.com", "CreateUser", "ApiErrorRateInsight", "AccessDenied"),
+ {"EventId": "malformed", "CloudTrailEvent": "not JSON"},
+ ]
+ }
+ )
+ ],
+ "DescribeTrails": [TRAILS],
+ "ListInsightsData": [
+ ScriptedResponse(json={"Events": [insight("s3.amazonaws.com", "GetObject", "ApiCallRateInsight")]}),
+ ScriptedResponse(status=400, json={"__type": "InvalidParameterException"}),
+ ],
+ "ListInsightsMetricData": [
+ ScriptedResponse(json={"Timestamps": [], "Values": []}),
+ ScriptedResponse(json={"Timestamps": [NOW.timestamp()], "Values": [1.0], "NextToken": "more"}),
+ ScriptedResponse(json={"Timestamps": [(NOW + dt.timedelta(hours=1)).timestamp()], "Values": [2.0]}),
+ ScriptedResponse(json={"Timestamps": [NOW.timestamp()], "Values": [3.0]}),
+ ],
+ }
+ ),
+ )
+ assert result.raised is None
+ assert calls(result.requests, "LookupEvents") == [{"MaxResults": 50, "EventCategory": "insight"}]
+ base = {"MaxResults": 21600, "Period": 3600}
+ assert calls(result.requests, "ListInsightsMetricData") == [
+ {**base, "EventSource": "iam.amazonaws.com", "EventName": "CreateUser", "InsightType": "ApiCallRateInsight"},
+ {
+ **base,
+ "EventSource": "iam.amazonaws.com",
+ "EventName": "CreateUser",
+ "InsightType": "ApiErrorRateInsight",
+ "ErrorCode": "AccessDenied",
+ },
+ {
+ **base,
+ "EventSource": "iam.amazonaws.com",
+ "EventName": "CreateUser",
+ "InsightType": "ApiErrorRateInsight",
+ "ErrorCode": "AccessDenied",
+ "NextToken": "more",
+ },
+ {
+ **base,
+ "EventSource": "s3.amazonaws.com",
+ "EventName": "GetObject",
+ "InsightType": "ApiCallRateInsight",
+ "TrailName": "arn:trail/a",
+ },
+ ]
+ assert [
+ (row["trail_arn"], row["insight_type"], row["error_code"], row["timestamp"], row["value"])
+ for row in result.rows
+ ] == [
+ ("", "ApiErrorRateInsight", "AccessDenied", NOW, 1.0),
+ ("", "ApiErrorRateInsight", "AccessDenied", NOW + dt.timedelta(hours=1), 2.0),
+ ("arn:trail/a", "ApiCallRateInsight", "", NOW, 3.0),
+ ]
+ assert result.response is not None and result.response.sort_mode is None
+
+
+ "last_value,start",
+ [(None, NOW - dt.timedelta(days=7)), (NOW - dt.timedelta(days=2), NOW - dt.timedelta(days=3))],
+)
+@time_machine.travel(NOW, tick=False)
+def test_lake_queries_describe_each_query_and_refresh_recent_statuses(
+ last_value: dt.datetime | None, start: dt.datetime
+) -> None:
+ query = {"QueryId": "q1", "QueryStatus": "RUNNING", "CreationTime": NOW.timestamp()}
+ result = driver().run(
+ "lake_queries",
+ by_operation(
+ {
+ "ListEventDataStores": [
+ ScriptedResponse(json={"EventDataStores": [{"EventDataStoreArn": "arn:store"}]})
+ ],
+ "ListQueries": [
+ ScriptedResponse(json={"Queries": [query, {**query, "QueryId": "q2"}], "NextToken": "p2"}),
+ ScriptedResponse(json={"Queries": [{**query, "QueryId": "q3"}]}),
+ ],
+ "DescribeQuery": [
+ ScriptedResponse(
+ json={"QueryId": "q1", "QueryStatus": "FINISHED", "QueryStatistics": {"BytesScanned": 10}}
+ ),
+ ScriptedResponse(status=400, json={"__type": "QueryIdNotFoundException"}),
+ ScriptedResponse(json={"QueryId": "q3", "QueryStatus": "FAILED", "ErrorMessage": "bad"}),
+ ],
+ }
+ ),
+ incremental_field="creation_time",
+ db_incremental_field_last_value=last_value,
+ )
+ assert result.raised is None
+ window = {
+ "MaxResults": 1000,
+ "EventDataStore": "arn:store",
+ "StartTime": start.timestamp(),
+ "EndTime": NOW.timestamp(),
+ }
+ assert calls(result.requests, "ListQueries") == [window, {**window, "NextToken": "p2"}]
+ assert calls(result.requests, "DescribeQuery") == [
+ {"EventDataStore": "arn:store", "QueryId": query_id} for query_id in ("q1", "q2", "q3")
+ ]
+ assert [
+ (row["event_data_store_arn"], row["query_id"], row["query_status"], row["creation_time"]) for row in result.rows
+ ] == [
+ ("arn:store", "q1", "FINISHED", NOW),
+ ("arn:store", "q2", "RUNNING", NOW),
+ ("arn:store", "q3", "FAILED", NOW),
+ ]
+ assert result.rows[0]["query_statistics"] == {"BytesScanned": 10}
+ # A hand-off after the first row of a page must read that page again.
+ assert [state.next_token for state in result.committed_states] == ["p2", None]