:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): implement amazon cloudwatch source (#118121)
Daniel RCc6daa2465365
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 94558929360b..cc1160ca8f1c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -76,6 +76,7 @@ the row lists both.| alguna | HTTP | requests + `rest_source.RESTClient` | ✅ || alpha_vantage | HTTP | requests | ✅ || amazon_ads | HTTP | requests | ✅ |+| amazon_cloudwatch | HTTP | requests + botocore (SigV4 signing, model-driven serializer) | ✅ || amazon_s3 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ || amazon_selling_partner | HTTP | requests + `rest_source.RESTClient` | ✅ || amplemarket | HTTP | requests + `rest_source.RESTClient` | ✅ |@@ -1036,7 +1037,6 @@ doesn't conflict with concurrent PRs.- allegro- alpaca_broker_api- amazon_ads-- amazon_cloudwatch- amazon_eventbridge- amazon_kinesis- amazon_snsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/amazon_cloudwatch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/amazon_cloudwatch.pynew file mode 100644index 000000000000..b36b792e4ae3--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/amazon_cloudwatch.py@@ -0,0 +1,871 @@+import re+import copy+import json+import hashlib+import datetime as dt+import dataclasses+from collections.abc import Iterator+from functools import cache+from typing import Any, Optional+from urllib.parse import urlencode++import requests+from botocore.auth import SigV4Auth+from botocore.awsrequest import AWSRequest+from botocore.credentials import Credentials+from botocore.loaders import Loader+from botocore.model import OperationModel, ServiceModel+from botocore.parsers import ResponseParserError, create_parser+from botocore.serialize import create_serializer+from botocore.validate import ParamValidator+from structlog.types import FilteringBoundLogger+from tenacity import (+ RetryCallState,+ retry,+ retry_if_exception_type,+ stop_after_attempt,+ stop_before_delay,+ wait_exponential_jitter,+)++from posthog.exceptions_capture import capture_exception++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.settings import (+ ALARMS_PAGE_SIZE,+ AMAZON_CLOUDWATCH_ENDPOINTS,+ LOG_EVENTS_PAGE_SIZE,+ LOG_GROUPS_PAGE_SIZE,+ LOGS_SERVICE,+ MAX_DISCOVERED_LOG_GROUPS,+ METRIC_DATA_MAX_DATAPOINTS,+ MONITORING_SERVICE,+ SERVICE_MODELS,+ STS_SERVICE,+ AwsService,+ CloudWatchEndpointConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import NO_RETRY+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse++# (connect, read) seconds for one try.+REQUEST_TIMEOUT = (10, 60)+MAX_ATTEMPTS = 6+# No try starts after this, so one call with its tries and waits stays under five minutes.+RETRY_BUDGET_SECONDS = 180++# AWS answers these as HTTP 400 with the code in the body, so the transport cannot see them.+THROTTLING_ERROR_CODES = frozenset(+ {+ "Throttling",+ "ThrottlingException",+ "RequestLimitExceeded",+ "TooManyRequestsException",+ "LimitExceededException",+ }+)++_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")++ALARM_TYPES = ["CompositeAlarm", "MetricAlarm"]+++class AmazonCloudWatchError(Exception):+ def __init__(self, message: str, code: str | None = None) -> None:+ super().__init__(message)+ self.code = code+++class AmazonCloudWatchTransientError(AmazonCloudWatchError):+ pass+++class AmazonCloudWatchThrottledError(AmazonCloudWatchTransientError):+ pass+++METRIC_QUERY_FORBIDDEN = "MetricQueryForbidden"+METRIC_QUERY_PARTIAL_DATA = "MetricQueryPartialData"+INVALID_METRIC_QUERIES = "Invalid metric queries"+INVALID_START_DATE = "Invalid start date"+# The cursor a slice's last page stages when no slice follows it, so a resumed attempt knows the run is done.+COMPLETED_SLICE = "__complete__"+METRIC_QUERY_INTERNAL_ERROR = "MetricQueryInternalError"+INVALID_REQUEST_ERROR_CODES = frozenset(+ {"InvalidParameterValue", "InvalidParameterCombination", "MissingParameter", "InvalidParameterException"}+)++[email protected](frozen=True)+class AmazonCloudWatchResumeConfig:+ """Where a previous attempt stopped: the slice (a log group, or a window start), its page token,+ and the time range of the run, so a resumed attempt reads the same range rather than one that+ moved with the clock."""++ slice_key: Optional[str] = None+ next_token: Optional[str] = None+ window_start: Optional[str] = None+ window_end: Optional[str] = None++[email protected](frozen=True)+class TimeWindow:+ start: dt.datetime+ end: dt.datetime+++def _snake(name: str) -> str:+ """`AlarmConfigurationUpdatedTimestamp` -> `alarm_configuration_updated_timestamp`."""+ return _CAMEL_BOUNDARY.sub("_", name).lower()+++def _snake_keys(obj: dict[str, Any]) -> dict[str, Any]:+ return {_snake(key): value for key, value in obj.items()}+++def _from_millis(value: Any) -> Optional[dt.datetime]:+ if value is None:+ return None+ return dt.datetime.fromtimestamp(int(value) / 1000, tz=dt.UTC)+++def _to_millis(value: dt.datetime) -> int:+ return int(value.timestamp() * 1000)+++def _as_utc(value: dt.datetime) -> dt.datetime:+ return value if value.tzinfo is not None else value.replace(tzinfo=dt.UTC)+++def coerce_datetime(value: Any) -> Optional[dt.datetime]:+ if isinstance(value, dt.datetime):+ return _as_utc(value)+ if isinstance(value, dt.date):+ return dt.datetime(value.year, value.month, value.day, tzinfo=dt.UTC)+ if isinstance(value, str) and value:+ try:+ return _as_utc(dt.datetime.fromisoformat(value.replace("Z", "+00:00")))+ except ValueError:+ return None+ return None+++@cache+def aws_dns_suffix(region: str) -> Optional[str]:+ """The DNS suffix of the AWS partition a region belongs to, or None for an unknown region.++ China regions live under `amazonaws.com.cn`, so the suffix comes from botocore's partition table.+ """+ for partition in Loader().load_data("partitions")["partitions"]:+ if region in partition.get("regions", {}) or re.match(partition["regionRegex"], region):+ return partition["outputs"]["dnsSuffix"]+ return None+++def is_valid_region(region: str) -> bool:+ return aws_dns_suffix(region.strip()) is not None+++def parse_log_group_names(raw: Optional[str]) -> list[str]:+ """One log group per line; blank lines and surrounding whitespace are ignored."""+ if not raw:+ return []+ names = [line.strip() for line in raw.splitlines()]+ return list(dict.fromkeys(name for name in names if name))+++def parse_metric_queries(raw: Optional[str]) -> list[dict[str, Any]]:+ """The metric queries field, as GetMetricData `MetricDataQueries`.++ Accepts the API shape (`Id`, `MetricStat`/`Expression`) as well as a flatter one+ (`id`, `namespace`, `metric_name`, `dimensions`, `stat`, `period`) that is easier to type.+ Raises `ValueError` with a message fit for the setup wizard.+ """+ if not raw or not raw.strip():+ return []+ try:+ parsed = json.loads(raw)+ except ValueError as error:+ raise ValueError(f"{INVALID_METRIC_QUERIES}: not valid JSON ({error}).") from error+ if isinstance(parsed, dict):+ parsed = [parsed]+ if not isinstance(parsed, list) or not all(isinstance(item, dict) for item in parsed):+ raise ValueError(f"{INVALID_METRIC_QUERIES}: expected a JSON list of query objects.")++ queries: list[dict[str, Any]] = []+ seen: set[str] = set()+ for index, item in enumerate(parsed, start=1):+ if "MetricStat" in item or "Expression" in item:+ query = dict(item)+ query.setdefault("Id", f"q{index}")+ else:+ query = _metric_query_from_flat(item, index)+ query_id = str(query["Id"])+ if query_id in seen:+ raise ValueError(f"{INVALID_METRIC_QUERIES}: the id '{query_id}' is used twice.")+ seen.add(query_id)+ query.setdefault("ReturnData", True)+ queries.append(query)++ input_shape = _operation(MONITORING_SERVICE, "GetMetricData").input_shape+ assert input_shape is not None # GetMetricData takes a request body+ report = ParamValidator().validate(+ {+ "MetricDataQueries": queries,+ "StartTime": dt.datetime(2000, 1, 1, tzinfo=dt.UTC),+ "EndTime": dt.datetime(2000, 1, 2, tzinfo=dt.UTC),+ },+ input_shape,+ )+ if report.has_errors():+ raise ValueError(f"{INVALID_METRIC_QUERIES}: they do not match the CloudWatch API. {report.generate_report()}")+ return queries+++def _period(value: Any, index: int, default: Optional[int]) -> Optional[int]:+ if value is None or value == "":+ return default+ try:+ return int(value)+ except (TypeError, ValueError) as error:+ raise ValueError(+ f"{INVALID_METRIC_QUERIES}: query {index} has a period that is not a whole number of seconds."+ ) from error+++def _metric_query_from_flat(item: dict[str, Any], index: int) -> dict[str, Any]:+ query_id = str(item.get("id") or f"q{index}")+ if "expression" in item:+ query: dict[str, Any] = {"Id": query_id, "Expression": item["expression"]}+ expression_period = _period(item.get("period"), index, None)+ if expression_period:+ query["Period"] = expression_period+ else:+ namespace = item.get("namespace")+ metric_name = item.get("metric_name") or item.get("metricName")+ if not namespace or not metric_name:+ raise ValueError(+ f"{INVALID_METRIC_QUERIES}: query {index} needs a namespace and a metric_name, or an expression."+ )+ dimension_list = _dimensions(item.get("dimensions") or {}, index)+ metric: dict[str, Any] = {"Namespace": namespace, "MetricName": metric_name}+ if dimension_list:+ metric["Dimensions"] = dimension_list+ query = {+ "Id": query_id,+ "MetricStat": {+ "Metric": metric,+ "Period": _period(item.get("period"), index, 300),+ "Stat": item.get("stat") or "Average",+ },+ }+ if item.get("unit"):+ query["MetricStat"]["Unit"] = item["unit"]+ if item.get("label"):+ query["Label"] = item["label"]+ return query+++def _dimensions(dimensions: Any, index: int) -> list[dict[str, str]]:+ if isinstance(dimensions, dict):+ return [{"Name": str(name), "Value": str(value)} for name, value in dimensions.items()]+ if not isinstance(dimensions, list):+ raise ValueError(+ f"{INVALID_METRIC_QUERIES}: query {index} needs dimensions as an object or a list of name and value pairs."+ )+ dimension_list: list[dict[str, str]] = []+ for dimension in dimensions:+ name = dimension.get("name") or dimension.get("Name") if isinstance(dimension, dict) else None+ value = dimension.get("value") or dimension.get("Value") if isinstance(dimension, dict) else None+ if not name or value is None:+ raise ValueError(f"{INVALID_METRIC_QUERIES}: query {index} has a dimension without a name and a value.")+ dimension_list.append({"Name": str(name), "Value": str(value)})+ return dimension_list+++@cache+def _service_model(service: AwsService) -> ServiceModel:+ model_name, api_version = SERVICE_MODELS[service]+ return ServiceModel(+ Loader().load_service_model(model_name, "service-2", api_version=api_version), service_name=model_name+ )+++def _operation(service: AwsService, operation: str) -> OperationModel:+ return _service_model(service).operation_model(operation)+++def make_session(secret_access_key: str, session_token: Optional[str]) -> requests.Session:+ redact = tuple(value for value in (secret_access_key, session_token) if value)+ return make_tracked_session(retry=NO_RETRY, redact_values=redact, allow_redirects=False)+++def _error_from(parsed: dict[str, Any], status_code: int) -> AmazonCloudWatchError:+ error = parsed.get("Error") or {}+ code = str(error.get("Code") or "").split("#")[-1]+ # botocore reports an empty or HTML error body by its status number alone.+ if not code or code.isdigit():+ code = f"HTTP {status_code}"+ message = str(error.get("Message") or "")[:500]+ text = f"Amazon CloudWatch request failed: {code} - {message}"+ if code in THROTTLING_ERROR_CODES or status_code == 429:+ return AmazonCloudWatchThrottledError(text, code)+ if status_code >= 500:+ return AmazonCloudWatchTransientError(text, code)+ return AmazonCloudWatchError(text, code)+++def _before_retry_wait(_state: RetryCallState) -> None:+ reach_safe_point()+++def _retry_wait(seconds: float) -> None:+ interruptible_wait(seconds)+++# The only retry layer: the session sends each try once, so tries and waits do not multiply.+@retry(+ retry=retry_if_exception_type((AmazonCloudWatchTransientError, requests.ConnectionError, requests.Timeout)),+ stop=stop_after_attempt(MAX_ATTEMPTS) | stop_before_delay(RETRY_BUDGET_SECONDS),+ wait=wait_exponential_jitter(initial=2, max=30),+ sleep=_retry_wait,+ before_sleep=_before_retry_wait,+ reraise=True,+)+def send_operation(+ session: requests.Session,+ credentials: Credentials,+ region: str,+ service: AwsService,+ operation: str,+ params: dict[str, Any],+) -> dict[str, Any]:+ """Serialize one API call from its botocore model, sign it with SigV4 and send it over the tracked session.++ The Logs API is JSON; the CloudWatch metrics API is the older query protocol (form-encoded+ request, XML response). The service model drives both, so neither needs hand-written parsing.+ """+ model = _service_model(service)+ operation_model = _operation(service, operation)+ serialized = create_serializer(model.protocol, include_validation=False).serialize_to_request(+ params, operation_model+ )+ body = serialized["body"]+ if isinstance(body, dict):+ body = urlencode(body).encode()+ elif isinstance(body, str):+ body = body.encode()+ suffix = aws_dns_suffix(region)+ if suffix is None:+ raise ValueError(f"Unknown AWS region: {region}")+ url = f"https://{model.endpoint_prefix}.{region}.{suffix}{serialized['url_path']}"++ aws_request = AWSRequest(method="POST", url=url, data=body, headers=dict(serialized["headers"]))+ SigV4Auth(credentials, model.signing_name, region).add_auth(aws_request)++ response = session.post(url, data=body, headers=dict(aws_request.headers.items()), timeout=REQUEST_TIMEOUT)++ output_shape = operation_model.output_shape+ assert output_shape is not None # every operation this source calls returns a body+ try:+ parsed = create_parser(model.protocol).parse(+ {"status_code": response.status_code, "headers": dict(response.headers), "body": response.content},+ output_shape,+ )+ except ResponseParserError:+ # A load balancer error page is not the protocol's format.+ parsed = {"Error": {}} if response.status_code >= 400 else {"Error": {"Message": "unreadable response"}}+ if response.status_code >= 400 or "Error" in parsed:+ raise _error_from(parsed, response.status_code)+ parsed.pop("ResponseMetadata", None)+ return parsed+++# AWS error code -> the message the setup wizard shows. A denial names the calling identity's+# IAM ARN, so the raw AWS text never reaches the user.+VALIDATION_ERROR_MESSAGES: dict[str, str] = {+ "UnrecognizedClientException": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",+ "InvalidClientTokenId": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",+ "SignatureDoesNotMatch": "AWS rejected the request signature. Re-enter the secret access key.",+ "InvalidSignatureException": "AWS rejected the request signature. If you are using temporary credentials, the session token has expired.",+ "ExpiredTokenException": "The AWS session token has expired. Reconnect with fresh credentials.",+ "ExpiredToken": "The AWS session token has expired. Reconnect with fresh credentials.",+ "AccessDeniedException": "These AWS credentials cannot read CloudWatch. Grant logs:DescribeLogGroups, logs:FilterLogEvents, cloudwatch:DescribeAlarms, cloudwatch:DescribeAlarmHistory and cloudwatch:GetMetricData to the IAM user or role.",+ "AccessDenied": "These AWS credentials cannot read CloudWatch. Grant logs:DescribeLogGroups, logs:FilterLogEvents, cloudwatch:DescribeAlarms, cloudwatch:DescribeAlarmHistory and cloudwatch:GetMetricData to the IAM user or role.",+ # A rejection without an error body reaches `_error_from` as its bare status.+ "HTTP 401": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",+ "HTTP 403": "These AWS credentials cannot read CloudWatch. Grant logs:DescribeLogGroups, logs:FilterLogEvents, cloudwatch:DescribeAlarms, cloudwatch:DescribeAlarmHistory and cloudwatch:GetMetricData to the IAM user or role.",+}++TRANSIENT_VALIDATION_ERROR = "Amazon CloudWatch is busy or temporarily unavailable. Wait a moment and try again."++GENERIC_VALIDATION_ERROR = (+ "AWS didn't accept the CloudWatch request. Check the access key is active and can read CloudWatch, then try again."+)+++def validate_credentials(+ aws_access_key_id: str,+ aws_secret_access_key: str,+ aws_session_token: Optional[str],+ aws_region: str,+ metric_queries: Optional[str],+ start_date: Optional[str] = None,+) -> tuple[bool, Optional[str]]:+ """One `GetCallerIdentity` probe, plus a local check of the metric queries and start date.++ STS answers any valid key without an IAM permission, so a key scoped to metrics or alarms only+ still connects. A missing permission for a table surfaces when that table syncs."""+ if not aws_access_key_id or not aws_secret_access_key:+ return False, "AWS access key ID and secret access key are required"+ if not aws_region or not is_valid_region(aws_region):+ return False, "Enter an AWS region code, for example us-east-1."+ try:+ parse_metric_queries(metric_queries)+ parse_start_date(start_date)+ except ValueError as error:+ return False, str(error)++ try:+ send_operation(+ make_session(aws_secret_access_key, aws_session_token),+ Credentials(aws_access_key_id, aws_secret_access_key, aws_session_token or None),+ aws_region.strip(),+ STS_SERVICE,+ "GetCallerIdentity",+ {},+ )+ except AmazonCloudWatchError as error:+ code = error.code or ""+ if code in VALIDATION_ERROR_MESSAGES:+ return False, VALIDATION_ERROR_MESSAGES[code]+ if isinstance(error, AmazonCloudWatchTransientError):+ return False, TRANSIENT_VALIDATION_ERROR+ capture_exception(error)+ return False, GENERIC_VALIDATION_ERROR+ except Exception:+ return False, "Could not reach Amazon CloudWatch. Check the region and try again."++ return True, None+++def parse_start_date(value: Optional[str]) -> Optional[dt.datetime]:+ """The configured start date, or None when it is empty. A value that is not a date raises,+ because falling back to the default range would drop the history the user asked for."""+ if not value or not value.strip():+ return None+ parsed = coerce_datetime(value.strip())+ if parsed is None:+ raise ValueError(f"{INVALID_START_DATE}: enter it as YYYY-MM-DD, for example 2024-01-31.")+ return parsed+++def resolve_window(+ endpoint_config: CloudWatchEndpointConfig,+ configured_start_date: Optional[str],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+ now: dt.datetime,+) -> TimeWindow:+ """The time range this run asks AWS for: the watermark on an incremental run, else the start+ date, else the endpoint's default lookback. Never before the configured start date.++ The default lookback only bounds a first run. A run after a long pause starts at its watermark,+ so the gap is read, not dropped."""+ configured = parse_start_date(configured_start_date)+ start = configured or (now - dt.timedelta(days=endpoint_config.lookback_days or 0))+ if should_use_incremental_field:+ watermark = coerce_datetime(db_incremental_field_last_value)+ if watermark is not None:+ start = max(configured, watermark) if configured else watermark+ return TimeWindow(start=min(start, now), end=now)+++# CloudWatch keeps datapoints at a coarser resolution as they age: 1-minute data for 15 days,+# 5-minute data for 63 days and 1-hour data for 455 days. A query asking for a finer period than+# the data still has returns nothing for that range.+_RESOLUTION_BY_AGE = (+ (dt.timedelta(days=63), 3600),+ (dt.timedelta(days=15), 300),+ (dt.timedelta(hours=3), 60),+)+++def period_for_window(period: int, window_start: dt.datetime, now: dt.datetime) -> int:+ """The query period to use for a window: the requested one, rounded up to the coarsest+ resolution CloudWatch still keeps for the oldest data in the window."""+ age = now - window_start+ resolution = next((seconds for threshold, seconds in _RESOLUTION_BY_AGE if age > threshold), 1)+ if period % resolution == 0:+ return period+ return (period // resolution + 1) * resolution+++def queries_for_window(+ queries: list[dict[str, Any]], window_start: dt.datetime, now: dt.datetime+) -> list[dict[str, Any]]:+ adapted = copy.deepcopy(queries)+ for query in adapted:+ if isinstance(query.get("MetricStat"), dict) and query["MetricStat"].get("Period"):+ query["MetricStat"]["Period"] = period_for_window(int(query["MetricStat"]["Period"]), window_start, now)+ elif query.get("Period"):+ query["Period"] = period_for_window(int(query["Period"]), window_start, now)+ return adapted+++def build_windows(window: TimeWindow, window_days: Optional[int]) -> list[TimeWindow]:+ if window_days is None:+ return [window]+ windows: list[TimeWindow] = []+ cursor = window.start+ step = dt.timedelta(days=window_days)+ while cursor < window.end:+ stop = min(cursor + step, window.end)+ windows.append(TimeWindow(start=cursor, end=stop))+ cursor = stop+ return windows+++class CloudWatchClient:+ def __init__(+ self,+ aws_access_key_id: str,+ aws_secret_access_key: str,+ aws_session_token: Optional[str],+ aws_region: str,+ ) -> None:+ self._session = make_session(aws_secret_access_key, aws_session_token)+ self._credentials = Credentials(aws_access_key_id, aws_secret_access_key, aws_session_token or None)+ self._region = aws_region.strip()++ def call(self, service: AwsService, operation: str, params: dict[str, Any]) -> dict[str, Any]:+ return send_operation(self._session, self._credentials, self._region, service, operation, params)++ def list_log_group_names(self, logger: FilteringBoundLogger, limit: int = MAX_DISCOVERED_LOG_GROUPS) -> list[str]:+ names: list[str] = []+ next_token: Optional[str] = None+ while len(names) < limit:+ params: dict[str, Any] = {"limit": min(LOG_GROUPS_PAGE_SIZE, limit - len(names))}+ if next_token:+ params["nextToken"] = next_token+ body = self.call(LOGS_SERVICE, "DescribeLogGroups", params)+ names.extend(+ group["logGroupName"]+ for group in body.get("logGroups") or []+ # FilterLogEvents reads Standard groups only; other classes would fail the whole table.+ if group.get("logGroupName") and group.get("logGroupClass", "STANDARD") == "STANDARD"+ )+ next_token = body.get("nextToken")+ if not next_token:+ return names+ logger.warning(+ f"Amazon CloudWatch has more than {limit} log groups, so only the first {limit} by name sync. "+ "List the log groups to sync on the source to choose them."+ )+ return names+++def log_group_rows(body: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for group in body.get("logGroups") or []:+ row = _snake_keys(group)+ row["creation_time"] = _from_millis(row.get("creation_time"))+ rows.append(row)+ return rows+++def log_event_rows(log_group_name: str, body: dict[str, Any]) -> list[dict[str, Any]]:+ return [+ {+ "log_group_name": log_group_name,+ "event_id": event.get("eventId"),+ "log_stream_name": event.get("logStreamName"),+ "timestamp": _from_millis(event.get("timestamp")),+ "ingestion_time": _from_millis(event.get("ingestionTime")),+ "message": event.get("message"),+ }+ for event in body.get("events") or []+ ]+++def alarm_rows(body: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for alarm_type, key in (("MetricAlarm", "MetricAlarms"), ("CompositeAlarm", "CompositeAlarms")):+ for alarm in body.get(key) or []:+ row = _snake_keys(alarm)+ row["alarm_type"] = alarm_type+ rows.append(row)+ return rows+++def alarm_history_rows(body: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for item in body.get("AlarmHistoryItems") or []:+ row = _snake_keys(item)+ row["history_id"] = hashlib.sha256(json.dumps(item, sort_keys=True, default=str).encode()).hexdigest()+ rows.append(row)+ return rows+++def check_metric_results(body: dict[str, Any]) -> None:+ """Raise for a query AWS answered with an error status inside an HTTP 200 response.++ `Forbidden` means the credentials cannot read that metric. `InternalError` is AWS's transient+ failure, retried with the attempt; the cursor of the page was not staged, so nothing is skipped.+ `PartialData` with no continuation token means the query produced more datapoints than one+ response holds, and the rest cannot be fetched.+ """+ for result in body.get("MetricDataResults") or []:+ status = result.get("StatusCode")+ if status == "PartialData" and not body.get("NextToken"):+ raise AmazonCloudWatchError(+ f"Amazon CloudWatch request failed: {METRIC_QUERY_PARTIAL_DATA} - metric query '{result.get('Id')}' "+ "returned more datapoints than one response holds",+ METRIC_QUERY_PARTIAL_DATA,+ )+ if status not in ("Forbidden", "InternalError"):+ continue+ detail = "; ".join(str(message.get("Value") or message.get("Code")) for message in result.get("Messages") or [])+ code = METRIC_QUERY_FORBIDDEN if status == "Forbidden" else METRIC_QUERY_INTERNAL_ERROR+ raise AmazonCloudWatchError(+ f"Amazon CloudWatch request failed: {code} - metric query '{result.get('Id')}': {detail}"[:600], code+ )+++def metric_data_rows(body: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for result in body.get("MetricDataResults") or []:+ timestamps = result.get("Timestamps") or []+ values = result.get("Values") or []+ for timestamp, value in zip(timestamps, values):+ rows.append(+ {+ "query_id": result.get("Id"),+ # One query returns several series for a SEARCH expression or a Metrics Insights+ # GROUP BY. The label names the series, so it is part of the row key.+ "label": result.get("Label") or result.get("Id"),+ "timestamp": _as_utc(timestamp),+ "value": value,+ "status_code": result.get("StatusCode"),+ }+ )+ return rows+++def _slices(+ endpoint_config: CloudWatchEndpointConfig,+ client: CloudWatchClient,+ log_group_names: list[str],+ window: TimeWindow,+ logger: FilteringBoundLogger,+) -> list[tuple[str, Optional[str]]]:+ """The request series of one run: (slice key, log group) pairs for logs, window starts otherwise."""+ if endpoint_config.operation == "FilterLogEvents":+ groups = log_group_names or client.list_log_group_names(logger)+ return [(name, name) for name in groups]+ if endpoint_config.lookback_days is None:+ return [("", None)]+ return [(piece.start.isoformat(), None) for piece in build_windows(window, endpoint_config.window_days)]+++def _params(+ endpoint_config: CloudWatchEndpointConfig,+ slice_key: str,+ log_group: Optional[str],+ window: TimeWindow,+ metric_queries: list[dict[str, Any]],+ next_token: Optional[str],+) -> dict[str, Any]:+ operation = endpoint_config.operation+ params: dict[str, Any]+ if operation == "DescribeLogGroups":+ params = {"limit": LOG_GROUPS_PAGE_SIZE}+ if next_token:+ params["nextToken"] = next_token+ return params+ if operation == "FilterLogEvents":+ params = {+ "logGroupName": log_group,+ "startTime": _to_millis(window.start),+ "endTime": _to_millis(window.end),+ "limit": LOG_EVENTS_PAGE_SIZE,+ }+ if next_token:+ params["nextToken"] = next_token+ return params+ if operation == "DescribeAlarms":+ params = {"AlarmTypes": ALARM_TYPES, "MaxRecords": ALARMS_PAGE_SIZE}+ elif operation == "DescribeAlarmHistory":+ params = {+ "AlarmTypes": ALARM_TYPES,+ "StartDate": window.start,+ "EndDate": window.end,+ "ScanBy": "TimestampAscending",+ "MaxRecords": ALARMS_PAGE_SIZE,+ }+ else:+ piece = next(+ (w for w in build_windows(window, endpoint_config.window_days) if w.start.isoformat() == slice_key),+ window,+ )+ params = {+ "MetricDataQueries": queries_for_window(metric_queries, piece.start, window.end),+ "StartTime": piece.start,+ "EndTime": piece.end,+ "ScanBy": "TimestampAscending",+ "MaxDatapoints": METRIC_DATA_MAX_DATAPOINTS,+ }+ if next_token:+ params["NextToken"] = next_token+ return params+++def _rows(+ endpoint_config: CloudWatchEndpointConfig, log_group: Optional[str], body: dict[str, Any]+) -> list[dict[str, Any]]:+ operation = endpoint_config.operation+ if operation == "DescribeLogGroups":+ return log_group_rows(body)+ if operation == "FilterLogEvents":+ return log_event_rows(log_group or "", body)+ if operation == "DescribeAlarms":+ return alarm_rows(body)+ if operation == "DescribeAlarmHistory":+ return alarm_history_rows(body)+ return metric_data_rows(body)+++def _next_token(endpoint_config: CloudWatchEndpointConfig, body: dict[str, Any]) -> Optional[str]:+ return body.get("nextToken") if endpoint_config.service == LOGS_SERVICE else body.get("NextToken")+++def get_rows(+ aws_access_key_id: str,+ aws_secret_access_key: str,+ aws_session_token: Optional[str],+ aws_region: str,+ log_group_names: Optional[str],+ metric_queries: Optional[str],+ start_date: Optional[str],+ endpoint: str,+ resumable_source_manager: ResumableSourceManager[AmazonCloudWatchResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ endpoint_config = AMAZON_CLOUDWATCH_ENDPOINTS[endpoint]+ client = CloudWatchClient(aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region)++ queries = parse_metric_queries(metric_queries) if endpoint_config.operation == "GetMetricData" else []+ if endpoint_config.operation == "GetMetricData" and not queries:+ logger.info("Amazon CloudWatch metric_data has no metric queries configured, nothing to sync")+ resumable_source_manager.clear_state()+ return++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ saved_start = coerce_datetime(resume.window_start) if resume else None+ saved_end = coerce_datetime(resume.window_end) if resume else None+ if saved_start is not None and saved_end is not None:+ window = TimeWindow(start=saved_start, end=saved_end)+ else:+ window = resolve_window(+ endpoint_config,+ start_date,+ should_use_incremental_field,+ db_incremental_field_last_value,+ dt.datetime.now(dt.UTC),+ )+ if resume is not None and resume.slice_key == COMPLETED_SLICE:+ resumable_source_manager.clear_state()+ return++ slices = _slices(endpoint_config, client, parse_log_group_names(log_group_names), window, logger)++ index, next_token = 0, None+ if resume is not None:+ # A saved slice that no longer exists (the window moved, a log group went away) restarts the run.+ for position, (slice_key, _) in enumerate(slices):+ if slice_key == resume.slice_key:+ index, next_token = position, resume.next_token+ logger.debug(f"Resuming Amazon CloudWatch sync. endpoint={endpoint}, slice={slice_key}")+ break++ for position in range(index, len(slices)):+ slice_key, log_group = slices[position]+ following_slice = slices[position + 1][0] if position + 1 < len(slices) else COMPLETED_SLICE+ while True:+ body = client.call(+ endpoint_config.service,+ endpoint_config.operation,+ _params(endpoint_config, slice_key, log_group, window, queries, next_token),+ )+ if endpoint_config.operation == "GetMetricData":+ check_metric_results(body)+ rows = _rows(endpoint_config, log_group, body)+ next_token = _next_token(endpoint_config, body)++ # Staged before the batch it covers: a crash after the write resumes at the next page,+ # or at the next slice once this one has no page left.+ resumable_source_manager.save_state(+ AmazonCloudWatchResumeConfig(+ slice_key=slice_key if next_token else following_slice,+ next_token=next_token,+ window_start=window.start.isoformat(),+ window_end=window.end.isoformat(),+ )+ )+ if rows:+ yield rows+ else:+ # FilterLogEvents often returns empty pages mid-walk; let the pipeline commit and hand off.+ resumable_source_manager.safe_point()++ if not next_token:+ break++ resumable_source_manager.clear_state()+++def amazon_cloudwatch_source(+ aws_access_key_id: str,+ aws_secret_access_key: str,+ aws_session_token: Optional[str],+ aws_region: str,+ log_group_names: Optional[str],+ metric_queries: Optional[str],+ start_date: Optional[str],+ endpoint: str,+ resumable_source_manager: ResumableSourceManager[AmazonCloudWatchResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+ logger: FilteringBoundLogger,+) -> SourceResponse:+ endpoint_config = AMAZON_CLOUDWATCH_ENDPOINTS[endpoint]++ return SourceResponse(+ name=endpoint,+ items=lambda: get_rows(+ aws_access_key_id=aws_access_key_id,+ aws_secret_access_key=aws_secret_access_key,+ aws_session_token=aws_session_token,+ aws_region=aws_region,+ log_group_names=log_group_names,+ metric_queries=metric_queries,+ start_date=start_date,+ endpoint=endpoint,+ resumable_source_manager=resumable_source_manager,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=db_incremental_field_last_value,+ logger=logger,+ ),+ primary_keys=endpoint_config.primary_key,+ partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,+ partition_mode="datetime" if endpoint_config.partition_key else None,+ partition_format=endpoint_config.partition_format,+ sort_mode=endpoint_config.sort_mode,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/canonical_descriptions.pynew file mode 100644index 000000000000..0643c54d4d25--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/canonical_descriptions.py@@ -0,0 +1,96 @@+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)++_LOGS_API = "https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/"+_MONITORING_API = "https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/"++CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {+ "log_groups": {+ "description": "A log group in the region. Log groups hold log streams that share retention, monitoring and access control settings.",+ "docs_url": f"{_LOGS_API}API_LogGroup.html",+ "columns": {+ "log_group_name": "Name of the log group.",+ "arn": "ARN of the log group, with a trailing `:*`.",+ "log_group_arn": "ARN of the log group without the trailing `:*`.",+ "creation_time": "When the log group was created.",+ "retention_in_days": "Days CloudWatch keeps events in this group. Empty means events never expire.",+ "stored_bytes": "Bytes of log data stored in the group.",+ "metric_filter_count": "Number of metric filters defined on the group.",+ "kms_key_id": "ARN of the KMS key encrypting the group, when one is set.",+ "log_group_class": "Storage class of the group: STANDARD, INFREQUENT_ACCESS or DELIVERY.",+ "data_protection_status": "Whether a data protection policy is active on the group.",+ },+ },+ "log_events": {+ "description": "A log event from one of the synced log groups.",+ "docs_url": f"{_LOGS_API}API_FilteredLogEvent.html",+ "columns": {+ "log_group_name": "Log group the event belongs to.",+ "event_id": "Identifier of the event, unique within its log group.",+ "log_stream_name": "Log stream the event was written to.",+ "timestamp": "When the event occurred, as reported by the writer.",+ "ingestion_time": "When CloudWatch Logs received the event.",+ "message": "The raw log line.",+ },+ },+ "alarms": {+ "description": "A metric or composite alarm with its current state and configuration.",+ "docs_url": f"{_MONITORING_API}API_MetricAlarm.html",+ "columns": {+ "alarm_arn": "ARN of the alarm.",+ "alarm_name": "Name of the alarm.",+ "alarm_type": "MetricAlarm or CompositeAlarm.",+ "alarm_description": "Description of the alarm.",+ "state_value": "Current state: OK, ALARM or INSUFFICIENT_DATA.",+ "state_reason": "Human-readable explanation of the current state.",+ "state_reason_data": "JSON explanation of the current state, as CloudWatch recorded it.",+ "state_updated_timestamp": "When the state was last updated.",+ "state_transitioned_timestamp": "When the state last changed value.",+ "alarm_configuration_updated_timestamp": "When the alarm configuration was last changed.",+ "actions_enabled": "Whether actions run when the alarm changes state.",+ "alarm_actions": "Actions taken when the alarm enters ALARM.",+ "ok_actions": "Actions taken when the alarm enters OK.",+ "insufficient_data_actions": "Actions taken when the alarm enters INSUFFICIENT_DATA.",+ "metric_name": "Metric the alarm watches, for a single-metric alarm.",+ "namespace": "Namespace of the watched metric.",+ "statistic": "Statistic applied to the metric (Average, Sum, ...).",+ "extended_statistic": "Percentile statistic applied to the metric, when one is used.",+ "dimensions": "Dimensions of the watched metric.",+ "period": "Seconds over which the statistic is applied.",+ "unit": "Unit of the watched metric.",+ "evaluation_periods": "Number of periods compared with the threshold.",+ "datapoints_to_alarm": "Number of breaching datapoints needed to enter ALARM.",+ "threshold": "Value the statistic is compared with.",+ "comparison_operator": "How the statistic is compared with the threshold.",+ "treat_missing_data": "How missing datapoints are treated.",+ "metrics": "Metric math queries, for an alarm built on an expression.",+ "alarm_rule": "Rule expression, for a composite alarm.",+ },+ },+ "alarm_history": {+ "description": "A change recorded for an alarm: a state transition, a configuration update or an action execution. CloudWatch keeps 30 days of history.",+ "docs_url": f"{_MONITORING_API}API_AlarmHistoryItem.html",+ "columns": {+ "alarm_name": "Name of the alarm.",+ "alarm_type": "MetricAlarm or CompositeAlarm.",+ "timestamp": "When the change happened.",+ "history_item_type": "ConfigurationUpdate, StateUpdate or Action.",+ "history_summary": "Summary of the change.",+ "history_data": "JSON detail of the change, as CloudWatch recorded it.",+ "alarm_contributor_id": "Contributor the item relates to, for alarms with contributors.",+ "history_id": "Hash of the whole history item. CloudWatch gives items no id, and an alarm with contributors records several at the same instant.",+ },+ },+ "metric_data": {+ "description": "A datapoint returned by one of the metric queries configured on the source.",+ "docs_url": f"{_MONITORING_API}API_GetMetricData.html",+ "columns": {+ "query_id": "Id of the metric query the datapoint belongs to.",+ "label": "Label of the query, defaulting to its id.",+ "timestamp": "Start of the period the datapoint covers.",+ "value": "Value of the statistic for the period.",+ "status_code": "Complete, or PartialData when CloudWatch sent the query's datapoints across several pages.",+ },+ },+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/settings.pynew file mode 100644index 000000000000..0718e2a8d591--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/settings.py@@ -0,0 +1,129 @@+from dataclasses import dataclass+from typing import Literal++from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType++# All three APIs are regional. The service name doubles as the SigV4 signing name and the endpoint+# prefix (`logs.<region>.amazonaws.com`, `monitoring.<region>.amazonaws.com`, `sts.<region>...`).+AwsService = Literal["logs", "monitoring", "sts"]++LOGS_SERVICE: AwsService = "logs"+MONITORING_SERVICE: AwsService = "monitoring"+STS_SERVICE: AwsService = "sts"+# botocore model name and API version of each service, pinned so a botocore upgrade cannot change them.+SERVICE_MODELS: dict[AwsService, tuple[str, str]] = {+ LOGS_SERVICE: ("logs", "2014-03-28"),+ MONITORING_SERVICE: ("cloudwatch", "2010-08-01"),+ STS_SERVICE: ("sts", "2011-06-15"),+}++# FilterLogEvents caps a page at 10,000 events or 1 MB. DescribeLogGroups caps at 50.+LOG_EVENTS_PAGE_SIZE = 10_000+LOG_GROUPS_PAGE_SIZE = 50+# DescribeAlarms and DescribeAlarmHistory cap at 100 records.+ALARMS_PAGE_SIZE = 100+# GetMetricData returns up to 100,800 datapoints per call before it needs a NextToken.+METRIC_DATA_MAX_DATAPOINTS = 100_800+# Log groups discovered when the user names none. Beyond this a user should list the groups.+MAX_DISCOVERED_LOG_GROUPS = 50+++@dataclass(frozen=True)+class CloudWatchEndpointConfig:+ name: str+ service: AwsService+ operation: str+ primary_key: list[str]+ # Days before today a first sync reaches back. Logs and metrics are high volume, so these+ # stay short; a user who wants more sets a start date on the source.+ lookback_days: int | None = None+ # Days covered by one request series. GetMetricData degrades to coarser datapoints the+ # further back it reads, and both time-bound operations paginate within the window.+ window_days: int | None = None+ partition_key: str | None = None+ partition_format: Literal["month", "week", "day", "hour"] | None = None+ # Seconds an incremental run re-reads behind its watermark, for rows that arrive late. The+ # merge key drops the rows read twice.+ incremental_lookback_seconds: int | None = None+ # `desc` makes the pipeline commit the highest timestamp only when the run completes. Log events+ # arrive per log group and datapoints per query, so neither is ordered across the run.+ sort_mode: Literal["asc", "desc"] = "asc"+++AMAZON_CLOUDWATCH_ENDPOINTS: dict[str, CloudWatchEndpointConfig] = {+ "log_groups": CloudWatchEndpointConfig(+ name="log_groups",+ service=LOGS_SERVICE,+ operation="DescribeLogGroups",+ primary_key=["log_group_name"],+ ),+ "log_events": CloudWatchEndpointConfig(+ name="log_events",+ service=LOGS_SERVICE,+ operation="FilterLogEvents",+ primary_key=["log_group_name", "event_id"],+ lookback_days=7,+ # CloudWatch Logs accepts events stamped in the past, and filters on that event time.+ incremental_lookback_seconds=3600,+ partition_key="timestamp",+ partition_format="day",+ sort_mode="desc",+ ),+ "alarms": CloudWatchEndpointConfig(+ name="alarms",+ service=MONITORING_SERVICE,+ operation="DescribeAlarms",+ primary_key=["alarm_arn"],+ ),+ "alarm_history": CloudWatchEndpointConfig(+ name="alarm_history",+ service=MONITORING_SERVICE,+ operation="DescribeAlarmHistory",+ # A history item has no id, and an alarm with contributors records several items per instant,+ # so the key is a hash of the whole item.+ primary_key=["history_id"],+ # CloudWatch keeps alarm history for 30 days.+ lookback_days=30,+ partition_key="timestamp",+ partition_format="month",+ ),+ "metric_data": CloudWatchEndpointConfig(+ name="metric_data",+ service=MONITORING_SERVICE,+ operation="GetMetricData",+ primary_key=["query_id", "label", "timestamp"],+ # One-minute datapoints are kept for 15 days; older reads come back at 5-minute resolution.+ lookback_days=14,+ window_days=14,+ # Recent datapoints are still being aggregated, and some services publish minutes late.+ incremental_lookback_seconds=3600,+ partition_key="timestamp",+ partition_format="day",+ sort_mode="desc",+ ),+}++ENDPOINTS = tuple(AMAZON_CLOUDWATCH_ENDPOINTS.keys())++# Every time-bound operation filters server-side on the event timestamp, so that is the cursor.+# Log groups and alarms are catalogs with no time filter and stay full refresh.+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ name: [+ {+ "label": "timestamp",+ "type": IncrementalFieldType.DateTime,+ "field": "timestamp",+ "field_type": IncrementalFieldType.DateTime,+ }+ ]+ for name, endpoint_config in AMAZON_CLOUDWATCH_ENDPOINTS.items()+ if endpoint_config.lookback_days is not None+}++ENDPOINT_DESCRIPTIONS: dict[str, str] = {+ "log_groups": "Log groups in the region, with retention, stored bytes and encryption settings.",+ "log_events": "Log events from the configured log groups (or every log group when none is set), one row per event.",+ "alarms": "Metric and composite alarms with their current state, thresholds and actions.",+ "alarm_history": "State transitions, configuration updates and action executions for every alarm.",+ "metric_data": "Datapoints for the metric queries configured on the source, one row per query and timestamp.",+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/source.pyindex e4dc33a37c7b..690defd4d21e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/source.py@@ -1,8 +1,43 @@-from typing import cast+from typing import Optional, cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.facade.codebase_detection import detect+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.amazon_cloudwatch import (+ INVALID_METRIC_QUERIES,+ INVALID_REQUEST_ERROR_CODES,+ INVALID_START_DATE,+ METRIC_QUERY_FORBIDDEN,+ METRIC_QUERY_INTERNAL_ERROR,+ METRIC_QUERY_PARTIAL_DATA,+ THROTTLING_ERROR_CODES,+ VALIDATION_ERROR_MESSAGES,+ AmazonCloudWatchResumeConfig,+ amazon_cloudwatch_source,+ validate_credentials as validate_amazon_cloudwatch_credentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.settings import (+ AMAZON_CLOUDWATCH_ENDPOINTS,+ ENDPOINT_DESCRIPTIONS,+ ENDPOINTS,+ INCREMENTAL_FIELDS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (+ SourceSchema,+ build_endpoint_schemas,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazoncloudwatch import (AmazonCloudWatchSourceConfig,)@@ -10,18 +45,213 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class AmazonCloudWatchSource(SimpleSource[AmazonCloudWatchSourceConfig]):+class AmazonCloudWatchSource(ResumableSource[AmazonCloudWatchSourceConfig, AmazonCloudWatchResumeConfig]):+ codebase_detection = detect(+ env=("^(AMAZON_CLOUDWATCH|CLOUDWATCH)_",),+ npm=("@aws-sdk/client-cloudwatch", "@aws-sdk/client-cloudwatch-logs"),+ cargo=("aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs"),+ go=(+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch",+ "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs",+ ),+ maven=("software.amazon.awssdk:cloudwatch", "software.amazon.awssdk:cloudwatchlogs"),+ rubygems=("aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs"),+ )++ lists_tables_without_credentials = True # static endpoint catalog — safe for public docs+ api_docs_url = "https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/Welcome.html"+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AMAZONCLOUDWATCH+ @property+ def connection_host_fields(self) -> list[str]:+ # The region picks the endpoint the signed requests, and so the stored key, are sent to.+ return ["aws_region"]++ def get_non_retryable_errors(self) -> dict[str, str | None]:+ errors: dict[str, str | None] = {+ f"Amazon CloudWatch request failed: {code}": message for code, message in VALIDATION_ERROR_MESSAGES.items()+ }+ errors["Amazon CloudWatch request failed: ResourceNotFoundException"] = (+ "A log group was not found in this region. Check the log group names on the source."+ )+ errors[INVALID_METRIC_QUERIES] = "The metric queries on the source are not valid. Fix them and retry the sync."+ errors[INVALID_START_DATE] = (+ "The start date on the source is not a date. Enter it as YYYY-MM-DD and retry the sync."+ )+ errors[f"Amazon CloudWatch request failed: {METRIC_QUERY_PARTIAL_DATA}"] = (+ "A metric query returns more datapoints than CloudWatch sends in one response. Use a longer period, "+ "or fewer metrics in the expression, then retry the sync."+ )+ for code in INVALID_REQUEST_ERROR_CODES:+ errors[f"Amazon CloudWatch request failed: {code}"] = (+ "AWS rejected a request built from the source settings, usually a metric query. "+ "Check the metric queries and log group names, then retry the sync."+ )+ errors[f"Amazon CloudWatch request failed: {METRIC_QUERY_FORBIDDEN}"] = (+ "These AWS credentials cannot read one of the configured metrics. Grant cloudwatch:GetMetricData "+ "for it, or remove the query from the source."+ )+ return errors++ def get_retryable_errors(self) -> set[str]:+ # `send_operation` already exhausts the transport retry and a bounded throttling retry+ # before these reach here; Temporal retries the whole activity next.+ return {f"Amazon CloudWatch request failed: {code}" for code in THROTTLING_ERROR_CODES} | {+ "Amazon CloudWatch request failed: InternalFailure",+ "Amazon CloudWatch request failed: ServiceUnavailable",+ "Amazon CloudWatch request failed: HTTP 5",+ f"Amazon CloudWatch request failed: {METRIC_QUERY_INTERNAL_ERROR}",+ }++ def get_canonical_descriptions(self) -> CanonicalDescriptions:+ from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.canonical_descriptions import (+ CANONICAL_DESCRIPTIONS,+ )++ return CANONICAL_DESCRIPTIONS++ def get_schemas(+ self,+ config: AmazonCloudWatchSourceConfig,+ team_id: int,+ with_counts: bool = False,+ names: list[str] | None = None,+ force_refresh: bool = False,+ api_version: str | None = None,+ ) -> list[SourceSchema]:+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ descriptions=ENDPOINT_DESCRIPTIONS,+ default_incremental_lookback_seconds={+ name: config.incremental_lookback_seconds+ for name, config in AMAZON_CLOUDWATCH_ENDPOINTS.items()+ if config.incremental_lookback_seconds is not None+ },+ )++ def validate_credentials(+ self,+ config: AmazonCloudWatchSourceConfig,+ team_id: int,+ schema_name: Optional[str] = None,+ api_version: str | None = None,+ ) -> tuple[bool, str | None]:+ return validate_amazon_cloudwatch_credentials(+ config.aws_access_key_id,+ config.aws_secret_access_key,+ config.aws_session_token,+ config.aws_region,+ config.metric_queries,+ config.start_date,+ )++ def get_resumable_source_manager(+ self, inputs: SourceInputs+ ) -> ResumableSourceManager[AmazonCloudWatchResumeConfig]:+ return ResumableSourceManager[AmazonCloudWatchResumeConfig](inputs, AmazonCloudWatchResumeConfig)++ def source_for_pipeline(+ self,+ config: AmazonCloudWatchSourceConfig,+ resumable_source_manager: ResumableSourceManager[AmazonCloudWatchResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:+ return amazon_cloudwatch_source(+ aws_access_key_id=config.aws_access_key_id,+ aws_secret_access_key=config.aws_secret_access_key,+ aws_session_token=config.aws_session_token,+ aws_region=config.aws_region,+ log_group_names=config.log_group_names,+ metric_queries=config.metric_queries,+ start_date=config.start_date,+ endpoint=inputs.schema_name,+ resumable_source_manager=resumable_source_manager,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,+ logger=inputs.logger,+ )+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(name=ExternalDataSourceType.AMAZONCLOUDWATCH,category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="Amazon CloudWatch",+ caption="""Sync CloudWatch log events, alarms, alarm history and metric datapoints from one AWS region into the PostHog Data warehouse.++Create an IAM user or role with the `logs:DescribeLogGroups`, `logs:FilterLogEvents`, `cloudwatch:DescribeAlarms`, `cloudwatch:DescribeAlarmHistory` and `cloudwatch:GetMetricData` permissions, then paste its access key ID and secret access key. Add a session token too if you are using temporary credentials.++Log events sync from the log groups you list, or from every log group in the region (up to 50) when the list is empty. A log group or metric query added later syncs from the next sync on; resync the table to backfill its history. Any log delivered to CloudWatch Logs works, including Bedrock model invocation logs and CloudFront standard logs. Metric datapoints sync for the metric queries you configure as JSON, for example `[{"id": "cpu", "namespace": "AWS/EC2", "metric_name": "CPUUtilization", "dimensions": {"InstanceId": "i-0123456789abcdef0"}, "stat": "Average", "period": 300}]`. S3 Storage Lens metrics live in the `AWS/S3/Storage-Lens` namespace. Older datapoints come back at the resolution CloudWatch keeps for them: 5 minutes after 15 days and 1 hour after 63 days.""",iconPath="/static/services/amazon_cloudwatch.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ docsUrl="https://posthog.com/docs/cdp/sources/amazon-cloudwatch",+ releaseStatus=ReleaseStatus.ALPHA,+ keywords=["aws", "cloudwatch", "logs", "metrics", "alarms"],+ fields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="aws_access_key_id",+ label="AWS access key ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="AKIA...",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="aws_secret_access_key",+ label="AWS secret access key",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="aws_session_token",+ label="AWS session token (only for temporary credentials)",+ type=SourceFieldInputConfigType.PASSWORD,+ required=False,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="aws_region",+ label="AWS region",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="us-east-1",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="log_group_names",+ label="Log groups to sync (one per line, empty syncs every log group)",+ type=SourceFieldInputConfigType.TEXTAREA,+ required=False,+ placeholder="/aws/lambda/my-function",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="metric_queries",+ label="Metric queries (JSON list)",+ type=SourceFieldInputConfigType.TEXTAREA,+ required=False,+ placeholder='[{"id": "cpu", "namespace": "AWS/EC2", "metric_name": "CPUUtilization", "stat": "Average", "period": 300}]',+ secret=False,+ ),+ SourceFieldInputConfig(+ name="start_date",+ label="Start date (defaults to 7 days back for logs, 14 for metrics, 30 for alarm history)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="2024-01-01",+ secret=False,+ ),+ ],+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/tests/test_amazon_cloudwatch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/tests/test_amazon_cloudwatch.pynew file mode 100644index 000000000000..cb38a0b8d058--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_cloudwatch/tests/test_amazon_cloudwatch.py@@ -0,0 +1,743 @@+import json+import datetime as dt+from collections.abc import Callable+from typing import Any+from urllib.parse import parse_qs++import pytest+import time_machine++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.amazon_cloudwatch import (+ GENERIC_VALIDATION_ERROR,+ MAX_ATTEMPTS,+ TRANSIENT_VALIDATION_ERROR,+ VALIDATION_ERROR_MESSAGES,+ AmazonCloudWatchResumeConfig,+ parse_metric_queries,+ validate_credentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_cloudwatch.source import (+ AmazonCloudWatchSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ always,+ scripted_network,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.responders import UnexpectedRequest+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazoncloudwatch import (+ AmazonCloudWatchSourceConfig,+)++NOW = "2024-03-10T12:00:00Z"+LOGS_ORIGIN = "https://logs.us-east-1.amazonaws.com"+MONITORING_ORIGIN = "https://monitoring.us-east-1.amazonaws.com"+XMLNS = "http://monitoring.amazonaws.com/doc/2010-08-01/"+++def config(**overrides: Any) -> AmazonCloudWatchSourceConfig:+ values: dict[str, Any] = {+ "aws_access_key_id": "AKIAEXAMPLE",+ "aws_secret_access_key": "secret",+ "aws_session_token": None,+ "aws_region": "us-east-1",+ "log_group_names": "/aws/lambda/one\n/aws/lambda/two\n",+ "metric_queries": None,+ "start_date": "2024-03-08",+ }+ values.update(overrides)+ return AmazonCloudWatchSourceConfig(**values)+++def driver(**overrides: Any) -> SourceDriver:+ return SourceDriver(AmazonCloudWatchSource(), config(**overrides))+++def by_origin(**queues: list[ScriptedResponse]) -> Callable[[RecordedRequest], ScriptedResponse]:+ """Both AWS APIs post to `/`, so answers are routed by host rather than by path."""+ table = {"logs": queues.get("logs", []), "monitoring": queues.get("monitoring", [])}++ def respond(request: RecordedRequest) -> ScriptedResponse:+ queue = table["logs"] if request.origin == LOGS_ORIGIN else table["monitoring"]+ if not queue:+ raise UnexpectedRequest(f"no answer left for {request.method} {request.url}")+ return queue.pop(0)++ return respond+++def logs_page(events: list[dict[str, Any]], next_token: str | None = None) -> ScriptedResponse:+ body: dict[str, Any] = {"events": events, "searchedLogStreams": []}+ if next_token:+ body["nextToken"] = next_token+ return ScriptedResponse(json=body)+++def event(event_id: str, timestamp: int = 1709900000000) -> dict[str, Any]:+ return {+ "eventId": event_id,+ "timestamp": timestamp,+ "ingestionTime": timestamp + 5,+ "message": "m",+ "logStreamName": "s",+ }+++def xml_response(action: str, result_xml: str, status: int = 200) -> ScriptedResponse:+ body = (+ f'<{action}Response xmlns="{XMLNS}"><{action}Result>{result_xml}</{action}Result>'+ f"<ResponseMetadata><RequestId>r</RequestId></ResponseMetadata></{action}Response>"+ )+ return ScriptedResponse(status=status, body=body.encode(), headers={"Content-Type": "text/xml"})+++CALLER_IDENTITY = ScriptedResponse(+ body=(+ b'<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/"><GetCallerIdentityResult>'+ b"<Arn>arn:aws:iam::000000000000:user/example</Arn><UserId>AIDAEXAMPLE</UserId><Account>000000000000</Account>"+ b"</GetCallerIdentityResult><ResponseMetadata><RequestId>r</RequestId></ResponseMetadata></GetCallerIdentityResponse>"+ ),+ headers={"Content-Type": "text/xml"},+)+++def xml_error(code: str, message: str, status: int = 400) -> ScriptedResponse:+ body = f'<ErrorResponse xmlns="{XMLNS}"><Error><Type>Sender</Type><Code>{code}</Code><Message>{message}</Message></Error><RequestId>r</RequestId></ErrorResponse>'+ return ScriptedResponse(status=status, body=body.encode(), headers={"Content-Type": "text/xml"})+++def metric_result(query_id: str, points: list[tuple[str, float]], label: str = "CPUUtilization") -> str:+ timestamps = "".join(f"<member>{ts}</member>" for ts, _ in points)+ values = "".join(f"<member>{value}</member>" for _, value in points)+ return (+ f"<member><Id>{query_id}</Id><Label>{label}</Label><Timestamps>{timestamps}</Timestamps>"+ f"<Values>{values}</Values><StatusCode>Complete</StatusCode></member>"+ )+++def form(request: RecordedRequest) -> dict[str, str]:+ return {key: values[0] for key, values in parse_qs((request.body or b"").decode()).items()}++[email protected](autouse=True)+def _frozen_clock():+ with time_machine.travel(NOW, tick=False):+ yield+++class TestLogEvents:+ def test_walks_every_page_of_each_log_group_and_stages_the_cursor_before_the_batch(self) -> None:+ result = driver().run(+ "log_events",+ by_origin(+ logs=[+ logs_page([event("a1")], next_token="p2"),+ logs_page([], next_token="p3"),+ logs_page([event("a2")]),+ logs_page([event("b1")]),+ ]+ ),+ )++ assert result.raised is None+ assert [request.headers["x-amz-target"] for request in result.requests] == ["Logs_20140328.FilterLogEvents"] * 4+ bodies = [request.json() for request in result.requests]+ assert [(body["logGroupName"], body.get("nextToken")) for body in bodies] == [+ ("/aws/lambda/one", None),+ ("/aws/lambda/one", "p2"),+ ("/aws/lambda/one", "p3"),+ ("/aws/lambda/two", None),+ ]+ assert bodies[0]["startTime"] == int(dt.datetime(2024, 3, 8, tzinfo=dt.UTC).timestamp() * 1000)+ assert bodies[0]["endTime"] == int(dt.datetime(2024, 3, 10, 12, tzinfo=dt.UTC).timestamp() * 1000)+ assert [row["event_id"] for row in result.rows] == ["a1", "a2", "b1"]+ assert result.rows[0]["log_group_name"] == "/aws/lambda/one"+ assert result.rows[0]["timestamp"] == dt.datetime(2024, 3, 8, 12, 13, 20, tzinfo=dt.UTC)+ # The cursor staged with the first page points at the second page, and persisted with it.+ first_cursor = AmazonCloudWatchResumeConfig(+ slice_key="/aws/lambda/one",+ next_token="p2",+ window_start="2024-03-08T00:00:00+00:00",+ window_end="2024-03-10T12:00:00+00:00",+ )+ assert result.saved_states[0] == first_cursor+ assert result.committed_states[0] == first_cursor+ # The last page of a group stages the next group, so a handoff there does not reread the group.+ assert result.saved_states[2] == AmazonCloudWatchResumeConfig(+ slice_key="/aws/lambda/two",+ next_token=None,+ window_start="2024-03-08T00:00:00+00:00",+ window_end="2024-03-10T12:00:00+00:00",+ )+ assert result.saved_states[3].slice_key == "__complete__"+ assert result.final_state is None+ # Events arrive per log group, so the watermark may only move once the whole run completes.+ assert result.response is not None and result.response.sort_mode == "desc"++ def test_resumes_at_the_saved_log_group_and_page(self) -> None:+ result = driver().run(+ "log_events",+ by_origin(logs=[logs_page([event("b2")])]),+ resume_state=AmazonCloudWatchResumeConfig(slice_key="/aws/lambda/two", next_token="p9"),+ )++ assert result.raised is None+ assert [(r.json()["logGroupName"], r.json().get("nextToken")) for r in result.requests] == [+ ("/aws/lambda/two", "p9")+ ]++ def test_a_cursor_staged_after_the_last_page_ends_the_run_without_a_request(self) -> None:+ result = driver().run(+ "log_events",+ by_origin(),+ resume_state=AmazonCloudWatchResumeConfig(+ slice_key="__complete__",+ window_start="2024-03-08T00:00:00+00:00",+ window_end="2024-03-10T12:00:00+00:00",+ ),+ )++ assert result.raised is None+ assert result.requests == []+ assert result.final_state is None++ def test_a_saved_log_group_no_longer_configured_restarts_from_the_first_group(self) -> None:+ result = driver().run(+ "log_events",+ by_origin(logs=[logs_page([]), logs_page([])]),+ resume_state=AmazonCloudWatchResumeConfig(slice_key="/aws/lambda/gone", next_token="p9"),+ )++ assert result.raised is None+ assert [r.json()["logGroupName"] for r in result.requests] == ["/aws/lambda/one", "/aws/lambda/two"]+ assert all("nextToken" not in r.json() for r in result.requests)++ @pytest.mark.parametrize(+ ("start_date", "incremental_field", "watermark", "expected_start"),+ [+ (+ "2024-03-08",+ "timestamp",+ dt.datetime(2024, 3, 9, 6, tzinfo=dt.UTC),+ dt.datetime(2024, 3, 9, 6, tzinfo=dt.UTC),+ ),+ # A watermark older than the configured start date never widens the window.+ ("2024-03-08", "timestamp", dt.datetime(2024, 1, 1, tzinfo=dt.UTC), dt.datetime(2024, 3, 8, tzinfo=dt.UTC)),+ ("2024-03-08", None, None, dt.datetime(2024, 3, 8, tzinfo=dt.UTC)),+ # The 7-day default only bounds a first run. After a longer pause the gap is still read.+ ("", "timestamp", dt.datetime(2024, 2, 20, tzinfo=dt.UTC), dt.datetime(2024, 2, 20, tzinfo=dt.UTC)),+ ("", None, None, dt.datetime(2024, 3, 3, 12, tzinfo=dt.UTC)),+ ],+ )+ def test_an_incremental_run_starts_at_the_watermark(+ self,+ start_date: str,+ incremental_field: str | None,+ watermark: dt.datetime | None,+ expected_start: dt.datetime,+ ) -> None:+ result = driver(log_group_names="/aws/lambda/one", start_date=start_date).run(+ "log_events",+ by_origin(logs=[logs_page([])]),+ incremental_field=incremental_field,+ db_incremental_field_last_value=watermark,+ )++ assert result.raised is None+ assert result.requests[0].json()["startTime"] == int(expected_start.timestamp() * 1000)++ def test_no_configured_log_groups_discovers_them_first(self) -> None:+ result = driver(log_group_names="").run(+ "log_events",+ by_origin(+ logs=[+ ScriptedResponse(+ json={+ "logGroups": [+ {"logGroupName": "/discovered"},+ # FilterLogEvents rejects these classes, which would fail the whole table.+ {"logGroupName": "/infrequent", "logGroupClass": "INFREQUENT_ACCESS"},+ {"logGroupName": "/delivery", "logGroupClass": "DELIVERY"},+ ]+ }+ ),+ logs_page([event("d1")]),+ ]+ ),+ )++ assert result.raised is None+ assert [r.headers["x-amz-target"] for r in result.requests] == [+ "Logs_20140328.DescribeLogGroups",+ "Logs_20140328.FilterLogEvents",+ ]+ assert result.requests[1].json()["logGroupName"] == "/discovered"++ def test_a_missing_log_group_fails_the_run_with_the_aws_code(self) -> None:+ result = driver(log_group_names="/aws/lambda/one").run(+ "log_events",+ by_origin(+ logs=[+ ScriptedResponse(+ status=400,+ headers={"x-amzn-ErrorType": "ResourceNotFoundException"},+ json={+ "__type": "ResourceNotFoundException",+ "message": "The specified log group does not exist.",+ },+ )+ ]+ ),+ )++ assert result.raised is not None+ assert str(result.raised).startswith("Amazon CloudWatch request failed: ResourceNotFoundException")+++class TestLogGroups:+ def test_converts_creation_time_and_follows_next_token(self) -> None:+ result = driver().run(+ "log_groups",+ by_origin(+ logs=[+ ScriptedResponse(+ json={+ "logGroups": [{"logGroupName": "/a", "creationTime": 1709900000000, "storedBytes": 10}],+ "nextToken": "n2",+ }+ ),+ ScriptedResponse(json={"logGroups": [{"logGroupName": "/b", "creationTime": 1709900000000}]}),+ ]+ ),+ )++ assert result.raised is None+ assert [r.json().get("nextToken") for r in result.requests] == [None, "n2"]+ assert result.rows[0] == {+ "log_group_name": "/a",+ "creation_time": dt.datetime(2024, 3, 8, 12, 13, 20, tzinfo=dt.UTC),+ "stored_bytes": 10,+ }+ assert [row["log_group_name"] for row in result.rows] == ["/a", "/b"]+++class TestMetricData:+ QUERIES = json.dumps(+ [+ {+ "id": "cpu",+ "namespace": "AWS/EC2",+ "metric_name": "CPUUtilization",+ "dimensions": {"InstanceId": "i-1"},+ "stat": "Average",+ "period": 300,+ },+ {"Id": "errors", "Expression": "SUM(METRICS())", "Period": 60},+ ]+ )++ def test_sends_the_queries_as_a_signed_form_request_and_reads_the_xml_datapoints(self) -> None:+ result = driver(metric_queries=self.QUERIES, start_date="2024-03-09").run(+ "metric_data",+ by_origin(+ monitoring=[+ xml_response(+ "GetMetricData",+ "<MetricDataResults>"+ + metric_result("cpu", [("2024-03-09T00:00:00Z", 1.5), ("2024-03-09T00:05:00Z", 2.0)])+ + metric_result("errors", [("2024-03-09T00:00:00Z", 3)], label="errors")+ + "</MetricDataResults><NextToken>tok2</NextToken>",+ ),+ xml_response("GetMetricData", "<MetricDataResults></MetricDataResults>"),+ ]+ ),+ )++ assert result.raised is None+ assert [r.origin for r in result.requests] == [MONITORING_ORIGIN] * 2+ first = form(result.requests[0])+ assert first["Action"] == "GetMetricData"+ assert first["Version"] == "2010-08-01"+ assert first["ScanBy"] == "TimestampAscending"+ assert first["StartTime"] == "2024-03-09T00:00:00Z"+ assert first["EndTime"] == "2024-03-10T12:00:00Z"+ assert first["MetricDataQueries.member.1.Id"] == "cpu"+ assert first["MetricDataQueries.member.1.MetricStat.Metric.Dimensions.member.1.Value"] == "i-1"+ assert first["MetricDataQueries.member.1.MetricStat.Period"] == "300"+ assert first["MetricDataQueries.member.2.Expression"] == "SUM(METRICS())"+ assert "NextToken" not in first+ assert form(result.requests[1])["NextToken"] == "tok2"+ assert "authorization" in result.requests[0].headers+ assert result.rows == [+ {+ "query_id": "cpu",+ "label": "CPUUtilization",+ "timestamp": dt.datetime(2024, 3, 9, 0, 0, tzinfo=dt.UTC),+ "value": 1.5,+ "status_code": "Complete",+ },+ {+ "query_id": "cpu",+ "label": "CPUUtilization",+ "timestamp": dt.datetime(2024, 3, 9, 0, 5, tzinfo=dt.UTC),+ "value": 2.0,+ "status_code": "Complete",+ },+ {+ "query_id": "errors",+ "label": "errors",+ "timestamp": dt.datetime(2024, 3, 9, 0, 0, tzinfo=dt.UTC),+ "value": 3.0,+ "status_code": "Complete",+ },+ ]++ @pytest.mark.parametrize(+ ("resume_state", "expected_windows"),+ [+ (+ None,+ [+ ("2024-02-01T00:00:00Z", "2024-02-15T00:00:00Z"),+ ("2024-02-15T00:00:00Z", "2024-02-29T00:00:00Z"),+ ("2024-02-29T00:00:00Z", "2024-03-10T12:00:00Z"),+ ],+ ),+ (+ AmazonCloudWatchResumeConfig(slice_key="2024-02-29T00:00:00+00:00", next_token=None),+ [("2024-02-29T00:00:00Z", "2024-03-10T12:00:00Z")],+ ),+ # The attempt that saved this cursor started earlier, so its range differs from one+ # computed now. The resumed attempt keeps the saved range and its windows.+ (+ AmazonCloudWatchResumeConfig(+ slice_key="2024-02-20T00:00:00+00:00",+ next_token="tok7",+ window_start="2024-02-06T00:00:00+00:00",+ window_end="2024-03-05T00:00:00+00:00",+ ),+ [("2024-02-20T00:00:00Z", "2024-03-05T00:00:00Z")],+ ),+ ],+ )+ def test_a_long_range_is_split_into_windows_and_a_saved_window_resumes_there(+ self, resume_state: AmazonCloudWatchResumeConfig | None, expected_windows: list[tuple[str, str]]+ ) -> None:+ empty = xml_response("GetMetricData", "<MetricDataResults></MetricDataResults>")+ result = driver(metric_queries=self.QUERIES, start_date="2024-02-01").run(+ "metric_data",+ by_origin(monitoring=[empty, empty, empty]),+ resume_state=resume_state,+ )++ assert result.raised is None+ assert [(form(r)["StartTime"], form(r)["EndTime"]) for r in result.requests] == expected_windows++ def test_each_window_asks_for_a_period_that_cloudwatch_still_keeps_for_data_that_old(self) -> None:+ queries = json.dumps([{"id": "cpu", "namespace": "AWS/EC2", "metric_name": "CPUUtilization", "period": 60}])+ empty = xml_response("GetMetricData", "<MetricDataResults></MetricDataResults>")+ result = driver(metric_queries=queries, start_date="2023-12-20").run("metric_data", always(empty))++ assert result.raised is None+ assert [+ (form(r)["StartTime"][:10], form(r)["MetricDataQueries.member.1.MetricStat.Period"])+ for r in result.requests+ ] == [+ ("2023-12-20", "3600"),+ ("2024-01-03", "3600"),+ ("2024-01-17", "300"),+ ("2024-01-31", "300"),+ ("2024-02-14", "300"),+ ("2024-02-28", "60"),+ ]++ @pytest.mark.parametrize(+ ("status", "message"),+ [+ (+ "Forbidden",+ "Amazon CloudWatch request failed: MetricQueryForbidden - metric query 'cpu': Access denied",+ ),+ (+ "InternalError",+ "Amazon CloudWatch request failed: MetricQueryInternalError - metric query 'cpu': Access denied",+ ),+ # Partial data with no continuation token cannot be completed, so the run must not end as a success.+ (+ "PartialData",+ "Amazon CloudWatch request failed: MetricQueryPartialData - metric query 'cpu' returned more "+ "datapoints than one response holds",+ ),+ ],+ )+ def test_a_query_failing_inside_a_successful_response_fails_the_run(self, status: str, message: str) -> None:+ failed = (+ "<member><Id>cpu</Id><Label>CPUUtilization</Label><Timestamps></Timestamps><Values></Values>"+ f"<StatusCode>{status}</StatusCode><Messages><member><Code>{status}</Code><Value>Access denied</Value></member></Messages></member>"+ )+ result = driver(metric_queries=self.QUERIES).run(+ "metric_data",+ by_origin(monitoring=[xml_response("GetMetricData", f"<MetricDataResults>{failed}</MetricDataResults>")]),+ )++ assert str(result.raised) == message+ # The failing page's cursor was never staged, so a retry asks for the same page again.+ assert result.saved_states == []++ def test_without_queries_nothing_is_requested(self) -> None:+ result = driver(metric_queries=" ").run("metric_data", by_origin())++ assert result.raised is None+ assert result.requests == []+ assert result.rows == []++ def test_an_aws_error_in_the_xml_body_fails_the_run_with_its_code(self) -> None:+ result = driver(metric_queries=self.QUERIES).run(+ "metric_data",+ by_origin(monitoring=[xml_error("InvalidParameterValue", "The value of Period is invalid.")]),+ )++ assert result.raised is not None+ assert str(result.raised) == (+ "Amazon CloudWatch request failed: InvalidParameterValue - The value of Period is invalid."+ )++ def test_a_throttled_call_is_retried_until_it_succeeds(self) -> None:+ result = driver(metric_queries=self.QUERIES).run(+ "metric_data",+ by_origin(+ monitoring=[+ xml_error("Throttling", "Rate exceeded"),+ xml_response("GetMetricData", "<MetricDataResults></MetricDataResults>"),+ ]+ ),+ )++ assert result.raised is None+ assert len(result.requests) == 2+++class TestAlarms:+ def test_metric_and_composite_alarms_land_in_one_table_tagged_by_type(self) -> None:+ result = driver().run(+ "alarms",+ by_origin(+ monitoring=[+ xml_response(+ "DescribeAlarms",+ "<MetricAlarms><member><AlarmName>cpu-high</AlarmName><AlarmArn>arn:1</AlarmArn>"+ "<StateValue>OK</StateValue><Threshold>80.0</Threshold>"+ "<Dimensions><member><Name>InstanceId</Name><Value>i-1</Value></member></Dimensions>"+ "</member></MetricAlarms><CompositeAlarms><member><AlarmName>site-down</AlarmName>"+ "<AlarmArn>arn:2</AlarmArn><AlarmRule>ALARM(cpu-high)</AlarmRule></member></CompositeAlarms>"+ "<NextToken>n2</NextToken>",+ ),+ xml_response("DescribeAlarms", "<MetricAlarms></MetricAlarms><CompositeAlarms></CompositeAlarms>"),+ ]+ ),+ )++ assert result.raised is None+ first = form(result.requests[0])+ assert first["Action"] == "DescribeAlarms"+ assert {first["AlarmTypes.member.1"], first["AlarmTypes.member.2"]} == {"CompositeAlarm", "MetricAlarm"}+ assert form(result.requests[1])["NextToken"] == "n2"+ assert [(row["alarm_arn"], row["alarm_type"]) for row in result.rows] == [+ ("arn:1", "MetricAlarm"),+ ("arn:2", "CompositeAlarm"),+ ]+ assert result.rows[0]["threshold"] == 80.0+ assert result.rows[0]["dimensions"] == [{"Name": "InstanceId", "Value": "i-1"}]+ assert result.rows[1]["alarm_rule"] == "ALARM(cpu-high)"+++class TestAlarmHistory:+ def test_an_incremental_run_asks_for_history_after_the_watermark_in_ascending_order(self) -> None:+ result = driver().run(+ "alarm_history",+ by_origin(+ monitoring=[+ xml_response(+ "DescribeAlarmHistory",+ "<AlarmHistoryItems>"+ + "".join(+ "<member><AlarmName>cpu-high</AlarmName><AlarmType>MetricAlarm</AlarmType>"+ "<Timestamp>2024-03-09T08:00:00.123Z</Timestamp>"+ "<HistoryItemType>AlarmContributorStateUpdate</HistoryItemType>"+ f"<AlarmContributorId>{contributor}</AlarmContributorId>"+ '<HistorySummary>Alarm updated from OK to ALARM</HistorySummary><HistoryData>{"x":1}</HistoryData>'+ "</member>"+ for contributor in ("c1", "c2")+ )+ + "</AlarmHistoryItems>",+ )+ ]+ ),+ incremental_field="timestamp",+ db_incremental_field_last_value=dt.datetime(2024, 3, 9, 6, tzinfo=dt.UTC),+ )++ assert result.raised is None+ params = form(result.requests[0])+ assert params["Action"] == "DescribeAlarmHistory"+ assert params["StartDate"] == "2024-03-09T06:00:00Z"+ assert params["EndDate"] == "2024-03-10T12:00:00Z"+ assert params["ScanBy"] == "TimestampAscending"+ # An alarm with contributors records one item per contributor at the same instant.+ assert len({row.pop("history_id") for row in result.rows}) == 2+ assert result.rows == [+ {+ "alarm_name": "cpu-high",+ "alarm_type": "MetricAlarm",+ "timestamp": dt.datetime(2024, 3, 9, 8, 0, 0, 123000, tzinfo=dt.UTC),+ "history_item_type": "AlarmContributorStateUpdate",+ "alarm_contributor_id": contributor,+ "history_summary": "Alarm updated from OK to ALARM",+ "history_data": '{"x":1}',+ }+ for contributor in ("c1", "c2")+ ]+ assert result.response is not None and result.response.sort_mode == "asc"+++class TestParseMetricQueries:+ def test_flat_queries_become_metric_data_queries(self) -> None:+ assert parse_metric_queries(+ '[{"namespace": "AWS/S3", "metric_name": "BucketSizeBytes", "dimensions": [{"name": "BucketName", "value": "b"}], "unit": "Bytes", "label": "size"}]'+ ) == [+ {+ "Id": "q1",+ "MetricStat": {+ "Metric": {+ "Namespace": "AWS/S3",+ "MetricName": "BucketSizeBytes",+ "Dimensions": [{"Name": "BucketName", "Value": "b"}],+ },+ "Period": 300,+ "Stat": "Average",+ "Unit": "Bytes",+ },+ "Label": "size",+ "ReturnData": True,+ }+ ]++ @pytest.mark.parametrize(+ ("raw", "message"),+ [+ ("not json", "Invalid metric queries: not valid JSON"),+ ('{"id": "a"}', "Invalid metric queries: query 1 needs a namespace and a metric_name"),+ (+ '[{"id": "a", "expression": "x"}, {"id": "a", "expression": "y"}]',+ "Invalid metric queries: the id 'a' is used twice",+ ),+ ("[1]", "Invalid metric queries: expected a JSON list"),+ (+ '[{"namespace": "AWS/EC2", "metric_name": "CPU", "dimensions": [1]}]',+ "Invalid metric queries: query 1 has a dimension without a name and a value",+ ),+ (+ '[{"namespace": "AWS/EC2", "metric_name": "CPU", "period": "five"}]',+ "Invalid metric queries: query 1 has a period that is not a whole number",+ ),+ ('[{"Id": "q1", "MetricStat": "invalid"}]', "Invalid metric queries: they do not match the CloudWatch API"),+ ],+ )+ def test_bad_queries_raise_a_message_for_the_wizard(self, raw: str, message: str) -> None:+ with pytest.raises(ValueError, match=message):+ parse_metric_queries(raw)+++class TestValidateCredentials:+ @pytest.mark.parametrize(+ ("overrides", "expected"),+ [+ ({"aws_access_key_id": ""}, "AWS access key ID and secret access key are required"),+ ({"aws_region": "mars"}, "Enter an AWS region code, for example us-east-1."),+ (+ {"metric_queries": "{"},+ "Invalid metric queries: not valid JSON (Expecting property name enclosed in double quotes: line 1 column 2 (char 1)).",+ ),+ # Falling back to the default range would drop the history the user asked for.+ ({"start_date": "2024-02-30"}, "Invalid start date: enter it as YYYY-MM-DD, for example 2024-01-31."),+ ],+ )+ def test_local_checks_fail_before_any_request(self, overrides: dict[str, Any], expected: str) -> None:+ c = config(**overrides)++ with scripted_network([]) as network:+ assert validate_credentials(+ c.aws_access_key_id,+ c.aws_secret_access_key,+ c.aws_session_token,+ c.aws_region,+ c.metric_queries,+ c.start_date,+ ) == (False, expected)++ assert network.requests == 0++ @pytest.mark.parametrize(+ ("response", "expected"),+ [+ (CALLER_IDENTITY, (True, None)),+ (+ xml_error("InvalidClientTokenId", "The security token included in the request is invalid."),+ (False, VALIDATION_ERROR_MESSAGES["InvalidClientTokenId"]),+ ),+ (+ xml_error("SignatureDoesNotMatch", "The request signature we calculated does not match."),+ (False, VALIDATION_ERROR_MESSAGES["SignatureDoesNotMatch"]),+ ),+ (xml_error("SomethingNewException", "?"), (False, GENERIC_VALIDATION_ERROR)),+ ],+ )+ def test_the_probe_result_is_translated_and_never_echoes_the_aws_text(+ self, response: ScriptedResponse, expected: tuple[bool, str | None]+ ) -> None:+ c = config()++ with scripted_network([response]) as network:+ assert (+ validate_credentials(+ c.aws_access_key_id, c.aws_secret_access_key, c.aws_session_token, c.aws_region, c.metric_queries+ )+ == expected+ )++ # STS answers any valid key without an IAM permission, so a key scoped to one table connects.+ probe = network.requests_log[0]+ assert probe.origin == "https://sts.us-east-1.amazonaws.com"+ assert b"Action=GetCallerIdentity" in (probe.body or b"")++ def test_a_china_region_is_reached_through_its_own_partition(self) -> None:+ c = config(aws_region="cn-north-1")++ with scripted_network([CALLER_IDENTITY]) as network:+ assert validate_credentials(+ c.aws_access_key_id, c.aws_secret_access_key, c.aws_session_token, c.aws_region, c.metric_queries+ ) == (True, None)++ assert network.requests_log[0].origin == "https://sts.cn-north-1.amazonaws.com.cn"++ @pytest.mark.parametrize(+ "response",+ [+ xml_error("Throttling", "Rate exceeded"),+ # A load balancer error page: botocore cannot parse it, and it carries no AWS code.+ ScriptedResponse(+ status=503, body=b"<html>Service Unavailable</html>", headers={"Content-Type": "text/html"}+ ),+ ],+ )+ def test_a_throttled_or_unavailable_probe_asks_the_user_to_retry(self, response: ScriptedResponse) -> None:+ c = config()++ with scripted_network(lambda _request: response) as network:+ assert validate_credentials(+ c.aws_access_key_id, c.aws_secret_access_key, c.aws_session_token, c.aws_region, c.metric_queries+ ) == (False, TRANSIENT_VALIDATION_ERROR)++ assert 1 < network.requests <= MAX_ATTEMPTSdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazoncloudwatch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazoncloudwatch.pyindex bc1330d82427..b4cc60df49f2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazoncloudwatch.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazoncloudwatch.py@@ -6,4 +6,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp@config.configclass AmazonCloudWatchSourceConfig(config.Config):- pass+ aws_access_key_id: str+ aws_secret_access_key: str+ aws_region: str+ aws_session_token: str | None = None+ log_group_names: str | None = None+ metric_queries: str | None = None+ start_date: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.pyindex 293288380e79..f600d0fa4cdf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py@@ -387,6 +387,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {"Adapty","AdobeAnalytics","Ahrefs",+ "AmazonCloudWatch","Anvil","Apollo","Arcade",diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex ddbed84fc07e..f413e9ce6327 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -13024,10 +13024,10 @@ class TestExternalDataSourceSetup(APIBaseTest):@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.capture_exception")def test_setup_rejects_source_without_schema_discovery(self, mock_capture_exception):- # AmazonCloudWatch doesn't implement get_schemas, so the base raises NotImplementedError.+ # AmazonEventBridge doesn't implement get_schemas, so the base raises NotImplementedError.response = self.client.post(f"/api/environments/{self.team.pk}/external_data_sources/setup/",- data={"source_type": "AmazonCloudWatch", "prefix": "scaffold_setup_test", "payload": {}},+ data={"source_type": "AmazonEventBridge", "prefix": "scaffold_setup_test", "payload": {}},)assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()assert "isn't available to connect yet" in response.json()["message"]