Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): implement aws cost and usage report source (#118143)

Daniel RC0f4939e31f6d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index 113657e4761f..84cf54fb57ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -116,6 +116,7 @@ the row lists both.
| aws_cloudtrail | HTTP | requests | ✅ |
| aws_compute_optimizer | HTTP | requests | ✅ |
| aws_config | HTTP | requests | ✅ |
+| aws_cost_and_usage_report | HTTP | requests + botocore (S3 SigV4) + pyarrow | ✅ |
| aws_cost_anomaly_detection | HTTP | requests | ✅ |
| aws_cost_explorer | HTTP | requests | ✅ |
| aws_glue_data_catalog | HTTP | requests | ✅ |
@@ -1056,7 +1057,6 @@ doesn't conflict with concurrent PRs.
- aws_athena
- aws_cloudformation
- aws_connect
-- aws_cost_and_usage_report
- aws_health
- aws_rds_performance_insights
- aws_support
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
new file mode 100644
index 000000000000..ff920e6dbdd6
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
@@ -0,0 +1,716 @@
+import re
+import json
+import decimal
+import datetime as dt
+import tempfile
+import dataclasses
+import email.utils
+from collections.abc import Callable, Iterator
+from functools import cache
+from typing import IO, Any, Optional
+from urllib.parse import quote, urlencode
+
+import orjson
+import pyarrow as pa
+import requests
+import pyarrow.parquet as pq
+from botocore.auth import S3SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.credentials import Credentials
+from botocore.loaders import Loader
+from defusedxml import ElementTree
+from structlog.types import FilteringBoundLogger
+
+from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
+ normalize_column_name as pipeline_column_name,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.settings import (
+ BILLING_PERIOD_DIRECTORY,
+ BILLING_PERIOD_INPUT,
+ BUCKET_NAME,
+ COST_AND_USAGE,
+ DATA_FILE_SUFFIXES,
+ DECIMAL_TYPE,
+ DOWNLOAD_CHUNK_BYTES,
+ LIST_PAGE_SIZE,
+ ROWS_PER_BATCH,
+ S3_NAMESPACE,
+ S3_SIGNING_NAME,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (
+ iter_file_rows,
+ normalize_column_name,
+ resolve_file_format,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+
+REQUEST_TIMEOUT_SECONDS = 120
+# A dropped connection mid-body escapes the transport's retry, which only covers the response
+# headers. One file is retried on its own so a late failure does not restart the whole refresh.
+DOWNLOAD_ATTEMPTS = 3
+NO_MANIFEST = "NoManifest"
+DELIVERY_IN_PROGRESS = "DeliveryInProgress"
+MANIFEST_UNREADABLE = "ManifestUnreadable"
+BILLING_PERIOD_COLUMN = "billing_period"
+ERROR_PREFIX = "AWS Cost and Usage Report request failed"
+WRONG_REGION_CODES = frozenset(
+ {"PermanentRedirect", "AuthorizationHeaderMalformed", "IllegalLocationConstraintException"}
+)
+
+
+class AwsCostAndUsageReportError(Exception):
+ def __init__(self, message: str, code: str | None = None, bucket_region: str | None = None) -> None:
+ super().__init__(message)
+ self.code = code
+ self.bucket_region = bucket_region
+
+
[email protected](frozen=True)
+class S3ListPage:
+ keys: list[str]
+ prefixes: list[str]
+ next_token: Optional[str]
+
+
[email protected](frozen=True)
+class Delivery:
+ """The latest complete delivery of one billing period, as its manifest describes it."""
+
+ keys: list[str]
+ # Column name -> the type the export declares for it, lowercased.
+ column_types: dict[str, str]
+ # When AWS wrote the manifest. An "overwrite" export rewrites its data files in place, so a
+ # file changed after this belongs to a newer delivery.
+ manifest_last_modified: Optional[str] = None
+
+
+@frozen
+class CostAndUsageReportResumeConfig:
+ """How far a full refresh got: the billing period, the delivery it read, and the next batch.
+
+ `manifest_last_modified` pins the delivery. If AWS delivered the period again since, the saved
+ file and batch numbers point into a different set of files.
+ """
+
+ billing_period: str
+ manifest_last_modified: Optional[str]
+ file_index: int
+ batch_index: int
+
+
[email protected](frozen=True)
+class S3Body:
+ content: bytes
+ last_modified: Optional[str]
+
+
+@cache
+def aws_dns_suffix(region: str) -> Optional[str]:
+ """The DNS suffix of the AWS partition a region belongs to, or None for an unknown region.
+
+ China regions live under `amazonaws.com.cn`, so the suffix comes from botocore's partition table.
+ """
+ for partition in Loader().load_data("partitions")["partitions"]:
+ if region in partition.get("regions", {}) or re.match(partition["regionRegex"], region):
+ return partition["outputs"]["dnsSuffix"]
+ return None
+
+
+def export_base(prefix: Optional[str], export_name: str) -> str:
+ """`<prefix>/<export-name>`, the directory Data Exports writes `data/` and `metadata/` under."""
+ parts = [part.strip("/") for part in (prefix or "", export_name) if part and part.strip("/")]
+ return "/".join(parts)
+
+
+def parse_billing_period(value: Optional[str]) -> Optional[dt.date]:
+ if not value or not value.strip():
+ return None
+ value = value.strip()
+ if not BILLING_PERIOD_INPUT.match(value):
+ raise ValueError("Enter the first billing period as YYYY-MM, for example 2024-01.")
+ year, month = value.split("-")
+ return dt.date(int(year), int(month), 1)
+
+
+def error_for_response(response: requests.Response) -> AwsCostAndUsageReportError:
+ code = f"HTTP {response.status_code}"
+ message = ""
+ if response.content:
+ try:
+ root = ElementTree.fromstring(response.content)
+ code = root.findtext("Code") or code
+ message = (root.findtext("Message") or "")[:500]
+ except ElementTree.ParseError:
+ message = response.text[:500]
+ region = response.headers.get("x-amz-bucket-region")
+ return AwsCostAndUsageReportError(f"{ERROR_PREFIX}: {code} - {message}", code, region)
+
+
+def parse_list_page(body: bytes) -> S3ListPage:
+ root = ElementTree.fromstring(body)
+ keys = [item.findtext(f"{S3_NAMESPACE}Key") or "" for item in root.findall(f"{S3_NAMESPACE}Contents")]
+ prefixes = [item.findtext(f"{S3_NAMESPACE}Prefix") or "" for item in root.findall(f"{S3_NAMESPACE}CommonPrefixes")]
+ truncated = (root.findtext(f"{S3_NAMESPACE}IsTruncated") or "").lower() == "true"
+ next_token = root.findtext(f"{S3_NAMESPACE}NextContinuationToken") if truncated else None
+ return S3ListPage(keys=keys, prefixes=[prefix for prefix in prefixes if prefix], next_token=next_token)
+
+
+class S3Client:
+ """Reads one bucket with SigV4-signed GETs over the tracked session."""
+
+ def __init__(
+ self,
+ aws_access_key_id: str,
+ aws_secret_access_key: str,
+ aws_session_token: Optional[str],
+ aws_region: str,
+ bucket: str,
+ ) -> None:
+ redact = tuple(value for value in (aws_secret_access_key, aws_session_token) if value)
+ self._session = make_tracked_session(redact_values=redact, allow_redirects=False)
+ # Data files run to hundreds of megabytes, so they stay out of the sampled request capture.
+ self._download_session = make_tracked_session(redact_values=redact, allow_redirects=False, capture=False)
+ self._credentials = Credentials(aws_access_key_id, aws_secret_access_key, aws_session_token or None)
+ self._region = aws_region.strip()
+ self._bucket = bucket.strip()
+
+ def _url(self, key: str = "", query: Optional[dict[str, str]] = None) -> str:
+ path = quote(key, safe="/~")
+ suffix = aws_dns_suffix(self._region) or "amazonaws.com"
+ # A bucket name with a dot breaks the TLS wildcard on the virtual-hosted name.
+ if "." in self._bucket:
+ url = f"https://s3.{self._region}.{suffix}/{self._bucket}/{path}"
+ else:
+ url = f"https://{self._bucket}.s3.{self._region}.{suffix}/{path}"
+ if query:
+ url = f"{url}?{urlencode(sorted(query.items()), quote_via=quote, safe='~')}"
+ return url
+
+ def _signed_headers(self, url: str, extra_headers: Optional[dict[str, str]] = None) -> dict[str, str]:
+ aws_request = AWSRequest(method="GET", url=url, headers=dict(extra_headers or {}))
+ S3SigV4Auth(self._credentials, S3_SIGNING_NAME, self._region).add_auth(aws_request)
+ return dict(aws_request.headers.items())
+
+ def _get(
+ self,
+ session: requests.Session,
+ url: str,
+ stream: bool = False,
+ extra_headers: Optional[dict[str, str]] = None,
+ ) -> requests.Response:
+ response = session.get(
+ url,
+ headers=self._signed_headers(url, extra_headers),
+ timeout=REQUEST_TIMEOUT_SECONDS,
+ stream=stream,
+ allow_redirects=False,
+ )
+ if response.status_code >= 300:
+ raise error_for_response(response)
+ return response
+
+ def list_page(
+ self, prefix: str, continuation_token: Optional[str] = None, max_keys: int = LIST_PAGE_SIZE
+ ) -> S3ListPage:
+ query = {"list-type": "2", "prefix": prefix, "delimiter": "/", "max-keys": str(max_keys)}
+ if continuation_token:
+ query["continuation-token"] = continuation_token
+ return parse_list_page(self._get(self._session, self._url(query=query)).content)
+
+ def list_all(self, prefix: str) -> S3ListPage:
+ keys: list[str] = []
+ prefixes: list[str] = []
+ token: Optional[str] = None
+ while True:
+ page = self.list_page(prefix, token)
+ keys.extend(page.keys)
+ prefixes.extend(page.prefixes)
+ token = page.next_token
+ reach_safe_point()
+ if not token:
+ return S3ListPage(keys=keys, prefixes=prefixes, next_token=None)
+
+ def get_object(self, key: str) -> Optional[S3Body]:
+ """The object body and its `Last-Modified`, or None when the key does not exist."""
+ try:
+ response = self._get(self._session, self._url(key))
+ return S3Body(content=response.content, last_modified=response.headers.get("Last-Modified"))
+ except AwsCostAndUsageReportError as error:
+ if error.code in ("NoSuchKey", "HTTP 404"):
+ return None
+ raise
+
+ def download(self, key: str, destination: IO[bytes], unmodified_since: Optional[str] = None) -> None:
+ conditions = {"If-Unmodified-Since": unmodified_since} if unmodified_since else None
+ for attempt in range(1, DOWNLOAD_ATTEMPTS + 1):
+ destination.seek(0)
+ destination.truncate()
+ try:
+ with self._get(
+ self._download_session, self._url(key), stream=True, extra_headers=conditions
+ ) as response:
+ for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_BYTES):
+ destination.write(chunk)
+ break
+ except (
+ requests.exceptions.ChunkedEncodingError,
+ requests.exceptions.ConnectionError,
+ requests.exceptions.Timeout,
+ ):
+ if attempt == DOWNLOAD_ATTEMPTS:
+ raise
+ interruptible_wait(2**attempt)
+ destination.flush()
+ destination.seek(0)
+
+
+def list_billing_periods(client: S3Client, base: str, first_period: Optional[dt.date]) -> list[dt.date]:
+ periods: set[dt.date] = set()
+ for prefix in client.list_all(f"{base}/data/").prefixes:
+ match = BILLING_PERIOD_DIRECTORY.search(prefix)
+ if match:
+ periods.add(dt.date(int(match.group(1)), int(match.group(2)), 1))
+ return sorted(period for period in periods if first_period is None or period >= first_period)
+
+
+def _strings(value: Any) -> Iterator[str]:
+ if isinstance(value, str):
+ yield value
+ elif isinstance(value, dict):
+ for item in value.values():
+ yield from _strings(item)
+ elif isinstance(value, list):
+ for item in value:
+ yield from _strings(item)
+
+
+def manifest_data_keys(manifest: Any, bucket: str) -> list[str]:
+ """The data files a manifest lists.
+
+ AWS documents what a manifest holds but not its field names, so every string that names a
+ data file counts, as an `s3://` URI in this bucket or as a bare key.
+ """
+ keys: list[str] = []
+ for value in _strings(manifest):
+ if not value.endswith(DATA_FILE_SUFFIXES):
+ continue
+ if value.startswith("s3://"):
+ value_bucket, _, key = value[len("s3://") :].partition("/")
+ if value_bucket != bucket:
+ continue
+ value = key
+ keys.append(value)
+ return list(dict.fromkeys(keys))
+
+
+def manifest_column_types(manifest: Any) -> dict[str, str]:
+ """The column types a manifest declares, from every object that carries a name and a type."""
+ types: dict[str, str] = {}
+
+ def walk(value: Any) -> None:
+ if isinstance(value, dict):
+ fields = {str(key).lower(): item for key, item in value.items()}
+ name, kind = fields.get("name"), fields.get("type")
+ if isinstance(name, str) and isinstance(kind, str):
+ types[name] = kind.lower()
+ for item in value.values():
+ walk(item)
+ elif isinstance(value, list):
+ for item in value:
+ walk(item)
+
+ walk(manifest)
+ return types
+
+
+def latest_delivery(client: S3Client, bucket: str, base: str, export_name: str, period: dt.date) -> Optional[Delivery]:
+ """The latest complete delivery of one billing period, or None while it has no manifest.
+
+ AWS writes the manifest only after every data file of a delivery has landed, so a partition
+ without one is still being delivered and its files would be incomplete.
+ """
+ partition = f"BILLING_PERIOD={period:%Y-%m}"
+ # An "overwrite" export writes `<export>-Manifest.json` and a "create new" one `Manifest.json`.
+ # An export switched between the two leaves the old one behind, so the newer one wins.
+ found: list[tuple[dt.datetime, str, S3Body]] = []
+ for manifest_key in (
+ f"{base}/metadata/{partition}/{export_name.strip('/')}-Manifest.json",
+ f"{base}/metadata/{partition}/Manifest.json",
+ ):
+ manifest_object = client.get_object(manifest_key)
+ reach_safe_point()
+ if manifest_object is not None:
+ found.append((_http_date(manifest_object.last_modified), manifest_key, manifest_object))
+ if not found:
+ return None
+ _, manifest_key, manifest_object = max(found, key=lambda item: item[0])
+ unreadable = AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {MANIFEST_UNREADABLE} - s3://{bucket}/{manifest_key} lists no data files",
+ MANIFEST_UNREADABLE,
+ )
+ try:
+ manifest = json.loads(manifest_object.content)
+ except ValueError as error:
+ raise unreadable from error
+ keys = manifest_data_keys(manifest, bucket)
+ if not keys:
+ raise unreadable
+ return Delivery(
+ keys=keys,
+ column_types=manifest_column_types(manifest),
+ manifest_last_modified=manifest_object.last_modified,
+ )
+
+
+def _http_date(value: Optional[str]) -> dt.datetime:
+ try:
+ return email.utils.parsedate_to_datetime(value) if value else dt.datetime.min.replace(tzinfo=dt.UTC)
+ except (TypeError, ValueError):
+ return dt.datetime.min.replace(tzinfo=dt.UTC)
+
+
+def _jsonable(value: Any, value_type: pa.DataType) -> Any:
+ if value is None:
+ return None
+ if pa.types.is_map(value_type):
+ return {str(key): _jsonable(item, value_type.item_type) for key, item in value}
+ if pa.types.is_struct(value_type):
+ return {field.name: _jsonable(value.get(field.name), field.type) for field in value_type}
+ if pa.types.is_list(value_type) or pa.types.is_large_list(value_type):
+ return [_jsonable(item, value_type.value_type) for item in value]
+ return value
+
+
+def _is_nested(value_type: pa.DataType) -> bool:
+ return (
+ pa.types.is_map(value_type)
+ or pa.types.is_struct(value_type)
+ or pa.types.is_list(value_type)
+ or pa.types.is_large_list(value_type)
+ )
+
+
+def shape_table(table: pa.Table, period: dt.date) -> pa.Table:
+ """JSON-encode nested columns (`resource_tags`, `product`, `cost_category`, ...) and add the billing period."""
+ for index, field in enumerate(table.schema):
+ if _is_nested(field.type):
+ encoded = [
+ None if value is None else orjson.dumps(_jsonable(value, field.type), default=str).decode()
+ for value in table.column(index).to_pylist()
+ ]
+ table = table.set_column(index, field.name, pa.array(encoded, type=pa.string()))
+
+ period_start = dt.datetime(period.year, period.month, 1, tzinfo=dt.UTC)
+ column = pa.array([period_start] * table.num_rows, type=pa.timestamp("us", tz="UTC"))
+ existing = table.schema.get_field_index(BILLING_PERIOD_COLUMN)
+ if existing >= 0:
+ return table.set_column(existing, BILLING_PERIOD_COLUMN, column)
+ return table.append_column(BILLING_PERIOD_COLUMN, column)
+
+
+def _parse_timestamp(value: str) -> dt.datetime:
+ parsed = dt.datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=dt.UTC)
+
+
+def _parse_boolean(value: str) -> bool:
+ return value.strip().lower() == "true"
+
+
[email protected](frozen=True)
+class CsvColumnType:
+ convert: Callable[[str], Any]
+ arrow_type: pa.DataType
+
+
+def csv_column_type(declared_type: str) -> Optional[CsvColumnType]:
+ """How to read a CSV cell of a declared column type, and the Arrow type it lands as. None keeps the text."""
+ if declared_type in ("double", "float", "real"):
+ return CsvColumnType(float, pa.float64())
+ if match := DECIMAL_TYPE.match(declared_type):
+ # Same type as the Parquet file of the same export, so a format switch keeps the column type.
+ precision, scale = int(match.group(1)), int(match.group(2))
+ step = decimal.Decimal(1).scaleb(-scale)
+ # The default context holds 28 digits, which a decimal(38, 30) value can exceed.
+ context = decimal.Context(prec=precision)
+ return CsvColumnType(
+ lambda text: decimal.Decimal(text).quantize(step, context=context), pa.decimal128(precision, scale)
+ )
+ if declared_type.startswith("decimal"):
+ return CsvColumnType(float, pa.float64())
+ if declared_type in ("bigint", "int", "integer", "smallint", "tinyint", "long"):
+ return CsvColumnType(int, pa.int64())
+ if declared_type.startswith("timestamp") or declared_type == "datetime":
+ return CsvColumnType(_parse_timestamp, pa.timestamp("us", tz="UTC"))
+ if declared_type == "date":
+ return CsvColumnType(dt.date.fromisoformat, pa.date32())
+ if declared_type in ("boolean", "bool"):
+ return CsvColumnType(_parse_boolean, pa.bool_())
+ return None
+
+
+def convert_csv_rows(rows: list[dict[str, Any]], converters: dict[str, Callable[[str], Any]]) -> int:
+ """Give CSV cells the types the export declares. Every cell arrives as text, so without this a
+ cost column would land as a string. An empty or unreadable cell becomes null.
+
+ Returns how many non-empty cells could not be read."""
+ unreadable = 0
+ for row in rows:
+ for column, convert in converters.items():
+ value = row.get(column)
+ if value is None:
+ continue
+ if value == "":
+ row[column] = None
+ continue
+ try:
+ row[column] = convert(value)
+ except (TypeError, ValueError, ArithmeticError):
+ row[column] = None
+ unreadable += 1
+ return unreadable
+
+
+def iter_data_file(
+ client: S3Client, key: str, period: dt.date, delivery: Delivery, logger: FilteringBoundLogger
+) -> Iterator[pa.Table]:
+ """Yield the file's rows as Arrow tables, whatever its format, so one run never mixes item types."""
+ with tempfile.NamedTemporaryFile(suffix=".download") as destination:
+ try:
+ client.download(key, destination, unmodified_since=delivery.manifest_last_modified)
+ except AwsCostAndUsageReportError as error:
+ if error.code != "PreconditionFailed":
+ raise
+ raise AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {DELIVERY_IN_PROGRESS} - AWS is rewriting billing period {period:%Y-%m}; "
+ "the next attempt reads its finished delivery",
+ DELIVERY_IN_PROGRESS,
+ ) from error
+ if key.endswith(".parquet"):
+ parquet_file = pq.ParquetFile(destination)
+ for batch in parquet_file.iter_batches(batch_size=ROWS_PER_BATCH):
+ table = pa.Table.from_batches([batch])
+ if table.num_rows:
+ yield shape_table(table, period)
+ return
+
+ period_start = dt.datetime(period.year, period.month, 1, tzinfo=dt.UTC)
+ # The CSV reader lowercases headers (`BilledCost` becomes `billedcost`), but the pipeline
+ # snake-cases Parquet columns (`billed_cost`). The manifest keeps the original names, so CSV
+ # columns take the Parquet name and an export that changes format keeps one column per measure.
+ column_names = {normalize_column_name(name): pipeline_column_name(name) for name in delivery.column_types}
+ column_types = {
+ pipeline_column_name(name): column_type
+ for name, kind in delivery.column_types.items()
+ if (column_type := csv_column_type(kind)) is not None
+ }
+ converters = {name: column_type.convert for name, column_type in column_types.items()}
+ unreadable = 0
+ for file_rows in iter_file_rows(
+ destination, resolve_file_format(key), key, chunk_size=ROWS_PER_BATCH, logger=logger
+ ):
+ if not file_rows:
+ continue
+ rows = [{column_names.get(name, name): value for name, value in row.items()} for row in file_rows]
+ unreadable += convert_csv_rows(rows, converters)
+ for row in rows:
+ row[BILLING_PERIOD_COLUMN] = period_start
+ schema = pa.schema(
+ [
+ pa.field(
+ name,
+ pa.timestamp("us", tz="UTC")
+ if name == BILLING_PERIOD_COLUMN
+ else (column_types[name].arrow_type if name in column_types else pa.string()),
+ )
+ for name in rows[0]
+ ]
+ )
+ yield pa.Table.from_pylist(rows, schema=schema)
+ if unreadable:
+ logger.warning(
+ f"{unreadable} cell(s) in {key} did not match the type the manifest declares and were set to null"
+ )
+
+
+def get_rows(
+ aws_access_key_id: str,
+ aws_secret_access_key: str,
+ aws_session_token: Optional[str],
+ aws_region: str,
+ bucket: str,
+ prefix: Optional[str],
+ export_name: str,
+ first_billing_period: Optional[str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[CostAndUsageReportResumeConfig],
+) -> Iterator[pa.Table]:
+ client = S3Client(aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region, bucket)
+ base = export_base(prefix, export_name)
+ periods = list_billing_periods(client, base, parse_billing_period(first_billing_period))
+ if not periods:
+ logger.warning(f"No billing periods found under s3://{bucket.strip()}/{base}/data/")
+ return
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ delivered = 1 if resume is not None else 0
+ for period in periods:
+ billing_period = f"{period:%Y-%m}"
+ if resume is not None and billing_period < resume.billing_period:
+ continue
+ delivery = latest_delivery(client, bucket.strip(), base, export_name, period)
+ if delivery is None:
+ logger.warning(f"Skipping billing period {billing_period}: AWS has not finished delivering it")
+ continue
+ first_file, first_batch = 0, 0
+ if resume is not None and billing_period == resume.billing_period:
+ if delivery.manifest_last_modified != resume.manifest_last_modified:
+ # The rows already read came from the older delivery, so only a fresh refresh is consistent.
+ resumable_source_manager.clear_state()
+ raise AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {DELIVERY_IN_PROGRESS} - AWS delivered billing period {billing_period} "
+ "again during this sync; the next attempt starts over",
+ DELIVERY_IN_PROGRESS,
+ )
+ first_file, first_batch = resume.file_index, resume.batch_index
+ else:
+ delivered += 1
+ logger.debug(f"Reading billing period {billing_period} from {len(delivery.keys)} file(s)")
+ for file_index, key in enumerate(delivery.keys):
+ if file_index < first_file:
+ continue
+ skip = first_batch if file_index == first_file else 0
+ for batch_index, table in enumerate(iter_data_file(client, key, period, delivery, logger)):
+ if batch_index < skip:
+ continue
+ resumable_source_manager.save_state(
+ CostAndUsageReportResumeConfig(
+ billing_period=billing_period,
+ manifest_last_modified=delivery.manifest_last_modified,
+ file_index=file_index,
+ batch_index=batch_index + 1,
+ )
+ )
+ yield table
+
+ if not delivered:
+ raise AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {NO_MANIFEST} - no billing period under s3://{bucket.strip()}/{base}/metadata/ has a manifest",
+ NO_MANIFEST,
+ )
+ # A retry after a failed finalization would otherwise resume past every batch and load nothing.
+ resumable_source_manager.clear_state()
+
+
+VALIDATION_ERROR_MESSAGES: dict[str, str] = {
+ "InvalidAccessKeyId": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",
+ "SignatureDoesNotMatch": "AWS rejected the request signature. Re-enter the secret access key.",
+ "ExpiredToken": "The AWS session token has expired. Reconnect with fresh credentials.",
+ "TokenRefreshRequired": "The AWS session token has expired. Reconnect with fresh credentials.",
+ "InvalidToken": "AWS rejected the session token. Re-enter it, or leave it empty for long-lived keys.",
+ "AccessDenied": "These AWS credentials cannot read the export. Grant s3:ListBucket on the bucket and s3:GetObject on the export prefix to the IAM user or role.",
+ "NoSuchBucket": "The bucket does not exist. Check the bucket name.",
+ # A rejection without an error body reaches the parser as its bare status.
+ "HTTP 401": "AWS rejected the access key. Check the access key ID and secret access key, and that the key is still active.",
+ "HTTP 403": "These AWS credentials cannot read the export. Grant s3:ListBucket on the bucket and s3:GetObject on the export prefix to the IAM user or role.",
+}
+
+WRONG_REGION_MESSAGE = "The bucket is in a different AWS region. Set the region the bucket was created in."
+
+NO_EXPORT_MESSAGE = (
+ "No billing periods were found for this export. Check the bucket, S3 path prefix and export name. "
+ "AWS can take up to 24 hours to deliver the first export after you create it."
+)
+
+GENERIC_VALIDATION_ERROR = "AWS didn't accept the request. Check the access key can read the bucket, then try again."
+
+
+def validate_credentials(
+ aws_access_key_id: str,
+ aws_secret_access_key: str,
+ aws_session_token: Optional[str],
+ aws_region: str,
+ bucket: str,
+ prefix: Optional[str],
+ export_name: str,
+ first_billing_period: Optional[str],
+) -> tuple[bool, Optional[str]]:
+ """Check the inputs, then list the export's partitions once: the call every sync starts with."""
+ if not aws_access_key_id or not aws_secret_access_key:
+ return False, "AWS access key ID and secret access key are required"
+ if not aws_region or aws_dns_suffix(aws_region.strip()) is None:
+ return False, "Enter an AWS region code, for example us-east-1."
+ if not bucket or not BUCKET_NAME.match(bucket.strip()):
+ return False, "Enter the bucket name only, without s3:// or a path."
+ if not export_name or not export_name.strip("/ "):
+ return False, "Enter the name of the data export."
+ try:
+ parse_billing_period(first_billing_period)
+ except ValueError as error:
+ return False, str(error)
+
+ client = S3Client(aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region, bucket)
+ try:
+ page = client.list_page(f"{export_base(prefix, export_name)}/data/", max_keys=1)
+ except AwsCostAndUsageReportError as error:
+ code = error.code or ""
+ if code in WRONG_REGION_CODES:
+ if error.bucket_region:
+ return False, f"The bucket is in {error.bucket_region}. Set the AWS region to {error.bucket_region}."
+ return False, WRONG_REGION_MESSAGE
+ if code in VALIDATION_ERROR_MESSAGES:
+ return False, VALIDATION_ERROR_MESSAGES[code]
+ capture_exception(error)
+ return False, GENERIC_VALIDATION_ERROR
+ except requests.RequestException:
+ return False, "Could not reach Amazon S3. Check the bucket name and region, then try again."
+ except Exception as error:
+ capture_exception(error)
+ return False, GENERIC_VALIDATION_ERROR
+
+ if not page.prefixes and not page.keys:
+ return False, NO_EXPORT_MESSAGE
+ return True, None
+
+
+def aws_cost_and_usage_report_source(
+ aws_access_key_id: str,
+ aws_secret_access_key: str,
+ aws_session_token: Optional[str],
+ aws_region: str,
+ bucket: str,
+ prefix: Optional[str],
+ export_name: str,
+ first_billing_period: Optional[str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[CostAndUsageReportResumeConfig],
+) -> SourceResponse:
+ return SourceResponse(
+ name=COST_AND_USAGE,
+ items=lambda: get_rows(
+ aws_access_key_id=aws_access_key_id,
+ aws_secret_access_key=aws_secret_access_key,
+ aws_session_token=aws_session_token,
+ aws_region=aws_region,
+ bucket=bucket,
+ prefix=prefix,
+ export_name=export_name,
+ first_billing_period=first_billing_period,
+ logger=logger,
+ resumable_source_manager=resumable_source_manager,
+ ),
+ primary_keys=None,
+ partition_keys=[BILLING_PERIOD_COLUMN],
+ partition_mode="datetime",
+ partition_format="month",
+ sort_mode=None,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/canonical_descriptions.py
new file mode 100644
index 000000000000..57abfc656002
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/canonical_descriptions.py
@@ -0,0 +1,42 @@
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
+
+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
+ "cost_and_usage": {
+ "description": "One row per line item of the AWS Data Exports export, from the latest delivery of each billing period. Columns follow the export's query, so a CUR 2.0 export and a FOCUS export carry different columns.",
+ "docs_url": "https://docs.aws.amazon.com/cur/latest/userguide/table-dictionary-cur2.html",
+ "columns": {
+ "billing_period": "First day of the billing period the row belongs to, taken from the export's BILLING_PERIOD partition.",
+ "bill_billing_period_start_date": "Start of the billing period the line item is billed in.",
+ "bill_billing_period_end_date": "End of the billing period the line item is billed in.",
+ "bill_payer_account_id": "Account ID of the management (payer) account that pays the bill.",
+ "bill_invoice_id": "ID of the invoice the line item appears on. Empty until the bill finalizes.",
+ "line_item_usage_account_id": "Account ID of the member account that used the resource.",
+ "line_item_line_item_type": "Kind of charge, for example Usage, Tax, Credit, Refund, SavingsPlanCoveredUsage or DiscountedUsage.",
+ "line_item_usage_start_date": "Start of the period the usage covers.",
+ "line_item_usage_end_date": "End of the period the usage covers.",
+ "line_item_product_code": "Code of the AWS service the line item belongs to, for example AmazonEC2.",
+ "line_item_usage_type": "Usage details, such as the instance type and region the usage ran in.",
+ "line_item_operation": "Operation that produced the usage, for example RunInstances.",
+ "line_item_resource_id": "ID of the resource, when resource IDs are included in the export.",
+ "line_item_usage_amount": "Amount of usage in the period, in the unit of the pricing dimension.",
+ "line_item_unblended_rate": "Rate the usage was charged at, before blending across the organization.",
+ "line_item_unblended_cost": "Usage amount multiplied by the unblended rate. The cost before amortization, as on the bill.",
+ "line_item_blended_cost": "Usage amount multiplied by the blended rate, the average rate across the organization.",
+ "line_item_net_unblended_cost": "Unblended cost after discounts. Present when the account has discounts.",
+ "line_item_currency_code": "Currency of the cost columns.",
+ "line_item_line_item_description": "Human-readable description of the line item.",
+ "identity_line_item_id": "ID of the line item. Unique within one delivery only: AWS assigns new IDs in later deliveries.",
+ "identity_time_interval": "Time interval the line item covers, in ISO 8601 form.",
+ "pricing_term": "Whether the usage was OnDemand, Reserved or Spot.",
+ "pricing_public_on_demand_cost": "Cost of the usage at public On-Demand rates.",
+ "savings_plan_savings_plan_effective_cost": "Share of the Savings Plans commitment allocated to the line item.",
+ "reservation_effective_cost": "Amortized upfront and recurring reservation fee for the line item.",
+ "product": "JSON object of the product attributes, such as region, instance type and operating system.",
+ "resource_tags": "JSON object of the user and AWS cost allocation tags on the resource.",
+ "cost_category": "JSON object of the cost category values assigned to the line item.",
+ "discount": "JSON object of the discount amounts applied to the line item.",
+ },
+ },
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/settings.py
new file mode 100644
index 000000000000..ca68695d6bb8
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/settings.py
@@ -0,0 +1,31 @@
+import re
+
+from products.warehouse_sources.backend.types import IncrementalField
+
+S3_NAMESPACE = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+S3_SIGNING_NAME = "s3"
+
+LIST_PAGE_SIZE = 1000
+# Bytes per read when streaming a data file to disk.
+DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024
+# Rows per batch handed to the pipeline, matching its own batcher.
+ROWS_PER_BATCH = 5000
+
+# Data Exports names each partition directory `BILLING_PERIOD=YYYY-MM`.
+BILLING_PERIOD_DIRECTORY = re.compile(r"BILLING_PERIOD=(\d{4})-(\d{2})/?$")
+BILLING_PERIOD_INPUT = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+DATA_FILE_SUFFIXES = (".parquet", ".csv.gz")
+BUCKET_NAME = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")
+DECIMAL_TYPE = re.compile(r"^decimal\((\d+),\s*(\d+)\)$")
+
+COST_AND_USAGE = "cost_and_usage"
+
+ENDPOINTS = (COST_AND_USAGE,)
+
+# AWS restates whole billing periods, and the line item id is not stable between deliveries, so
+# there is no key to merge on. The table is rebuilt from the latest delivery of each period.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+
+ENDPOINT_DESCRIPTIONS: dict[str, str] = {
+ COST_AND_USAGE: "Every line item of the export, from the latest delivery of each billing period in range.",
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
index e7f41d1e083a..8922f293da2c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/source.py
@@ -1,8 +1,40 @@
-from typing import cast
+from typing import Optional, cast
-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.backend.facade.source_config import (
+ DataWarehouseSourceCategory,
+ ReleaseStatus,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.aws_cost_and_usage_report import (
+ DELIVERY_IN_PROGRESS,
+ ERROR_PREFIX,
+ MANIFEST_UNREADABLE,
+ VALIDATION_ERROR_MESSAGES,
+ WRONG_REGION_CODES,
+ WRONG_REGION_MESSAGE,
+ CostAndUsageReportResumeConfig,
+ aws_cost_and_usage_report_source,
+ validate_credentials as validate_cost_and_usage_report_credentials,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.settings import (
+ ENDPOINT_DESCRIPTIONS,
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
+ SourceSchema,
+ build_endpoint_schemas,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awscostandusagereport import (
AwsCostAndUsageReportSourceConfig,
)
@@ -10,18 +42,195 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class AwsCostAndUsageReportSource(SimpleSource[AwsCostAndUsageReportSourceConfig]):
+class AwsCostAndUsageReportSource(ResumableSource[AwsCostAndUsageReportSourceConfig, CostAndUsageReportResumeConfig]):
+ codebase_detection = detect(
+ npm=("@aws-sdk/client-bcm-data-exports", "@aws-sdk/client-cost-and-usage-report-service"),
+ cargo=("aws-sdk-bcmdataexports", "aws-sdk-costandusagereport"),
+ go=(
+ "github.com/aws/aws-sdk-go-v2/service/bcmdataexports",
+ "github.com/aws/aws-sdk-go-v2/service/costandusagereportservice",
+ ),
+ maven=("software.amazon.awssdk:bcmdataexports", "software.amazon.awssdk:costandusagereport"),
+ rubygems=("aws-sdk-bcmdataexports", "aws-sdk-costandusagereportservice"),
+ )
+
+ lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
+ api_docs_url = "https://docs.aws.amazon.com/cur/latest/userguide/dataexports-export-delivery.html"
+
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.AWSCOSTANDUSAGEREPORT
+ @property
+ def connection_host_fields(self) -> list[str]:
+ # The region and bucket decide where the signed requests, and so the stored key, are sent.
+ return ["aws_region", "bucket"]
+
+ def get_non_retryable_errors(self) -> dict[str, str | None]:
+ errors: dict[str, str | None] = {
+ f"{ERROR_PREFIX}: {code}": message for code, message in VALIDATION_ERROR_MESSAGES.items()
+ }
+ errors.update({f"{ERROR_PREFIX}: {code}": WRONG_REGION_MESSAGE for code in WRONG_REGION_CODES})
+ errors["Enter the first billing period as YYYY-MM"] = (
+ "The first billing period on the source is not in YYYY-MM form. Fix it and retry the sync."
+ )
+ return errors
+
+ def get_retryable_errors(self) -> set[str]:
+ # An "overwrite" export is being rewritten; the next attempt reads the finished delivery.
+ # An unreadable manifest is replaced by the next delivery.
+ return {f"{ERROR_PREFIX}: {DELIVERY_IN_PROGRESS}", f"{ERROR_PREFIX}: {MANIFEST_UNREADABLE}"}
+
+ def resume_covers_run(self, *, incremental_or_append: bool, schema_name: str | None = None) -> bool:
+ return not incremental_or_append
+
+ def get_resumable_source_manager(
+ self, inputs: SourceInputs
+ ) -> ResumableSourceManager[CostAndUsageReportResumeConfig]:
+ return ResumableSourceManager[CostAndUsageReportResumeConfig](inputs, CostAndUsageReportResumeConfig)
+
+ def get_canonical_descriptions(self) -> CanonicalDescriptions:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.canonical_descriptions import (
+ CANONICAL_DESCRIPTIONS,
+ )
+
+ return CANONICAL_DESCRIPTIONS
+
+ def get_schemas(
+ self,
+ config: AwsCostAndUsageReportSourceConfig,
+ team_id: int,
+ with_counts: bool = False,
+ names: list[str] | None = None,
+ force_refresh: bool = False,
+ api_version: str | None = None,
+ ) -> list[SourceSchema]:
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, descriptions=ENDPOINT_DESCRIPTIONS)
+
+ def validate_credentials(
+ self,
+ config: AwsCostAndUsageReportSourceConfig,
+ team_id: int,
+ schema_name: Optional[str] = None,
+ api_version: str | None = None,
+ ) -> tuple[bool, str | None]:
+ return validate_cost_and_usage_report_credentials(
+ config.aws_access_key_id,
+ config.aws_secret_access_key,
+ config.aws_session_token,
+ config.aws_region,
+ config.bucket,
+ config.prefix,
+ config.export_name,
+ config.first_billing_period,
+ )
+
+ def source_for_pipeline(
+ self,
+ config: AwsCostAndUsageReportSourceConfig,
+ resumable_source_manager: ResumableSourceManager[CostAndUsageReportResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
+ if inputs.schema_name not in ENDPOINTS:
+ raise ValueError(f"Unknown AWS Cost and Usage Report table: {inputs.schema_name}")
+ # A reset's first attempt reads from the start. Later attempts of the run keep its progress.
+ if inputs.reset_pipeline and inputs.activity_attempt == 1:
+ resumable_source_manager.clear_state()
+ return aws_cost_and_usage_report_source(
+ aws_access_key_id=config.aws_access_key_id,
+ aws_secret_access_key=config.aws_secret_access_key,
+ aws_session_token=config.aws_session_token,
+ aws_region=config.aws_region,
+ bucket=config.bucket,
+ prefix=config.prefix,
+ export_name=config.export_name,
+ first_billing_period=config.first_billing_period,
+ logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
+ )
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.AWSCOSTANDUSAGEREPORT,
category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,
- label="Amazon Web Services (AWS Cost and Usage Report)",
+ label="AWS Cost and Usage Report",
+ caption="""Sync the line items of an AWS Data Exports export (Cost and Usage Report 2.0 or FOCUS) from its S3 bucket into the PostHog Data warehouse.
+
+Create the export in the AWS Billing console under **Data Exports**, delivered to S3 as Parquet or gzip CSV. Parquet keeps the column types of the export; gzip CSV takes them from the export's manifest. Then create an IAM user or role with `s3:ListBucket` on the bucket and `s3:GetObject` on the export's prefix, and paste its access key ID and secret access key.
+
+AWS rewrites a whole billing period on each delivery and restates the previous month for up to two weeks after it ends, so every sync rebuilds the table from the latest delivery of each billing period. Each sync reads every month from the first billing period on, and each read counts as synced rows, so start at the earliest month you need.""",
iconPath="/static/services/aws_cost_and_usage_report.png",
- fields=cast(list[FieldType], []),
- unreleasedSource=True,
+ docsUrl="https://posthog.com/docs/cdp/sources/aws-cost-and-usage-report",
+ releaseStatus=ReleaseStatus.ALPHA,
+ keywords=["aws", "cur", "finops", "cloud spend", "data exports", "focus"],
+ fields=cast(
+ list[FieldType],
+ [
+ SourceFieldInputConfig(
+ name="aws_access_key_id",
+ label="AWS access key ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="AKIA...",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="aws_secret_access_key",
+ label="AWS secret access key",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="",
+ secret=True,
+ ),
+ SourceFieldInputConfig(
+ name="aws_session_token",
+ label="AWS session token (only for temporary credentials)",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=False,
+ placeholder="",
+ secret=True,
+ ),
+ SourceFieldInputConfig(
+ name="aws_region",
+ label="Bucket region",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="us-east-1",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="bucket",
+ label="S3 bucket",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="my-billing-exports",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="prefix",
+ label="S3 path prefix (as set on the export, empty when none)",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="billing/cur",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="export_name",
+ label="Export name",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="my-cur-export",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="first_billing_period",
+ label="First billing period to sync (YYYY-MM)",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="2024-01",
+ secret=False,
+ ),
+ ],
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
new file mode 100644
index 000000000000..ada41662da9b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
@@ -0,0 +1,525 @@
+import io
+import gzip
+import datetime as dt
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any
+from urllib.parse import unquote
+
+import pytest
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.aws_cost_and_usage_report import (
+ NO_EXPORT_MESSAGE,
+ VALIDATION_ERROR_MESSAGES,
+ WRONG_REGION_MESSAGE,
+ CostAndUsageReportResumeConfig,
+ validate_credentials,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.aws_cost_and_usage_report.source import (
+ AwsCostAndUsageReportSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ scripted_network,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.responders import UnexpectedRequest
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awscostandusagereport import (
+ AwsCostAndUsageReportSourceConfig,
+)
+
+BUCKET = "billing-exports"
+ORIGIN = f"https://{BUCKET}.s3.us-east-1.amazonaws.com"
+BASE = "cur/my-export"
+MARCH = dt.datetime(2024, 3, 1, tzinfo=dt.UTC)
+APRIL = dt.datetime(2024, 4, 1, tzinfo=dt.UTC)
+
+
+def config(**overrides: Any) -> AwsCostAndUsageReportSourceConfig:
+ values: dict[str, Any] = {
+ "aws_access_key_id": "AKIAEXAMPLE",
+ "aws_secret_access_key": "secret",
+ "aws_session_token": None,
+ "aws_region": "us-east-1",
+ "bucket": BUCKET,
+ "prefix": "/cur/",
+ "export_name": "my-export",
+ "first_billing_period": "2024-03",
+ }
+ values.update(overrides)
+ return AwsCostAndUsageReportSourceConfig(**values)
+
+
+def listing(
+ keys: Sequence[tuple[str, int]] = (),
+ prefixes: Sequence[str] = (),
+ next_token: str | None = None,
+) -> ScriptedResponse:
+ contents = "".join(f"<Contents><Key>{key}</Key><Size>{size}</Size></Contents>" for key, size in keys)
+ common = "".join(f"<CommonPrefixes><Prefix>{prefix}</Prefix></CommonPrefixes>" for prefix in prefixes)
+ truncation = (
+ f"<IsTruncated>true</IsTruncated><NextContinuationToken>{next_token}</NextContinuationToken>"
+ if next_token
+ else "<IsTruncated>false</IsTruncated>"
+ )
+ body = (
+ '<?xml version="1.0" encoding="UTF-8"?>'
+ f'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>{BUCKET}</Name>'
+ f"{truncation}{contents}{common}</ListBucketResult>"
+ )
+ return ScriptedResponse(body=body.encode(), headers={"Content-Type": "application/xml"})
+
+
+def s3_error(code: str, status: int, headers: Mapping[str, str] | None = None) -> ScriptedResponse:
+ body = f'<?xml version="1.0" encoding="UTF-8"?><Error><Code>{code}</Code><Message>{code} message</Message></Error>'
+ return ScriptedResponse(
+ status=status, body=body.encode(), headers={"Content-Type": "application/xml", **(headers or {})}
+ )
+
+
+def parquet_file(rows: list[dict[str, Any]]) -> ScriptedResponse:
+ table = pa.table(
+ {
+ "identity_line_item_id": [row["id"] for row in rows],
+ "line_item_unblended_cost": [row["cost"] for row in rows],
+ "resource_tags": pa.array([row["tags"] for row in rows], type=pa.map_(pa.string(), pa.string())),
+ }
+ )
+ buffer = io.BytesIO()
+ pq.write_table(table, buffer)
+ return ScriptedResponse(body=buffer.getvalue(), headers={"Content-Type": "application/octet-stream"})
+
+
+def csv_gz_file(text: str) -> ScriptedResponse:
+ return ScriptedResponse(body=gzip.compress(text.encode()), headers={"Content-Type": "application/octet-stream"})
+
+
+def s3(
+ lists: Mapping[str, list[ScriptedResponse]],
+ objects: Mapping[str, ScriptedResponse | list[ScriptedResponse]],
+) -> Callable[[RecordedRequest], ScriptedResponse]:
+ """Answer ListObjectsV2 by its prefix, in order, and GetObject by its key. A key missing from
+ `objects` is a 404, and a list answers the same key's requests in turn."""
+ queues = {prefix: list(answers) for prefix, answers in lists.items()}
+ object_queues = {key: list(answer) for key, answer in objects.items() if isinstance(answer, list)}
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ assert request.origin == ORIGIN
+ if request.param("list-type") == "2":
+ queue = queues.get(request.param("prefix") or "")
+ if not queue:
+ raise UnexpectedRequest(f"no listing scripted for {request.url}")
+ return queue.pop(0)
+ key = unquote(request.path).lstrip("/")
+ if key in object_queues:
+ return object_queues[key].pop(0)
+ answer = objects.get(key)
+ return answer if isinstance(answer, ScriptedResponse) else s3_error("NoSuchKey", 404)
+
+ return respond
+
+
+def table_rows(items: list[Any]) -> list[dict[str, Any]]:
+ """Rows of the Arrow tables a Parquet file yields; the driver only flattens lists of rows."""
+ return [row for table in items for row in table.to_pylist()]
+
+
+def requested_keys(requests: list[RecordedRequest]) -> list[str]:
+ return [unquote(request.path).lstrip("/") for request in requests if request.param("list-type") is None]
+
+
+class TestSync:
+ def test_reads_the_files_each_manifest_lists_for_every_period_from_the_first_one(self) -> None:
+ manifest = {
+ "dataFiles": [
+ f"s3://{BUCKET}/{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",
+ "s3://another-bucket/elsewhere/my-export-00001.snappy.parquet",
+ ],
+ "additionalOutputFiles": [{"path": f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-create-table.sql"}],
+ }
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {
+ f"{BASE}/data/": [
+ listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-02/"], next_token="t2"),
+ listing(
+ prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-04/", f"{BASE}/data/BILLING_PERIOD=2024-03/"]
+ ),
+ ],
+ },
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),
+ f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet": parquet_file(
+ [
+ {"id": "a", "cost": 1.5, "tags": [("user:team", "web")]},
+ {"id": "b", "cost": 2.0, "tags": None},
+ ]
+ ),
+ f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"]}
+ ),
+ f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet": parquet_file(
+ [{"id": "c", "cost": 0.25, "tags": []}]
+ ),
+ },
+ ),
+ )
+
+ assert result.raised is None
+ first_list = result.requests[0]
+ assert (first_list.param("prefix"), first_list.param("delimiter")) == (f"{BASE}/data/", "/")
+ assert result.requests[1].param("continuation-token") == "t2"
+ # February predates the first billing period; April's manifest uses the bare name.
+ assert requested_keys(result.requests) == [
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json",
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json",
+ f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",
+ f"{BASE}/metadata/BILLING_PERIOD=2024-04/my-export-Manifest.json",
+ f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json",
+ f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet",
+ ]
+ assert table_rows(result.items) == [
+ {
+ "identity_line_item_id": "a",
+ "line_item_unblended_cost": 1.5,
+ "resource_tags": '{"user:team":"web"}',
+ "billing_period": MARCH,
+ },
+ {
+ "identity_line_item_id": "b",
+ "line_item_unblended_cost": 2.0,
+ "resource_tags": None,
+ "billing_period": MARCH,
+ },
+ {
+ "identity_line_item_id": "c",
+ "line_item_unblended_cost": 0.25,
+ "resource_tags": "{}",
+ "billing_period": APRIL,
+ },
+ ]
+ assert "authorization" in first_list.headers
+ assert result.response is not None and result.response.partition_keys == ["billing_period"]
+
+ def test_a_period_still_being_delivered_is_skipped_and_one_with_no_manifest_anywhere_fails(self) -> None:
+ march_file = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
+ periods = listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/", f"{BASE}/data/BILLING_PERIOD=2024-04/"])
+
+ # April has data files but no manifest yet: AWS is still delivering it.
+ partly = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {f"{BASE}/data/": [periods]},
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [march_file]}
+ ),
+ march_file: parquet_file([{"id": "x", "cost": 1.0, "tags": None}]),
+ },
+ ),
+ )
+
+ assert partly.raised is None
+ assert [row["billing_period"] for row in table_rows(partly.items)] == [MARCH]
+ assert not any("BILLING_PERIOD=2024-04/" in key and "/data/" in key for key in requested_keys(partly.requests))
+
+ periods = listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/", f"{BASE}/data/BILLING_PERIOD=2024-04/"])
+ nothing = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage", s3({f"{BASE}/data/": [periods]}, {})
+ )
+
+ assert str(nothing.raised).startswith("AWS Cost and Usage Report request failed: NoManifest")
+
+ def test_gzip_csv_cells_take_the_names_and_types_a_parquet_file_would_have(self) -> None:
+ # A FOCUS export names columns in PascalCase. The pipeline snake-cases Parquet columns, so CSV
+ # columns must land under the same names or one measure splits across two columns.
+ key = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.csv.gz"
+ manifest = {
+ "dataFiles": [key],
+ "columns": [
+ {"name": "BillingAccountId", "type": "string"},
+ {"name": "BilledCost", "type": "double"},
+ # 30 decimal places exceed the 28-digit default decimal context.
+ {"name": "ConsumedQuantity", "type": "decimal(38,30)"},
+ {"name": "x_LeaseContractLength", "type": "bigint"},
+ {"name": "ChargePeriodStart", "type": "timestamp"},
+ {"name": "x_UsageDate", "type": "date"},
+ ],
+ }
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])]},
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),
+ key: csv_gz_file(
+ "BillingAccountId,BilledCost,ConsumedQuantity,x_LeaseContractLength,ChargePeriodStart,x_UsageDate\n"
+ "012345678901,1.5,24,3,2024-03-01T00:00:00Z,2024-03-01\n"
+ "012345678901,,not-a-number,,,\n"
+ ),
+ },
+ ),
+ )
+
+ assert result.raised is None
+ assert table_rows(result.items) == [
+ {
+ "billing_account_id": "012345678901",
+ "billed_cost": 1.5,
+ "consumed_quantity": 24.0,
+ "x_lease_contract_length": 3,
+ "charge_period_start": MARCH,
+ "x_usage_date": dt.date(2024, 3, 1),
+ "billing_period": MARCH,
+ },
+ {
+ "billing_account_id": "012345678901",
+ "billed_cost": None,
+ "consumed_quantity": None,
+ "x_lease_contract_length": None,
+ "charge_period_start": None,
+ "x_usage_date": None,
+ "billing_period": MARCH,
+ },
+ ]
+
+ def test_a_file_rewritten_after_its_manifest_stops_the_run_instead_of_mixing_deliveries(self) -> None:
+ key = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
+ manifest_written = "Wed, 06 Mar 2024 10:00:00 GMT"
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])]},
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(
+ json={"dataFiles": [key]}, headers={"Last-Modified": manifest_written}
+ ),
+ # AWS overwrote the file after the manifest we read, as an "overwrite" export does.
+ key: s3_error("PreconditionFailed", 412),
+ },
+ ),
+ )
+
+ download = next(request for request in result.requests if unquote(request.path).endswith(".parquet"))
+ assert download.headers["if-unmodified-since"] == manifest_written
+ assert str(result.raised).startswith("AWS Cost and Usage Report request failed: DeliveryInProgress")
+
+ def test_a_run_mixing_parquet_and_csv_periods_yields_arrow_tables_only(self) -> None:
+ march = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.csv.gz"
+ april = f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {
+ f"{BASE}/data/": [
+ listing(
+ prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/", f"{BASE}/data/BILLING_PERIOD=2024-04/"]
+ )
+ ]
+ },
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [march], "columns": [{"name": "line_item_unblended_cost", "type": "double"}]}
+ ),
+ march: csv_gz_file("identity_line_item_id,line_item_unblended_cost\na,1.5\n"),
+ f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [april]}
+ ),
+ april: parquet_file([{"id": "b", "cost": 2.0, "tags": None}]),
+ },
+ ),
+ )
+
+ assert result.raised is None
+ assert all(isinstance(item, pa.Table) for item in result.items)
+ assert [
+ (row["identity_line_item_id"], row["line_item_unblended_cost"]) for row in table_rows(result.items)
+ ] == [
+ ("a", 1.5),
+ ("b", 2.0),
+ ]
+
+ def test_a_download_cut_off_mid_body_is_retried_for_that_file_only(self) -> None:
+ key = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
+ # A chunked body with no terminating chunk fails after the headers, past the transport's retry.
+ cut_off = ScriptedResponse(body=b"5\r\nPAR1", headers={"Transfer-Encoding": "chunked"})
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3(
+ {f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])]},
+ {
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [key]}
+ ),
+ key: [cut_off, parquet_file([{"id": "x", "cost": 1.0, "tags": None}])],
+ },
+ ),
+ )
+
+ assert result.raised is None
+ assert requested_keys(result.requests).count(key) == 2
+ assert [row["identity_line_item_id"] for row in table_rows(result.items)] == ["x"]
+
+ def test_a_denied_read_fails_the_run_with_the_s3_code(self) -> None:
+ result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage",
+ s3({f"{BASE}/data/": [s3_error("AccessDenied", 403)]}, {}),
+ )
+
+ assert str(result.raised) == "AWS Cost and Usage Report request failed: AccessDenied - AccessDenied message"
+
+
+def probe(c: AwsCostAndUsageReportSourceConfig) -> tuple[bool, str | None]:
+ return validate_credentials(
+ c.aws_access_key_id,
+ c.aws_secret_access_key,
+ c.aws_session_token,
+ c.aws_region,
+ c.bucket,
+ c.prefix,
+ c.export_name,
+ c.first_billing_period,
+ )
+
+
+class TestDeliveries:
+ MANIFEST_KEY = f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json"
+ FIRST = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
+ SECOND = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00002.snappy.parquet"
+ WRITTEN = "Wed, 06 Mar 2024 10:00:00 GMT"
+
+ def _run(self, objects: Mapping[str, ScriptedResponse], resume_state: Any = None) -> Any:
+ periods = listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"])
+ return SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage", s3({f"{BASE}/data/": [periods]}, objects), resume_state=resume_state
+ )
+
+ @pytest.mark.parametrize(
+ "saved_written, expected_ids, raised",
+ [
+ ("Wed, 06 Mar 2024 10:00:00 GMT", ["b"], None),
+ # AWS delivered the month again, so the saved file and batch point into other files.
+ ("Tue, 05 Mar 2024 10:00:00 GMT", [], "DeliveryInProgress"),
+ ],
+ )
+ def test_a_resumed_run_continues_after_its_last_batch_only_within_the_same_delivery(
+ self, saved_written: str, expected_ids: list[str], raised: str | None
+ ) -> None:
+ manifest = ScriptedResponse(
+ json={"dataFiles": [self.FIRST, self.SECOND]}, headers={"Last-Modified": self.WRITTEN}
+ )
+ result = self._run(
+ {
+ self.MANIFEST_KEY: manifest,
+ self.FIRST: parquet_file([{"id": "a", "cost": 1.0, "tags": []}]),
+ self.SECOND: parquet_file([{"id": "b", "cost": 2.0, "tags": []}]),
+ },
+ resume_state=CostAndUsageReportResumeConfig(
+ billing_period="2024-03", manifest_last_modified=saved_written, file_index=0, batch_index=1
+ ),
+ )
+
+ assert [row["identity_line_item_id"] for row in table_rows(result.items)] == expected_ids
+ if raised is None:
+ assert result.raised is None
+ assert result.saved_states[-1] == CostAndUsageReportResumeConfig(
+ billing_period="2024-03", manifest_last_modified=self.WRITTEN, file_index=1, batch_index=1
+ )
+ # A retry after a failed finalization must read again, not resume past every batch.
+ assert result.final_state is None
+ else:
+ assert raised in str(result.raised)
+ assert result.final_state is None
+
+ @pytest.mark.parametrize(
+ "manifest",
+ [
+ ScriptedResponse(body=b"{not json", headers={"Content-Type": "application/json"}),
+ ScriptedResponse(json={"dataFiles": []}),
+ ],
+ )
+ def test_a_manifest_with_no_readable_file_list_fails_instead_of_dropping_the_month(
+ self, manifest: ScriptedResponse
+ ) -> None:
+ result = self._run({self.MANIFEST_KEY: manifest})
+
+ assert str(result.raised).startswith("AWS Cost and Usage Report request failed: ManifestUnreadable")
+ assert result.items == []
+
+ def test_the_newer_manifest_wins_when_an_export_left_both_names_behind(self) -> None:
+ result = self._run(
+ {
+ self.MANIFEST_KEY: ScriptedResponse(
+ json={"dataFiles": [self.FIRST]}, headers={"Last-Modified": "Mon, 04 Mar 2024 10:00:00 GMT"}
+ ),
+ f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [self.SECOND]}, headers={"Last-Modified": self.WRITTEN}
+ ),
+ self.SECOND: parquet_file([{"id": "b", "cost": 2.0, "tags": []}]),
+ }
+ )
+
+ assert result.raised is None
+ assert [row["identity_line_item_id"] for row in table_rows(result.items)] == ["b"]
+
+
+class TestValidateCredentials:
+ @pytest.mark.parametrize(
+ ("overrides", "expected"),
+ [
+ ({"aws_secret_access_key": ""}, "AWS access key ID and secret access key are required"),
+ ({"aws_region": "Frankfurt"}, "Enter an AWS region code, for example us-east-1."),
+ ({"bucket": "s3://billing-exports/cur"}, "Enter the bucket name only, without s3:// or a path."),
+ ({"export_name": " / "}, "Enter the name of the data export."),
+ ({"first_billing_period": "2024-13"}, "Enter the first billing period as YYYY-MM, for example 2024-01."),
+ ],
+ )
+ def test_local_checks_fail_before_any_request(self, overrides: dict[str, Any], expected: str) -> None:
+ with scripted_network([]) as network:
+ assert probe(config(**overrides)) == (False, expected)
+
+ assert network.requests == 0
+
+ @pytest.mark.parametrize(
+ ("answer", "expected"),
+ [
+ (listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/"]), (True, None)),
+ (listing(), (False, NO_EXPORT_MESSAGE)),
+ (s3_error("AccessDenied", 403), (False, VALIDATION_ERROR_MESSAGES["AccessDenied"])),
+ (s3_error("NoSuchBucket", 404), (False, VALIDATION_ERROR_MESSAGES["NoSuchBucket"])),
+ (
+ s3_error("PermanentRedirect", 301, {"x-amz-bucket-region": "eu-west-1"}),
+ (False, "The bucket is in eu-west-1. Set the AWS region to eu-west-1."),
+ ),
+ (s3_error("AuthorizationHeaderMalformed", 400), (False, WRONG_REGION_MESSAGE)),
+ ],
+ )
+ def test_the_probe_lists_the_export_and_translates_s3_errors(
+ self, answer: ScriptedResponse, expected: tuple[bool, str | None]
+ ) -> None:
+ with scripted_network([answer]) as network:
+ assert probe(config()) == expected
+
+ request = network.requests_log[0]
+ assert request.origin == ORIGIN
+ assert (request.param("prefix"), request.param("max-keys")) == (f"{BASE}/data/", "1")
+
+ @pytest.mark.parametrize(
+ ("bucket", "region", "origin", "path"),
+ [
+ ("billing.exports", "us-east-1", "https://s3.us-east-1.amazonaws.com", "/billing.exports/"),
+ (BUCKET, "cn-north-1", f"https://{BUCKET}.s3.cn-north-1.amazonaws.com.cn", "/"),
+ ],
+ )
+ def test_the_endpoint_follows_the_bucket_name_and_the_aws_partition(
+ self, bucket: str, region: str, origin: str, path: str
+ ) -> None:
+ with scripted_network([listing(prefixes=["x/"])]) as network:
+ assert probe(config(bucket=bucket, aws_region=region)) == (True, None)
+
+ request = network.requests_log[0]
+ assert (request.origin, request.path) == (origin, path)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awscostandusagereport.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awscostandusagereport.py
index c47f377580b8..0d9ec7458824 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awscostandusagereport.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/awscostandusagereport.py
@@ -6,4 +6,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
@config.config
class AwsCostAndUsageReportSourceConfig(config.Config):
- pass
+ aws_access_key_id: str
+ aws_secret_access_key: str
+ aws_region: str
+ bucket: str
+ export_name: str
+ first_billing_period: str
+ aws_session_token: str | None = None
+ prefix: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/contract_harness.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/contract_harness.py
index eecc4da7e692..6d1ffcf9f046 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/contract_harness.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/contract_harness.py
@@ -207,6 +207,7 @@ _PLACEHOLDERS_BY_NAME = (
("domain", "contract.example.com"),
("region", "us-east-1"),
("date", "2024-01-01"),
+ ("billing_period", "2024-01"),
)
_UUID = "00000000-0000-4000-8000-000000000000"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
index 458f2f89981e..614c3892a4d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
@@ -395,6 +395,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {
"Avalara",
"AwsBudgets",
"AwsComputeOptimizer",
+ "AwsCostAndUsageReport",
"AzureApplicationInsights",
"Bettermode",
"BillCom",