Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): read lowercase billing_period partitions in aws cur source (#129961)

Ben Smith4ab5edc062e2
diff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
index d8814f127215..137892959647 100644
--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py
@@ -54,6 +54,7 @@ REQUEST_TIMEOUT_SECONDS = 120
# headers. One file is retried on its own so a late failure does not restart the whole refresh.
DOWNLOAD_ATTEMPTS = 3
NO_MANIFEST = "NoManifest"
+NO_BILLING_PERIODS = "NoBillingPeriods"
DELIVERY_IN_PROGRESS = "DeliveryInProgress"
MANIFEST_UNREADABLE = "ManifestUnreadable"
UNSUPPORTED_COMPRESSION = "UnsupportedCompression"
@@ -270,13 +271,16 @@ class S3Client:
destination.seek(0)
-def list_billing_periods(client: S3Client, base: str, first_period: Optional[dt.date]) -> list[dt.date]:
- periods: set[dt.date] = set()
+def list_billing_periods(client: S3Client, base: str, first_period: Optional[dt.date]) -> dict[dt.date, str]:
+ """Each billing period in range, oldest first, with the name of its partition directory."""
+ partitions: dict[dt.date, str] = {}
for prefix in client.list_all(f"{base}/data/").prefixes:
match = BILLING_PERIOD_DIRECTORY.search(prefix)
if match:
- periods.add(dt.date(int(match.group(1)), int(match.group(2)), 1))
- return sorted(period for period in periods if first_period is None or period >= first_period)
+ partitions[dt.date(int(match.group(2)), int(match.group(3)), 1)] = match.group(1)
+ return {
+ period: partitions[period] for period in sorted(partitions) if first_period is None or period >= first_period
+ }
def _strings(value: Any) -> Iterator[str]:
@@ -329,13 +333,12 @@ def manifest_column_types(manifest: Any) -> dict[str, str]:
return types
-def latest_delivery(client: S3Client, bucket: str, base: str, export_name: str, period: dt.date) -> Optional[Delivery]:
+def latest_delivery(client: S3Client, bucket: str, base: str, export_name: str, partition: str) -> Optional[Delivery]:
"""The latest complete delivery of one billing period, or None while it has no manifest.
AWS writes the manifest only after every data file of a delivery has landed, so a partition
without one is still being delivered and its files would be incomplete.
"""
- partition = f"BILLING_PERIOD={period:%Y-%m}"
# An "overwrite" export writes `<export>-Manifest.json` and a "create new" one `Manifest.json`.
# An export switched between the two leaves the old one behind, so the newer one wins.
found: list[tuple[dt.datetime, str, S3Body]] = []
@@ -560,16 +563,22 @@ def get_rows(
base = export_base(prefix, export_name)
periods = list_billing_periods(client, base, parse_billing_period(first_billing_period))
if not periods:
- logger.warning(f"No billing periods found under s3://{bucket.strip()}/{base}/data/")
- return
+ in_range = (
+ f" from {first_billing_period.strip()} on" if first_billing_period and first_billing_period.strip() else ""
+ )
+ raise AwsCostAndUsageReportError(
+ f"{ERROR_PREFIX}: {NO_BILLING_PERIODS} - no billing period{in_range} under s3://{bucket.strip()}/{base}/data/. "
+ "Check the S3 path prefix, export name and first billing period of the source.",
+ NO_BILLING_PERIODS,
+ )
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
delivered = 1 if resume is not None else 0
- for period in periods:
+ for period, partition in periods.items():
billing_period = f"{period:%Y-%m}"
if resume is not None and billing_period < resume.billing_period:
continue
- delivery = latest_delivery(client, bucket.strip(), base, export_name, period)
+ delivery = latest_delivery(client, bucket.strip(), base, export_name, partition)
if delivery is None:
logger.warning(f"Skipping billing period {billing_period}: AWS has not finished delivering it")
continue
diff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py
index 3094e935074b..7828cdcd5ab6 100644
--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py
+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py
@@ -11,8 +11,10 @@ DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024
# Rows per batch handed to the pipeline, matching its own batcher.
ROWS_PER_BATCH = 5000
-# Data Exports names each partition directory `BILLING_PERIOD=YYYY-MM`.
-BILLING_PERIOD_DIRECTORY = re.compile(r"BILLING_PERIOD=(\d{4})-(\d{2})/?$")
+# Data Exports names each partition directory `BILLING_PERIOD=YYYY-MM`. A CUR 2.0 export writes the
+# name in upper case and a FOCUS export can write it in lower case, so the match ignores case and the
+# directory name found under data/ is reused for the metadata/ path.
+BILLING_PERIOD_DIRECTORY = re.compile(r"(BILLING_PERIOD=(\d{4})-(\d{2}))/?$", re.IGNORECASE)
BILLING_PERIOD_INPUT = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
DATA_FILE_SUFFIXES = (".parquet", ".csv.gz")
BUCKET_NAME = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")
diff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
index c21b7e459588..dd5b2ef7fb13 100644
--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py
@@ -133,37 +133,36 @@ def requested_keys(requests: list[RecordedRequest]) -> list[str]:
class TestSync:
- def test_reads_the_files_each_manifest_lists_for_every_period_from_the_first_one(self) -> None:
+ @pytest.mark.parametrize("partition", ["BILLING_PERIOD", "billing_period"])
+ def test_reads_the_files_each_manifest_lists_for_every_period_from_the_first_one(self, partition: str) -> None:
manifest = {
"dataFiles": [
- f"s3://{BUCKET}/{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",
+ f"s3://{BUCKET}/{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet",
"s3://another-bucket/elsewhere/my-export-00001.snappy.parquet",
],
- "additionalOutputFiles": [{"path": f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-create-table.sql"}],
+ "additionalOutputFiles": [{"path": f"{BASE}/metadata/{partition}=2024-03/my-export-create-table.sql"}],
}
result = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
"cost_and_usage",
s3(
{
f"{BASE}/data/": [
- listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-02/"], next_token="t2"),
- listing(
- prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-04/", f"{BASE}/data/BILLING_PERIOD=2024-03/"]
- ),
+ listing(prefixes=[f"{BASE}/data/{partition}=2024-02/"], next_token="t2"),
+ listing(prefixes=[f"{BASE}/data/{partition}=2024-04/", f"{BASE}/data/{partition}=2024-03/"]),
],
},
{
- f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),
- f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet": parquet_file(
+ f"{BASE}/metadata/{partition}=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),
+ f"{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet": parquet_file(
[
{"id": "a", "cost": 1.5, "tags": [("user:team", "web")]},
{"id": "b", "cost": 2.0, "tags": None},
]
),
- f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json": ScriptedResponse(
- json={"dataFiles": [f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"]}
+ f"{BASE}/metadata/{partition}=2024-04/Manifest.json": ScriptedResponse(
+ json={"dataFiles": [f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet"]}
),
- f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet": parquet_file(
+ f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet": parquet_file(
[{"id": "c", "cost": 0.25, "tags": []}]
),
},
@@ -176,12 +175,12 @@ class TestSync:
assert result.requests[1].param("continuation-token") == "t2"
# February predates the first billing period; April's manifest uses the bare name.
assert requested_keys(result.requests) == [
- f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json",
- f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json",
- f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",
- f"{BASE}/metadata/BILLING_PERIOD=2024-04/my-export-Manifest.json",
- f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json",
- f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet",
+ f"{BASE}/metadata/{partition}=2024-03/my-export-Manifest.json",
+ f"{BASE}/metadata/{partition}=2024-03/Manifest.json",
+ f"{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet",
+ f"{BASE}/metadata/{partition}=2024-04/my-export-Manifest.json",
+ f"{BASE}/metadata/{partition}=2024-04/Manifest.json",
+ f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet",
]
assert table_rows(result.items) == [
{
@@ -206,7 +205,7 @@ class TestSync:
assert "authorization" in first_list.headers
assert result.response is not None and result.response.partition_keys == ["billing_period"]
- def test_a_period_still_being_delivered_is_skipped_and_one_with_no_manifest_anywhere_fails(self) -> None:
+ def test_a_period_still_being_delivered_is_skipped_and_an_export_with_nothing_to_read_fails(self) -> None:
march_file = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"
periods = listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/", f"{BASE}/data/BILLING_PERIOD=2024-04/"])
@@ -235,6 +234,15 @@ class TestSync:
assert str(nothing.raised).startswith("AWS Cost and Usage Report request failed: NoManifest")
+ before_first = SourceDriver(AwsCostAndUsageReportSource(), config()).run(
+ "cost_and_usage", s3({f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-02/"])]}, {})
+ )
+
+ assert str(before_first.raised).startswith(
+ f"AWS Cost and Usage Report request failed: NoBillingPeriods - no billing period from 2024-03 on under s3://{BUCKET}/{BASE}/data/"
+ )
+ assert before_first.items == []
+
def test_gzip_csv_cells_take_the_names_and_types_a_parquet_file_would_have(self) -> None:
# A FOCUS export names columns in PascalCase. The pipeline snake-cases Parquet columns, so CSV
# columns must land under the same names or one measure splits across two columns.