:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): read lowercase billing_period partitions in aws cur source (#129961)
Ben Smith4ab5edc062e2
diff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.pyindex d8814f127215..137892959647 100644--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/aws_cost_and_usage_report.py@@ -54,6 +54,7 @@ REQUEST_TIMEOUT_SECONDS = 120# headers. One file is retried on its own so a late failure does not restart the whole refresh.DOWNLOAD_ATTEMPTS = 3NO_MANIFEST = "NoManifest"+NO_BILLING_PERIODS = "NoBillingPeriods"DELIVERY_IN_PROGRESS = "DeliveryInProgress"MANIFEST_UNREADABLE = "ManifestUnreadable"UNSUPPORTED_COMPRESSION = "UnsupportedCompression"@@ -270,13 +271,16 @@ class S3Client:destination.seek(0)-def list_billing_periods(client: S3Client, base: str, first_period: Optional[dt.date]) -> list[dt.date]:- periods: set[dt.date] = set()+def list_billing_periods(client: S3Client, base: str, first_period: Optional[dt.date]) -> dict[dt.date, str]:+ """Each billing period in range, oldest first, with the name of its partition directory."""+ partitions: dict[dt.date, str] = {}for prefix in client.list_all(f"{base}/data/").prefixes:match = BILLING_PERIOD_DIRECTORY.search(prefix)if match:- periods.add(dt.date(int(match.group(1)), int(match.group(2)), 1))- return sorted(period for period in periods if first_period is None or period >= first_period)+ partitions[dt.date(int(match.group(2)), int(match.group(3)), 1)] = match.group(1)+ return {+ period: partitions[period] for period in sorted(partitions) if first_period is None or period >= first_period+ }def _strings(value: Any) -> Iterator[str]:@@ -329,13 +333,12 @@ def manifest_column_types(manifest: Any) -> dict[str, str]:return types-def latest_delivery(client: S3Client, bucket: str, base: str, export_name: str, period: dt.date) -> Optional[Delivery]:+def latest_delivery(client: S3Client, bucket: str, base: str, export_name: str, partition: str) -> Optional[Delivery]:"""The latest complete delivery of one billing period, or None while it has no manifest.AWS writes the manifest only after every data file of a delivery has landed, so a partitionwithout one is still being delivered and its files would be incomplete."""- partition = f"BILLING_PERIOD={period:%Y-%m}"# An "overwrite" export writes `<export>-Manifest.json` and a "create new" one `Manifest.json`.# An export switched between the two leaves the old one behind, so the newer one wins.found: list[tuple[dt.datetime, str, S3Body]] = []@@ -560,16 +563,22 @@ def get_rows(base = export_base(prefix, export_name)periods = list_billing_periods(client, base, parse_billing_period(first_billing_period))if not periods:- logger.warning(f"No billing periods found under s3://{bucket.strip()}/{base}/data/")- return+ in_range = (+ f" from {first_billing_period.strip()} on" if first_billing_period and first_billing_period.strip() else ""+ )+ raise AwsCostAndUsageReportError(+ f"{ERROR_PREFIX}: {NO_BILLING_PERIODS} - no billing period{in_range} under s3://{bucket.strip()}/{base}/data/. "+ "Check the S3 path prefix, export name and first billing period of the source.",+ NO_BILLING_PERIODS,+ )resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Nonedelivered = 1 if resume is not None else 0- for period in periods:+ for period, partition in periods.items():billing_period = f"{period:%Y-%m}"if resume is not None and billing_period < resume.billing_period:continue- delivery = latest_delivery(client, bucket.strip(), base, export_name, period)+ delivery = latest_delivery(client, bucket.strip(), base, export_name, partition)if delivery is None:logger.warning(f"Skipping billing period {billing_period}: AWS has not finished delivering it")continuediff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.pyindex 3094e935074b..7828cdcd5ab6 100644--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/settings.py@@ -11,8 +11,10 @@ DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024# Rows per batch handed to the pipeline, matching its own batcher.ROWS_PER_BATCH = 5000-# Data Exports names each partition directory `BILLING_PERIOD=YYYY-MM`.-BILLING_PERIOD_DIRECTORY = re.compile(r"BILLING_PERIOD=(\d{4})-(\d{2})/?$")+# Data Exports names each partition directory `BILLING_PERIOD=YYYY-MM`. A CUR 2.0 export writes the+# name in upper case and a FOCUS export can write it in lower case, so the match ignores case and the+# directory name found under data/ is reused for the metadata/ path.+BILLING_PERIOD_DIRECTORY = re.compile(r"(BILLING_PERIOD=(\d{4})-(\d{2}))/?$", re.IGNORECASE)BILLING_PERIOD_INPUT = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")DATA_FILE_SUFFIXES = (".parquet", ".csv.gz")BUCKET_NAME = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")diff --git a/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py b/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.pyindex c21b7e459588..dd5b2ef7fb13 100644--- a/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py+++ b/products/warehouse_sources/sources/aws_cost_and_usage_report/tests/test_aws_cost_and_usage_report.py@@ -133,37 +133,36 @@ def requested_keys(requests: list[RecordedRequest]) -> list[str]:class TestSync:- def test_reads_the_files_each_manifest_lists_for_every_period_from_the_first_one(self) -> None:+ @pytest.mark.parametrize("partition", ["BILLING_PERIOD", "billing_period"])+ def test_reads_the_files_each_manifest_lists_for_every_period_from_the_first_one(self, partition: str) -> None:manifest = {"dataFiles": [- f"s3://{BUCKET}/{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",+ f"s3://{BUCKET}/{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet","s3://another-bucket/elsewhere/my-export-00001.snappy.parquet",],- "additionalOutputFiles": [{"path": f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-create-table.sql"}],+ "additionalOutputFiles": [{"path": f"{BASE}/metadata/{partition}=2024-03/my-export-create-table.sql"}],}result = SourceDriver(AwsCostAndUsageReportSource(), config()).run("cost_and_usage",s3({f"{BASE}/data/": [- listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-02/"], next_token="t2"),- listing(- prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-04/", f"{BASE}/data/BILLING_PERIOD=2024-03/"]- ),+ listing(prefixes=[f"{BASE}/data/{partition}=2024-02/"], next_token="t2"),+ listing(prefixes=[f"{BASE}/data/{partition}=2024-04/", f"{BASE}/data/{partition}=2024-03/"]),],},{- f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),- f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet": parquet_file(+ f"{BASE}/metadata/{partition}=2024-03/my-export-Manifest.json": ScriptedResponse(json=manifest),+ f"{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet": parquet_file([{"id": "a", "cost": 1.5, "tags": [("user:team", "web")]},{"id": "b", "cost": 2.0, "tags": None},]),- f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json": ScriptedResponse(- json={"dataFiles": [f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet"]}+ f"{BASE}/metadata/{partition}=2024-04/Manifest.json": ScriptedResponse(+ json={"dataFiles": [f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet"]}),- f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet": parquet_file(+ f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet": parquet_file([{"id": "c", "cost": 0.25, "tags": []}]),},@@ -176,12 +175,12 @@ class TestSync:assert result.requests[1].param("continuation-token") == "t2"# February predates the first billing period; April's manifest uses the bare name.assert requested_keys(result.requests) == [- f"{BASE}/metadata/BILLING_PERIOD=2024-03/my-export-Manifest.json",- f"{BASE}/metadata/BILLING_PERIOD=2024-03/Manifest.json",- f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet",- f"{BASE}/metadata/BILLING_PERIOD=2024-04/my-export-Manifest.json",- f"{BASE}/metadata/BILLING_PERIOD=2024-04/Manifest.json",- f"{BASE}/data/BILLING_PERIOD=2024-04/my-export-00001.snappy.parquet",+ f"{BASE}/metadata/{partition}=2024-03/my-export-Manifest.json",+ f"{BASE}/metadata/{partition}=2024-03/Manifest.json",+ f"{BASE}/data/{partition}=2024-03/my-export-00001.snappy.parquet",+ f"{BASE}/metadata/{partition}=2024-04/my-export-Manifest.json",+ f"{BASE}/metadata/{partition}=2024-04/Manifest.json",+ f"{BASE}/data/{partition}=2024-04/my-export-00001.snappy.parquet",]assert table_rows(result.items) == [{@@ -206,7 +205,7 @@ class TestSync:assert "authorization" in first_list.headersassert result.response is not None and result.response.partition_keys == ["billing_period"]- def test_a_period_still_being_delivered_is_skipped_and_one_with_no_manifest_anywhere_fails(self) -> None:+ def test_a_period_still_being_delivered_is_skipped_and_an_export_with_nothing_to_read_fails(self) -> None:march_file = f"{BASE}/data/BILLING_PERIOD=2024-03/my-export-00001.snappy.parquet"periods = listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-03/", f"{BASE}/data/BILLING_PERIOD=2024-04/"])@@ -235,6 +234,15 @@ class TestSync:assert str(nothing.raised).startswith("AWS Cost and Usage Report request failed: NoManifest")+ before_first = SourceDriver(AwsCostAndUsageReportSource(), config()).run(+ "cost_and_usage", s3({f"{BASE}/data/": [listing(prefixes=[f"{BASE}/data/BILLING_PERIOD=2024-02/"])]}, {})+ )++ assert str(before_first.raised).startswith(+ f"AWS Cost and Usage Report request failed: NoBillingPeriods - no billing period from 2024-03 on under s3://{BUCKET}/{BASE}/data/"+ )+ assert before_first.items == []+def test_gzip_csv_cells_take_the_names_and_types_a_parquet_file_would_have(self) -> None:# A FOCUS export names columns in PascalCase. The pipeline snake-cases Parquet columns, so CSV# columns must land under the same names or one measure splits across two columns.