:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): smooth s3 source urls, delta defaults, and reimports (#124785)
Tom Owers81a3475363f4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.pyindex 56b9cf4a37f1..22706f23e6df 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py@@ -497,6 +497,16 @@ def normalize_table_column_names(table: pa.Table) -> pa.Table:INTERNAL_COLUMN_NAMES = frozenset({"_ph_debug", PARTITION_KEY, "_dlt_id", "_dlt_load_id"})+def drop_internal_columns(table: pa.Table) -> pa.Table:+ """Drop the columns the pipeline adds itself from a source batch.++ A source can return them when it reads a table PostHog wrote, such as a Delta table in a bucket.+ Kept, they would exist twice in the batch once the pipeline adds its own.+ """+ internal = [name for name in table.column_names if name in INTERNAL_COLUMN_NAMES]+ return table.drop_columns(internal) if internal else table++def _fold_column_name_for_match(name: str) -> str:try:return NamingConvention.normalize_identifier(name)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 439d0b3a05f3..c33ebbf4097f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -77,6 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.sfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (_append_debug_column_to_pyarrows_table,_handle_null_columns_with_definitions,+ drop_internal_columns,evolve_pyarrow_schema,merge_observed_columns_into_schema_metadata,normalize_column_name,@@ -1161,7 +1162,7 @@ class PipelineV3(Generic[ResumableData]):cleanup_memory(pa_memory_pool, py_table if "py_table" in locals() else None)async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:- pa_table = _append_debug_column_to_pyarrows_table(pa_table, self._load_id)+ pa_table = _append_debug_column_to_pyarrows_table(drop_internal_columns(pa_table), self._load_id)if not self._resource.preserve_column_names:pa_table = normalize_table_column_names(pa_table)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex a277eb6f56cb..9237242f11d0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1391,13 +1391,21 @@ class TestSingleTableRunIsUntouched:class TestColumnNamesReachTheWrite:@pytest.mark.parametrize(- "preserve, expected",+ "preserve, internal_columns, expected",[- (False, ["order_id", "total_amount", "_ph_debug"]),- (True, ["Order ID", "totalAmount", "_ph_debug"]),+ (False, {}, ["order_id", "total_amount", "_ph_debug"]),+ (True, {}, ["Order ID", "totalAmount", "_ph_debug"]),+ # A Delta table PostHog wrote, read back from a bucket, already holds the columns the pipeline adds.+ (+ False,+ {"_ph_debug": '{"load_id": 1}', "_ph_partition_key": "2024-05"},+ ["order_id", "total_amount", "_ph_debug"],+ ),],)- async def test_preserving_names_skips_the_snake_case_rename(self, preserve: bool, expected: list[str]) -> None:+ async def test_preserving_names_skips_the_snake_case_rename(+ self, preserve: bool, internal_columns: dict[str, str], expected: list[str]+ ) -> None:# A self-managed table moved into a source keeps the column names its queries use.pipeline = _make_pipeline()pipeline._resource = SourceResponse(@@ -1406,7 +1414,13 @@ class TestColumnNamesReachTheWrite:cast(MagicMock, pipeline._schema).configure_mock(incremental_field=None, enabled_columns=None)pipeline._last_incremental_field_value = Nonepipeline._earliest_incremental_field_value = None- table = pa.table({"Order ID": pa.array([1], pa.int64()), "totalAmount": pa.array([2.5], pa.float64())})+ table = pa.table(+ {+ "Order ID": pa.array([1], pa.int64()),+ "totalAmount": pa.array([2.5], pa.float64()),+ **{name: pa.array([value], pa.string()) for name, value in internal_columns.items()},+ }+ )with (patch.object(PipelineV3, "_stage_batch", AsyncMock(return_value=1)),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.pyindex 5ae8d3a1512e..80aff19b1aaa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py@@ -1,3 +1,4 @@+import dataclassesfrom typing import Any, castfrom products.warehouse_sources.backend.facade.codebase_detection import detect@@ -22,6 +23,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objtext_field,url_field,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ lookup_s3_bucket_region,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (S3_EXAMPLE_URL,parse_s3_url,@@ -65,9 +69,12 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):return ["url_pattern"]def object_store_config(self, config: AmazonS3SourceConfig) -> ObjectStoreConfig:+ url = parse_s3_url(config.url_pattern)+ if url.region is None:+ url = dataclasses.replace(url, region=lookup_s3_bucket_region(url.bucket))return ObjectStoreConfig.from_url(ObjectStoreProvider.S3,- parse_s3_url(config.url_pattern),+ url,access_key_id=config.access_key_id.strip(),secret_access_key=config.secret_access_key.strip(),)@@ -100,7 +107,10 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):fields=cast(list[FieldType],[- url_field(S3_EXAMPLE_URL),+ url_field(+ S3_EXAMPLE_URL,+ "Paste the S3 URI (s3://...) or the object URL from the AWS console.",+ ),text_field("access_key_id", "Access key ID", "AKIA..."),secret_field("secret_access_key", "Secret access key"),*file_option_fields(),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.pyindex f0de6a2b06b7..d22e65c410bd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py@@ -6,7 +6,7 @@ from pathlib import Pathfrom typing import Any, castimport pytest-from unittest.mock import patch+from unittest.mock import call, patchimport pyarrow as paimport structlog@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objNO_SUCH_BUCKET_ERROR,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStore+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import build_default_sync_settingsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (@@ -178,13 +179,57 @@ def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -assert rows[0]["c1"] == "1"+def test_a_delta_table_defaults_to_incremental_and_a_file_table_to_full_refresh(tmp_path: Path) -> None:+ write(tmp_path, "exports/orders/a.csv", b"id\n1\n")+ write(tmp_path, "exports/sales/_delta_log/00000000000000000000.json", b"")+ write(tmp_path, "exports/sales/part-0.parquet", b"")++ schemas = _LocalAmazonS3Source(tmp_path).get_schemas(make_config(), team_id=1)++ assert {schema.name: build_default_sync_settings(schema) for schema in schemas} == {+ "orders": {"sync_type": "full_refresh"},+ "sales": {+ "sync_type": "incremental",+ "incremental_field": "_delta_version",+ "incremental_field_type": IncrementalFieldType.Integer,+ },+ }+++ "url_pattern, looks_up_region",+ [+ ("https://customer-data.s3.us-east-1.amazonaws.com/exports/", False),+ ("https://customer-data.s3.amazonaws.com/exports/", True),+ ("https://s3.amazonaws.com/customer-data/exports/", True),+ ("s3://customer-data/exports/", True),+ ],+)+def test_a_url_without_a_region_uses_the_bucket_region_from_s3(+ tmp_path: Path, url_pattern: str, looks_up_region: bool+) -> None:+ write(tmp_path, "exports/a.csv", b"id\n1\n")++ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source.lookup_s3_bucket_region",+ return_value="us-east-1",+ ) as lookup:+ valid, message = _LocalAmazonS3Source(tmp_path).validate_credentials(+ make_config(url_pattern=url_pattern), team_id=1+ )++ # `_LocalAmazonS3Source.open_store` checks that the store gets the bucket's region.+ assert (valid, message) == (True, None)+ assert lookup.called is looks_up_region+ assert all(lookup_call == call("customer-data") for lookup_call in lookup.call_args_list)++@pytest.mark.parametrize("files, config_overrides, fs_error, expected",[({"exports/a.csv": b"id\n1\n"}, {}, None, None),({"exports/notes.md": b"# hi"}, {}, None, NO_IMPORTABLE_FILES_ERROR),({}, {"url_pattern": "https://s3.us-east-1.amazonaws.com/Not_A_Bucket/exports/"}, None, INVALID_CONFIG_ERROR),- ({}, {"url_pattern": "https://customer-data.s3.amazonaws.com/exports/"}, None, INVALID_CONFIG_ERROR),({}, {}, PermissionError("Access Denied"), ACCESS_DENIED_ERROR),({}, {}, FileNotFoundError("NoSuchBucket"), NO_SUCH_BUCKET_ERROR),],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.pyindex a1beb2d2fd3e..18de7ff43c70 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py@@ -34,7 +34,7 @@ GCS_ENDPOINT_URL = "https://storage.googleapis.com"# a path into the URLs built from it. GCS also allows underscores._BUCKET_NAMES: dict[ObjectStoreProvider, re.Pattern[str]] = {ObjectStoreProvider.S3: re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"),- # GCS allows up to 222 characters in a dotted name; `_valid_bucket` checks the length of each part.+ # GCS allows up to 222 characters in a dotted name; `valid_bucket_name` checks the length of each part.ObjectStoreProvider.GCS: re.compile(r"[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]"),ObjectStoreProvider.R2: re.compile(r"[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"),}@@ -48,7 +48,7 @@ _AZURE_CONTAINER_NAME = re.compile(r"[a-z0-9][a-z0-9-]{1,61}[a-z0-9]")_AZURE_ACCOUNT_KEY = re.compile(r"[A-Za-z0-9+/]+={0,2}")-def _valid_bucket(provider: ObjectStoreProvider, bucket: str) -> bool:+def valid_bucket_name(provider: ObjectStoreProvider, bucket: str) -> bool:if not _BUCKET_NAMES[provider].fullmatch(bucket) or ".." in bucket:return False# A GCS name without dots is at most 63 characters, and so is each dot-separated part of one with dots.@@ -104,7 +104,7 @@ class ObjectStoreConfig:)def __post_init__(self) -> None:- if not _valid_bucket(self.provider, self.bucket):+ if not valid_bucket_name(self.provider, self.bucket):raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.bucket}' in the URL isn't a valid bucket name.")if self.provider == ObjectStoreProvider.S3:if not _REGION_NAME.fullmatch(self.region):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.pyindex f5a699a40c14..306a69c00c70 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py@@ -60,17 +60,18 @@ def source_caption(store: str) -> str:)-def url_field(placeholder: str) -> FieldType:+def url_field(placeholder: str, hint: str | None = None) -> FieldType:+ caption = (+ "Use * to match any part of a file or folder name, and ** to match across folders. A URL "+ "without a wildcard imports the folder or file it points to."+ )return SourceFieldInputConfig(name="url_pattern",label="Files URL",type=SourceFieldInputConfigType.TEXT,required=True,placeholder=placeholder,- caption=(- "Use * to match any part of a file or folder name, and ** to match across folders. A URL "- "without a wildcard imports the folder or file it points to."- ),+ caption=f"{hint} {caption}" if hint else caption,secret=False,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.pyindex b21457ad007c..ec5627e55c64 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py@@ -1,4 +1,6 @@+import functoolsimport dataclasses+from collections.abc import Mappingfrom datetime import UTC, datetimefrom typing import IO, Any, cast@@ -7,6 +9,7 @@ import boto3from adlfs import AzureBlobFileSystemfrom azure.core.exceptions import ClientAuthenticationError, HttpResponseError, ResourceNotFoundErrorfrom azure.storage.blob import ContainerClient+from botocore import UNSIGNEDfrom botocore.config import Config as BotoConfigfrom botocore.exceptions import ClientErrorfrom fsspec import AbstractFileSystem@@ -17,12 +20,19 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (AzureBlobConfig,ObjectStoreConfig,+ ObjectStoreProvider,StoreConfig,+ valid_bucket_name,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ INVALID_CONFIG_ERROR,+ NO_SUCH_BUCKET_ERROR,TOO_MANY_FILES_ERROR,ObjectStorageError,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (+ S3_EXAMPLE_URL,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (CONNECT_TIMEOUT_SECONDS,MAX_LISTED_KEYS,@@ -32,6 +42,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objS3_MAX_PAGE_SIZE = 1000AZURE_PAGE_SIZE = 1000+# S3 names a bucket's region in this header of every answer about the bucket, a refusal included.+S3_BUCKET_REGION_HEADER = "x-amz-bucket-region"@frozen@@ -229,6 +241,46 @@ def build_s3_client(config: ObjectStoreConfig) -> Any:)[email protected]_cache(maxsize=1024)+def lookup_s3_bucket_region(bucket: str) -> str:+ """The region of an AWS S3 bucket, for a files URL that doesn't name it.++ The request is anonymous, so no stored credential goes with it. S3 names the region also when it+ refuses the request, so this works for a private bucket. A bucket keeps its region, so the+ answer is cached.+ """+ if not valid_bucket_name(ObjectStoreProvider.S3, bucket):+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{bucket}' in the URL isn't a valid bucket name.")+ client = boto3.client(+ "s3",+ region_name="us-east-1",+ config=BotoConfig(+ signature_version=UNSIGNED,+ connect_timeout=CONNECT_TIMEOUT_SECONDS,+ read_timeout=READ_TIMEOUT_SECONDS,+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},+ ),+ )+ try:+ response = client.head_bucket(Bucket=bucket)+ except ClientError as e:+ if (region := _bucket_region(e.response)) is not None:+ return region+ if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404:+ raise ObjectStorageError(f"{NO_SUCH_BUCKET_ERROR}: '{bucket}'. Check the bucket name.") from e+ raise+ if (region := _bucket_region(response)) is None:+ raise ObjectStorageError(+ f"{INVALID_CONFIG_ERROR}: couldn't find the region of '{bucket}'. Use the regional URL, such as "+ f"{S3_EXAMPLE_URL}"+ )+ return region+++def _bucket_region(response: Mapping[str, Any]) -> str | None:+ return response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get(S3_BUCKET_REGION_HEADER)++def open_s3_store(config: ObjectStoreConfig) -> S3ObjectStore:return S3ObjectStore(fs=build_s3_filesystem(config),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.pyindex f065a066e5cf..44e66972572e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py@@ -1,7 +1,8 @@"""Parse the files URL of an object storage source into a bucket and the key glob after it.Each provider accepts only its own hosts. The endpoint the stored credentials go to is rebuilt from-the parsed bucket, region and account, never taken from the URL's host as written.+the parsed bucket, region and account, never taken from the URL's host as written. An S3 URL can+leave out the region, and the source then looks the region up from S3."""import re@@ -30,7 +31,7 @@ _HOST = re.compile(r"[a-z0-9._-]+")_REGION = r"[a-z]{2}(?:-[a-z0-9]+)+-\d+"_S3_VIRTUAL_HOSTED = re.compile(rf"(?P<bucket>[a-z0-9.-]+)\.s3\.(?P<region>{_REGION})\.amazonaws\.com")_S3_PATH_STYLE = re.compile(rf"s3\.(?P<region>{_REGION})\.amazonaws\.com")-_S3_WITHOUT_REGION = re.compile(r"(?:[a-z0-9.-]+\.)?s3\.amazonaws\.com")+_S3_WITHOUT_REGION = re.compile(r"(?:(?P<bucket>[a-z0-9.-]+)\.)?s3\.amazonaws\.com")_GCS_HOST = "storage.googleapis.com"_R2_HOST = re.compile(r"(?P<account>[0-9a-f]{32})\.r2\.cloudflarestorage\.com")_AZURE_HOST = re.compile(r"(?P<account>[a-z0-9]{3,24})\.blob\.core\.windows\.net")@@ -112,7 +113,10 @@ def _path_style(path: str, example: str, region: str | None = None, account: strdef parse_s3_url(url: str) -> StorageUrl:- parts = _split(url, ("https",), S3_EXAMPLE_URL)+ """Parse an S3 object URL, or an `s3://` URI. A URL with no region gives a `StorageUrl` with no region."""+ parts = _split(url, ("https", "s3"), S3_EXAMPLE_URL)+ if parts.scheme == "s3":+ return _path_style(f"/{parts.host}{parts.path}", S3_EXAMPLE_URL)host = parts.hostif (virtual_hosted := _S3_VIRTUAL_HOSTED.fullmatch(host)) is not None:return StorageUrl(@@ -122,8 +126,10 @@ def parse_s3_url(url: str) -> StorageUrl:)if (path_style := _S3_PATH_STYLE.fullmatch(host)) is not None:return _path_style(parts.path, S3_EXAMPLE_URL, region=path_style.group("region"))- if _S3_WITHOUT_REGION.fullmatch(host):- raise _invalid(f"the URL doesn't name the bucket's region. Use the regional URL, such as {S3_EXAMPLE_URL}")+ if (without_region := _S3_WITHOUT_REGION.fullmatch(host)) is not None:+ if (bucket := without_region.group("bucket")) is not None:+ return StorageUrl(bucket=bucket, key=_key(parts.path.removeprefix("/")))+ return _path_style(parts.path, S3_EXAMPLE_URL)raise _invalid(f"'{host}' isn't an Amazon S3 URL. Use a URL such as {S3_EXAMPLE_URL}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 56acc9068c71..eb4df59d4a84 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -363,7 +363,9 @@ def _source_schema(table: TableDefinition) -> SourceSchema:if is_fileselse incremental_field(DELTA_VERSION_COLUMN, IncrementalFieldType.Integer)],- prefers_full_refresh=True,+ # An incremental sync keeps a Delta table equal to its source, so only file tables need a full+ # refresh to drop the rows of deleted files.+ prefers_full_refresh=is_files,schema_metadata=table.schema_metadata(),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.pyindex 66e5baf0a3e4..16f08acf2874 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py@@ -72,6 +72,24 @@ class TestParseUrl:"HTTPS://Acme-Exports.S3.us-east-1.amazonaws.com/my%20files/a.csv",StorageUrl(bucket="acme-exports", key="my files/a.csv", region="us-east-1"),),+ (+ "s3_without_region_virtual_hosted",+ parse_s3_url,+ "https://acme-exports.s3.amazonaws.com/raw/orders/",+ StorageUrl(bucket="acme-exports", key="raw/orders/"),+ ),+ (+ "s3_without_region_path_style",+ parse_s3_url,+ "https://s3.amazonaws.com/acme-exports/raw/*.csv",+ StorageUrl(bucket="acme-exports", key="raw/*.csv"),+ ),+ (+ "s3_uri",+ parse_s3_url,+ "s3://acme-exports/raw/orders/**/*.parquet",+ StorageUrl(bucket="acme-exports", key="raw/orders/**/*.parquet"),+ ),("gcs_https",parse_gcs_url,@@ -103,8 +121,6 @@ class TestParseUrl:@parameterized.expand([- ("s3_without_region_virtual_hosted", parse_s3_url, "https://acme.s3.amazonaws.com/a.csv", "region"),- ("s3_without_region_path_style", parse_s3_url, "https://s3.amazonaws.com/acme/a.csv", "region"),("s3_other_host", parse_s3_url, "https://acme.example.com/a.csv", "isn't an Amazon S3 URL"),("s3_host_suffix",@@ -124,6 +140,10 @@ class TestParseUrl:("s3_empty_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a//b.csv", "'//'"),("s3_control_character", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a%0A.csv", "control"),("s3_glob_in_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme-*/a.csv", "exact"),+ ("s3_uri_glob_in_bucket", parse_s3_url, "s3://acme-*/a.csv", "special characters"),+ ("s3_uri_parent_folder", parse_s3_url, "s3://acme/a/../b.csv", "'..'"),+ ("s3_gs_scheme", parse_s3_url, "gs://acme/a.csv", "s3://"),+ ("s3_without_region_no_bucket", parse_s3_url, "https://s3.amazonaws.com/", "doesn't name a bucket"),("s3_encoded_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme%2Fx/a.csv", "exact"),("s3_no_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/", "doesn't name a bucket"),("s3_not_a_url", parse_s3_url, "acme-exports/a.csv", "full URL"),@@ -145,10 +165,6 @@ class TestParseUrl:assert message.startswith(INVALID_CONFIG_ERROR)assert detail in message- def test_a_url_without_a_region_gives_an_example(self) -> None:- with pytest.raises(ObjectStorageError, match=r"such as https://[a-z-]+\.s3\.us-east-1\.amazonaws\.com/"):- parse_s3_url("https://acme-exports.s3.amazonaws.com/a.csv")-class TestSelectKeys:@parameterized.expand(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 7efcfb01d6b6..b51fdf63931f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -16,6 +16,7 @@ import boto3import pyarrow as paimport structlogimport pyarrow.parquet as pq+from botocore.exceptions import ClientErrorfrom botocore.stub import Stubberfrom deltalake import DeltaTable, WriterProperties, write_deltalakefrom fsspec.implementations.local import LocalFileSystem@@ -23,6 +24,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import (delta,+ filesystem,formats,source_base,)@@ -58,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objObjectStore,S3ObjectStore,StoredObject,+ lookup_s3_bucket_region,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (iter_csv_tables,@@ -1182,3 +1185,75 @@ class TestS3Listing:with pytest.raises(PermissionError):_s3_store(client).list_objects("", 10)+++class TestS3BucketRegion:+ @pytest.fixture(autouse=True)+ def _clear_cache(self) -> Any:+ lookup_s3_bucket_region.cache_clear()+ yield+ lookup_s3_bucket_region.cache_clear()++ def _stubbed_client(self, monkeypatch: pytest.MonkeyPatch) -> tuple[Any, Stubber]:+ client = boto3.client("s3", region_name="us-east-1")+ monkeypatch.setattr(filesystem.boto3, "client", lambda *_args, **_kwargs: client)+ return client, Stubber(client)++ def test_a_readable_bucket_names_its_region(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _, stubber = self._stubbed_client(monkeypatch)+ stubber.add_response(+ "head_bucket",+ {"ResponseMetadata": {"HTTPStatusCode": 200, "HTTPHeaders": {"x-amz-bucket-region": "eu-central-1"}}},+ {"Bucket": "customer-data"},+ )++ with stubber:+ assert lookup_s3_bucket_region("customer-data") == "eu-central-1"++ @pytest.mark.parametrize(+ "status, headers, expected",+ [+ # A private bucket refuses the anonymous request but still names its region.+ (403, {"x-amz-bucket-region": "eu-central-1"}, "eu-central-1"),+ (301, {"x-amz-bucket-region": "ap-southeast-2"}, "ap-southeast-2"),+ ],+ )+ def test_an_error_answer_still_names_the_region(+ self, monkeypatch: pytest.MonkeyPatch, status: int, headers: dict[str, str], expected: str+ ) -> None:+ _, stubber = self._stubbed_client(monkeypatch)+ stubber.add_client_error(+ "head_bucket",+ service_error_code=str(status),+ http_status_code=status,+ response_meta={"HTTPHeaders": headers},+ )++ with stubber:+ assert lookup_s3_bucket_region("customer-data") == expected++ @pytest.mark.parametrize(+ "status, error, match",+ [+ (404, ObjectStorageError, NO_SUCH_BUCKET_ERROR),+ # An outage stays a retryable error, not a missing bucket.+ (500, ClientError, "500"),+ ],+ )+ def test_an_answer_without_a_region(+ self, monkeypatch: pytest.MonkeyPatch, status: int, error: type[Exception], match: str+ ) -> None:+ _, stubber = self._stubbed_client(monkeypatch)+ stubber.add_client_error("head_bucket", service_error_code=str(status), http_status_code=status)++ with stubber, pytest.raises(error, match=match):+ lookup_s3_bucket_region("customer-data")++ def test_an_invalid_bucket_sends_no_request(self, monkeypatch: pytest.MonkeyPatch) -> None:+ def refuse(*_args: Any, **_kwargs: Any) -> Any:+ raise AssertionError("no client should be built")++ monkeypatch.setattr(filesystem.boto3, "client", refuse)++ with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):+ lookup_s3_bucket_region("Not_A_Bucket")