Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): smooth s3 source urls, delta defaults, and reimports (#124785)

Tom Owers81a3475363f4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
index 56b9cf4a37f1..22706f23e6df 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
@@ -497,6 +497,16 @@ def normalize_table_column_names(table: pa.Table) -> pa.Table:
INTERNAL_COLUMN_NAMES = frozenset({"_ph_debug", PARTITION_KEY, "_dlt_id", "_dlt_load_id"})
+def drop_internal_columns(table: pa.Table) -> pa.Table:
+ """Drop the columns the pipeline adds itself from a source batch.
+
+ A source can return them when it reads a table PostHog wrote, such as a Delta table in a bucket.
+ Kept, they would exist twice in the batch once the pipeline adds its own.
+ """
+ internal = [name for name in table.column_names if name in INTERNAL_COLUMN_NAMES]
+ return table.drop_columns(internal) if internal else table
+
+
def _fold_column_name_for_match(name: str) -> str:
try:
return NamingConvention.normalize_identifier(name)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 439d0b3a05f3..c33ebbf4097f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -77,6 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.s
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
_append_debug_column_to_pyarrows_table,
_handle_null_columns_with_definitions,
+ drop_internal_columns,
evolve_pyarrow_schema,
merge_observed_columns_into_schema_metadata,
normalize_column_name,
@@ -1161,7 +1162,7 @@ class PipelineV3(Generic[ResumableData]):
cleanup_memory(pa_memory_pool, py_table if "py_table" in locals() else None)
async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:
- pa_table = _append_debug_column_to_pyarrows_table(pa_table, self._load_id)
+ pa_table = _append_debug_column_to_pyarrows_table(drop_internal_columns(pa_table), self._load_id)
if not self._resource.preserve_column_names:
pa_table = normalize_table_column_names(pa_table)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index a277eb6f56cb..9237242f11d0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1391,13 +1391,21 @@ class TestSingleTableRunIsUntouched:
class TestColumnNamesReachTheWrite:
@pytest.mark.parametrize(
- "preserve, expected",
+ "preserve, internal_columns, expected",
[
- (False, ["order_id", "total_amount", "_ph_debug"]),
- (True, ["Order ID", "totalAmount", "_ph_debug"]),
+ (False, {}, ["order_id", "total_amount", "_ph_debug"]),
+ (True, {}, ["Order ID", "totalAmount", "_ph_debug"]),
+ # A Delta table PostHog wrote, read back from a bucket, already holds the columns the pipeline adds.
+ (
+ False,
+ {"_ph_debug": '{"load_id": 1}', "_ph_partition_key": "2024-05"},
+ ["order_id", "total_amount", "_ph_debug"],
+ ),
],
)
- async def test_preserving_names_skips_the_snake_case_rename(self, preserve: bool, expected: list[str]) -> None:
+ async def test_preserving_names_skips_the_snake_case_rename(
+ self, preserve: bool, internal_columns: dict[str, str], expected: list[str]
+ ) -> None:
# A self-managed table moved into a source keeps the column names its queries use.
pipeline = _make_pipeline()
pipeline._resource = SourceResponse(
@@ -1406,7 +1414,13 @@ class TestColumnNamesReachTheWrite:
cast(MagicMock, pipeline._schema).configure_mock(incremental_field=None, enabled_columns=None)
pipeline._last_incremental_field_value = None
pipeline._earliest_incremental_field_value = None
- table = pa.table({"Order ID": pa.array([1], pa.int64()), "totalAmount": pa.array([2.5], pa.float64())})
+ table = pa.table(
+ {
+ "Order ID": pa.array([1], pa.int64()),
+ "totalAmount": pa.array([2.5], pa.float64()),
+ **{name: pa.array([value], pa.string()) for name, value in internal_columns.items()},
+ }
+ )
with (
patch.object(PipelineV3, "_stage_batch", AsyncMock(return_value=1)),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
index 5ae8d3a1512e..80aff19b1aaa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
@@ -1,3 +1,4 @@
+import dataclasses
from typing import Any, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
@@ -22,6 +23,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
text_field,
url_field,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ lookup_s3_bucket_region,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
S3_EXAMPLE_URL,
parse_s3_url,
@@ -65,9 +69,12 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
return ["url_pattern"]
def object_store_config(self, config: AmazonS3SourceConfig) -> ObjectStoreConfig:
+ url = parse_s3_url(config.url_pattern)
+ if url.region is None:
+ url = dataclasses.replace(url, region=lookup_s3_bucket_region(url.bucket))
return ObjectStoreConfig.from_url(
ObjectStoreProvider.S3,
- parse_s3_url(config.url_pattern),
+ url,
access_key_id=config.access_key_id.strip(),
secret_access_key=config.secret_access_key.strip(),
)
@@ -100,7 +107,10 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
fields=cast(
list[FieldType],
[
- url_field(S3_EXAMPLE_URL),
+ url_field(
+ S3_EXAMPLE_URL,
+ "Paste the S3 URI (s3://...) or the object URL from the AWS console.",
+ ),
text_field("access_key_id", "Access key ID", "AKIA..."),
secret_field("secret_access_key", "Secret access key"),
*file_option_fields(),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
index f0de6a2b06b7..d22e65c410bd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -6,7 +6,7 @@ from pathlib import Path
from typing import Any, cast
import pytest
-from unittest.mock import patch
+from unittest.mock import call, patch
import pyarrow as pa
import structlog
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
NO_SUCH_BUCKET_ERROR,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStore
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import build_default_sync_settings
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputs
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (
@@ -178,13 +179,57 @@ def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -
assert rows[0]["c1"] == "1"
+def test_a_delta_table_defaults_to_incremental_and_a_file_table_to_full_refresh(tmp_path: Path) -> None:
+ write(tmp_path, "exports/orders/a.csv", b"id\n1\n")
+ write(tmp_path, "exports/sales/_delta_log/00000000000000000000.json", b"")
+ write(tmp_path, "exports/sales/part-0.parquet", b"")
+
+ schemas = _LocalAmazonS3Source(tmp_path).get_schemas(make_config(), team_id=1)
+
+ assert {schema.name: build_default_sync_settings(schema) for schema in schemas} == {
+ "orders": {"sync_type": "full_refresh"},
+ "sales": {
+ "sync_type": "incremental",
+ "incremental_field": "_delta_version",
+ "incremental_field_type": IncrementalFieldType.Integer,
+ },
+ }
+
+
+ "url_pattern, looks_up_region",
+ [
+ ("https://customer-data.s3.us-east-1.amazonaws.com/exports/", False),
+ ("https://customer-data.s3.amazonaws.com/exports/", True),
+ ("https://s3.amazonaws.com/customer-data/exports/", True),
+ ("s3://customer-data/exports/", True),
+ ],
+)
+def test_a_url_without_a_region_uses_the_bucket_region_from_s3(
+ tmp_path: Path, url_pattern: str, looks_up_region: bool
+) -> None:
+ write(tmp_path, "exports/a.csv", b"id\n1\n")
+
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source.lookup_s3_bucket_region",
+ return_value="us-east-1",
+ ) as lookup:
+ valid, message = _LocalAmazonS3Source(tmp_path).validate_credentials(
+ make_config(url_pattern=url_pattern), team_id=1
+ )
+
+ # `_LocalAmazonS3Source.open_store` checks that the store gets the bucket's region.
+ assert (valid, message) == (True, None)
+ assert lookup.called is looks_up_region
+ assert all(lookup_call == call("customer-data") for lookup_call in lookup.call_args_list)
+
+
@pytest.mark.parametrize(
"files, config_overrides, fs_error, expected",
[
({"exports/a.csv": b"id\n1\n"}, {}, None, None),
({"exports/notes.md": b"# hi"}, {}, None, NO_IMPORTABLE_FILES_ERROR),
({}, {"url_pattern": "https://s3.us-east-1.amazonaws.com/Not_A_Bucket/exports/"}, None, INVALID_CONFIG_ERROR),
- ({}, {"url_pattern": "https://customer-data.s3.amazonaws.com/exports/"}, None, INVALID_CONFIG_ERROR),
({}, {}, PermissionError("Access Denied"), ACCESS_DENIED_ERROR),
({}, {}, FileNotFoundError("NoSuchBucket"), NO_SUCH_BUCKET_ERROR),
],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
index a1beb2d2fd3e..18de7ff43c70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
@@ -34,7 +34,7 @@ GCS_ENDPOINT_URL = "https://storage.googleapis.com"
# a path into the URLs built from it. GCS also allows underscores.
_BUCKET_NAMES: dict[ObjectStoreProvider, re.Pattern[str]] = {
ObjectStoreProvider.S3: re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]"),
- # GCS allows up to 222 characters in a dotted name; `_valid_bucket` checks the length of each part.
+ # GCS allows up to 222 characters in a dotted name; `valid_bucket_name` checks the length of each part.
ObjectStoreProvider.GCS: re.compile(r"[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]"),
ObjectStoreProvider.R2: re.compile(r"[a-z0-9][a-z0-9-]{1,61}[a-z0-9]"),
}
@@ -48,7 +48,7 @@ _AZURE_CONTAINER_NAME = re.compile(r"[a-z0-9][a-z0-9-]{1,61}[a-z0-9]")
_AZURE_ACCOUNT_KEY = re.compile(r"[A-Za-z0-9+/]+={0,2}")
-def _valid_bucket(provider: ObjectStoreProvider, bucket: str) -> bool:
+def valid_bucket_name(provider: ObjectStoreProvider, bucket: str) -> bool:
if not _BUCKET_NAMES[provider].fullmatch(bucket) or ".." in bucket:
return False
# A GCS name without dots is at most 63 characters, and so is each dot-separated part of one with dots.
@@ -104,7 +104,7 @@ class ObjectStoreConfig:
)
def __post_init__(self) -> None:
- if not _valid_bucket(self.provider, self.bucket):
+ if not valid_bucket_name(self.provider, self.bucket):
raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.bucket}' in the URL isn't a valid bucket name.")
if self.provider == ObjectStoreProvider.S3:
if not _REGION_NAME.fullmatch(self.region):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
index f5a699a40c14..306a69c00c70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
@@ -60,17 +60,18 @@ def source_caption(store: str) -> str:
)
-def url_field(placeholder: str) -> FieldType:
+def url_field(placeholder: str, hint: str | None = None) -> FieldType:
+ caption = (
+ "Use * to match any part of a file or folder name, and ** to match across folders. A URL "
+ "without a wildcard imports the folder or file it points to."
+ )
return SourceFieldInputConfig(
name="url_pattern",
label="Files URL",
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder=placeholder,
- caption=(
- "Use * to match any part of a file or folder name, and ** to match across folders. A URL "
- "without a wildcard imports the folder or file it points to."
- ),
+ caption=f"{hint} {caption}" if hint else caption,
secret=False,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
index b21457ad007c..ec5627e55c64 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
@@ -1,4 +1,6 @@
+import functools
import dataclasses
+from collections.abc import Mapping
from datetime import UTC, datetime
from typing import IO, Any, cast
@@ -7,6 +9,7 @@ import boto3
from adlfs import AzureBlobFileSystem
from azure.core.exceptions import ClientAuthenticationError, HttpResponseError, ResourceNotFoundError
from azure.storage.blob import ContainerClient
+from botocore import UNSIGNED
from botocore.config import Config as BotoConfig
from botocore.exceptions import ClientError
from fsspec import AbstractFileSystem
@@ -17,12 +20,19 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
AzureBlobConfig,
ObjectStoreConfig,
+ ObjectStoreProvider,
StoreConfig,
+ valid_bucket_name,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ INVALID_CONFIG_ERROR,
+ NO_SUCH_BUCKET_ERROR,
TOO_MANY_FILES_ERROR,
ObjectStorageError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ S3_EXAMPLE_URL,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
CONNECT_TIMEOUT_SECONDS,
MAX_LISTED_KEYS,
@@ -32,6 +42,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
S3_MAX_PAGE_SIZE = 1000
AZURE_PAGE_SIZE = 1000
+# S3 names a bucket's region in this header of every answer about the bucket, a refusal included.
+S3_BUCKET_REGION_HEADER = "x-amz-bucket-region"
@frozen
@@ -229,6 +241,46 @@ def build_s3_client(config: ObjectStoreConfig) -> Any:
)
[email protected]_cache(maxsize=1024)
+def lookup_s3_bucket_region(bucket: str) -> str:
+ """The region of an AWS S3 bucket, for a files URL that doesn't name it.
+
+ The request is anonymous, so no stored credential goes with it. S3 names the region also when it
+ refuses the request, so this works for a private bucket. A bucket keeps its region, so the
+ answer is cached.
+ """
+ if not valid_bucket_name(ObjectStoreProvider.S3, bucket):
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{bucket}' in the URL isn't a valid bucket name.")
+ client = boto3.client(
+ "s3",
+ region_name="us-east-1",
+ config=BotoConfig(
+ signature_version=UNSIGNED,
+ connect_timeout=CONNECT_TIMEOUT_SECONDS,
+ read_timeout=READ_TIMEOUT_SECONDS,
+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},
+ ),
+ )
+ try:
+ response = client.head_bucket(Bucket=bucket)
+ except ClientError as e:
+ if (region := _bucket_region(e.response)) is not None:
+ return region
+ if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404:
+ raise ObjectStorageError(f"{NO_SUCH_BUCKET_ERROR}: '{bucket}'. Check the bucket name.") from e
+ raise
+ if (region := _bucket_region(response)) is None:
+ raise ObjectStorageError(
+ f"{INVALID_CONFIG_ERROR}: couldn't find the region of '{bucket}'. Use the regional URL, such as "
+ f"{S3_EXAMPLE_URL}"
+ )
+ return region
+
+
+def _bucket_region(response: Mapping[str, Any]) -> str | None:
+ return response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get(S3_BUCKET_REGION_HEADER)
+
+
def open_s3_store(config: ObjectStoreConfig) -> S3ObjectStore:
return S3ObjectStore(
fs=build_s3_filesystem(config),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py
index f065a066e5cf..44e66972572e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py
@@ -1,7 +1,8 @@
"""Parse the files URL of an object storage source into a bucket and the key glob after it.
Each provider accepts only its own hosts. The endpoint the stored credentials go to is rebuilt from
-the parsed bucket, region and account, never taken from the URL's host as written.
+the parsed bucket, region and account, never taken from the URL's host as written. An S3 URL can
+leave out the region, and the source then looks the region up from S3.
"""
import re
@@ -30,7 +31,7 @@ _HOST = re.compile(r"[a-z0-9._-]+")
_REGION = r"[a-z]{2}(?:-[a-z0-9]+)+-\d+"
_S3_VIRTUAL_HOSTED = re.compile(rf"(?P<bucket>[a-z0-9.-]+)\.s3\.(?P<region>{_REGION})\.amazonaws\.com")
_S3_PATH_STYLE = re.compile(rf"s3\.(?P<region>{_REGION})\.amazonaws\.com")
-_S3_WITHOUT_REGION = re.compile(r"(?:[a-z0-9.-]+\.)?s3\.amazonaws\.com")
+_S3_WITHOUT_REGION = re.compile(r"(?:(?P<bucket>[a-z0-9.-]+)\.)?s3\.amazonaws\.com")
_GCS_HOST = "storage.googleapis.com"
_R2_HOST = re.compile(r"(?P<account>[0-9a-f]{32})\.r2\.cloudflarestorage\.com")
_AZURE_HOST = re.compile(r"(?P<account>[a-z0-9]{3,24})\.blob\.core\.windows\.net")
@@ -112,7 +113,10 @@ def _path_style(path: str, example: str, region: str | None = None, account: str
def parse_s3_url(url: str) -> StorageUrl:
- parts = _split(url, ("https",), S3_EXAMPLE_URL)
+ """Parse an S3 object URL, or an `s3://` URI. A URL with no region gives a `StorageUrl` with no region."""
+ parts = _split(url, ("https", "s3"), S3_EXAMPLE_URL)
+ if parts.scheme == "s3":
+ return _path_style(f"/{parts.host}{parts.path}", S3_EXAMPLE_URL)
host = parts.host
if (virtual_hosted := _S3_VIRTUAL_HOSTED.fullmatch(host)) is not None:
return StorageUrl(
@@ -122,8 +126,10 @@ def parse_s3_url(url: str) -> StorageUrl:
)
if (path_style := _S3_PATH_STYLE.fullmatch(host)) is not None:
return _path_style(parts.path, S3_EXAMPLE_URL, region=path_style.group("region"))
- if _S3_WITHOUT_REGION.fullmatch(host):
- raise _invalid(f"the URL doesn't name the bucket's region. Use the regional URL, such as {S3_EXAMPLE_URL}")
+ if (without_region := _S3_WITHOUT_REGION.fullmatch(host)) is not None:
+ if (bucket := without_region.group("bucket")) is not None:
+ return StorageUrl(bucket=bucket, key=_key(parts.path.removeprefix("/")))
+ return _path_style(parts.path, S3_EXAMPLE_URL)
raise _invalid(f"'{host}' isn't an Amazon S3 URL. Use a URL such as {S3_EXAMPLE_URL}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 56acc9068c71..eb4df59d4a84 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -363,7 +363,9 @@ def _source_schema(table: TableDefinition) -> SourceSchema:
if is_files
else incremental_field(DELTA_VERSION_COLUMN, IncrementalFieldType.Integer)
],
- prefers_full_refresh=True,
+ # An incremental sync keeps a Delta table equal to its source, so only file tables need a full
+ # refresh to drop the rows of deleted files.
+ prefers_full_refresh=is_files,
schema_metadata=table.schema_metadata(),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py
index 66e5baf0a3e4..16f08acf2874 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py
@@ -72,6 +72,24 @@ class TestParseUrl:
"HTTPS://Acme-Exports.S3.us-east-1.amazonaws.com/my%20files/a.csv",
StorageUrl(bucket="acme-exports", key="my files/a.csv", region="us-east-1"),
),
+ (
+ "s3_without_region_virtual_hosted",
+ parse_s3_url,
+ "https://acme-exports.s3.amazonaws.com/raw/orders/",
+ StorageUrl(bucket="acme-exports", key="raw/orders/"),
+ ),
+ (
+ "s3_without_region_path_style",
+ parse_s3_url,
+ "https://s3.amazonaws.com/acme-exports/raw/*.csv",
+ StorageUrl(bucket="acme-exports", key="raw/*.csv"),
+ ),
+ (
+ "s3_uri",
+ parse_s3_url,
+ "s3://acme-exports/raw/orders/**/*.parquet",
+ StorageUrl(bucket="acme-exports", key="raw/orders/**/*.parquet"),
+ ),
(
"gcs_https",
parse_gcs_url,
@@ -103,8 +121,6 @@ class TestParseUrl:
@parameterized.expand(
[
- ("s3_without_region_virtual_hosted", parse_s3_url, "https://acme.s3.amazonaws.com/a.csv", "region"),
- ("s3_without_region_path_style", parse_s3_url, "https://s3.amazonaws.com/acme/a.csv", "region"),
("s3_other_host", parse_s3_url, "https://acme.example.com/a.csv", "isn't an Amazon S3 URL"),
(
"s3_host_suffix",
@@ -124,6 +140,10 @@ class TestParseUrl:
("s3_empty_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a//b.csv", "'//'"),
("s3_control_character", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a%0A.csv", "control"),
("s3_glob_in_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme-*/a.csv", "exact"),
+ ("s3_uri_glob_in_bucket", parse_s3_url, "s3://acme-*/a.csv", "special characters"),
+ ("s3_uri_parent_folder", parse_s3_url, "s3://acme/a/../b.csv", "'..'"),
+ ("s3_gs_scheme", parse_s3_url, "gs://acme/a.csv", "s3://"),
+ ("s3_without_region_no_bucket", parse_s3_url, "https://s3.amazonaws.com/", "doesn't name a bucket"),
("s3_encoded_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme%2Fx/a.csv", "exact"),
("s3_no_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/", "doesn't name a bucket"),
("s3_not_a_url", parse_s3_url, "acme-exports/a.csv", "full URL"),
@@ -145,10 +165,6 @@ class TestParseUrl:
assert message.startswith(INVALID_CONFIG_ERROR)
assert detail in message
- def test_a_url_without_a_region_gives_an_example(self) -> None:
- with pytest.raises(ObjectStorageError, match=r"such as https://[a-z-]+\.s3\.us-east-1\.amazonaws\.com/"):
- parse_s3_url("https://acme-exports.s3.amazonaws.com/a.csv")
-
class TestSelectKeys:
@parameterized.expand(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 7efcfb01d6b6..b51fdf63931f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -16,6 +16,7 @@ import boto3
import pyarrow as pa
import structlog
import pyarrow.parquet as pq
+from botocore.exceptions import ClientError
from botocore.stub import Stubber
from deltalake import DeltaTable, WriterProperties, write_deltalake
from fsspec.implementations.local import LocalFileSystem
@@ -23,6 +24,7 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import (
delta,
+ filesystem,
formats,
source_base,
)
@@ -58,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
ObjectStore,
S3ObjectStore,
StoredObject,
+ lookup_s3_bucket_region,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (
iter_csv_tables,
@@ -1182,3 +1185,75 @@ class TestS3Listing:
with pytest.raises(PermissionError):
_s3_store(client).list_objects("", 10)
+
+
+class TestS3BucketRegion:
+ @pytest.fixture(autouse=True)
+ def _clear_cache(self) -> Any:
+ lookup_s3_bucket_region.cache_clear()
+ yield
+ lookup_s3_bucket_region.cache_clear()
+
+ def _stubbed_client(self, monkeypatch: pytest.MonkeyPatch) -> tuple[Any, Stubber]:
+ client = boto3.client("s3", region_name="us-east-1")
+ monkeypatch.setattr(filesystem.boto3, "client", lambda *_args, **_kwargs: client)
+ return client, Stubber(client)
+
+ def test_a_readable_bucket_names_its_region(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _, stubber = self._stubbed_client(monkeypatch)
+ stubber.add_response(
+ "head_bucket",
+ {"ResponseMetadata": {"HTTPStatusCode": 200, "HTTPHeaders": {"x-amz-bucket-region": "eu-central-1"}}},
+ {"Bucket": "customer-data"},
+ )
+
+ with stubber:
+ assert lookup_s3_bucket_region("customer-data") == "eu-central-1"
+
+ @pytest.mark.parametrize(
+ "status, headers, expected",
+ [
+ # A private bucket refuses the anonymous request but still names its region.
+ (403, {"x-amz-bucket-region": "eu-central-1"}, "eu-central-1"),
+ (301, {"x-amz-bucket-region": "ap-southeast-2"}, "ap-southeast-2"),
+ ],
+ )
+ def test_an_error_answer_still_names_the_region(
+ self, monkeypatch: pytest.MonkeyPatch, status: int, headers: dict[str, str], expected: str
+ ) -> None:
+ _, stubber = self._stubbed_client(monkeypatch)
+ stubber.add_client_error(
+ "head_bucket",
+ service_error_code=str(status),
+ http_status_code=status,
+ response_meta={"HTTPHeaders": headers},
+ )
+
+ with stubber:
+ assert lookup_s3_bucket_region("customer-data") == expected
+
+ @pytest.mark.parametrize(
+ "status, error, match",
+ [
+ (404, ObjectStorageError, NO_SUCH_BUCKET_ERROR),
+ # An outage stays a retryable error, not a missing bucket.
+ (500, ClientError, "500"),
+ ],
+ )
+ def test_an_answer_without_a_region(
+ self, monkeypatch: pytest.MonkeyPatch, status: int, error: type[Exception], match: str
+ ) -> None:
+ _, stubber = self._stubbed_client(monkeypatch)
+ stubber.add_client_error("head_bucket", service_error_code=str(status), http_status_code=status)
+
+ with stubber, pytest.raises(error, match=match):
+ lookup_s3_bucket_region("customer-data")
+
+ def test_an_invalid_bucket_sends_no_request(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ def refuse(*_args: Any, **_kwargs: Any) -> Any:
+ raise AssertionError("no client should be built")
+
+ monkeypatch.setattr(filesystem.boto3, "client", refuse)
+
+ with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):
+ lookup_s3_bucket_region("Not_A_Bucket")