:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): check only a migrated table's path when syncing turns on (#129165)
Tom Owerscf5535c1cf01
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.pyindex 59cb9ffce821..263fe85bd193 100644--- a/products/warehouse_sources/backend/facade/models.py+++ b/products/warehouse_sources/backend/facade/models.py@@ -42,6 +42,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (discovered_schema_finder,get_all_schemas_for_source_id,get_schemas_for_direct_reconciliation,+ is_pinned_schema,mark_schema_running_unless_halted,resolve_sync_type,self_managed_table_ref,@@ -113,6 +114,7 @@ __all__ = ["asave_datawarehousetable","auto_enable_new_schemas","discovered_schema_finder",+ "is_pinned_schema","self_managed_table_ref","create_datawarehousetable","get_all_schemas_for_source_id",diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex f4d3d3178a24..ad2ce8551e59 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -56,6 +56,7 @@ from products.warehouse_sources.backend.facade.models import (ExternalDataSource,UnsupportedSyncTypeError,discovered_schema_finder,+ is_pinned_schema,mark_schema_running_unless_halted,resolve_destinations,resolve_sync_type,@@ -70,8 +71,10 @@ from products.warehouse_sources.backend.facade.source_management import (BUFFER_LANE,CDC_RESET_PENDING_KEY,AnySource,+ Config,RowFilterValidationError,SourceRegistry,+ SourceSchema,WebhookSource,add_table_failure_message,filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,@@ -193,6 +196,41 @@ def manual_sync_refusal(source_model: ExternalDataSource) -> str | None:return source.manual_sync_refusal(source_model)+def _discovery_error_response(source_impl: AnySource, error: Exception) -> Response:+ """A 400 with the error's message. Only an error the source doesn't classify goes to error tracking.++ A failure that the source itself classifies is an expected customer or upstream condition rather+ than a bug, and must not flood error tracking. Both maps count: a non-retryable match names+ something only the customer can fix, such as bad credentials, and a retryable match names a+ transient failure `get_retryable_errors` already exists to keep out of error tracking.+ Mirrors `refresh_schemas`'s `_classify_refresh_schemas_error`.+ """+ error_text = str(error)+ expected_patterns = (*source_impl.get_non_retryable_errors(), *source_impl.get_retryable_errors())+ if not any(pattern and pattern in error_text for pattern in expected_patterns):+ capture_exception(error)+ return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": error_text})+++def describe_pinned_schema(source_impl: AnySource, config: Config, schema: ExternalDataSchema) -> SourceSchema | None:+ """The source's own description of a pinned schema, or None when the schema isn't pinned or the+ source can't describe it. The source also checks that the credentials can read the schema's data."""+ if not is_pinned_schema(schema):+ return None+ return source_impl.describe_pinned_schema(config, schema.team_id, schema.name, schema.schema_metadata or {})+++def discover_existing_schema(+ source_impl: AnySource, config: Config, schema: ExternalDataSchema, api_version: str | None+) -> list[SourceSchema]:+ """What the source reports for one stored schema. Discovery can't report a pinned schema, so the+ source describes a pinned schema from its stored definition when it can."""+ pinned = describe_pinned_schema(source_impl, config, schema)+ if pinned is not None:+ return [pinned]+ return source_impl.get_schemas(config, schema.team_id, names=[schema.name], api_version=api_version)++def source_row_filter_columns(source_type: str, schema_name: str) -> tuple[Any, ...] | None:"""The columns a row filter on this schema may use, or None when any column of the table may.@@ -1850,10 +1888,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersreturn Falsetry:config = source_impl.parse_config(source_inputs)- source_schemas = source_impl.get_schemas(+ source_schemas = discover_existing_schema(+ source_impl,config,- schema.team_id,- names=[schema.name],+ schema,api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),)except Exception:@@ -1875,10 +1913,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializerssource_type = known_source_key(source.source_type)source_impl = SourceRegistry.get_source(source_type)config = source_impl.parse_config(source_inputs)- source_schemas = source_impl.get_schemas(+ source_schemas = discover_existing_schema(+ source_impl,config,- schema.team_id,- names=[schema.name],+ schema,api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),)except Exception:@@ -1905,10 +1943,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersreturn unsupportedtry:config = source_impl.parse_config(source_inputs)- source_schemas = source_impl.get_schemas(+ source_schemas = discover_existing_schema(+ source_impl,config,- schema.team_id,- names=[schema.name],+ schema,api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),)except Exception as e:@@ -2674,56 +2712,52 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):effective_api_version = new_source.resolve_api_version(instance.api_version or source.api_version)try:- credentials_valid, credentials_error = new_source.validate_credentials(- config, self.team_id, instance.name, api_version=effective_api_version- )+ # Discovery can't report a pinned schema, and a source-wide check can fail where the sync+ # works, for example when the key can't list the bucket root. The source checks access to+ # the schema's own data instead.+ pinned_schema = describe_pinned_schema(new_source, config, instance)except Exception as e:- credentials_valid, credentials_error = source_helpers._credentials_validation_failed(- new_source, self.team_id, e- )- if not credentials_valid:- return Response(- status=status.HTTP_400_BAD_REQUEST,- data={"message": credentials_error or "Invalid credentials"},- )+ return _discovery_error_response(new_source, e)+ if pinned_schema is not None:+ schemas = [pinned_schema]+ else:+ try:+ credentials_valid, credentials_error = new_source.validate_credentials(+ config, self.team_id, instance.name, api_version=effective_api_version+ )+ except Exception as e:+ credentials_valid, credentials_error = source_helpers._credentials_validation_failed(+ new_source, self.team_id, e+ )+ if not credentials_valid:+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": credentials_error or "Invalid credentials"},+ )- stored_resource_id = (- (instance.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY)- if new_source.uses_stable_schema_resource_ids- else None- )- try:- schemas = new_source.get_schemas(- config, self.team_id, names=[instance.name], api_version=effective_api_version+ stored_resource_id = (+ (instance.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY)+ if new_source.uses_stable_schema_resource_ids+ else None)- # A stored resource id can outlive the stored name after an upstream rename. Discover every- # schema only when the name lookup misses that id, so an unrenamed schema keeps the cheap- # filtered discovery.- if stored_resource_id is not None:- found = discovered_schema_finder(schemas)(instance.name, instance.schema_metadata)- found_resource_id = (- (found.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY) if found else None+ try:+ schemas = new_source.get_schemas(+ config, self.team_id, names=[instance.name], api_version=effective_api_version)- if str(found_resource_id) != str(stored_resource_id):- schemas = new_source.get_schemas(- config, self.team_id, names=None, api_version=effective_api_version+ # A stored resource id can outlive the stored name after an upstream rename. Discover every+ # schema only when the name lookup misses that id, so an unrenamed schema keeps the cheap+ # filtered discovery.+ if stored_resource_id is not None:+ found = discovered_schema_finder(schemas)(instance.name, instance.schema_metadata)+ found_resource_id = (+ (found.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY) if found else None)- except Exception as e:- # `validate_credentials` above just probed the same connection successfully, so a- # failure here that the source itself classifies is an expected customer or upstream- # condition rather than a bug, and must not flood error tracking. Both maps count: a- # non-retryable match names something only the customer can fix, such as bad- # credentials, and a retryable match names a transient failure `get_retryable_errors`- # already exists to keep out of error tracking.- # Mirrors `refresh_schemas`'s `_classify_refresh_schemas_error`.- error_text = str(e)- expected_patterns = (*new_source.get_non_retryable_errors(), *new_source.get_retryable_errors())- if not any(pattern and pattern in error_text for pattern in expected_patterns):- capture_exception(e)- return Response(- status=status.HTTP_400_BAD_REQUEST,- data={"message": str(e)},- )+ if str(found_resource_id) != str(stored_resource_id):+ schemas = new_source.get_schemas(+ config, self.team_id, names=None, api_version=effective_api_version+ )+ except Exception as e:+ return _discovery_error_response(new_source, e)if not schemas:return Response(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.pyindex 4f29397ef900..1f42a08c5371 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py@@ -441,6 +441,19 @@ class _BaseSource(ABC, Generic[ConfigType]):"""raise NotImplementedError()+ def describe_pinned_schema(+ self, config: ConfigType, team_id: int, name: str, schema_metadata: Mapping[str, Any]+ ) -> SourceSchema | None:+ """Describe a pinned schema from the definition stored on it, without discovery.++ Discovery can't report a pinned schema (see `SCHEMA_PINNED_METADATA_KEY`), so callers ask+ this first. A source that can rebuild the schema from `schema_metadata` checks that the+ credentials can read the schema's own data, raises the errors a sync raises when they+ can't, and returns the schema. `None` means the caller uses `validate_credentials` and+ `get_schemas`, as for any other schema.+ """+ return None+def _placeholder_config(self) -> ConfigType:"""Build a credential-free config instance for static, no-network schema listing.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pyindex b4d62e145cb2..77a0bd020729 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -282,3 +282,9 @@ def iter_delta_tables() -> Iterator[pa.Table]:"""Stream one whole Delta snapshot."""yield from plan_delta_read(uri, storage_options, table_name, logger=logger).iter_tables(batch_rows)+++def check_delta_table(uri: str, storage_options: dict[str, str], table_name: str) -> None:+ """Load the table's `_delta_log`, which needs the same read access as a sync, without reading data."""+ with _delta_errors(table_name):+ DeltaTable(uri, storage_options=storage_options, without_files=True)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 1440fcae8f4f..554002071627 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -1,5 +1,5 @@from abc import abstractmethod-from collections.abc import Iterator+from collections.abc import Iterator, Mappingfrom datetime import UTC, datetimefrom typing import Any, ClassVar, Generic, NoReturn, Optional@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (DELTA_VERSION_COLUMN,ObjectStorageDeltaCursor,+ check_delta_table,iter_delta_tables,plan_delta_read,)@@ -243,6 +244,32 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDeif requested is None or table.name in requested]+ def describe_pinned_schema(+ self, config: ConfigType, team_id: int, name: str, schema_metadata: Mapping[str, Any]+ ) -> SourceSchema | None:+ """Describe a table from its stored definition, and check that the key can read the table.++ Only the table's own location is read. A key that can read the table but can't list the+ bucket root passes, and a large bucket isn't walked. Discovery also names a table after its+ folder, and a pinned table, such as a migrated self-managed table, can have another name.+ """+ table = TableDefinition.from_schema_metadata(name, schema_metadata)+ if table is None:+ return None+ store = self.open_store(self.object_store_config(config))+ location = store.path(table.path)+ try:+ if table.kind == "delta":+ check_delta_table(store.delta_uri(table.path), store.delta_storage_options, table.name)+ else:+ resolve_table_files(store, table)+ except OSError as e:+ # As in `validate_credentials`: s3fs gives a bare 401 or 403 as a plain OSError.+ if _http_status(e) in (401, 403):+ raise translate_storage_error(PermissionError(), location) from e+ _raise_translated(e, location)+ return _source_schema(table)+def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> SourceResponse:store = self.open_store(self.object_store_config(config))table = TableDefinition.from_schema_metadata(inputs.schema_name, inputs.schema_metadata) or self._find_table(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.pyindex 867b54d89c71..69925f12c314 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py@@ -1,6 +1,10 @@import dataclasses-from collections.abc import Callable-from typing import Any+from collections.abc import Callable, Iterator+from datetime import UTC, datetime+from typing import Any, cast++from botocore.exceptions import ClientError+from fsspec.implementations.local import LocalFileSystemfrom products.warehouse_sources.backend.facade.source_config import SourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.config import Config@@ -11,6 +15,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objObjectStoreProvider,StoreConfig,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ S3ObjectStore,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (parse_azure_url,parse_gcs_url,@@ -95,3 +102,46 @@ class FakeObjectStorageSource(ObjectStorageSource[FakeObjectStorageConfig]):self, location: SelfManagedLocation, credential: SelfManagedCredential) -> dict[str, Any]:return self._connection_inputs+++class PrefixScopedS3Client:+ """A boto3 S3 client for a key that may list only keys under one prefix, as a prefix-scoped IAM policy allows."""++ def __init__(self, keys: list[str], allowed_prefix: str) -> None:+ self.keys = keys+ self.allowed_prefix = allowed_prefix+ self.listed_prefixes: list[str] = []++ def get_paginator(self, operation: str) -> "PrefixScopedS3Client":+ assert operation == "list_objects_v2"+ return self++ def paginate(self, Bucket: str, Prefix: str, PaginationConfig: dict[str, Any]) -> Iterator[dict[str, Any]]:+ self.listed_prefixes.append(Prefix)+ if not Prefix.startswith(self.allowed_prefix):+ raise ClientError(+ cast(+ Any,+ {+ "Error": {"Code": "AccessDenied", "Message": "Access Denied"},+ "ResponseMetadata": {"HTTPStatusCode": 403},+ },+ ),+ "ListObjectsV2",+ )+ modified_at = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)+ contents = [{"Key": key, "Size": 5, "LastModified": modified_at} for key in self.keys if key.startswith(Prefix)]+ yield {"Contents": contents, "IsTruncated": False}+++def bucket_root_store(client: PrefixScopedS3Client, bucket: str = "example-bucket") -> S3ObjectStore:+ """A store rooted at the bucket, as for a source that holds the migrated tables of one bucket."""+ return S3ObjectStore(+ fs=LocalFileSystem(),+ root=bucket,+ delta_uri_prefix="s3://",+ delta_storage_options={},+ client=client,+ bucket=bucket,+ prefix="",+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 0e6d357b2694..24aca3b7ffd1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -76,6 +76,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objlisting_digest,plan_mirror,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ parse_self_managed_url,+ table_definition_for,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (JSONL_CHUNK_ROWS,MAX_TABLES,@@ -85,6 +89,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objiter_table,validate_store_location,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.tests.fake_source import (+ FakeObjectStorageConfig,+ FakeObjectStorageSource,+ PrefixScopedS3Client,+ bucket_root_store,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScopeMODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)@@ -1374,3 +1384,111 @@ class TestS3BucketRegion:with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):lookup_s3_bucket_region("Not_A_Bucket")+++def _source_with_store(store: ObjectStore) -> FakeObjectStorageSource:+ source = FakeObjectStorageSource()+ source.open_store = MagicMock(return_value=store) # type: ignore[method-assign]+ return source+++BUCKET_ROOT_CONFIG = FakeObjectStorageConfig(+ values={+ "url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "c2VjcmV0",+ }+)+ORDERS_KEYS = ["tables/orders/2024-01.csv", "tables/orders/2024-02.csv", "tables/other/x.csv"]+MIGRATED_ORDERS = table_definition_for(+ "orders",+ parse_self_managed_url("https://example-bucket.s3.us-east-1.amazonaws.com/tables/orders/*.csv"),+ "CSVWithNames",+ {},+ None,+)+DISCOVERED_ORDERS = TableDefinition(+ name="orders",+ kind="files",+ path="tables/orders",+ scope="folder",+ file_format="csv",+ csv=CsvOptions(),+ pattern=None,+ source_format="infer",+)+++class TestDescribePinnedSchema:+ @parameterized.expand([("migrated_glob", MIGRATED_ORDERS), ("folder", DISCOVERED_ORDERS)])+ def test_a_key_scoped_to_the_table_prefix_reads_the_table_without_listing_the_bucket_root(+ self, _name: str, table: TableDefinition+ ) -> None:+ client = PrefixScopedS3Client(ORDERS_KEYS, allowed_prefix="tables/orders/")+ source = _source_with_store(bucket_root_store(client))++ schema = source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", table.schema_metadata())++ assert schema is not None+ assert (schema.name, schema.schema_metadata) == ("orders", table.schema_metadata())+ assert client.listed_prefixes == ["tables/orders/"]+ # Discovery lists the bucket root, which the same key can't do.+ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)+ assert not valid and message is not None and message.startswith(ACCESS_DENIED_ERROR)++ @parameterized.expand(+ [+ ("denied", ORDERS_KEYS, "tables/other/", f"{ACCESS_DENIED_ERROR} 'example-bucket/tables/orders'"),+ ("no_files", ["tables/other/x.csv"], "tables/", f"{NO_TABLE_FILES_ERROR} 'orders'"),+ ]+ )+ def test_a_table_the_key_can_not_read_fails_with_the_sync_error(+ self, _name: str, keys: list[str], allowed_prefix: str, expected_start: str+ ) -> None:+ source = _source_with_store(bucket_root_store(PrefixScopedS3Client(keys, allowed_prefix)))++ with pytest.raises(ObjectStorageError) as error:+ source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", MIGRATED_ORDERS.schema_metadata())++ assert str(error.value).startswith(expected_start)++ @parameterized.expand([("empty", {}), ("unknown_kind", {"kind": "iceberg", "path": "x"})])+ def test_metadata_without_a_table_definition_is_not_described(self, _name: str, metadata: dict) -> None:+ source = _source_with_store(MagicMock())++ assert source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", metadata) is None+ source.open_store.assert_not_called() # type: ignore[attr-defined]++ def test_a_delta_table_is_described_from_its_log(self, tmp_path: Path) -> None:+ write_deltalake(str(tmp_path / "tables" / "events"), pa.table({"id": [1]}))+ metadata: dict[str, Any] = {+ "kind": "delta",+ "path": "tables/events",+ "format": "delta",+ "pattern": None,+ "csv": {},+ }+ source = _source_with_store(local_store(tmp_path))++ schema = source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "events", metadata)++ assert schema is not None+ assert [field["field"] for field in schema.incremental_fields] == ["_delta_version"]++ @pytest.mark.parametrize(+ "raised, error_prefix",+ [+ pytest.param(None, NO_TABLE_FILES_ERROR, id="missing"),+ pytest.param(OSError("Generic S3 error: 403 Forbidden"), ACCESS_DENIED_ERROR, id="denied"),+ ],+ )+ def test_an_unreadable_delta_log_fails_with_the_sync_error(+ self, raised: Exception | None, error_prefix: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ if raised is not None:+ monkeypatch.setattr(delta, "DeltaTable", MagicMock(side_effect=raised))+ metadata: dict[str, Any] = {"kind": "delta", "path": "events", "format": "delta", "pattern": None, "csv": {}}+ source = _source_with_store(local_store(tmp_path))++ with pytest.raises(ObjectStorageError, match=f"^{error_prefix}"):+ source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "events", metadata)diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 1ea83871ac2b..c998f87ea99b 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -46,6 +46,7 @@ from products.warehouse_sources.backend.facade.models import (update_sync_type_config_keys,)from products.warehouse_sources.backend.facade.types import ExternalDataSourceType+from products.warehouse_sources.backend.models.external_data_schema import SCHEMA_PINNED_METADATA_KEYfrom products.warehouse_sources.backend.presentation.views.external_data_schema import (UNREADABLE_SOURCE_CONFIG_ERROR,ExternalDataSchemaSerializer,@@ -61,6 +62,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.basWebhookCreationResult,WebhookSyncResult,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_MODIFIED_AT_COLUMN+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ACCESS_DENIED_ERROR,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ parse_self_managed_url,+ table_definition_for,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.tests.fake_source import (+ PrefixScopedS3Client,+ bucket_root_store,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchemafrom products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.stripe.constants import (@@ -69,6 +82,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con)from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.source import StripeSourcefrom products.warehouse_sources.backend.tests.api.utils import create_external_data_source_ok+from products.warehouse_sources.sources.amazon_s3.source import AmazonS3Source_PH_HISTORY_NAMING_FLAG = ("products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"@@ -5629,3 +5643,64 @@ class TestExternalDataSchemaIntrospection(SimpleTestCase):serializer = view.get_serializer()assert isinstance(serializer, ExternalDataSchemaSerializer)assert "table_id" in serializer.fields+++MIGRATED_ORDERS_METADATA = table_definition_for(+ "orders",+ parse_self_managed_url("https://example-bucket.s3.us-east-1.amazonaws.com/tables/orders/*.csv"),+ "CSVWithNames",+ {},+ None,+).schema_metadata()+++class TestPinnedSchemaIncrementalFields(APIBaseTest):+ @parameterized.expand(+ [+ # The key can list the table's own prefix but not the bucket root.+ ("pinned_key_scoped_to_the_table", True, "tables/orders/", None),+ ("pinned_key_denied_the_table", True, "tables/other/", "'example-bucket/tables/orders'"),+ # A schema without the pin still checks the whole bucket, as before.+ ("unpinned", False, "tables/orders/", "'example-bucket'"),+ ]+ )+ @mock.patch("products.warehouse_sources.backend.presentation.views.external_data_schema.capture_exception")+ def test_a_pinned_schema_is_checked_at_its_own_location(+ self, _name: str, pinned: bool, allowed_prefix: str, denied_location: str | None, mock_capture: mock.MagicMock+ ) -> None:+ source = ExternalDataSource.objects.create(+ team=self.team,+ source_type=ExternalDataSourceType.AMAZONS3,+ job_inputs={+ "url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "c2VjcmV0",+ "file_format": "infer",+ },+ )+ metadata = (+ {**MIGRATED_ORDERS_METADATA, SCHEMA_PINNED_METADATA_KEY: True} if pinned else MIGRATED_ORDERS_METADATA+ )+ schema = ExternalDataSchema.objects.create(+ name="orders",+ team=self.team,+ source=source,+ should_sync=False,+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,+ sync_type_config={"schema_metadata": metadata},+ )+ client = PrefixScopedS3Client(["tables/orders/2024-01.csv", "tables/other/x.csv"], allowed_prefix)++ with mock.patch.object(AmazonS3Source, "open_store", return_value=bucket_root_store(client)):+ response = self.client.post(+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/incremental_fields",+ )++ if denied_location is None:+ assert response.status_code == status.HTTP_200_OK, response.json()+ assert [field["field"] for field in response.json()["incremental_fields"]] == [FILE_MODIFIED_AT_COLUMN]+ assert client.listed_prefixes == ["tables/orders/"]+ else:+ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert response.json()["message"].startswith(f"{ACCESS_DENIED_ERROR} {denied_location}.")+ mock_capture.assert_not_called()