Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): check only a migrated table's path when syncing turns on (#129165)

Tom Owerscf5535c1cf01
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.py
index 59cb9ffce821..263fe85bd193 100644
--- a/products/warehouse_sources/backend/facade/models.py
+++ b/products/warehouse_sources/backend/facade/models.py
@@ -42,6 +42,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
discovered_schema_finder,
get_all_schemas_for_source_id,
get_schemas_for_direct_reconciliation,
+ is_pinned_schema,
mark_schema_running_unless_halted,
resolve_sync_type,
self_managed_table_ref,
@@ -113,6 +114,7 @@ __all__ = [
"asave_datawarehousetable",
"auto_enable_new_schemas",
"discovered_schema_finder",
+ "is_pinned_schema",
"self_managed_table_ref",
"create_datawarehousetable",
"get_all_schemas_for_source_id",
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index f4d3d3178a24..ad2ce8551e59 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -56,6 +56,7 @@ from products.warehouse_sources.backend.facade.models import (
ExternalDataSource,
UnsupportedSyncTypeError,
discovered_schema_finder,
+ is_pinned_schema,
mark_schema_running_unless_halted,
resolve_destinations,
resolve_sync_type,
@@ -70,8 +71,10 @@ from products.warehouse_sources.backend.facade.source_management import (
BUFFER_LANE,
CDC_RESET_PENDING_KEY,
AnySource,
+ Config,
RowFilterValidationError,
SourceRegistry,
+ SourceSchema,
WebhookSource,
add_table_failure_message,
filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,
@@ -193,6 +196,41 @@ def manual_sync_refusal(source_model: ExternalDataSource) -> str | None:
return source.manual_sync_refusal(source_model)
+def _discovery_error_response(source_impl: AnySource, error: Exception) -> Response:
+ """A 400 with the error's message. Only an error the source doesn't classify goes to error tracking.
+
+ A failure that the source itself classifies is an expected customer or upstream condition rather
+ than a bug, and must not flood error tracking. Both maps count: a non-retryable match names
+ something only the customer can fix, such as bad credentials, and a retryable match names a
+ transient failure `get_retryable_errors` already exists to keep out of error tracking.
+ Mirrors `refresh_schemas`'s `_classify_refresh_schemas_error`.
+ """
+ error_text = str(error)
+ expected_patterns = (*source_impl.get_non_retryable_errors(), *source_impl.get_retryable_errors())
+ if not any(pattern and pattern in error_text for pattern in expected_patterns):
+ capture_exception(error)
+ return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": error_text})
+
+
+def describe_pinned_schema(source_impl: AnySource, config: Config, schema: ExternalDataSchema) -> SourceSchema | None:
+ """The source's own description of a pinned schema, or None when the schema isn't pinned or the
+ source can't describe it. The source also checks that the credentials can read the schema's data."""
+ if not is_pinned_schema(schema):
+ return None
+ return source_impl.describe_pinned_schema(config, schema.team_id, schema.name, schema.schema_metadata or {})
+
+
+def discover_existing_schema(
+ source_impl: AnySource, config: Config, schema: ExternalDataSchema, api_version: str | None
+) -> list[SourceSchema]:
+ """What the source reports for one stored schema. Discovery can't report a pinned schema, so the
+ source describes a pinned schema from its stored definition when it can."""
+ pinned = describe_pinned_schema(source_impl, config, schema)
+ if pinned is not None:
+ return [pinned]
+ return source_impl.get_schemas(config, schema.team_id, names=[schema.name], api_version=api_version)
+
+
def source_row_filter_columns(source_type: str, schema_name: str) -> tuple[Any, ...] | None:
"""The columns a row filter on this schema may use, or None when any column of the table may.
@@ -1850,10 +1888,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
return False
try:
config = source_impl.parse_config(source_inputs)
- source_schemas = source_impl.get_schemas(
+ source_schemas = discover_existing_schema(
+ source_impl,
config,
- schema.team_id,
- names=[schema.name],
+ schema,
api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),
)
except Exception:
@@ -1875,10 +1913,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
source_type = known_source_key(source.source_type)
source_impl = SourceRegistry.get_source(source_type)
config = source_impl.parse_config(source_inputs)
- source_schemas = source_impl.get_schemas(
+ source_schemas = discover_existing_schema(
+ source_impl,
config,
- schema.team_id,
- names=[schema.name],
+ schema,
api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),
)
except Exception:
@@ -1905,10 +1943,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
return unsupported
try:
config = source_impl.parse_config(source_inputs)
- source_schemas = source_impl.get_schemas(
+ source_schemas = discover_existing_schema(
+ source_impl,
config,
- schema.team_id,
- names=[schema.name],
+ schema,
api_version=source_impl.resolve_api_version(self._effective_probe_api_version(schema, source_impl)),
)
except Exception as e:
@@ -2674,56 +2712,52 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
effective_api_version = new_source.resolve_api_version(instance.api_version or source.api_version)
try:
- credentials_valid, credentials_error = new_source.validate_credentials(
- config, self.team_id, instance.name, api_version=effective_api_version
- )
+ # Discovery can't report a pinned schema, and a source-wide check can fail where the sync
+ # works, for example when the key can't list the bucket root. The source checks access to
+ # the schema's own data instead.
+ pinned_schema = describe_pinned_schema(new_source, config, instance)
except Exception as e:
- credentials_valid, credentials_error = source_helpers._credentials_validation_failed(
- new_source, self.team_id, e
- )
- if not credentials_valid:
- return Response(
- status=status.HTTP_400_BAD_REQUEST,
- data={"message": credentials_error or "Invalid credentials"},
- )
+ return _discovery_error_response(new_source, e)
+ if pinned_schema is not None:
+ schemas = [pinned_schema]
+ else:
+ try:
+ credentials_valid, credentials_error = new_source.validate_credentials(
+ config, self.team_id, instance.name, api_version=effective_api_version
+ )
+ except Exception as e:
+ credentials_valid, credentials_error = source_helpers._credentials_validation_failed(
+ new_source, self.team_id, e
+ )
+ if not credentials_valid:
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": credentials_error or "Invalid credentials"},
+ )
- stored_resource_id = (
- (instance.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY)
- if new_source.uses_stable_schema_resource_ids
- else None
- )
- try:
- schemas = new_source.get_schemas(
- config, self.team_id, names=[instance.name], api_version=effective_api_version
+ stored_resource_id = (
+ (instance.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY)
+ if new_source.uses_stable_schema_resource_ids
+ else None
)
- # A stored resource id can outlive the stored name after an upstream rename. Discover every
- # schema only when the name lookup misses that id, so an unrenamed schema keeps the cheap
- # filtered discovery.
- if stored_resource_id is not None:
- found = discovered_schema_finder(schemas)(instance.name, instance.schema_metadata)
- found_resource_id = (
- (found.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY) if found else None
+ try:
+ schemas = new_source.get_schemas(
+ config, self.team_id, names=[instance.name], api_version=effective_api_version
)
- if str(found_resource_id) != str(stored_resource_id):
- schemas = new_source.get_schemas(
- config, self.team_id, names=None, api_version=effective_api_version
+ # A stored resource id can outlive the stored name after an upstream rename. Discover every
+ # schema only when the name lookup misses that id, so an unrenamed schema keeps the cheap
+ # filtered discovery.
+ if stored_resource_id is not None:
+ found = discovered_schema_finder(schemas)(instance.name, instance.schema_metadata)
+ found_resource_id = (
+ (found.schema_metadata or {}).get(SCHEMA_RESOURCE_ID_METADATA_KEY) if found else None
)
- except Exception as e:
- # `validate_credentials` above just probed the same connection successfully, so a
- # failure here that the source itself classifies is an expected customer or upstream
- # condition rather than a bug, and must not flood error tracking. Both maps count: a
- # non-retryable match names something only the customer can fix, such as bad
- # credentials, and a retryable match names a transient failure `get_retryable_errors`
- # already exists to keep out of error tracking.
- # Mirrors `refresh_schemas`'s `_classify_refresh_schemas_error`.
- error_text = str(e)
- expected_patterns = (*new_source.get_non_retryable_errors(), *new_source.get_retryable_errors())
- if not any(pattern and pattern in error_text for pattern in expected_patterns):
- capture_exception(e)
- return Response(
- status=status.HTTP_400_BAD_REQUEST,
- data={"message": str(e)},
- )
+ if str(found_resource_id) != str(stored_resource_id):
+ schemas = new_source.get_schemas(
+ config, self.team_id, names=None, api_version=effective_api_version
+ )
+ except Exception as e:
+ return _discovery_error_response(new_source, e)
if not schemas:
return Response(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
index 4f29397ef900..1f42a08c5371 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
@@ -441,6 +441,19 @@ class _BaseSource(ABC, Generic[ConfigType]):
"""
raise NotImplementedError()
+ def describe_pinned_schema(
+ self, config: ConfigType, team_id: int, name: str, schema_metadata: Mapping[str, Any]
+ ) -> SourceSchema | None:
+ """Describe a pinned schema from the definition stored on it, without discovery.
+
+ Discovery can't report a pinned schema (see `SCHEMA_PINNED_METADATA_KEY`), so callers ask
+ this first. A source that can rebuild the schema from `schema_metadata` checks that the
+ credentials can read the schema's own data, raises the errors a sync raises when they
+ can't, and returns the schema. `None` means the caller uses `validate_credentials` and
+ `get_schemas`, as for any other schema.
+ """
+ return None
+
def _placeholder_config(self) -> ConfigType:
"""Build a credential-free config instance for static, no-network schema listing.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
index b4d62e145cb2..77a0bd020729 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -282,3 +282,9 @@ def iter_delta_tables(
) -> Iterator[pa.Table]:
"""Stream one whole Delta snapshot."""
yield from plan_delta_read(uri, storage_options, table_name, logger=logger).iter_tables(batch_rows)
+
+
+def check_delta_table(uri: str, storage_options: dict[str, str], table_name: str) -> None:
+ """Load the table's `_delta_log`, which needs the same read access as a sync, without reading data."""
+ with _delta_errors(table_name):
+ DeltaTable(uri, storage_options=storage_options, without_files=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 1440fcae8f4f..554002071627 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from datetime import UTC, datetime
from typing import Any, ClassVar, Generic, NoReturn, Optional
@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (
DELTA_VERSION_COLUMN,
ObjectStorageDeltaCursor,
+ check_delta_table,
iter_delta_tables,
plan_delta_read,
)
@@ -243,6 +244,32 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
if requested is None or table.name in requested
]
+ def describe_pinned_schema(
+ self, config: ConfigType, team_id: int, name: str, schema_metadata: Mapping[str, Any]
+ ) -> SourceSchema | None:
+ """Describe a table from its stored definition, and check that the key can read the table.
+
+ Only the table's own location is read. A key that can read the table but can't list the
+ bucket root passes, and a large bucket isn't walked. Discovery also names a table after its
+ folder, and a pinned table, such as a migrated self-managed table, can have another name.
+ """
+ table = TableDefinition.from_schema_metadata(name, schema_metadata)
+ if table is None:
+ return None
+ store = self.open_store(self.object_store_config(config))
+ location = store.path(table.path)
+ try:
+ if table.kind == "delta":
+ check_delta_table(store.delta_uri(table.path), store.delta_storage_options, table.name)
+ else:
+ resolve_table_files(store, table)
+ except OSError as e:
+ # As in `validate_credentials`: s3fs gives a bare 401 or 403 as a plain OSError.
+ if _http_status(e) in (401, 403):
+ raise translate_storage_error(PermissionError(), location) from e
+ _raise_translated(e, location)
+ return _source_schema(table)
+
def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> SourceResponse:
store = self.open_store(self.object_store_config(config))
table = TableDefinition.from_schema_metadata(inputs.schema_name, inputs.schema_metadata) or self._find_table(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py
index 867b54d89c71..69925f12c314 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/fake_source.py
@@ -1,6 +1,10 @@
import dataclasses
-from collections.abc import Callable
-from typing import Any
+from collections.abc import Callable, Iterator
+from datetime import UTC, datetime
+from typing import Any, cast
+
+from botocore.exceptions import ClientError
+from fsspec.implementations.local import LocalFileSystem
from products.warehouse_sources.backend.facade.source_config import SourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.config import Config
@@ -11,6 +15,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
ObjectStoreProvider,
StoreConfig,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ S3ObjectStore,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
parse_azure_url,
parse_gcs_url,
@@ -95,3 +102,46 @@ class FakeObjectStorageSource(ObjectStorageSource[FakeObjectStorageConfig]):
self, location: SelfManagedLocation, credential: SelfManagedCredential
) -> dict[str, Any]:
return self._connection_inputs
+
+
+class PrefixScopedS3Client:
+ """A boto3 S3 client for a key that may list only keys under one prefix, as a prefix-scoped IAM policy allows."""
+
+ def __init__(self, keys: list[str], allowed_prefix: str) -> None:
+ self.keys = keys
+ self.allowed_prefix = allowed_prefix
+ self.listed_prefixes: list[str] = []
+
+ def get_paginator(self, operation: str) -> "PrefixScopedS3Client":
+ assert operation == "list_objects_v2"
+ return self
+
+ def paginate(self, Bucket: str, Prefix: str, PaginationConfig: dict[str, Any]) -> Iterator[dict[str, Any]]:
+ self.listed_prefixes.append(Prefix)
+ if not Prefix.startswith(self.allowed_prefix):
+ raise ClientError(
+ cast(
+ Any,
+ {
+ "Error": {"Code": "AccessDenied", "Message": "Access Denied"},
+ "ResponseMetadata": {"HTTPStatusCode": 403},
+ },
+ ),
+ "ListObjectsV2",
+ )
+ modified_at = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)
+ contents = [{"Key": key, "Size": 5, "LastModified": modified_at} for key in self.keys if key.startswith(Prefix)]
+ yield {"Contents": contents, "IsTruncated": False}
+
+
+def bucket_root_store(client: PrefixScopedS3Client, bucket: str = "example-bucket") -> S3ObjectStore:
+ """A store rooted at the bucket, as for a source that holds the migrated tables of one bucket."""
+ return S3ObjectStore(
+ fs=LocalFileSystem(),
+ root=bucket,
+ delta_uri_prefix="s3://",
+ delta_storage_options={},
+ client=client,
+ bucket=bucket,
+ prefix="",
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 0e6d357b2694..24aca3b7ffd1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -76,6 +76,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
listing_digest,
plan_mirror,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ parse_self_managed_url,
+ table_definition_for,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
JSONL_CHUNK_ROWS,
MAX_TABLES,
@@ -85,6 +89,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
iter_table,
validate_store_location,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.tests.fake_source import (
+ FakeObjectStorageConfig,
+ FakeObjectStorageSource,
+ PrefixScopedS3Client,
+ bucket_root_store,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ReplaceScope
MODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)
@@ -1374,3 +1384,111 @@ class TestS3BucketRegion:
with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):
lookup_s3_bucket_region("Not_A_Bucket")
+
+
+def _source_with_store(store: ObjectStore) -> FakeObjectStorageSource:
+ source = FakeObjectStorageSource()
+ source.open_store = MagicMock(return_value=store) # type: ignore[method-assign]
+ return source
+
+
+BUCKET_ROOT_CONFIG = FakeObjectStorageConfig(
+ values={
+ "url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "c2VjcmV0",
+ }
+)
+ORDERS_KEYS = ["tables/orders/2024-01.csv", "tables/orders/2024-02.csv", "tables/other/x.csv"]
+MIGRATED_ORDERS = table_definition_for(
+ "orders",
+ parse_self_managed_url("https://example-bucket.s3.us-east-1.amazonaws.com/tables/orders/*.csv"),
+ "CSVWithNames",
+ {},
+ None,
+)
+DISCOVERED_ORDERS = TableDefinition(
+ name="orders",
+ kind="files",
+ path="tables/orders",
+ scope="folder",
+ file_format="csv",
+ csv=CsvOptions(),
+ pattern=None,
+ source_format="infer",
+)
+
+
+class TestDescribePinnedSchema:
+ @parameterized.expand([("migrated_glob", MIGRATED_ORDERS), ("folder", DISCOVERED_ORDERS)])
+ def test_a_key_scoped_to_the_table_prefix_reads_the_table_without_listing_the_bucket_root(
+ self, _name: str, table: TableDefinition
+ ) -> None:
+ client = PrefixScopedS3Client(ORDERS_KEYS, allowed_prefix="tables/orders/")
+ source = _source_with_store(bucket_root_store(client))
+
+ schema = source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", table.schema_metadata())
+
+ assert schema is not None
+ assert (schema.name, schema.schema_metadata) == ("orders", table.schema_metadata())
+ assert client.listed_prefixes == ["tables/orders/"]
+ # Discovery lists the bucket root, which the same key can't do.
+ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)
+ assert not valid and message is not None and message.startswith(ACCESS_DENIED_ERROR)
+
+ @parameterized.expand(
+ [
+ ("denied", ORDERS_KEYS, "tables/other/", f"{ACCESS_DENIED_ERROR} 'example-bucket/tables/orders'"),
+ ("no_files", ["tables/other/x.csv"], "tables/", f"{NO_TABLE_FILES_ERROR} 'orders'"),
+ ]
+ )
+ def test_a_table_the_key_can_not_read_fails_with_the_sync_error(
+ self, _name: str, keys: list[str], allowed_prefix: str, expected_start: str
+ ) -> None:
+ source = _source_with_store(bucket_root_store(PrefixScopedS3Client(keys, allowed_prefix)))
+
+ with pytest.raises(ObjectStorageError) as error:
+ source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", MIGRATED_ORDERS.schema_metadata())
+
+ assert str(error.value).startswith(expected_start)
+
+ @parameterized.expand([("empty", {}), ("unknown_kind", {"kind": "iceberg", "path": "x"})])
+ def test_metadata_without_a_table_definition_is_not_described(self, _name: str, metadata: dict) -> None:
+ source = _source_with_store(MagicMock())
+
+ assert source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "orders", metadata) is None
+ source.open_store.assert_not_called() # type: ignore[attr-defined]
+
+ def test_a_delta_table_is_described_from_its_log(self, tmp_path: Path) -> None:
+ write_deltalake(str(tmp_path / "tables" / "events"), pa.table({"id": [1]}))
+ metadata: dict[str, Any] = {
+ "kind": "delta",
+ "path": "tables/events",
+ "format": "delta",
+ "pattern": None,
+ "csv": {},
+ }
+ source = _source_with_store(local_store(tmp_path))
+
+ schema = source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "events", metadata)
+
+ assert schema is not None
+ assert [field["field"] for field in schema.incremental_fields] == ["_delta_version"]
+
+ @pytest.mark.parametrize(
+ "raised, error_prefix",
+ [
+ pytest.param(None, NO_TABLE_FILES_ERROR, id="missing"),
+ pytest.param(OSError("Generic S3 error: 403 Forbidden"), ACCESS_DENIED_ERROR, id="denied"),
+ ],
+ )
+ def test_an_unreadable_delta_log_fails_with_the_sync_error(
+ self, raised: Exception | None, error_prefix: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ if raised is not None:
+ monkeypatch.setattr(delta, "DeltaTable", MagicMock(side_effect=raised))
+ metadata: dict[str, Any] = {"kind": "delta", "path": "events", "format": "delta", "pattern": None, "csv": {}}
+ source = _source_with_store(local_store(tmp_path))
+
+ with pytest.raises(ObjectStorageError, match=f"^{error_prefix}"):
+ source.describe_pinned_schema(BUCKET_ROOT_CONFIG, 1, "events", metadata)
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 1ea83871ac2b..c998f87ea99b 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -46,6 +46,7 @@ from products.warehouse_sources.backend.facade.models import (
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
+from products.warehouse_sources.backend.models.external_data_schema import SCHEMA_PINNED_METADATA_KEY
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
UNREADABLE_SOURCE_CONFIG_ERROR,
ExternalDataSchemaSerializer,
@@ -61,6 +62,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
WebhookCreationResult,
WebhookSyncResult,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FILE_MODIFIED_AT_COLUMN
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ACCESS_DENIED_ERROR,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ parse_self_managed_url,
+ table_definition_for,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.tests.fake_source import (
+ PrefixScopedS3Client,
+ bucket_root_store,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSource
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.constants import (
@@ -69,6 +82,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
)
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.source import StripeSource
from products.warehouse_sources.backend.tests.api.utils import create_external_data_source_ok
+from products.warehouse_sources.sources.amazon_s3.source import AmazonS3Source
_PH_HISTORY_NAMING_FLAG = (
"products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"
@@ -5629,3 +5643,64 @@ class TestExternalDataSchemaIntrospection(SimpleTestCase):
serializer = view.get_serializer()
assert isinstance(serializer, ExternalDataSchemaSerializer)
assert "table_id" in serializer.fields
+
+
+MIGRATED_ORDERS_METADATA = table_definition_for(
+ "orders",
+ parse_self_managed_url("https://example-bucket.s3.us-east-1.amazonaws.com/tables/orders/*.csv"),
+ "CSVWithNames",
+ {},
+ None,
+).schema_metadata()
+
+
+class TestPinnedSchemaIncrementalFields(APIBaseTest):
+ @parameterized.expand(
+ [
+ # The key can list the table's own prefix but not the bucket root.
+ ("pinned_key_scoped_to_the_table", True, "tables/orders/", None),
+ ("pinned_key_denied_the_table", True, "tables/other/", "'example-bucket/tables/orders'"),
+ # A schema without the pin still checks the whole bucket, as before.
+ ("unpinned", False, "tables/orders/", "'example-bucket'"),
+ ]
+ )
+ @mock.patch("products.warehouse_sources.backend.presentation.views.external_data_schema.capture_exception")
+ def test_a_pinned_schema_is_checked_at_its_own_location(
+ self, _name: str, pinned: bool, allowed_prefix: str, denied_location: str | None, mock_capture: mock.MagicMock
+ ) -> None:
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_type=ExternalDataSourceType.AMAZONS3,
+ job_inputs={
+ "url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "c2VjcmV0",
+ "file_format": "infer",
+ },
+ )
+ metadata = (
+ {**MIGRATED_ORDERS_METADATA, SCHEMA_PINNED_METADATA_KEY: True} if pinned else MIGRATED_ORDERS_METADATA
+ )
+ schema = ExternalDataSchema.objects.create(
+ name="orders",
+ team=self.team,
+ source=source,
+ should_sync=False,
+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,
+ sync_type_config={"schema_metadata": metadata},
+ )
+ client = PrefixScopedS3Client(["tables/orders/2024-01.csv", "tables/other/x.csv"], allowed_prefix)
+
+ with mock.patch.object(AmazonS3Source, "open_store", return_value=bucket_root_store(client)):
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/incremental_fields",
+ )
+
+ if denied_location is None:
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ assert [field["field"] for field in response.json()["incremental_fields"]] == [FILE_MODIFIED_AT_COLUMN]
+ assert client.listed_prefixes == ["tables/orders/"]
+ else:
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert response.json()["message"].startswith(f"{ACCESS_DENIED_ERROR} {denied_location}.")
+ mock_capture.assert_not_called()