:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add amazon s3 import source on a shared object storage backend (#116382)
Tom Owers6602226bf2f6
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py b/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.pyindex 82c455c4817c..1eb1ebff5f8f 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py@@ -20,7 +20,7 @@ from posthog.models.user import Userfrom products.warehouse_sources.backend.facade.models import PendingSourceCredentialfrom products.warehouse_sources.backend.facade.types import ExternalDataSourceType-from . import base+from . import base, helpersclass SourceConnectLinkSerializer(serializers.Serializer):@@ -165,6 +165,11 @@ class ExternalDataSourceCredentialStoreMixin(base.ExternalDataSourceViewSetBase)source_type_model = ExternalDataSourceType(source_type)source = base.SourceRegistry.get_source(source_type_model)+ if helpers.source_feature_flag_missing(source, self.team):+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": helpers._source_unavailable_message(source_type)},+ )error_response, _ = self._validate_source_config_and_credentials(source, source_type_model, payload)if error_response is not None:diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.pyindex 93681481743c..2706f6ee8ac6 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py@@ -4,13 +4,14 @@ from __future__ import annotationsimport refrom collections.abc import Iterable, Mapping-from typing import Any+from typing import TYPE_CHECKING, Anyfrom urllib.parse import unquote, urlsplitfrom django.db.models import Q, QuerySetfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception+from posthog.ph_client import feature_enabled_or_falsefrom products.data_warehouse.backend.facade.api import (DirectQueryEngine,@@ -42,6 +43,9 @@ from products.warehouse_sources.backend.facade.source_management import ()from products.warehouse_sources.backend.facade.types import ExternalDataSourceType, ManagedWarehouseSQLMode+if TYPE_CHECKING:+ from posthog.models.team import Team+REFRESH_SCHEMAS_FALLBACK_ERROR_MESSAGE = "Could not fetch schemas from source."RESERVED_SOURCE_NAME_MESSAGE = "This source name is reserved by PostHog."@@ -70,6 +74,20 @@ def _source_unavailable_message(source_type: str) -> str:)+def source_feature_flag_missing(source: AnySource, team: Team) -> bool:+ """Whether the team lacks the feature flag the source requires before the API serves it."""+ flag = source.requires_team_feature_flag+ if flag is None:+ return False+ organization_id = str(team.organization_id)+ return not feature_enabled_or_false(+ flag,+ organization_id,+ groups={"organization": organization_id, "project": str(team.id)},+ group_properties={"organization": {"id": organization_id}},+ )++def _canonical_legacy_managed_warehouse_source(queryset: QuerySet[ExternalDataSource],) -> ExternalDataSource | None:diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.pyindex 05613ccae919..5980ad393242 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py@@ -370,6 +370,11 @@ class ExternalDataSourceSchemaOperationsMixin(base.ExternalDataSourceViewSetBasedata={"message": f"Unknown source_type '{source_type}'"},)source = base.SourceRegistry.get_source(source_type_model)+ if helpers.source_feature_flag_missing(source, self.team):+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": helpers._source_unavailable_message(source_type)},+ )is_valid, errors = source.validate_config(request.data)if not is_valid:return Response(diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex 720f68df1af8..d3b39f6b071e 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -1184,6 +1184,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):).data,status=status.HTTP_400_BAD_REQUEST,)+ if helpers.source_feature_flag_missing(source, self.team):+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": helpers._source_unavailable_message(source_type)},+ )max_instances = source.max_instances_per_teamif max_instances is not None and helpers.count_active_sources(self.team_id, source_type_model) >= max_instances:return Response(@@ -2018,6 +2023,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):source_type_model = ExternalDataSourceType(source_type)source = base.SourceRegistry.get_source(source_type_model)+ if helpers.source_feature_flag_missing(source, self.team):+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": helpers._source_unavailable_message(source_type)},+ )error_response, source_config = self._validate_source_config_and_credentials(source, source_type_model, payload)if error_response is not None or source_config is None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 009c2b7f64eb..b3cb0ff97225 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -76,6 +76,7 @@ the row lists both.| alguna | HTTP | requests + `rest_source.RESTClient` | ✅ || alpha_vantage | HTTP | requests | ✅ || amazon_ads | HTTP | requests | ✅ |+| amazon_s3 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ || amplemarket | HTTP | requests + `rest_source.RESTClient` | ✅ || amplitude | HTTP | requests | ✅ || anthropic | HTTP | requests | ✅ |@@ -964,6 +965,10 @@ the row lists both.- **linkedin_ads** uses `linkedin-api`'s `RestliClient`, which constructs its own internal `requests.Session`.We don't yet have a session-injection seam on it, so outbound calls bypass the tracked transport. (The fileimports `requests` only for exception types — those references are expected and don't need a pragma.)+- **amazon_s3** lists through a boto3 client, reads files through `s3fs` (aiobotocore over aiohttp), and reads+ Delta tables through delta-rs, which has its own Rust object store client. None of the three exposes a+ `requests` session or an interceptor seam, so outbound traffic bypasses the tracked transport. Each client sets+ its own connect and read timeouts and a bounded retry count.- **temporalio** talks gRPC, but the Python `temporalio` SDK runs its entire gRPC stack in a Rust core(`temporalio.bridge`, a PyO3 module) — service RPCs dispatch through `ServiceClient._rpc_call` into thebridge, so there is no Python `grpc.Channel` or client-interceptor seam for the tracked gRPC transport to@@ -1018,7 +1023,6 @@ doesn't conflict with concurrent PRs.- amazon_cloudwatch- amazon_eventbridge- amazon_kinesis-- amazon_s3- amazon_selling_partner- amazon_sns- amazon_sqsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.pyindex d7a2a3678d5e..6399b6159f14 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py@@ -1,28 +1,195 @@from typing import cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.facade.codebase_detection import detect+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,+ SourceFieldSelectConfig,+ SourceFieldSelectConfigConverter,+ SourceFieldSelectConfigOption,+ SourceFieldSwitchGroupConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ DiscoveryOptions,+ ObjectStoreConfig,+ ObjectStoreProvider,+ normalize_prefix,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (+ ObjectStorageSource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistryfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (AmazonS3SourceConfig,)from products.warehouse_sources.backend.types import ExternalDataSourceType+FEATURE_FLAG = "dwh-object-storage-sources"+@SourceRegistry.register-class AmazonS3Source(SimpleSource[AmazonS3SourceConfig]):+class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):+ codebase_detection = detect(+ npm=("@aws-sdk/client-s3",),+ pypi=("s3fs",),+ cargo=("aws-sdk-s3",),+ go=("github.com/aws/aws-sdk-go-v2/service/s3",),+ maven=("software.amazon.awssdk:s3",),+ )++ api_docs_url = "https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html"+ requires_team_feature_flag = FEATURE_FLAG+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AMAZONS3+ @property+ def connection_host_fields(self) -> list[str]:+ # The bucket and region pick where the stored key is sent, so changing either must re-require it.+ return ["bucket", "region"]++ def object_store_config(self, config: AmazonS3SourceConfig) -> ObjectStoreConfig:+ return ObjectStoreConfig(+ provider=ObjectStoreProvider.S3,+ bucket=config.bucket.strip(),+ region=config.region.strip(),+ access_key_id=config.access_key_id.strip(),+ secret_access_key=config.secret_access_key.strip(),+ prefix=normalize_prefix(config.path),+ )++ def discovery_options(self, config: AmazonS3SourceConfig) -> DiscoveryOptions:+ combine = config.combine_files+ return DiscoveryOptions(+ file_format=config.file_format,+ file_pattern=(config.file_pattern or "").strip() or None,+ allow_double_quotes=config.csv_allow_double_quotes,+ combined_table_name=combine.table_name if combine is not None and combine.enabled else None,+ )+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(name=ExternalDataSourceType.AMAZONS3,category=DataWarehouseSourceCategory.FILE_STORAGE,- keywords=["s3"],+ keywords=["s3", "aws", "bucket", "parquet", "delta lake"],label="Amazon S3",+ caption=(+ "Import CSV, JSON Lines and Parquet files, and Delta Lake tables, from an S3 bucket. PostHog "+ "creates one table per folder, one per file at the top of the folder you choose, and one per "+ "Delta table. Every sync reads the files in full, so each table matches the bucket."+ ),iconPath="/static/services/aws-s3.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ releaseStatus=ReleaseStatus.ALPHA,+ featureFlag=FEATURE_FLAG,+ fields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="bucket",+ label="Bucket",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="my-company-exports",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="region",+ label="Region",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="us-east-1",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="path",+ label="Folder (optional)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="exports/",+ caption="Leave it empty to import from the whole bucket.",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="file_pattern",+ label="File pattern (optional)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="*.parquet",+ caption=(+ "A glob matched against each file's path inside the folder, where * also matches "+ "across folders. Leave it empty to import every file."+ ),+ secret=False,+ ),+ SourceFieldInputConfig(+ name="access_key_id",+ label="Access key ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="AKIA...",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="secret_access_key",+ label="Secret access key",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="",+ secret=True,+ ),+ SourceFieldSelectConfig(+ name="file_format",+ label="File format",+ required=True,+ defaultValue="infer",+ options=[+ SourceFieldSelectConfigOption(label="Detect from file extension", value="infer"),+ SourceFieldSelectConfigOption(label="CSV with a header row", value="csv"),+ SourceFieldSelectConfigOption(label="CSV without a header row", value="csv_no_header"),+ SourceFieldSelectConfigOption(label="JSON Lines", value="jsonl"),+ SourceFieldSelectConfigOption(label="Parquet", value="parquet"),+ SourceFieldSelectConfigOption(label="Delta Lake", value="delta"),+ ],+ ),+ SourceFieldSelectConfig(+ name="csv_allow_double_quotes",+ label="Do double quotes enclose CSV values?",+ required=True,+ defaultValue="true",+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,+ options=[+ SourceFieldSelectConfigOption(label="Yes", value="true"),+ SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),+ ],+ ),+ SourceFieldSwitchGroupConfig(+ name="combine_files",+ label="Combine every file into one table?",+ caption=(+ "Turn this on when the folder holds files that share the same columns, such as a "+ "daily export. Delta tables still get a table each."+ ),+ default=False,+ fields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="table_name",+ label="Table name",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="orders",+ secret=False,+ ),+ ],+ ),+ ),+ ],+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.pynew file mode 100644index 000000000000..ad7f346ca97b--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py@@ -0,0 +1,129 @@+import json+from collections.abc import Iterable+from pathlib import Path+from typing import Any, cast++import pytest++import pyarrow as pa+from fsspec import AbstractFileSystem+from fsspec.implementations.local import LocalFileSystem++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ ObjectStoreConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ACCESS_DENIED_ERROR,+ INVALID_CONFIG_ERROR,+ NO_IMPORTABLE_FILES_ERROR,+ NO_SUCH_BUCKET_ERROR,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStore+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputs+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (+ AmazonS3SourceConfig,+)+++class _DeniedFileSystem(LocalFileSystem):+ def __init__(self, error: Exception) -> None:+ super().__init__()+ self.error = error++ def find(self, path: str, *args: Any, **kwargs: Any) -> Any:+ raise self.error+++class _LocalAmazonS3Source(AmazonS3Source):+ """Serves the bucket from a local folder: `<root>/<bucket>/<prefix>`."""++ def __init__(self, root: Path, fs: AbstractFileSystem | None = None) -> None:+ self.root = root.resolve()+ self.fs = fs or LocalFileSystem()++ def open_store(self, store_config: ObjectStoreConfig) -> ObjectStore:+ folder = self.root / store_config.bucket / store_config.prefix+ return ObjectStore(fs=self.fs, root=str(folder).rstrip("/"), delta_uri_prefix="", delta_storage_options={})+++def make_config(**overrides: Any) -> AmazonS3SourceConfig:+ values: dict[str, Any] = {+ "bucket": "customer-data",+ "region": "us-east-1",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "secret",+ "path": "exports",+ **overrides,+ }+ return AmazonS3SourceConfig.from_dict(values)+++def write(root: Path, relative: str, content: bytes) -> None:+ path = root / "customer-data" / relative+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(content)+++def read_rows(source: AmazonS3Source, config: AmazonS3SourceConfig, schema_name: str, metadata: Any) -> list[dict]:+ inputs = source_inputs(schema_name)+ inputs.schema_metadata = metadata+ response = source.source_for_pipeline(config, inputs)+ return [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]++[email protected]("with_metadata", [True, False])+def test_sync_reads_the_table_discovery_offered(tmp_path: Path, with_metadata: bool) -> None:+ write(tmp_path, "exports/orders/a.csv", b"id,total\n1,10\n")+ write(tmp_path, "exports/orders/b.csv", b"id,total\n2,20\n")+ write(tmp_path, "exports/orders/archive/c.csv", b"id,total\n3,30\n")+ source = _LocalAmazonS3Source(tmp_path)+ config = make_config()+ schemas = {schema.name: schema for schema in source.get_schemas(config, team_id=1)}+ assert set(schemas) == {"orders", "orders_archive"}+ # The pipeline stores the metadata as JSON on the schema row.+ metadata = json.loads(json.dumps(schemas["orders"].schema_metadata)) if with_metadata else None++ rows = read_rows(source, config, "orders", metadata)++ assert [(row["id"], row["_file_name"]) for row in rows] == [("1", "orders/a.csv"), ("2", "orders/b.csv")]+++def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -> None:+ write(tmp_path, "exports/raw/a.txt", b"1,2\n")+ source = _LocalAmazonS3Source(tmp_path)+ [schema] = source.get_schemas(make_config(file_format="csv_no_header"), team_id=1)++ rows = read_rows(source, make_config(file_format="infer"), schema.name, schema.schema_metadata)++ assert rows[0]["c1"] == "1"+++ "files, config_overrides, fs_error, expected",+ [+ ({"exports/a.csv": b"id\n1\n"}, {}, None, None),+ ({"exports/notes.md": b"# hi"}, {}, None, NO_IMPORTABLE_FILES_ERROR),+ ({}, {"bucket": "Not A Bucket"}, None, INVALID_CONFIG_ERROR),+ ({}, {}, PermissionError("Access Denied"), ACCESS_DENIED_ERROR),+ ({}, {}, FileNotFoundError("NoSuchBucket"), NO_SUCH_BUCKET_ERROR),+ ],+)+def test_validate_credentials(+ tmp_path: Path,+ files: dict[str, bytes],+ config_overrides: dict[str, str],+ fs_error: Exception | None,+ expected: str | None,+) -> None:+ for relative, content in files.items():+ write(tmp_path, relative, content)+ (tmp_path / "customer-data" / "exports").mkdir(parents=True, exist_ok=True)+ fs = _DeniedFileSystem(fs_error) if fs_error is not None else None+ source = _LocalAmazonS3Source(tmp_path, fs)++ valid, message = source.validate_credentials(make_config(**config_overrides), team_id=1)++ assert valid is (expected is None)+ if expected is not None:+ assert message is not None and message.startswith(expected)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.pyindex 07d61b416e61..10459a18844c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py@@ -473,6 +473,11 @@ class _BaseSource(ABC, Generic[ConfigType]):incoming and existing inputs so a source can gate on context. Default: none."""return []+ # Feature flag a team must have before the API lets it discover tables for or create a source of+ # this type. `SourceConfig.featureFlag` only hides the source in the UI, so a source that must+ # not be reachable through the API without the flag sets this as well.+ requires_team_feature_flag: str | None = None+# Cap on how many active sources of this type a team may have (``None`` = unlimited). Custom# sources are capped; everything else is open. The API enforces this at create.max_instances_per_team: int | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.pynew file mode 100644index 000000000000..3d5c22961a4e--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py@@ -0,0 +1,97 @@+import re+import dataclasses+from enum import StrEnum+from typing import Literal, get_args++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ INVALID_CONFIG_ERROR,+ ObjectStorageError,+)++FileFormatChoice = Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"]+# The format one file is read with. A CSV file's header choice lives in `CsvOptions`.+FileFormat = Literal["csv", "jsonl", "parquet"]++# S3 bucket naming rules: 3-63 characters of lowercase letters, digits, dots and hyphens. Holding+# the bucket to this keeps it from injecting a host or a path into the URLs built from it.+_BUCKET_NAME = re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]")+_REGION_NAME = re.compile(r"[a-z]{2}(-[a-z0-9]+)+-\d+")+++class ObjectStoreProvider(StrEnum):+ S3 = "s3"+++@frozen+class ObjectStoreConfig:+ """Where a bucket lives and the credentials to read it.++ `endpoint_url` stays `None` for AWS S3. S3-compatible stores that take HMAC keys (GCS, R2) set it.+ """++ provider: ObjectStoreProvider+ bucket: str+ region: str+ access_key_id: str = dataclasses.field(repr=False)+ secret_access_key: str = dataclasses.field(repr=False)+ prefix: str = ""+ endpoint_url: str | None = None++ def __post_init__(self) -> None:+ if not _BUCKET_NAME.fullmatch(self.bucket) or ".." in self.bucket:+ raise ObjectStorageError(+ f"{INVALID_CONFIG_ERROR}: '{self.bucket}' isn't a valid bucket name. Enter only the bucket "+ "name, without s3:// or a folder."+ )+ if not _REGION_NAME.fullmatch(self.region):+ raise ObjectStorageError(+ f"{INVALID_CONFIG_ERROR}: '{self.region}' isn't a valid region, such as us-east-1."+ )+ # Without both keys the SDKs fall back to the worker's own AWS credentials, which must+ # never be used to read a customer-chosen bucket.+ if not self.access_key_id.strip() or not self.secret_access_key.strip():+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: enter both the access key ID and the secret access key.")+ if any(segment in (".", "..") for segment in self.prefix.split("/")):+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the folder can't contain '.' or '..' segments.")++ @property+ def location_url(self) -> str:+ """The https URL of the folder, in the form the warehouse table URL checks accept."""+ path = f"/{self.prefix}/" if self.prefix else "/"+ if self.endpoint_url:+ return f"{self.endpoint_url.rstrip('/')}/{self.bucket}{path}"+ return f"https://{self.bucket}.s3.{self.region}.amazonaws.com{path}"++ @property+ def display_location(self) -> str:+ return f"{self.bucket}/{self.prefix}" if self.prefix else self.bucket+++@frozen+class CsvOptions:+ has_header: bool = True+ # Mirrors ClickHouse's `format_csv_allow_double_quotes`: off means a double quote is an ordinary+ # character and never encloses a field.+ allow_double_quotes: bool = True+++@frozen+class DiscoveryOptions:+ """The settings that decide which files exist as tables, and how each one is read."""++ file_format: FileFormatChoice = "infer"+ file_pattern: str | None = None+ allow_double_quotes: bool = True+ combined_table_name: str | None = None++ def __post_init__(self) -> None:+ # The API checks the shape of a payload, not that a `Literal` field holds one of its values.+ if self.file_format not in get_args(FileFormatChoice):+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.file_format}' isn't a supported file format.")+++def normalize_prefix(path: str | None) -> str:+ """Drop empty segments only. A '.' or '..' segment stays, so `ObjectStoreConfig` rejects it."""+ return "/".join(segment for segment in (path or "").strip().split("/") if segment)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.pynew file mode 100644index 000000000000..df2bb5bac436--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py@@ -0,0 +1,78 @@+import re+from collections.abc import Iterator++import pyarrow as pa+from deltalake import DeltaTable+from deltalake.exceptions import DeltaError, DeltaProtocolError, TableNotFoundError+from structlog.types import FilteringBoundLogger++from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ACCESS_DENIED_ERROR,+ DELTA_UNSUPPORTED_ERROR,+ FILE_PARSE_ERROR,+ NO_TABLE_FILES_ERROR,+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ DELTA_BATCH_ROWS,+)++# Whole words only, so a status code doesn't match inside an object key such as `sales403/part.parquet`.+_ACCESS_DENIED_MARKERS = re.compile(r"\b(403|AccessDenied|Forbidden|InvalidAccessKeyId|SignatureDoesNotMatch)\b")+++def _raise_if_access_denied(error: Exception, table_name: str) -> None:+ # delta-rs reports object store failures as text, so a rejected credential shows only as the+ # HTTP status or the S3 error code inside the message.+ if _ACCESS_DENIED_MARKERS.search(str(error)):+ raise ObjectStorageError(+ f"{ACCESS_DENIED_ERROR} for the Delta table '{table_name}'. Check that the key can read its folder."+ ) from error+++def iter_delta_tables(+ uri: str,+ storage_options: dict[str, str],+ table_name: str,+ batch_rows: int = DELTA_BATCH_ROWS,+ logger: FilteringBoundLogger | None = None,+) -> Iterator[pa.Table]:+ """Stream one Delta snapshot.++ The version is pinned when the table opens, so commits that land during the read don't mix two+ versions into one import. The scan reads one fragment at a time on one thread to bound memory.+ """+ try:+ table = DeltaTable(uri, storage_options=storage_options)+ version = table.version()+ dataset = table.to_pyarrow_dataset()+ except DeltaProtocolError as e:+ raise ObjectStorageError(f"{DELTA_UNSUPPORTED_ERROR}: {e}") from e+ except TableNotFoundError as e:+ raise ObjectStorageError(+ f"{NO_TABLE_FILES_ERROR} '{table_name}'. The Delta table is no longer at its folder in the bucket."+ ) from e+ except (DeltaError, OSError) as e:+ _raise_if_access_denied(e, table_name)+ raise++ if logger is not None:+ logger.info("Reading Delta table snapshot", table=table_name, delta_version=version)++ scanner = dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)+ yielded = False+ try:+ for batch in scanner.to_batches():+ if batch.num_rows:+ yielded = True+ yield pa.Table.from_batches([batch])+ except pa.ArrowInvalid as e:+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} a data file of the Delta table '{table_name}': {e}") from e+ except (DeltaError, OSError) as e:+ # The scan reads the data files lazily, so a key that can read the log but not the data fails here.+ _raise_if_access_denied(e, table_name)+ raise+ if not yielded:+ # An empty snapshot still has to reach the pipeline as a schema, or the table is dropped and+ # never recreated.+ yield dataset.schema.empty_table()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.pynew file mode 100644index 000000000000..a6090e29726b--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py@@ -0,0 +1,341 @@+import re+import fnmatch+import posixpath+import dataclasses+from collections.abc import Iterable, Mapping+from typing import Any, Literal, cast, get_args++from structlog.types import FilteringBoundLogger++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ CsvOptions,+ DiscoveryOptions,+ FileFormat,+ FileFormatChoice,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ NO_TABLE_FILES_ERROR,+ TOO_MANY_FILES_ERROR,+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ ObjectStore,+ StoredObject,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ MAX_TABLE_OBJECTS,+ MAX_TABLES,+)++TableKind = Literal["files", "delta"]+# Which files a `files` table reads: one file, a folder and its hive partitions, or every file.+FileScope = Literal["file", "folder", "all"]++DELTA_LOG_DIRECTORY = "_delta_log"+COMPRESSED_SUFFIX = ".gz"++EXTENSION_FORMATS: dict[str, FileFormat] = {+ ".csv": "csv",+ ".tsv": "csv",+ ".jsonl": "jsonl",+ ".ndjson": "jsonl",+ # Exports that land in buckets (Firehose, Spark, ClickHouse JSONEachRow) write one object per line+ # under a .json name, so .json files are read as JSON Lines.+ ".json": "jsonl",+ ".parquet": "parquet",+}++_FILE_FORMAT_CHOICES: frozenset[str] = frozenset(get_args(FileFormatChoice))+++@frozen+class TableDefinition:+ name: str+ kind: TableKind+ # Relative to the store root. For a `file` scope it is the file itself.+ path: str+ scope: FileScope | None+ # "delta" for a Delta table. For files: the format the files are read with, or "infer" when one+ # combined table mixes formats and each file is read by its extension.+ file_format: FileFormatChoice+ csv: CsvOptions+ pattern: str | None+ # The source's configured format at discovery, replayed at sync so the same files group together.+ source_format: FileFormatChoice+ files: tuple[StoredObject, ...] = ()++ def same_location(self, other: "TableDefinition") -> bool:+ return (+ self.kind == other.kind+ and self.scope == other.scope+ and self.path == other.path+ and self.file_format == other.file_format+ )++ def schema_metadata(self) -> dict[str, Any]:+ return {+ "kind": self.kind,+ "scope": self.scope,+ "path": self.path,+ "pattern": self.pattern,+ "format": self.file_format,+ "source_format": self.source_format,+ "csv": {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes},+ }++ @classmethod+ def from_schema_metadata(cls, name: str, metadata: Mapping[str, Any] | None) -> "TableDefinition | None":+ """Rebuild a table from what `schema_metadata` persisted, or None when the keys are missing."""+ if not metadata:+ return None+ kind = metadata.get("kind")+ scope = metadata.get("scope")+ path = metadata.get("path")+ file_format = metadata.get("format")+ source_format = metadata.get("source_format", file_format)+ pattern = metadata.get("pattern")+ csv = metadata.get("csv") or {}+ if (+ kind not in ("files", "delta")+ or (kind == "files" and scope not in ("file", "folder", "all"))+ or not isinstance(path, str)+ or file_format not in _FILE_FORMAT_CHOICES+ or source_format not in _FILE_FORMAT_CHOICES+ or not (pattern is None or isinstance(pattern, str))+ or not isinstance(csv, dict)+ ):+ return None+ return cls(+ name=name,+ kind=cast(TableKind, kind),+ path=path,+ scope=cast(FileScope, scope) if kind == "files" else None,+ file_format=cast(FileFormatChoice, file_format),+ csv=CsvOptions(+ has_header=bool(csv.get("has_header", True)),+ allow_double_quotes=bool(csv.get("allow_double_quotes", True)),+ ),+ pattern=pattern,+ source_format=cast(FileFormatChoice, source_format),+ )+++def sanitize_table_name(name: str) -> str:+ return re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_").lower()+++def _strip_compression(path: str) -> str:+ return path[: -len(COMPRESSED_SUFFIX)] if path.lower().endswith(COMPRESSED_SUFFIX) else path+++def is_compressed(path: str) -> bool:+ return path.lower().endswith(COMPRESSED_SUFFIX)+++def infer_file_format(path: str) -> FileFormat | None:+ extension = posixpath.splitext(_strip_compression(path).lower())[1]+ return EXTENSION_FORMATS.get(extension)+++def resolve_file_format(path: str, choice: FileFormatChoice) -> FileFormat | None:+ """The format one file is read with, or None when `infer` can't tell from its extension."""+ if choice in ("csv", "csv_no_header"):+ return "csv"+ if choice in ("jsonl", "parquet"):+ return cast(FileFormat, choice)+ return infer_file_format(path)+++def _table_format(path: str, choice: FileFormatChoice) -> FileFormatChoice | None:+ if choice == "infer":+ return infer_file_format(path)+ return choice+++def _is_hidden(segments: Iterable[str]) -> bool:+ # `_SUCCESS`, `_committed_*`, `.crc` and `_temporary/` are Spark and Hadoop bookkeeping, not data.+ return any(segment.startswith(("_", ".")) for segment in segments)+++def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:+ roots: set[str] = set()+ for stored in objects:+ segments = stored.path.split("/")+ if DELTA_LOG_DIRECTORY in segments[:-1]:+ root_segments = segments[: segments.index(DELTA_LOG_DIRECTORY)]+ if not _is_hidden(root_segments):+ roots.add("/".join(root_segments))+ return sorted(roots)+++def _is_under(path: str, roots: frozenset[str]) -> bool:+ if not roots:+ return False+ if "" in roots:+ return True+ segments = path.split("/")[:-1]+ return any("/".join(segments[:depth]) in roots for depth in range(1, len(segments) + 1))+++def _table_folder(path: str) -> str | None:+ """The folder a file's table is named after, or None for a file directly under the root.++ Hive partition folders (`year=2024/month=01`) belong to the table above them, so the folder ends+ at the first `key=value` segment.+ """+ directories = path.split("/")[:-1]+ if not directories:+ return None+ folder: list[str] = []+ for directory in directories:+ if "=" in directory:+ break+ folder.append(directory)+ return "/".join(folder)+++def _format_suffix(file_format: FileFormatChoice) -> str:+ return "csv" if file_format == "csv_no_header" else file_format+++def group_tables(+ objects: Iterable[StoredObject],+ options: DiscoveryOptions,+ root_name: str,+ logger: FilteringBoundLogger | None = None,+ max_tables: int | None = MAX_TABLES,+) -> list[TableDefinition]:+ """Turn a bucket listing into tables.++ - A folder holding `_delta_log/` is one Delta table, and none of its files are offered again.+ - Other files group by their folder, with hive partition folders folded into the table above.+ - A file directly under the root is a table of its own.+ - `combined_table_name` puts every file into one table.++ The same listing and options always give the same tables, which is what lets a sync find its+ table again from `schema_metadata`.+ """+ ordered = sorted(objects, key=lambda stored: stored.path)+ delta_roots = frozenset(_delta_roots(ordered))+ choice = options.file_format+ pattern = options.file_pattern or None+ combined_name = (options.combined_table_name or "").strip() or None++ def definition(+ name: str,+ kind: TableKind,+ path: str,+ scope: FileScope | None,+ file_format: FileFormatChoice,+ files: list[StoredObject],+ ) -> TableDefinition:+ return TableDefinition(+ name=name,+ kind=kind,+ path=path,+ scope=scope,+ file_format=file_format,+ csv=CsvOptions(has_header=file_format != "csv_no_header", allow_double_quotes=options.allow_double_quotes),+ pattern=pattern,+ source_format=choice,+ files=tuple(files),+ )++ candidates: list[TableDefinition] = []+ if choice in ("infer", "delta"):+ for root in sorted(delta_roots):+ candidates.append(definition(root.replace("/", "_") or root_name, "delta", root, None, "delta", []))++ if choice != "delta":+ data_files: list[tuple[StoredObject, FileFormatChoice]] = []+ for stored in ordered:+ segments = stored.path.split("/")+ if not segments[-1] or _is_hidden(segments) or _is_under(stored.path, delta_roots):+ continue+ if pattern is not None and not fnmatch.fnmatchcase(stored.path, pattern):+ continue+ file_format = _table_format(stored.path, choice)+ if file_format is not None:+ data_files.append((stored, file_format))++ if combined_name is not None:+ if data_files:+ formats = {file_format for _, file_format in data_files}+ combined_format = formats.pop() if len(formats) == 1 else "infer"+ candidates.append(+ definition(combined_name, "files", "", "all", combined_format, [f for f, _ in data_files])+ )+ else:+ folders: dict[tuple[str, FileFormatChoice], list[StoredObject]] = {}+ for stored, file_format in data_files:+ folder = _table_folder(stored.path)+ if folder is None:+ stem = posixpath.splitext(_strip_compression(stored.path))[0]+ candidates.append(definition(stem, "files", stored.path, "file", file_format, [stored]))+ else:+ folders.setdefault((folder, file_format), []).append(stored)+ formats_per_folder: dict[str, int] = {}+ for folder, _ in folders:+ formats_per_folder[folder] = formats_per_folder.get(folder, 0) + 1+ for (folder, file_format), files in folders.items():+ base = folder.replace("/", "_") or root_name+ name = f"{base}_{_format_suffix(file_format)}" if formats_per_folder[folder] > 1 else base+ candidates.append(definition(name, "files", folder, "folder", file_format, files))++ tables: list[TableDefinition] = []+ taken: set[str] = set()+ next_suffix: dict[str, int] = {}+ for candidate in candidates:+ base = sanitize_table_name(candidate.name) or "data"+ name = base+ suffix = next_suffix.get(base, 2)+ while name in taken:+ name = f"{base}_{suffix}"+ suffix += 1+ next_suffix[base] = suffix+ taken.add(name)+ tables.append(dataclasses.replace(candidate, name=name))++ if max_tables is not None and len(tables) > max_tables:+ if logger is not None:+ logger.warning("Reached the object storage table limit; later tables are ignored", max_tables=max_tables)+ tables = tables[:max_tables]+ return tables+++def resolve_table_files(+ store: ObjectStore,+ table: TableDefinition,+ max_objects: int = MAX_TABLE_OBJECTS,+ logger: FilteringBoundLogger | None = None,+) -> TableDefinition:+ """List a `files` table's folder again and return the table with the files it holds now.++ The listing runs through `group_tables` with the options the table was discovered with, so a+ subfolder that is a table of its own, or a file another format claims, stays out.+ """+ if table.kind != "files":+ return table+ listing_prefix = "" if table.scope == "all" else table.path+ objects = store.list_objects(listing_prefix, max_objects + 1)+ if len(objects) > max_objects:+ raise ObjectStorageError(+ f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{listing_prefix or '/'}'. Point the source at a "+ "narrower folder, or set a file pattern that matches fewer files."+ )+ options = DiscoveryOptions(+ file_format=table.source_format,+ file_pattern=table.pattern,+ allow_double_quotes=table.csv.allow_double_quotes,+ combined_table_name=table.name if table.scope == "all" else None,+ )+ for candidate in group_tables(objects, options, root_name=table.name, logger=logger, max_tables=None):+ # A combined table keeps its identity when a new file changes which formats it mixes.+ if candidate.same_location(table) or (table.scope == "all" and candidate.scope == "all"):+ return dataclasses.replace(candidate, name=table.name)+ raise ObjectStorageError(+ f"{NO_TABLE_FILES_ERROR} '{table.name}'. The files it was discovered from may have been renamed, moved, "+ "or removed from the bucket."+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.pynew file mode 100644index 000000000000..d2a96f7e3488--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.py@@ -0,0 +1,48 @@+# Error message prefixes. `get_non_retryable_errors` matches on these, so they have to stay stable+# and stay out of the interpolated part of each message.+INVALID_CONFIG_ERROR = "The storage settings are invalid"+BLOCKED_LOCATION_ERROR = "PostHog can't import from this storage location"+ACCESS_DENIED_ERROR = "The storage provider denied access to the bucket"+NO_SUCH_BUCKET_ERROR = "The bucket doesn't exist"+WRONG_REGION_ERROR = "The bucket is in a different region"+NO_IMPORTABLE_FILES_ERROR = "No importable files were found"+NO_TABLE_FILES_ERROR = "No files are available for table"+TOO_MANY_FILES_ERROR = "The table has more files than PostHog can import"+FILE_PARSE_ERROR = "Couldn't parse the file"+DELTA_UNSUPPORTED_ERROR = "The Delta table uses features PostHog can't read"+DELTA_READ_ERROR = "Couldn't read the Delta table"++# S3 error codes, and the message S3 sends with a 301, that mean the request went to the wrong+# regional endpoint for this bucket. s3fs keeps only the message when it translates the error.+_WRONG_REGION_MARKERS = (+ "PermanentRedirect",+ "AuthorizationHeaderMalformed",+ "IllegalLocationConstraint",+ "must be addressed using the specified endpoint",+)+++class ObjectStorageError(Exception):+ """A problem the user has to fix: rejected credentials, a missing bucket, an unreadable file."""+++def translate_storage_error(error: Exception, location: str) -> Exception:+ """Map a filesystem error to an `ObjectStorageError` with a stable prefix, or return it unchanged.++ s3fs turns S3 error codes into builtin exceptions: `AccessDenied`, `InvalidAccessKeyId` and+ `SignatureDoesNotMatch` become `PermissionError`, and `NoSuchBucket` becomes `FileNotFoundError`.+ Anything else stays as it is, so a transient failure keeps its retryable message.+ """+ if isinstance(error, ObjectStorageError):+ return error+ if isinstance(error, PermissionError):+ return ObjectStorageError(+ f"{ACCESS_DENIED_ERROR} '{location}'. Check the access key and secret, and that the key can list "+ "and read objects in this bucket."+ )+ if isinstance(error, FileNotFoundError):+ return ObjectStorageError(f"{NO_SUCH_BUCKET_ERROR}: '{location}'. Check the bucket name.")+ message = str(error)+ if any(marker in message for marker in _WRONG_REGION_MARKERS):+ return ObjectStorageError(f"{WRONG_REGION_ERROR} from the one configured for '{location}'. Fix the region.")+ return errordiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.pynew file mode 100644index 000000000000..087825c3b01c--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py@@ -0,0 +1,205 @@+import dataclasses+from datetime import UTC, datetime+from typing import IO, Any, cast++import s3fs+import boto3+from botocore.config import Config as BotoConfig+from botocore.exceptions import ClientError+from fsspec import AbstractFileSystem+from s3fs.errors import translate_boto_error++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ ObjectStoreConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ TOO_MANY_FILES_ERROR,+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ CONNECT_TIMEOUT_SECONDS,+ MAX_LISTED_KEYS,+ MAX_REQUEST_ATTEMPTS,+ READ_TIMEOUT_SECONDS,+)++S3_MAX_PAGE_SIZE = 1000+++@frozen+class StoredObject:+ # Relative to the store root, '/'-separated.+ path: str+ size: int+ modified_at: datetime+++@frozen+class ObjectStore:+ """A filesystem rooted at the configured bucket folder, plus what deltalake needs to read it.++ Delta tables are read by delta-rs, which does not use the fsspec filesystem, so the store also+ carries the URI prefix and the storage options for it.+ """++ fs: AbstractFileSystem+ # The fsspec path of the folder: `bucket/prefix` for S3.+ root: str+ delta_uri_prefix: str+ delta_storage_options: dict[str, str] = dataclasses.field(repr=False)++ def path(self, relative: str) -> str:+ return f"{self.root}/{relative}" if relative else self.root++ def delta_uri(self, relative: str) -> str:+ return f"{self.delta_uri_prefix}{self.path(relative)}"++ def list_objects(self, relative_prefix: str, max_objects: int) -> list[StoredObject]:+ """List up to `max_objects` files under the prefix, in path order.++ The prefix is a folder, or one file. This generic version walks the whole fsspec tree, so+ stores with a large remote listing override it with a paginated, capped one.+ """+ found = cast(dict[str, dict[str, Any]], self.fs.find(self.path(relative_prefix), detail=True))+ root_prefix = f"{self.root}/"+ objects: list[StoredObject] = []+ for name in sorted(found):+ info = found[name]+ if info.get("type") != "file" or not name.startswith(root_prefix):+ continue+ objects.append(+ StoredObject(+ path=name[len(root_prefix) :], size=int(info.get("size") or 0), modified_at=_modified_at(info)+ )+ )+ if len(objects) >= max_objects:+ break+ return objects++ def open(self, relative: str) -> IO[bytes]:+ return cast(IO[bytes], self.fs.open(self.path(relative), "rb"))+++def _modified_at(info: dict[str, Any]) -> datetime:+ last_modified = info.get("LastModified")+ if isinstance(last_modified, datetime):+ return last_modified if last_modified.tzinfo else last_modified.replace(tzinfo=UTC)+ return datetime.fromtimestamp(float(info.get("mtime") or 0), tz=UTC)+++def build_s3_filesystem(config: ObjectStoreConfig) -> s3fs.S3FileSystem:+ client_kwargs: dict[str, Any] = {"region_name": config.region}+ return s3fs.S3FileSystem(+ key=config.access_key_id,+ secret=config.secret_access_key,+ endpoint_url=config.endpoint_url,+ client_kwargs=client_kwargs,+ config_kwargs={+ "connect_timeout": CONNECT_TIMEOUT_SECONDS,+ "read_timeout": READ_TIMEOUT_SECONDS,+ "retries": {"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},+ },+ # fsspec caches instances by their arguments, which would keep every customer's credentials+ # alive in the worker. Directory listings are re-read on every run anyway.+ skip_instance_cache=True,+ use_listings_cache=False,+ )+++def delta_storage_options(config: ObjectStoreConfig) -> dict[str, str]:+ options = {+ "AWS_ACCESS_KEY_ID": config.access_key_id,+ "AWS_SECRET_ACCESS_KEY": config.secret_access_key,+ "AWS_REGION": config.region,+ "connect_timeout": f"{CONNECT_TIMEOUT_SECONDS}s",+ "timeout": f"{READ_TIMEOUT_SECONDS}s",+ }+ if config.endpoint_url:+ options["AWS_ENDPOINT_URL"] = config.endpoint_url+ return options+++@frozen+class S3ObjectStore(ObjectStore):+ """Lists through a boto3 client and reads through s3fs.++ s3fs caps a listing by asking for a page of twice the cap, which S3-compatible stores reject+ above their own page limit. The boto3 paginator keeps pages at the S3 maximum and stops at the cap.+ """++ client: Any = dataclasses.field(repr=False)+ bucket: str+ prefix: str++ def list_objects(self, relative_prefix: str, max_objects: int) -> list[StoredObject]:+ key = "/".join(part for part in (self.prefix, relative_prefix) if part)+ objects = self._list(f"{key}/" if key else "", max_objects)+ if not objects and relative_prefix:+ # The prefix may name one file rather than a folder.+ objects = [stored for stored in self._list(key, 1) if stored.path == relative_prefix]+ return objects++ def _list(self, key_prefix: str, max_objects: int) -> list[StoredObject]:+ root_key = f"{self.prefix}/" if self.prefix else ""+ paginator = self.client.get_paginator("list_objects_v2")+ # The file cap counts files, not keys: folder marker keys are skipped, so stopping after a fixed+ # number of keys could hide that the folder holds more files than the cap. A separate cap on+ # the keys walked keeps a prefix full of marker keys from running the listing without end.+ pages = paginator.paginate(+ Bucket=self.bucket, Prefix=key_prefix, PaginationConfig={"PageSize": S3_MAX_PAGE_SIZE}+ )+ objects: list[StoredObject] = []+ walked_keys = 0+ try:+ for page in pages:+ for item in page.get("Contents", []):+ walked_keys += 1+ if walked_keys > MAX_LISTED_KEYS:+ raise ObjectStorageError(+ f"{TOO_MANY_FILES_ERROR} ({MAX_LISTED_KEYS:,} keys) under '{key_prefix or '/'}'. Point "+ "the source at a narrower folder."+ )+ object_key: str = item["Key"]+ if not object_key.startswith(root_key) or object_key.endswith("/"):+ continue+ objects.append(+ StoredObject(+ path=object_key[len(root_key) :],+ size=int(item.get("Size") or 0),+ modified_at=_modified_at(item),+ )+ )+ if len(objects) >= max_objects:+ return objects+ except ClientError as e:+ raise translate_boto_error(e) from e+ return objects+++def build_s3_client(config: ObjectStoreConfig) -> Any:+ return boto3.client(+ "s3",+ aws_access_key_id=config.access_key_id,+ aws_secret_access_key=config.secret_access_key,+ region_name=config.region,+ endpoint_url=config.endpoint_url,+ config=BotoConfig(+ connect_timeout=CONNECT_TIMEOUT_SECONDS,+ read_timeout=READ_TIMEOUT_SECONDS,+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},+ ),+ )+++def open_s3_store(config: ObjectStoreConfig) -> S3ObjectStore:+ return S3ObjectStore(+ fs=build_s3_filesystem(config),+ root=f"{config.bucket}/{config.prefix}" if config.prefix else config.bucket,+ delta_uri_prefix="s3://",+ delta_storage_options=delta_storage_options(config),+ client=build_s3_client(config),+ bucket=config.bucket,+ prefix=config.prefix,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pynew file mode 100644index 000000000000..ef63b282aa4b--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -0,0 +1,330 @@+import io+import csv+import gzip+import json+import zlib+import tempfile+from collections.abc import Iterator+from typing import IO, Any, Literal, cast++import pyarrow as pa+import pyarrow.csv as pa_csv+import pyarrow.parquet as pq+from structlog.types import FilteringBoundLogger++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (+ FILE_MODIFIED_AT_COLUMN,+ FILE_PATH_COLUMN,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ CsvOptions,+ FileFormat,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressed+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ FILE_PARSE_ERROR,+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import StoredObject+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ CSV_BLOCK_SIZE_BYTES,+ JSONL_CHUNK_BYTES,+ JSONL_CHUNK_ROWS,+ MAX_FIRST_CSV_RECORD_BYTES,+ MAX_RECORD_BYTES,+ MAX_UNPACKED_PARQUET_BYTES,+ PARQUET_BATCH_ROWS,+)++TSV_SUFFIXES = (".tsv", ".tsv.gz")+UTF8_BOM = b"\xef\xbb\xbf"+SPOOL_MEMORY_BYTES = 8 * 1024 * 1024+COPY_CHUNK_BYTES = 1024 * 1024+++class _PrefixedStream(io.RawIOBase):+ """Replays bytes already read from a stream before the rest of it.++ The first CSV line is read ahead to name the columns. A headerless file still needs that line+ parsed as data, so it goes back in front of the stream.+ """++ def __init__(self, prefix: bytes, rest: IO[bytes]) -> None:+ self._prefix = prefix+ self._rest = rest++ def readable(self) -> bool:+ return True++ def readinto(self, buffer: Any) -> int:+ view = memoryview(buffer)+ if self._prefix:+ count = min(len(view), len(self._prefix))+ view[:count] = self._prefix[:count]+ self._prefix = self._prefix[count:]+ return count+ data = self._rest.read(len(view))+ view[: len(data)] = data+ return len(data)+++def csv_column_names(cells: list[str], has_header: bool) -> list[str]:+ """Header names as written, made unique. A headerless file gets ClickHouse's `c1`, `c2`, ... names."""+ if not has_header:+ return [f"c{index}" for index in range(1, len(cells) + 1)]+ names: list[str] = []+ seen: set[str] = set()+ next_suffix: dict[str, int] = {}+ for index, cell in enumerate(cells, start=1):+ base = cell.strip() or f"c{index}"+ name = base+ suffix = next_suffix.get(base, 2)+ while name in seen:+ name = f"{base}_{suffix}"+ suffix += 1+ next_suffix[base] = suffix+ seen.add(name)+ names.append(name)+ return names+++def _read_bounded_line(stream: IO[bytes], file_path: str, kind: str) -> bytes:+ """Read one line, refusing a line longer than `MAX_RECORD_BYTES` rather than holding it in memory."""+ line = stream.readline(MAX_RECORD_BYTES + 1)+ if len(line) > MAX_RECORD_BYTES:+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{file_path}' as {kind}: a line is longer than {MAX_RECORD_BYTES:,} bytes."+ )+ return line+++def _read_first_csv_record(+ stream: IO[bytes], delimiter: str, allow_double_quotes: bool, file_path: str+) -> tuple[bytes, list[str]]:+ """The first non-blank record as the bytes it spans and its fields.++ The CSV reader pulls lines until the record is complete, so a quoted field that spans lines is+ one record, and a stray quote inside an unquoted field is not mistaken for the start of one.+ """+ consumed = bytearray()++ def lines() -> Iterator[str]:+ while True:+ line = _read_bounded_line(stream, file_path, "CSV")+ if not line:+ return+ if not consumed:+ if not line.strip():+ continue+ line = line.removeprefix(UTF8_BOM)+ if len(consumed) + len(line) > MAX_FIRST_CSV_RECORD_BYTES:+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{file_path}' as CSV: the first record is longer than "+ f"{MAX_FIRST_CSV_RECORD_BYTES:,} bytes."+ )+ consumed.extend(line)+ yield line.decode("utf-8", errors="replace")++ if allow_double_quotes:+ reader = csv.reader(lines(), delimiter=delimiter)+ else:+ reader = csv.reader(lines(), delimiter=delimiter, quoting=csv.QUOTE_NONE)+ try:+ fields = next(reader, [])+ except csv.Error as e:+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: the first record is invalid ({e}).") from e+ return bytes(consumed), fields+++def iter_csv_tables(+ stream: IO[bytes],+ options: CsvOptions,+ delimiter: str = ",",+ file_path: str = "",+ logger: FilteringBoundLogger | None = None,+) -> Iterator[pa.Table]:+ """Stream a CSV file as string columns.++ Every column is read as text: Arrow fixes a column's type from the first block it sees, so a+ later value that does not fit would fail the whole file mid-stream, and two files of one table+ could disagree on a type. Empty unquoted fields are null.+ """+ first_record, first_fields = _read_first_csv_record(stream, delimiter, options.allow_double_quotes, file_path)+ if not first_record:+ return+ names = csv_column_names(first_fields, options.has_header)+ body: IO[bytes] = stream if options.has_header else cast(IO[bytes], _PrefixedStream(first_record, stream))++ skipped_rows = 0++ def skip_invalid_row(_row: pa_csv.InvalidRow) -> Literal["skip", "error"]:+ nonlocal skipped_rows+ skipped_rows += 1+ return "skip"++ yielded = False+ try:+ reader = pa_csv.open_csv(+ body,+ read_options=pa_csv.ReadOptions(column_names=names, block_size=CSV_BLOCK_SIZE_BYTES, use_threads=False),+ parse_options=pa_csv.ParseOptions(+ delimiter=delimiter,+ quote_char='"' if options.allow_double_quotes else False,+ newlines_in_values=options.allow_double_quotes,+ ignore_empty_lines=True,+ invalid_row_handler=skip_invalid_row,+ ),+ convert_options=pa_csv.ConvertOptions(+ column_types=dict.fromkeys(names, pa.string()),+ null_values=[""],+ strings_can_be_null=True,+ quoted_strings_can_be_null=False,+ ),+ )+ for batch in reader:+ if batch.num_rows:+ yielded = True+ yield pa.Table.from_batches([batch])+ except pa.ArrowInvalid as e:+ if options.has_header and not yielded and "Empty CSV file" in str(e):+ # The file holds only its header. It is a table with no rows, not a broken file.+ yield pa.table({name: pa.array([], pa.string()) for name in names})+ return+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: {e}") from e++ if skipped_rows and logger is not None:+ # A row whose field count differs from the header is malformed, or the file uses another+ # delimiter. Skipping it keeps its values out of the wrong columns.+ logger.warning(+ "Skipped CSV rows whose field count differs from the header",+ file=file_path,+ skipped_rows=skipped_rows,+ expected=len(names),+ )+++def _as_rows(value: Any) -> Iterator[dict[str, Any]]:+ # A generator, so a line holding a huge array is wrapped into rows a batch at a time.+ if isinstance(value, dict):+ yield value+ elif isinstance(value, list):+ for item in value:+ yield item if isinstance(item, dict) else {"value": item}+ else:+ yield {"value": value}+++def iter_jsonl_tables(+ stream: IO[bytes],+ file_path: str = "",+ max_rows: int = JSONL_CHUNK_ROWS,+ max_bytes: int = JSONL_CHUNK_BYTES,+) -> Iterator[pa.Table]:+ rows: list[dict[str, Any]] = []+ pending_bytes = 0+ line_number = 0+ while True:+ line = _read_bounded_line(stream, file_path, "JSON Lines")+ if not line:+ break+ line_number += 1+ if not line.strip():+ continue+ try:+ parsed = json.loads(line)+ except (ValueError, RecursionError) as e:+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{file_path}' as JSON Lines: line {line_number} isn't valid JSON ({e})."+ ) from e+ pending_bytes += len(line)+ for row in _as_rows(parsed):+ rows.append(row)+ if len(rows) >= max_rows:+ yield table_from_py_list(rows)+ rows = []+ pending_bytes = 0+ if rows and pending_bytes >= max_bytes:+ yield table_from_py_list(rows)+ rows = []+ pending_bytes = 0+ if rows:+ yield table_from_py_list(rows)+++def iter_parquet_tables(+ stream: IO[bytes], file_path: str = "", batch_rows: int = PARQUET_BATCH_ROWS+) -> Iterator[pa.Table]:+ try:+ parquet_file = pq.ParquetFile(stream)+ yielded = False+ for batch in parquet_file.iter_batches(batch_size=batch_rows, use_threads=False):+ if batch.num_rows:+ yielded = True+ yield pa.Table.from_batches([batch])+ if not yielded:+ # A file with no rows still carries a schema, which the pipeline needs to create the table.+ yield parquet_file.schema_arrow.empty_table()+ except pa.ArrowInvalid as e:+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from e+++def with_file_columns(table: pa.Table, stored: StoredObject) -> pa.Table:+ """Tag every row with the file it came from, so a table built from many files stays traceable."""+ for column in (FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN):+ if column in table.column_names:+ table = table.drop_columns([column])+ rows = table.num_rows+ table = table.append_column(FILE_PATH_COLUMN, pa.repeat(pa.scalar(stored.path, pa.string()), rows))+ return table.append_column(+ FILE_MODIFIED_AT_COLUMN,+ pa.repeat(pa.scalar(stored.modified_at, pa.timestamp("us", tz="UTC")), rows),+ )+++def _iter_gzipped_parquet(stream: IO[bytes], stored: StoredObject) -> Iterator[pa.Table]:+ """Parquet readers seek to the footer, which a gzip stream can't, so the file is unpacked to disk first."""+ with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES) as unpacked:+ try:+ with gzip.GzipFile(fileobj=stream) as compressed:+ unpacked_bytes = 0+ while chunk := compressed.read(COPY_CHUNK_BYTES):+ unpacked_bytes += len(chunk)+ if unpacked_bytes > MAX_UNPACKED_PARQUET_BYTES:+ raise ObjectStorageError(+ f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: it unpacks to more than "+ f"{MAX_UNPACKED_PARQUET_BYTES:,} bytes."+ )+ unpacked.write(chunk)+ except (gzip.BadGzipFile, zlib.error, EOFError) as e:+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: {e}") from e+ unpacked.seek(0)+ for table in iter_parquet_tables(cast(IO[bytes], unpacked), stored.path):+ yield with_file_columns(table, stored)+++def iter_file_tables(+ stream: IO[bytes],+ stored: StoredObject,+ file_format: FileFormat,+ csv_options: CsvOptions,+ logger: FilteringBoundLogger | None = None,+) -> Iterator[pa.Table]:+ """Read one file as Arrow tables of bounded size, each carrying the `_file_*` columns."""+ if file_format == "parquet":+ if is_compressed(stored.path):+ yield from _iter_gzipped_parquet(stream, stored)+ return+ tables = iter_parquet_tables(stream, stored.path)+ else:+ binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else stream+ if file_format == "csv":+ delimiter = "\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ","+ tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)+ else:+ tables = iter_jsonl_tables(binary, stored.path)+ try:+ for table in tables:+ yield with_file_columns(table, stored)+ except (gzip.BadGzipFile, zlib.error, EOFError) as e:+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: {e}") from ediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.pynew file mode 100644index 000000000000..c092feffe954--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py@@ -0,0 +1,30 @@+# A bucket listing is one flat, paginated walk (1,000 keys per request), so these caps bound both the+# request count and the memory a listing holds.+MAX_DISCOVERY_OBJECTS = 10_000+# A sync lists only the table's own folder. The cap is higher than discovery's because a sync that+# silently skipped files would replace the table with a partial copy, so going past it fails the sync.+MAX_TABLE_OBJECTS = 50_000+# A listing also walks folder marker keys (ending in `/`), which are never files, so the walk has its own cap.+MAX_LISTED_KEYS = 100_000+# One table per root-level file can turn a large flat folder into thousands of tables.+MAX_TABLES = 1_000++CONNECT_TIMEOUT_SECONDS = 10+READ_TIMEOUT_SECONDS = 60+MAX_REQUEST_ATTEMPTS = 3++CSV_BLOCK_SIZE_BYTES = 4 * 1024 * 1024+JSONL_CHUNK_ROWS = 5_000+JSONL_CHUNK_BYTES = 64 * 1024 * 1024+# One JSON Lines record, or the first line of a CSV file, is held whole in memory to be parsed. A gzip+# file can expand far past its stored size, so reads stop at this many decompressed bytes.+MAX_RECORD_BYTES = 32 * 1024 * 1024+# The first record of a CSV file names the columns, and is read whole before the streaming reader starts.+MAX_FIRST_CSV_RECORD_BYTES = 1024 * 1024+# A gzipped Parquet file is unpacked to local disk to be read, so the unpacked size is capped too.+MAX_UNPACKED_PARQUET_BYTES = 2 * 1024 * 1024 * 1024+PARQUET_BATCH_ROWS = 10_000+DELTA_BATCH_ROWS = 16_384++# Arrow batches from wide files can be large, so the pipeline's batcher flushes before this many bytes.+CHUNK_SIZE_BYTES = 100 * 1024 * 1024diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pynew file mode 100644index 000000000000..b0a377d642a5--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -0,0 +1,205 @@+from abc import abstractmethod+from collections.abc import Iterator+from typing import Generic, NoReturn, Optional++import pyarrow as pa+from structlog.types import FilteringBoundLogger++from products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, SimpleSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ DiscoveryOptions,+ ObjectStoreConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import iter_delta_tables+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (+ TableDefinition,+ group_tables,+ resolve_file_format,+ resolve_table_files,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ACCESS_DENIED_ERROR,+ BLOCKED_LOCATION_ERROR,+ DELTA_UNSUPPORTED_ERROR,+ FILE_PARSE_ERROR,+ INVALID_CONFIG_ERROR,+ NO_IMPORTABLE_FILES_ERROR,+ NO_SUCH_BUCKET_ERROR,+ NO_TABLE_FILES_ERROR,+ TOO_MANY_FILES_ERROR,+ WRONG_REGION_ERROR,+ ObjectStorageError,+ translate_storage_error,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ ObjectStore,+ open_s3_store,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (+ iter_file_tables,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ CHUNK_SIZE_BYTES,+ MAX_DISCOVERY_OBJECTS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse+++def _raise_translated(error: Exception, location: str) -> NoReturn:+ translated = translate_storage_error(error, location)+ if translated is error:+ raise error+ raise translated from error+++def validate_store_location(config: ObjectStoreConfig) -> None:+ """Apply the checks a self-managed warehouse table URL gets: https, no internal hosts, and no+ bucket that PostHog itself owns."""+ is_valid, error = validate_warehouse_table_url_pattern(config.location_url)+ if not is_valid:+ raise ObjectStorageError(f"{BLOCKED_LOCATION_ERROR}: {error}")+++class ObjectStorageSource(SimpleSource[ConfigType], Generic[ConfigType]):+ """Imports the files in a bucket folder: CSV, JSON Lines and Parquet files, and Delta tables.++ Every sync reads each table in full, so the table matches the bucket at the time of the sync.+ """++ @abstractmethod+ def object_store_config(self, config: ConfigType) -> ObjectStoreConfig: ...++ @abstractmethod+ def discovery_options(self, config: ConfigType) -> DiscoveryOptions: ...++ def open_store(self, store_config: ObjectStoreConfig) -> ObjectStore:+ """Re-check the location every time the stored credentials are about to be sent to it."""+ validate_store_location(store_config)+ return open_s3_store(store_config)++ def get_non_retryable_errors(self) -> dict[str, str | None]:+ return {+ INVALID_CONFIG_ERROR: None,+ BLOCKED_LOCATION_ERROR: None,+ ACCESS_DENIED_ERROR: (+ "The storage provider rejected these credentials. Check the access key and secret, and that "+ "the key can list and read the bucket, then reconnect the source."+ ),+ NO_SUCH_BUCKET_ERROR: "The bucket doesn't exist. Check the bucket name, then reconnect the source.",+ WRONG_REGION_ERROR: "The bucket is in a different region. Fix the region, then reconnect the source.",+ NO_TABLE_FILES_ERROR: (+ "The files behind this table are no longer in the bucket. Refresh the source's tables, or "+ "restore the files."+ ),+ TOO_MANY_FILES_ERROR: None,+ FILE_PARSE_ERROR: None,+ DELTA_UNSUPPORTED_ERROR: None,+ }++ def _discover(+ self, store: ObjectStore, config: ConfigType, logger: FilteringBoundLogger | None = None+ ) -> list[TableDefinition]:+ store_config = self.object_store_config(config)+ try:+ objects = store.list_objects("", MAX_DISCOVERY_OBJECTS)+ except Exception as e:+ _raise_translated(e, store_config.display_location)+ if len(objects) >= MAX_DISCOVERY_OBJECTS and logger is not None:+ logger.warning("Reached the object storage listing limit; later objects are ignored")+ root_name = store_config.prefix.rsplit("/", 1)[-1] if store_config.prefix else store_config.bucket+ return group_tables(objects, self.discovery_options(config), root_name=root_name, logger=logger)++ def validate_credentials(+ self,+ config: ConfigType,+ team_id: int,+ schema_name: Optional[str] = None,+ api_version: str | None = None,+ ) -> tuple[bool, str | None]:+ try:+ store_config = self.object_store_config(config)+ tables = self._discover(self.open_store(store_config), config)+ except ObjectStorageError as e:+ return False, str(e)+ if not tables:+ return False, (+ f"{NO_IMPORTABLE_FILES_ERROR} in '{store_config.display_location}'. PostHog can read the bucket but "+ "found nothing to import there. Check the folder, the file pattern, and the file format."+ )+ return True, None++ def get_schemas(+ self,+ config: ConfigType,+ team_id: int,+ with_counts: bool = False,+ names: list[str] | None = None,+ force_refresh: bool = False,+ api_version: str | None = None,+ ) -> list[SourceSchema]:+ store = self.open_store(self.object_store_config(config))+ requested = set(names) if names is not None else None+ return [+ SourceSchema(+ name=table.name,+ supports_incremental=False,+ supports_append=False,+ schema_metadata=table.schema_metadata(),+ )+ for table in self._discover(store, config)+ if requested is None or table.name in requested+ ]++ def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> SourceResponse:+ store = self.open_store(self.object_store_config(config))+ table = TableDefinition.from_schema_metadata(inputs.schema_name, inputs.schema_metadata)++ def items() -> Iterator[pa.Table]:+ resolved = (+ table if table is not None else self._find_table(store, config, inputs.schema_name, inputs.logger)+ )+ yield from iter_table(store, resolved, inputs.logger)++ return SourceResponse(+ name=inputs.schema_name,+ items=items,+ # Files carry no stable row identity, so tables are replaced wholesale on every sync.+ primary_keys=None,+ chunk_size_bytes=CHUNK_SIZE_BYTES,+ )++ def _find_table(+ self, store: ObjectStore, config: ConfigType, schema_name: str, logger: FilteringBoundLogger+ ) -> TableDefinition:+ """Find a table by name when its schema predates `schema_metadata`."""+ for table in self._discover(store, config, logger):+ if table.name == schema_name:+ return table+ raise ObjectStorageError(+ f"{NO_TABLE_FILES_ERROR} '{schema_name}'. The files it was discovered from may have been renamed, "+ "moved, or removed from the bucket."+ )+++def iter_table(store: ObjectStore, table: TableDefinition, logger: FilteringBoundLogger) -> Iterator[pa.Table]:+ if table.kind == "delta":+ yield from iter_delta_tables(+ store.delta_uri(table.path), store.delta_storage_options, table.name, logger=logger+ )+ return++ try:+ resolved = resolve_table_files(store, table, logger=logger)+ except Exception as e:+ _raise_translated(e, store.root)+ for stored in resolved.files:+ file_format = resolve_file_format(stored.path, resolved.file_format)+ if file_format is None:+ continue+ try:+ with store.open(stored.path) as stream:+ yield from iter_file_tables(stream, stored, file_format, resolved.csv, logger)+ except PermissionError as e:+ _raise_translated(e, store.root)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pynew file mode 100644index 000000000000..6624d8efb173--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -0,0 +1,668 @@+import io+import csv+import gzip+from datetime import UTC, datetime+from pathlib import Path+from typing import Any++import pytest++from django.test import override_settings++import boto3+import pyarrow as pa+import structlog+import pyarrow.parquet as pq+from botocore.stub import Stubber+from deltalake import write_deltalake+from fsspec.implementations.local import LocalFileSystem+from parameterized import parameterized++from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import formats+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ CsvOptions,+ DiscoveryOptions,+ ObjectStoreConfig,+ ObjectStoreProvider,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import iter_delta_tables+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (+ TableDefinition,+ group_tables,+ resolve_table_files,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ACCESS_DENIED_ERROR,+ BLOCKED_LOCATION_ERROR,+ FILE_PARSE_ERROR,+ INVALID_CONFIG_ERROR,+ NO_SUCH_BUCKET_ERROR,+ NO_TABLE_FILES_ERROR,+ TOO_MANY_FILES_ERROR,+ WRONG_REGION_ERROR,+ ObjectStorageError,+ translate_storage_error,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ ObjectStore,+ S3ObjectStore,+ StoredObject,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (+ iter_csv_tables,+ iter_jsonl_tables,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import MAX_TABLES+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (+ iter_table,+ validate_store_location,+)++MODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)+LOGGER = structlog.get_logger()+++def stored(path: str) -> StoredObject:+ return StoredObject(path=path, size=10, modified_at=MODIFIED_AT)+++def local_store(root: Path) -> ObjectStore:+ return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})+++def summarize(tables: list[TableDefinition]) -> list[tuple[str, str, str | None, str, str, list[str]]]:+ return [+ (table.name, table.kind, table.scope, table.path, table.file_format, [f.path for f in table.files])+ for table in tables+ ]+++def read_rows(store: ObjectStore, table: TableDefinition) -> list[dict]:+ return [row for batch in iter_table(store, table, LOGGER) for row in batch.to_pylist()]+++class TestGroupTables:+ @parameterized.expand(+ [+ (+ "delta_folder_is_one_table_and_hides_its_files",+ [+ "sales/_delta_log/00000000000000000000.json",+ "sales/part-0.parquet",+ "events/a.csv",+ ],+ DiscoveryOptions(),+ [+ ("sales", "delta", None, "sales", "delta", []),+ ("events", "files", "folder", "events", "csv", ["events/a.csv"]),+ ],+ ),+ (+ "root_files_are_one_table_each",+ ["orders.csv.gz", "users.jsonl", "notes.txt"],+ DiscoveryOptions(),+ [+ ("orders", "files", "file", "orders.csv.gz", "csv", ["orders.csv.gz"]),+ ("users", "files", "file", "users.jsonl", "jsonl", ["users.jsonl"]),+ ],+ ),+ (+ "hive_partitions_fold_into_the_table_above",+ ["events/year=2024/month=01/a.parquet", "events/year=2024/month=02/b.parquet"],+ DiscoveryOptions(),+ [+ (+ "events",+ "files",+ "folder",+ "events",+ "parquet",+ ["events/year=2024/month=01/a.parquet", "events/year=2024/month=02/b.parquet"],+ ),+ ],+ ),+ (+ "hive_partitions_under_the_root_name_the_table_after_the_root",+ ["year=2024/a.parquet"],+ DiscoveryOptions(),+ [("exports", "files", "folder", "", "parquet", ["year=2024/a.parquet"])],+ ),+ (+ "combine_puts_every_file_in_one_table_and_keeps_delta_tables",+ ["a.csv", "daily/b.csv", "sales/_delta_log/0.json"],+ DiscoveryOptions(combined_table_name="All Orders"),+ [+ ("sales", "delta", None, "sales", "delta", []),+ ("all_orders", "files", "all", "", "csv", ["a.csv", "daily/b.csv"]),+ ],+ ),+ (+ "combine_with_mixed_formats_reads_each_file_by_extension",+ ["a.csv", "b.parquet"],+ DiscoveryOptions(combined_table_name="mixed"),+ [("mixed", "files", "all", "", "infer", ["a.csv", "b.parquet"])],+ ),+ (+ "delta_format_offers_only_delta_tables",+ ["a.csv", "sales/_delta_log/0.json", "sales/p.parquet"],+ DiscoveryOptions(file_format="delta"),+ [("sales", "delta", None, "sales", "delta", [])],+ ),+ (+ "explicit_format_reads_any_extension_and_skips_delta_tables",+ ["exports/a.txt", "exports/b", "sales/_delta_log/0.json", "sales/p.parquet"],+ DiscoveryOptions(file_format="csv_no_header"),+ [("exports", "files", "folder", "exports", "csv_no_header", ["exports/a.txt", "exports/b"])],+ ),+ (+ "spark_bookkeeping_files_are_skipped",+ ["out/_SUCCESS", "out/.part-0.crc", "out/_temporary/x.csv", "out/part-0.csv"],+ DiscoveryOptions(),+ [("out", "files", "folder", "out", "csv", ["out/part-0.csv"])],+ ),+ (+ "pattern_matches_across_folders",+ ["a/x.csv", "a/y.json", "b/z.csv"],+ DiscoveryOptions(file_pattern="*.csv"),+ [+ ("a", "files", "folder", "a", "csv", ["a/x.csv"]),+ ("b", "files", "folder", "b", "csv", ["b/z.csv"]),+ ],+ ),+ (+ "a_folder_with_two_formats_names_a_table_per_format",+ ["data/a.csv", "data/b.parquet"],+ DiscoveryOptions(),+ [+ ("data_csv", "files", "folder", "data", "csv", ["data/a.csv"]),+ ("data_parquet", "files", "folder", "data", "parquet", ["data/b.parquet"]),+ ],+ ),+ (+ "names_that_sanitize_alike_get_a_suffix",+ ["a-b.csv", "a_b.csv"],+ DiscoveryOptions(),+ [+ ("a_b", "files", "file", "a-b.csv", "csv", ["a-b.csv"]),+ ("a_b_2", "files", "file", "a_b.csv", "csv", ["a_b.csv"]),+ ],+ ),+ ]+ )+ def test_group_tables(+ self,+ _name: str,+ paths: list[str],+ options: DiscoveryOptions,+ expected: list[tuple[str, str, str | None, str, str, list[str]]],+ ) -> None:+ tables = group_tables([stored(path) for path in paths], options, root_name="exports")++ assert summarize(tables) == expected++ def test_metadata_round_trip_rebuilds_the_same_table(self) -> None:+ options = DiscoveryOptions(file_format="csv_no_header", file_pattern="*.txt", allow_double_quotes=False)+ [table] = group_tables([stored("exports/a.txt")], options, root_name="root")++ rebuilt = TableDefinition.from_schema_metadata(table.name, table.schema_metadata())++ assert rebuilt is not None+ assert rebuilt.same_location(table)+ assert (rebuilt.csv, rebuilt.pattern, rebuilt.source_format) == (+ CsvOptions(has_header=False, allow_double_quotes=False),+ "*.txt",+ "csv_no_header",+ )++ @parameterized.expand(+ [+ ("missing", None),+ ("unknown_kind", {"kind": "folder", "path": "a", "format": "csv"}),+ ("files_without_scope", {"kind": "files", "path": "a", "format": "csv"}),+ ("unknown_format", {"kind": "files", "scope": "folder", "path": "a", "format": "xml"}),+ ]+ )+ def test_unusable_metadata_falls_back_to_discovery(self, _name: str, metadata: dict | None) -> None:+ assert TableDefinition.from_schema_metadata("a", metadata) is None+++class TestTableLimit:+ def test_names_that_sanitize_alike_get_increasing_suffixes(self) -> None:+ objects = [stored(f"{name}.csv") for name in ("a-b", "a_b", "a b", "a.b")]++ names = [table.name for table in group_tables(objects, DiscoveryOptions(), root_name="root")]++ assert sorted(names) == ["a_b", "a_b_2", "a_b_3", "a_b_4"]++ def test_an_unknown_file_format_is_an_invalid_config(self) -> None:+ with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):+ DiscoveryOptions(file_format="avro") # type: ignore[arg-type]++ def test_discovery_stops_at_the_table_limit_but_resolving_a_table_does_not(self) -> None:+ objects = [stored(f"file_{index:05d}.csv") for index in range(MAX_TABLES + 1)]++ assert len(group_tables(objects, DiscoveryOptions(), root_name="root")) == MAX_TABLES+ assert len(group_tables(objects, DiscoveryOptions(), root_name="root", max_tables=None)) == MAX_TABLES + 1+++class TestResolveTableFiles:+ def test_a_folder_table_leaves_out_subfolders_that_are_tables_of_their_own(self, tmp_path: Path) -> None:+ for path in ("events/a.csv", "events/year=2024/b.csv", "events/clicks/c.csv"):+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / path).write_text("id\n1\n")+ store = local_store(tmp_path)+ table = TableDefinition.from_schema_metadata(+ "events",+ {"kind": "files", "scope": "folder", "path": "events", "format": "csv", "source_format": "infer"},+ )+ assert table is not None++ resolved = resolve_table_files(store, table)++ assert [f.path for f in resolved.files] == ["events/a.csv", "events/year=2024/b.csv"]+++ "paths, max_objects, expected",+ [+ (["t/a.csv", "t/b.csv", "t/c.csv"], 2, TOO_MANY_FILES_ERROR),+ (["other/a.csv"], 10, NO_TABLE_FILES_ERROR),+ ],+)+def test_sync_refuses_a_partial_or_missing_table(+ tmp_path: Path, paths: list[str], max_objects: int, expected: str+) -> None:+ for path in paths:+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / path).write_text("id\n1\n")+ table = TableDefinition.from_schema_metadata(+ "t", {"kind": "files", "scope": "folder", "path": "t", "format": "csv", "source_format": "infer"}+ )+ assert table is not None++ with pytest.raises(ObjectStorageError, match=expected):+ resolve_table_files(local_store(tmp_path), table, max_objects=max_objects)+++class TestCsv:+ @parameterized.expand(+ [+ (+ "header_with_quoted_delimiter",+ b'id,name\n1,"Smith, Jo"\n',+ CsvOptions(),+ [{"id": "1", "name": "Smith, Jo"}],+ ),+ (+ "no_header_uses_clickhouse_names",+ b"1,a\n2,b\n",+ CsvOptions(has_header=False),+ [+ {"c1": "1", "c2": "a"},+ {"c1": "2", "c2": "b"},+ ],+ ),+ (+ "double_quotes_off_keeps_quotes_as_text",+ b'id,name\n"1,2"\n',+ CsvOptions(allow_double_quotes=False),+ [{"id": '"1', "name": '2"'}],+ ),+ (+ "quoted_value_spans_lines",+ b'id,note\n1,"line one\nline two"\n',+ CsvOptions(),+ [{"id": "1", "note": "line one\nline two"}],+ ),+ (+ "empty_unquoted_is_null_and_empty_quoted_is_text",+ b'a,b\n,""\n',+ CsvOptions(),+ [{"a": None, "b": ""}],+ ),+ (+ "values_stay_text_when_later_rows_change_type",+ b"n\n1\nnot a number\n",+ CsvOptions(),+ [{"n": "1"}, {"n": "not a number"}],+ ),+ (+ "rows_with_the_wrong_field_count_are_skipped",+ b"a,b\n1,2\n1,2,3\n4,5\n",+ CsvOptions(),+ [{"a": "1", "b": "2"}, {"a": "4", "b": "5"}],+ ),+ (+ "byte_order_mark_and_duplicate_headers",+ b"\xef\xbb\xbfid,id,\n1,2,3\n",+ CsvOptions(),+ [{"id": "1", "id_2": "2", "c3": "3"}],+ ),+ ("empty_file", b"", CsvOptions(), []),+ (+ "headerless_quote_inside_an_unquoted_field_does_not_open_a_record",+ b'1,12" pipe,3\n4,5,6\n',+ CsvOptions(has_header=False),+ [{"c1": "1", "c2": '12" pipe', "c3": "3"}, {"c1": "4", "c2": "5", "c3": "6"}],+ ),+ (+ "repeated_headers_get_increasing_suffixes",+ b"a,a,a,a_2\n1,2,3,4\n",+ CsvOptions(),+ [{"a": "1", "a_2": "2", "a_3": "3", "a_2_2": "4"}],+ ),+ (+ "headerless_first_record_spans_lines",+ b'1,"line one\nline two",3\n4,5,6\n',+ CsvOptions(has_header=False),+ [{"c1": "1", "c2": "line one\nline two", "c3": "3"}, {"c1": "4", "c2": "5", "c3": "6"}],+ ),+ ]+ )+ def test_csv_rows(self, _name: str, content: bytes, options: CsvOptions, expected: list[dict]) -> None:+ rows = [row for table in iter_csv_tables(io.BytesIO(content), options) for row in table.to_pylist()]++ assert rows == expected++ def test_an_unterminated_quote_in_the_first_record_stays_within_the_record_limit(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(formats, "MAX_FIRST_CSV_RECORD_BYTES", 64)+ content = b'1,"open\n' + b"line\n" * 100++ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):+ list(iter_csv_tables(io.BytesIO(content), CsvOptions(has_header=False), file_path="x.csv"))++ def test_a_header_only_file_is_a_table_with_no_rows(self) -> None:+ tables = list(iter_csv_tables(io.BytesIO(b"id,name\n"), CsvOptions()))++ assert [(table.column_names, table.num_rows) for table in tables] == [(["id", "name"], 0)]++ def test_an_oversized_first_row_field_is_a_parse_error(self) -> None:+ # Other code in the process may have raised the csv module's global field limit.+ previous_limit = csv.field_size_limit(1024)+ try:+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):+ content = io.BytesIO(b"a," + b"x" * 2048 + b"\n")+ list(iter_csv_tables(content, CsvOptions(has_header=False), file_path="big.csv"))+ finally:+ csv.field_size_limit(previous_limit)+++class TestJsonl:+ def test_a_line_holding_a_large_array_is_chunked_by_row_count(self) -> None:+ tables = list(iter_jsonl_tables(io.BytesIO(b"[" + b",".join(b"1" for _ in range(25)) + b"]\n"), max_rows=10))++ assert [table.num_rows for table in tables] == [10, 10, 5]++ def test_chunks_by_row_count_and_wraps_scalars(self) -> None:+ content = b'{"id": 1}\n\n{"id": 2}\n[{"id": 3}, 4]\n"five"\n'++ tables = list(iter_jsonl_tables(io.BytesIO(content), max_rows=2))++ assert [table.num_rows for table in tables] == [2, 2, 1]+ assert [row for table in tables for row in table.to_pylist()] == [+ {"id": 1},+ {"id": 2},+ {"id": 3, "value": None},+ {"id": None, "value": 4},+ {"value": "five"},+ ]++ def test_a_too_deeply_nested_record_is_a_parse_error(self) -> None:+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):+ list(iter_jsonl_tables(io.BytesIO(b"[" * 100_000 + b"]" * 100_000 + b"\n"), "deep.jsonl"))++ def test_invalid_line_names_the_file_and_line(self) -> None:+ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'x.jsonl' as JSON Lines: line 2"):+ list(iter_jsonl_tables(io.BytesIO(b'{"id": 1}\n{oops\n'), "x.jsonl"))++ def test_a_line_over_the_record_limit_is_a_parse_error(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(formats, "MAX_RECORD_BYTES", 16)++ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'x.jsonl' as JSON Lines"):+ list(iter_jsonl_tables(io.BytesIO(b'{"id": 1}\n{"id": "' + b"x" * 64 + b'"}\n'), "x.jsonl"))+++class TestReadTable:+ def test_reads_each_format_and_tags_rows_with_their_file(self, tmp_path: Path) -> None:+ (tmp_path / "mixed").mkdir()+ (tmp_path / "mixed" / "a.csv.gz").write_bytes(gzip.compress(b"id\n1\n"))+ (tmp_path / "mixed" / "b.jsonl").write_bytes(b'{"id": "2"}\n')+ pq.write_table(pa.table({"id": ["3"]}), tmp_path / "mixed" / "c.parquet")+ store = local_store(tmp_path)+ [table] = group_tables(+ store.list_objects("", 100), DiscoveryOptions(combined_table_name="mixed"), root_name="root"+ )++ rows = read_rows(store, table)++ assert [(row["id"], row["_file_name"]) for row in rows] == [+ ("1", "mixed/a.csv.gz"),+ ("2", "mixed/b.jsonl"),+ ("3", "mixed/c.parquet"),+ ]+ assert all(row["_file_modified_at"].tzinfo is not None for row in rows)++ def test_gzipped_parquet_is_unpacked_before_it_is_read(self, tmp_path: Path) -> None:+ buffer = io.BytesIO()+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)+ (tmp_path / "orders.parquet.gz").write_bytes(gzip.compress(buffer.getvalue()))+ store = local_store(tmp_path)+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")++ assert [row["id"] for row in read_rows(store, table)] == ["1", "2"]++ def test_gzipped_parquet_over_the_unpacked_size_limit_is_a_parse_error(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(formats, "MAX_UNPACKED_PARQUET_BYTES", 16)+ (tmp_path / "big.parquet.gz").write_bytes(gzip.compress(b"x" * 1024))+ store = local_store(tmp_path)+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")++ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):+ read_rows(store, table)++ def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:+ pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")+ store = local_store(tmp_path)+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")++ batches = list(iter_table(store, table, LOGGER))++ assert [(batch.column_names[0], batch.num_rows) for batch in batches] == [("id", 0)]++ def test_corrupt_gzip_is_a_parse_error(self, tmp_path: Path) -> None:+ (tmp_path / "bad.csv.gz").write_bytes(b"not gzip")+ store = local_store(tmp_path)+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")++ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):+ read_rows(store, table)++ def test_delta_read_stays_on_the_version_it_opened(self, tmp_path: Path) -> None:+ table_path = tmp_path / "sales"+ write_deltalake(str(table_path), pa.table({"id": [1]}))+ write_deltalake(str(table_path), pa.table({"id": [2]}), mode="append")+ store = local_store(tmp_path)+ [table] = group_tables(store.list_objects("", 100), DiscoveryOptions(), root_name="root")+ assert table.kind == "delta"++ batches = iter_delta_tables(store.delta_uri(table.path), {}, table.name, batch_rows=1)+ first = next(batches)+ write_deltalake(str(table_path), pa.table({"id": [7, 8, 9]}), mode="overwrite")+ rest = list(batches)++ assert sorted(row["id"] for batch in [first, *rest] for row in batch.to_pylist()) == [1, 2]++ def test_an_empty_delta_snapshot_still_yields_its_schema(self, tmp_path: Path) -> None:+ table_path = tmp_path / "sales"+ write_deltalake(str(table_path), pa.table({"id": [1]}))+ write_deltalake(str(table_path), pa.table({"id": pa.array([], pa.int64())}), mode="overwrite")+ store = local_store(tmp_path)++ tables = list(iter_delta_tables(store.delta_uri("sales"), {}, "sales"))++ assert [(table.column_names, table.num_rows) for table in tables] == [(["id"], 0)]+++class TestLocationAndErrors:+ @parameterized.expand(+ [+ ("bucket_with_a_host", {"bucket": "evil.com/x"}, INVALID_CONFIG_ERROR),+ ("bucket_with_a_scheme", {"bucket": "s3://data"}, INVALID_CONFIG_ERROR),+ ("region_with_a_host", {"region": "us-east-1.evil.com"}, INVALID_CONFIG_ERROR),+ ("missing_secret", {"secret_access_key": " "}, INVALID_CONFIG_ERROR),+ ("parent_folder", {"prefix": "a/../b"}, INVALID_CONFIG_ERROR),+ ]+ )+ def test_config_rejects_values_that_could_retarget_the_request(+ self, _name: str, overrides: dict, expected: str+ ) -> None:+ values = {+ "provider": ObjectStoreProvider.S3,+ "bucket": "customer-data",+ "region": "us-east-1",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "secret",+ **overrides,+ }++ with pytest.raises(ObjectStorageError, match=expected):+ ObjectStoreConfig(**values)++ def test_posthog_owned_bucket_is_blocked(self) -> None:+ config = ObjectStoreConfig(+ provider=ObjectStoreProvider.S3,+ bucket="posthog-owned",+ region="us-east-1",+ access_key_id="AKIAEXAMPLE",+ secret_access_key="secret",+ )++ with override_settings(DATAWAREHOUSE_BUCKET="posthog-owned"):+ with pytest.raises(ObjectStorageError, match=BLOCKED_LOCATION_ERROR):+ validate_store_location(config)++ @parameterized.expand(+ [+ ("access_denied", PermissionError("Access Denied"), ACCESS_DENIED_ERROR),+ ("no_such_bucket", FileNotFoundError("The specified bucket does not exist"), NO_SUCH_BUCKET_ERROR),+ (+ "wrong_region",+ OSError("An error occurred (PermanentRedirect) when calling the ListObjectsV2 operation"),+ WRONG_REGION_ERROR,+ ),+ ]+ )+ def test_storage_errors_map_to_stable_prefixes(self, _name: str, error: Exception, expected: str) -> None:+ assert str(translate_storage_error(error, "customer-data")).startswith(expected)++ def test_unknown_storage_errors_stay_retryable(self) -> None:+ error = ConnectionResetError("reset by peer")++ assert translate_storage_error(error, "customer-data") is error+++def _s3_store(client: Any) -> S3ObjectStore:+ return S3ObjectStore(+ fs=LocalFileSystem(),+ root="customer-data/exports",+ delta_uri_prefix="s3://",+ delta_storage_options={},+ client=client,+ bucket="customer-data",+ prefix="exports",+ )+++def _listing(*keys: str, next_token: str | None = None) -> dict[str, Any]:+ response: dict[str, Any] = {+ "Contents": [{"Key": key, "Size": 5, "LastModified": MODIFIED_AT} for key in keys],+ "IsTruncated": next_token is not None,+ }+ if next_token is not None:+ response["NextContinuationToken"] = next_token+ return response+++class TestS3Listing:+ @parameterized.expand(+ [+ (+ "pages_until_the_cap",+ "",+ 3,+ [+ ({"Prefix": "exports/"}, _listing("exports/a.csv", "exports/dir/", next_token="t1")),+ (+ {"Prefix": "exports/", "ContinuationToken": "t1"},+ _listing("exports/b.csv", "exports/c.csv", "exports/d.csv"),+ ),+ ],+ ["a.csv", "b.csv", "c.csv"],+ ),+ (+ "a_prefix_that_names_one_file",+ "orders.csv",+ 10,+ [+ ({"Prefix": "exports/orders.csv/"}, _listing()),+ ({"Prefix": "exports/orders.csv", "MaxKeys": 1000}, _listing("exports/orders.csv")),+ ],+ ["orders.csv"],+ ),+ ]+ )+ def test_list_objects(+ self,+ _name: str,+ relative_prefix: str,+ max_objects: int,+ pages: list[tuple[dict[str, Any], dict[str, Any]]],+ expected: list[str],+ ) -> None:+ client = boto3.client(+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"+ )+ with Stubber(client) as stubber:+ for params, response in pages:+ stubber.add_response(+ "list_objects_v2", response, {"Bucket": "customer-data", "MaxKeys": 1000, **params}+ )++ listed = _s3_store(client).list_objects(relative_prefix, max_objects)++ assert [stored.path for stored in listed] == expected++ def test_a_prefix_of_folder_markers_fails_instead_of_walking_every_key(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem."+ "MAX_LISTED_KEYS",+ 3,+ )+ client = boto3.client(+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"+ )+ with Stubber(client) as stubber:+ stubber.add_response(+ "list_objects_v2",+ _listing("exports/a/", "exports/b/", "exports/c/", "exports/d/"),+ {"Bucket": "customer-data", "MaxKeys": 1000, "Prefix": "exports/"},+ )++ with pytest.raises(ObjectStorageError, match=TOO_MANY_FILES_ERROR):+ _s3_store(client).list_objects("", 10)++ def test_rejected_credentials_become_an_access_error(self) -> None:+ client = boto3.client(+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"+ )+ with Stubber(client) as stubber:+ stubber.add_client_error("list_objects_v2", service_error_code="InvalidAccessKeyId", http_status_code=403)++ with pytest.raises(PermissionError):+ _s3_store(client).list_objects("", 10)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.pyindex 68b0163449da..9fb8f23589dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py@@ -1,9 +1,25 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.+from typing import Literal+from products.warehouse_sources.backend.temporal.data_imports.sources.common import config+class AmazonS3CombineFilesConfig(config.Config):+ table_name: str+ enabled: bool = config.value(converter=config.str_to_bool, default=False)++@config.configclass AmazonS3SourceConfig(config.Config):- pass+ bucket: str+ region: str+ access_key_id: str+ secret_access_key: str+ path: str | None = None+ file_pattern: str | None = None+ file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)+ combine_files: AmazonS3CombineFilesConfig | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex f1ff82120f6a..fe778d40d5f0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -3,6 +3,7 @@ Adyen fail:safe-pointAirbrake fail:rate-limit-budget,stall-budgetAlphaVantage fail:rate-limit-budget,stall-budgetAmazonAds fail:rate-limit-budget,stall-budget+AmazonS3 not-checkable:schemasAmplitude fail:rate-limit-budget,stall-budgetAnthropic fail:safe-pointApollo fail:rate-limit-budget,stall-budgetdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.pyindex e30f7d15e0a8..9d5422caaa02 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py@@ -49,6 +49,7 @@ PASS = "pass"# `config`: `get_schemas` rejects every placeholder config.# `empty`: `get_schemas` returns no schema for a placeholder config.SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {+ "AmazonS3": "network","BigQuery": "driver","Baserow": "network","CheckoutCom": "network",@@ -94,6 +95,7 @@ SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {# `request`: the extraction sends a request before it checks the schema, or the vendor supplies the list.# `webhook`: the schemas are webhook-only, so the extraction accepts any name and yields no rows.UNKNOWN_SCHEMA_NOT_CHECKABLE: dict[str, str] = {+ "AmazonS3": "request","Attentive": "webhook","Baserow": "request","BigQuery": "driver",diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex a4ce99e4352f..2ffa77aaaa9f 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -161,6 +161,7 @@ def _configure_source_mock_versioning(mock_get_source) -> None:mock_get_source.return_value.default_version = "v1"mock_get_source.return_value.get_version_deprecation.return_value = Nonemock_get_source.return_value.max_instances_per_team = None+ mock_get_source.return_value.requires_team_feature_flag = Nonemock_get_source.return_value.connection_host_fields = []mock_get_source.return_value.detects_primary_keys = Falsemock_get_source.return_value.server_managed_job_input_fields.return_value = []@@ -5972,6 +5973,7 @@ class TestExternalDataSource(APIBaseTest):]mock_source.get_endpoint_permissions.return_value = {}mock_source.detects_primary_keys = False+ mock_source.requires_team_feature_flag = Noneresponse = self.client.post(f"/api/environments/{self.team.pk}/external_data_sources/database_schema/",@@ -9324,6 +9326,41 @@ class TestExternalDataSource(APIBaseTest):else:assert response.json()["message"] != limit_message+ @parameterized.expand(+ [+ ("create_without_flag", "", False),+ ("create_with_flag", "", True),+ ("database_schema_without_flag", "database_schema/", False),+ ("database_schema_with_flag", "database_schema/", True),+ ("setup_without_flag", "setup/", False),+ ("setup_with_flag", "setup/", True),+ ("store_credentials_without_flag", "store_credentials/", False),+ ("store_credentials_with_flag", "store_credentials/", True),+ ]+ )+ def test_source_behind_a_team_feature_flag(self, _name: str, path: str, flag_enabled: bool) -> None:+ payload = {+ "bucket": "Not A Bucket",+ "region": "us-east-1",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "secret",+ }+ with patch(+ "products.warehouse_sources.backend.presentation.views.external_data_source.helpers.feature_enabled_or_false",+ return_value=flag_enabled,+ ):+ response = self.client.post(+ f"/api/environments/{self.team.pk}/external_data_sources/{path}",+ data={"source_type": "AmazonS3", "prefix": "s3_", "payload": payload, **payload},+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ # With the flag the request reaches credential validation, which rejects the bucket name+ # before any network call.+ expected = "The storage settings are invalid" if flag_enabled else "isn't available to connect yet"+ assert expected in response.json()["message"]+ assert not ExternalDataSource.objects.filter(team_id=self.team.pk, source_type="AmazonS3").exists()+def test_revenue_analytics_config_created_automatically(self):source = self._create_external_data_source()@@ -12985,10 +13022,10 @@ class TestExternalDataSourceSetup(APIBaseTest):@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.capture_exception")def test_setup_rejects_source_without_schema_discovery(self, mock_capture_exception):- # AmazonS3 doesn't implement get_schemas, so the base raises NotImplementedError.+ # AmazonCloudWatch doesn't implement get_schemas, so the base raises NotImplementedError.response = self.client.post(f"/api/environments/{self.team.pk}/external_data_sources/setup/",- data={"source_type": "AmazonS3", "prefix": "s3_setup_test", "payload": {}},+ data={"source_type": "AmazonCloudWatch", "prefix": "scaffold_setup_test", "payload": {}},)assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()assert "isn't available to connect yet" in response.json()["message"]