Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add amazon s3 import source on a shared object storage backend (#116382)

Tom Owers6602226bf2f6
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py b/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py
index 82c455c4817c..1eb1ebff5f8f 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/credential_store.py
@@ -20,7 +20,7 @@ from posthog.models.user import User
from products.warehouse_sources.backend.facade.models import PendingSourceCredential
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
-from . import base
+from . import base, helpers
class SourceConnectLinkSerializer(serializers.Serializer):
@@ -165,6 +165,11 @@ class ExternalDataSourceCredentialStoreMixin(base.ExternalDataSourceViewSetBase)
source_type_model = ExternalDataSourceType(source_type)
source = base.SourceRegistry.get_source(source_type_model)
+ if helpers.source_feature_flag_missing(source, self.team):
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": helpers._source_unavailable_message(source_type)},
+ )
error_response, _ = self._validate_source_config_and_credentials(source, source_type_model, payload)
if error_response is not None:
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
index 93681481743c..2706f6ee8ac6 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
@@ -4,13 +4,14 @@ from __future__ import annotations
import re
from collections.abc import Iterable, Mapping
-from typing import Any
+from typing import TYPE_CHECKING, Any
from urllib.parse import unquote, urlsplit
from django.db.models import Q, QuerySet
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
+from posthog.ph_client import feature_enabled_or_false
from products.data_warehouse.backend.facade.api import (
DirectQueryEngine,
@@ -42,6 +43,9 @@ from products.warehouse_sources.backend.facade.source_management import (
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType, ManagedWarehouseSQLMode
+if TYPE_CHECKING:
+ from posthog.models.team import Team
+
REFRESH_SCHEMAS_FALLBACK_ERROR_MESSAGE = "Could not fetch schemas from source."
RESERVED_SOURCE_NAME_MESSAGE = "This source name is reserved by PostHog."
@@ -70,6 +74,20 @@ def _source_unavailable_message(source_type: str) -> str:
)
+def source_feature_flag_missing(source: AnySource, team: Team) -> bool:
+ """Whether the team lacks the feature flag the source requires before the API serves it."""
+ flag = source.requires_team_feature_flag
+ if flag is None:
+ return False
+ organization_id = str(team.organization_id)
+ return not feature_enabled_or_false(
+ flag,
+ organization_id,
+ groups={"organization": organization_id, "project": str(team.id)},
+ group_properties={"organization": {"id": organization_id}},
+ )
+
+
def _canonical_legacy_managed_warehouse_source(
queryset: QuerySet[ExternalDataSource],
) -> ExternalDataSource | None:
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
index 05613ccae919..5980ad393242 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
@@ -370,6 +370,11 @@ class ExternalDataSourceSchemaOperationsMixin(base.ExternalDataSourceViewSetBase
data={"message": f"Unknown source_type '{source_type}'"},
)
source = base.SourceRegistry.get_source(source_type_model)
+ if helpers.source_feature_flag_missing(source, self.team):
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": helpers._source_unavailable_message(source_type)},
+ )
is_valid, errors = source.validate_config(request.data)
if not is_valid:
return Response(
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 720f68df1af8..d3b39f6b071e 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -1184,6 +1184,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
).data,
status=status.HTTP_400_BAD_REQUEST,
)
+ if helpers.source_feature_flag_missing(source, self.team):
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": helpers._source_unavailable_message(source_type)},
+ )
max_instances = source.max_instances_per_team
if max_instances is not None and helpers.count_active_sources(self.team_id, source_type_model) >= max_instances:
return Response(
@@ -2018,6 +2023,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
source_type_model = ExternalDataSourceType(source_type)
source = base.SourceRegistry.get_source(source_type_model)
+ if helpers.source_feature_flag_missing(source, self.team):
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": helpers._source_unavailable_message(source_type)},
+ )
error_response, source_config = self._validate_source_config_and_credentials(source, source_type_model, payload)
if error_response is not None or source_config is None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index 009c2b7f64eb..b3cb0ff97225 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -76,6 +76,7 @@ the row lists both.
| alguna | HTTP | requests + `rest_source.RESTClient` | ✅ |
| alpha_vantage | HTTP | requests | ✅ |
| amazon_ads | HTTP | requests | ✅ |
+| amazon_s3 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ |
| amplemarket | HTTP | requests + `rest_source.RESTClient` | ✅ |
| amplitude | HTTP | requests | ✅ |
| anthropic | HTTP | requests | ✅ |
@@ -964,6 +965,10 @@ the row lists both.
- **linkedin_ads** uses `linkedin-api`'s `RestliClient`, which constructs its own internal `requests.Session`.
We don't yet have a session-injection seam on it, so outbound calls bypass the tracked transport. (The file
imports `requests` only for exception types — those references are expected and don't need a pragma.)
+- **amazon_s3** lists through a boto3 client, reads files through `s3fs` (aiobotocore over aiohttp), and reads
+ Delta tables through delta-rs, which has its own Rust object store client. None of the three exposes a
+ `requests` session or an interceptor seam, so outbound traffic bypasses the tracked transport. Each client sets
+ its own connect and read timeouts and a bounded retry count.
- **temporalio** talks gRPC, but the Python `temporalio` SDK runs its entire gRPC stack in a Rust core
(`temporalio.bridge`, a PyO3 module) — service RPCs dispatch through `ServiceClient._rpc_call` into the
bridge, so there is no Python `grpc.Channel` or client-interceptor seam for the tracked gRPC transport to
@@ -1018,7 +1023,6 @@ doesn't conflict with concurrent PRs.
- amazon_cloudwatch
- amazon_eventbridge
- amazon_kinesis
-- amazon_s3
- amazon_selling_partner
- amazon_sns
- amazon_sqs
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
index d7a2a3678d5e..6399b6159f14 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
@@ -1,28 +1,195 @@
from typing import cast
-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.backend.facade.source_config import (
+ DataWarehouseSourceCategory,
+ ReleaseStatus,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigConverter,
+ SourceFieldSelectConfigOption,
+ SourceFieldSwitchGroupConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ DiscoveryOptions,
+ ObjectStoreConfig,
+ ObjectStoreProvider,
+ normalize_prefix,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
+ ObjectStorageSource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (
AmazonS3SourceConfig,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+FEATURE_FLAG = "dwh-object-storage-sources"
+
@SourceRegistry.register
-class AmazonS3Source(SimpleSource[AmazonS3SourceConfig]):
+class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
+ codebase_detection = detect(
+ npm=("@aws-sdk/client-s3",),
+ pypi=("s3fs",),
+ cargo=("aws-sdk-s3",),
+ go=("github.com/aws/aws-sdk-go-v2/service/s3",),
+ maven=("software.amazon.awssdk:s3",),
+ )
+
+ api_docs_url = "https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html"
+ requires_team_feature_flag = FEATURE_FLAG
+
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.AMAZONS3
+ @property
+ def connection_host_fields(self) -> list[str]:
+ # The bucket and region pick where the stored key is sent, so changing either must re-require it.
+ return ["bucket", "region"]
+
+ def object_store_config(self, config: AmazonS3SourceConfig) -> ObjectStoreConfig:
+ return ObjectStoreConfig(
+ provider=ObjectStoreProvider.S3,
+ bucket=config.bucket.strip(),
+ region=config.region.strip(),
+ access_key_id=config.access_key_id.strip(),
+ secret_access_key=config.secret_access_key.strip(),
+ prefix=normalize_prefix(config.path),
+ )
+
+ def discovery_options(self, config: AmazonS3SourceConfig) -> DiscoveryOptions:
+ combine = config.combine_files
+ return DiscoveryOptions(
+ file_format=config.file_format,
+ file_pattern=(config.file_pattern or "").strip() or None,
+ allow_double_quotes=config.csv_allow_double_quotes,
+ combined_table_name=combine.table_name if combine is not None and combine.enabled else None,
+ )
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.AMAZONS3,
category=DataWarehouseSourceCategory.FILE_STORAGE,
- keywords=["s3"],
+ keywords=["s3", "aws", "bucket", "parquet", "delta lake"],
label="Amazon S3",
+ caption=(
+ "Import CSV, JSON Lines and Parquet files, and Delta Lake tables, from an S3 bucket. PostHog "
+ "creates one table per folder, one per file at the top of the folder you choose, and one per "
+ "Delta table. Every sync reads the files in full, so each table matches the bucket."
+ ),
iconPath="/static/services/aws-s3.png",
- fields=cast(list[FieldType], []),
- unreleasedSource=True,
+ releaseStatus=ReleaseStatus.ALPHA,
+ featureFlag=FEATURE_FLAG,
+ fields=cast(
+ list[FieldType],
+ [
+ SourceFieldInputConfig(
+ name="bucket",
+ label="Bucket",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="my-company-exports",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="region",
+ label="Region",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="us-east-1",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="path",
+ label="Folder (optional)",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="exports/",
+ caption="Leave it empty to import from the whole bucket.",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="file_pattern",
+ label="File pattern (optional)",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="*.parquet",
+ caption=(
+ "A glob matched against each file's path inside the folder, where * also matches "
+ "across folders. Leave it empty to import every file."
+ ),
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="access_key_id",
+ label="Access key ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="AKIA...",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="secret_access_key",
+ label="Secret access key",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="",
+ secret=True,
+ ),
+ SourceFieldSelectConfig(
+ name="file_format",
+ label="File format",
+ required=True,
+ defaultValue="infer",
+ options=[
+ SourceFieldSelectConfigOption(label="Detect from file extension", value="infer"),
+ SourceFieldSelectConfigOption(label="CSV with a header row", value="csv"),
+ SourceFieldSelectConfigOption(label="CSV without a header row", value="csv_no_header"),
+ SourceFieldSelectConfigOption(label="JSON Lines", value="jsonl"),
+ SourceFieldSelectConfigOption(label="Parquet", value="parquet"),
+ SourceFieldSelectConfigOption(label="Delta Lake", value="delta"),
+ ],
+ ),
+ SourceFieldSelectConfig(
+ name="csv_allow_double_quotes",
+ label="Do double quotes enclose CSV values?",
+ required=True,
+ defaultValue="true",
+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
+ options=[
+ SourceFieldSelectConfigOption(label="Yes", value="true"),
+ SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),
+ ],
+ ),
+ SourceFieldSwitchGroupConfig(
+ name="combine_files",
+ label="Combine every file into one table?",
+ caption=(
+ "Turn this on when the folder holds files that share the same columns, such as a "
+ "daily export. Delta tables still get a table each."
+ ),
+ default=False,
+ fields=cast(
+ list[FieldType],
+ [
+ SourceFieldInputConfig(
+ name="table_name",
+ label="Table name",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="orders",
+ secret=False,
+ ),
+ ],
+ ),
+ ),
+ ],
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
new file mode 100644
index 000000000000..ad7f346ca97b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -0,0 +1,129 @@
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, cast
+
+import pytest
+
+import pyarrow as pa
+from fsspec import AbstractFileSystem
+from fsspec.implementations.local import LocalFileSystem
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ ObjectStoreConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ACCESS_DENIED_ERROR,
+ INVALID_CONFIG_ERROR,
+ NO_IMPORTABLE_FILES_ERROR,
+ NO_SUCH_BUCKET_ERROR,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import ObjectStore
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import source_inputs
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazons3 import (
+ AmazonS3SourceConfig,
+)
+
+
+class _DeniedFileSystem(LocalFileSystem):
+ def __init__(self, error: Exception) -> None:
+ super().__init__()
+ self.error = error
+
+ def find(self, path: str, *args: Any, **kwargs: Any) -> Any:
+ raise self.error
+
+
+class _LocalAmazonS3Source(AmazonS3Source):
+ """Serves the bucket from a local folder: `<root>/<bucket>/<prefix>`."""
+
+ def __init__(self, root: Path, fs: AbstractFileSystem | None = None) -> None:
+ self.root = root.resolve()
+ self.fs = fs or LocalFileSystem()
+
+ def open_store(self, store_config: ObjectStoreConfig) -> ObjectStore:
+ folder = self.root / store_config.bucket / store_config.prefix
+ return ObjectStore(fs=self.fs, root=str(folder).rstrip("/"), delta_uri_prefix="", delta_storage_options={})
+
+
+def make_config(**overrides: Any) -> AmazonS3SourceConfig:
+ values: dict[str, Any] = {
+ "bucket": "customer-data",
+ "region": "us-east-1",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "secret",
+ "path": "exports",
+ **overrides,
+ }
+ return AmazonS3SourceConfig.from_dict(values)
+
+
+def write(root: Path, relative: str, content: bytes) -> None:
+ path = root / "customer-data" / relative
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(content)
+
+
+def read_rows(source: AmazonS3Source, config: AmazonS3SourceConfig, schema_name: str, metadata: Any) -> list[dict]:
+ inputs = source_inputs(schema_name)
+ inputs.schema_metadata = metadata
+ response = source.source_for_pipeline(config, inputs)
+ return [row for table in cast(Iterable[pa.Table], response.items()) for row in table.to_pylist()]
+
+
[email protected]("with_metadata", [True, False])
+def test_sync_reads_the_table_discovery_offered(tmp_path: Path, with_metadata: bool) -> None:
+ write(tmp_path, "exports/orders/a.csv", b"id,total\n1,10\n")
+ write(tmp_path, "exports/orders/b.csv", b"id,total\n2,20\n")
+ write(tmp_path, "exports/orders/archive/c.csv", b"id,total\n3,30\n")
+ source = _LocalAmazonS3Source(tmp_path)
+ config = make_config()
+ schemas = {schema.name: schema for schema in source.get_schemas(config, team_id=1)}
+ assert set(schemas) == {"orders", "orders_archive"}
+ # The pipeline stores the metadata as JSON on the schema row.
+ metadata = json.loads(json.dumps(schemas["orders"].schema_metadata)) if with_metadata else None
+
+ rows = read_rows(source, config, "orders", metadata)
+
+ assert [(row["id"], row["_file_name"]) for row in rows] == [("1", "orders/a.csv"), ("2", "orders/b.csv")]
+
+
+def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -> None:
+ write(tmp_path, "exports/raw/a.txt", b"1,2\n")
+ source = _LocalAmazonS3Source(tmp_path)
+ [schema] = source.get_schemas(make_config(file_format="csv_no_header"), team_id=1)
+
+ rows = read_rows(source, make_config(file_format="infer"), schema.name, schema.schema_metadata)
+
+ assert rows[0]["c1"] == "1"
+
+
+ "files, config_overrides, fs_error, expected",
+ [
+ ({"exports/a.csv": b"id\n1\n"}, {}, None, None),
+ ({"exports/notes.md": b"# hi"}, {}, None, NO_IMPORTABLE_FILES_ERROR),
+ ({}, {"bucket": "Not A Bucket"}, None, INVALID_CONFIG_ERROR),
+ ({}, {}, PermissionError("Access Denied"), ACCESS_DENIED_ERROR),
+ ({}, {}, FileNotFoundError("NoSuchBucket"), NO_SUCH_BUCKET_ERROR),
+ ],
+)
+def test_validate_credentials(
+ tmp_path: Path,
+ files: dict[str, bytes],
+ config_overrides: dict[str, str],
+ fs_error: Exception | None,
+ expected: str | None,
+) -> None:
+ for relative, content in files.items():
+ write(tmp_path, relative, content)
+ (tmp_path / "customer-data" / "exports").mkdir(parents=True, exist_ok=True)
+ fs = _DeniedFileSystem(fs_error) if fs_error is not None else None
+ source = _LocalAmazonS3Source(tmp_path, fs)
+
+ valid, message = source.validate_credentials(make_config(**config_overrides), team_id=1)
+
+ assert valid is (expected is None)
+ if expected is not None:
+ assert message is not None and message.startswith(expected)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
index 07d61b416e61..10459a18844c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
@@ -473,6 +473,11 @@ class _BaseSource(ABC, Generic[ConfigType]):
incoming and existing inputs so a source can gate on context. Default: none."""
return []
+ # Feature flag a team must have before the API lets it discover tables for or create a source of
+ # this type. `SourceConfig.featureFlag` only hides the source in the UI, so a source that must
+ # not be reachable through the API without the flag sets this as well.
+ requires_team_feature_flag: str | None = None
+
# Cap on how many active sources of this type a team may have (``None`` = unlimited). Custom
# sources are capped; everything else is open. The API enforces this at create.
max_instances_per_team: int | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
new file mode 100644
index 000000000000..3d5c22961a4e
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
@@ -0,0 +1,97 @@
+import re
+import dataclasses
+from enum import StrEnum
+from typing import Literal, get_args
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ INVALID_CONFIG_ERROR,
+ ObjectStorageError,
+)
+
+FileFormatChoice = Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"]
+# The format one file is read with. A CSV file's header choice lives in `CsvOptions`.
+FileFormat = Literal["csv", "jsonl", "parquet"]
+
+# S3 bucket naming rules: 3-63 characters of lowercase letters, digits, dots and hyphens. Holding
+# the bucket to this keeps it from injecting a host or a path into the URLs built from it.
+_BUCKET_NAME = re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]")
+_REGION_NAME = re.compile(r"[a-z]{2}(-[a-z0-9]+)+-\d+")
+
+
+class ObjectStoreProvider(StrEnum):
+ S3 = "s3"
+
+
+@frozen
+class ObjectStoreConfig:
+ """Where a bucket lives and the credentials to read it.
+
+ `endpoint_url` stays `None` for AWS S3. S3-compatible stores that take HMAC keys (GCS, R2) set it.
+ """
+
+ provider: ObjectStoreProvider
+ bucket: str
+ region: str
+ access_key_id: str = dataclasses.field(repr=False)
+ secret_access_key: str = dataclasses.field(repr=False)
+ prefix: str = ""
+ endpoint_url: str | None = None
+
+ def __post_init__(self) -> None:
+ if not _BUCKET_NAME.fullmatch(self.bucket) or ".." in self.bucket:
+ raise ObjectStorageError(
+ f"{INVALID_CONFIG_ERROR}: '{self.bucket}' isn't a valid bucket name. Enter only the bucket "
+ "name, without s3:// or a folder."
+ )
+ if not _REGION_NAME.fullmatch(self.region):
+ raise ObjectStorageError(
+ f"{INVALID_CONFIG_ERROR}: '{self.region}' isn't a valid region, such as us-east-1."
+ )
+ # Without both keys the SDKs fall back to the worker's own AWS credentials, which must
+ # never be used to read a customer-chosen bucket.
+ if not self.access_key_id.strip() or not self.secret_access_key.strip():
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: enter both the access key ID and the secret access key.")
+ if any(segment in (".", "..") for segment in self.prefix.split("/")):
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the folder can't contain '.' or '..' segments.")
+
+ @property
+ def location_url(self) -> str:
+ """The https URL of the folder, in the form the warehouse table URL checks accept."""
+ path = f"/{self.prefix}/" if self.prefix else "/"
+ if self.endpoint_url:
+ return f"{self.endpoint_url.rstrip('/')}/{self.bucket}{path}"
+ return f"https://{self.bucket}.s3.{self.region}.amazonaws.com{path}"
+
+ @property
+ def display_location(self) -> str:
+ return f"{self.bucket}/{self.prefix}" if self.prefix else self.bucket
+
+
+@frozen
+class CsvOptions:
+ has_header: bool = True
+ # Mirrors ClickHouse's `format_csv_allow_double_quotes`: off means a double quote is an ordinary
+ # character and never encloses a field.
+ allow_double_quotes: bool = True
+
+
+@frozen
+class DiscoveryOptions:
+ """The settings that decide which files exist as tables, and how each one is read."""
+
+ file_format: FileFormatChoice = "infer"
+ file_pattern: str | None = None
+ allow_double_quotes: bool = True
+ combined_table_name: str | None = None
+
+ def __post_init__(self) -> None:
+ # The API checks the shape of a payload, not that a `Literal` field holds one of its values.
+ if self.file_format not in get_args(FileFormatChoice):
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.file_format}' isn't a supported file format.")
+
+
+def normalize_prefix(path: str | None) -> str:
+ """Drop empty segments only. A '.' or '..' segment stays, so `ObjectStoreConfig` rejects it."""
+ return "/".join(segment for segment in (path or "").strip().split("/") if segment)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
new file mode 100644
index 000000000000..df2bb5bac436
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/delta.py
@@ -0,0 +1,78 @@
+import re
+from collections.abc import Iterator
+
+import pyarrow as pa
+from deltalake import DeltaTable
+from deltalake.exceptions import DeltaError, DeltaProtocolError, TableNotFoundError
+from structlog.types import FilteringBoundLogger
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ACCESS_DENIED_ERROR,
+ DELTA_UNSUPPORTED_ERROR,
+ FILE_PARSE_ERROR,
+ NO_TABLE_FILES_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ DELTA_BATCH_ROWS,
+)
+
+# Whole words only, so a status code doesn't match inside an object key such as `sales403/part.parquet`.
+_ACCESS_DENIED_MARKERS = re.compile(r"\b(403|AccessDenied|Forbidden|InvalidAccessKeyId|SignatureDoesNotMatch)\b")
+
+
+def _raise_if_access_denied(error: Exception, table_name: str) -> None:
+ # delta-rs reports object store failures as text, so a rejected credential shows only as the
+ # HTTP status or the S3 error code inside the message.
+ if _ACCESS_DENIED_MARKERS.search(str(error)):
+ raise ObjectStorageError(
+ f"{ACCESS_DENIED_ERROR} for the Delta table '{table_name}'. Check that the key can read its folder."
+ ) from error
+
+
+def iter_delta_tables(
+ uri: str,
+ storage_options: dict[str, str],
+ table_name: str,
+ batch_rows: int = DELTA_BATCH_ROWS,
+ logger: FilteringBoundLogger | None = None,
+) -> Iterator[pa.Table]:
+ """Stream one Delta snapshot.
+
+ The version is pinned when the table opens, so commits that land during the read don't mix two
+ versions into one import. The scan reads one fragment at a time on one thread to bound memory.
+ """
+ try:
+ table = DeltaTable(uri, storage_options=storage_options)
+ version = table.version()
+ dataset = table.to_pyarrow_dataset()
+ except DeltaProtocolError as e:
+ raise ObjectStorageError(f"{DELTA_UNSUPPORTED_ERROR}: {e}") from e
+ except TableNotFoundError as e:
+ raise ObjectStorageError(
+ f"{NO_TABLE_FILES_ERROR} '{table_name}'. The Delta table is no longer at its folder in the bucket."
+ ) from e
+ except (DeltaError, OSError) as e:
+ _raise_if_access_denied(e, table_name)
+ raise
+
+ if logger is not None:
+ logger.info("Reading Delta table snapshot", table=table_name, delta_version=version)
+
+ scanner = dataset.scanner(batch_size=batch_rows, batch_readahead=1, fragment_readahead=1, use_threads=False)
+ yielded = False
+ try:
+ for batch in scanner.to_batches():
+ if batch.num_rows:
+ yielded = True
+ yield pa.Table.from_batches([batch])
+ except pa.ArrowInvalid as e:
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} a data file of the Delta table '{table_name}': {e}") from e
+ except (DeltaError, OSError) as e:
+ # The scan reads the data files lazily, so a key that can read the log but not the data fails here.
+ _raise_if_access_denied(e, table_name)
+ raise
+ if not yielded:
+ # An empty snapshot still has to reach the pipeline as a schema, or the table is dropped and
+ # never recreated.
+ yield dataset.schema.empty_table()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
new file mode 100644
index 000000000000..a6090e29726b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
@@ -0,0 +1,341 @@
+import re
+import fnmatch
+import posixpath
+import dataclasses
+from collections.abc import Iterable, Mapping
+from typing import Any, Literal, cast, get_args
+
+from structlog.types import FilteringBoundLogger
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ CsvOptions,
+ DiscoveryOptions,
+ FileFormat,
+ FileFormatChoice,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ NO_TABLE_FILES_ERROR,
+ TOO_MANY_FILES_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ ObjectStore,
+ StoredObject,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ MAX_TABLE_OBJECTS,
+ MAX_TABLES,
+)
+
+TableKind = Literal["files", "delta"]
+# Which files a `files` table reads: one file, a folder and its hive partitions, or every file.
+FileScope = Literal["file", "folder", "all"]
+
+DELTA_LOG_DIRECTORY = "_delta_log"
+COMPRESSED_SUFFIX = ".gz"
+
+EXTENSION_FORMATS: dict[str, FileFormat] = {
+ ".csv": "csv",
+ ".tsv": "csv",
+ ".jsonl": "jsonl",
+ ".ndjson": "jsonl",
+ # Exports that land in buckets (Firehose, Spark, ClickHouse JSONEachRow) write one object per line
+ # under a .json name, so .json files are read as JSON Lines.
+ ".json": "jsonl",
+ ".parquet": "parquet",
+}
+
+_FILE_FORMAT_CHOICES: frozenset[str] = frozenset(get_args(FileFormatChoice))
+
+
+@frozen
+class TableDefinition:
+ name: str
+ kind: TableKind
+ # Relative to the store root. For a `file` scope it is the file itself.
+ path: str
+ scope: FileScope | None
+ # "delta" for a Delta table. For files: the format the files are read with, or "infer" when one
+ # combined table mixes formats and each file is read by its extension.
+ file_format: FileFormatChoice
+ csv: CsvOptions
+ pattern: str | None
+ # The source's configured format at discovery, replayed at sync so the same files group together.
+ source_format: FileFormatChoice
+ files: tuple[StoredObject, ...] = ()
+
+ def same_location(self, other: "TableDefinition") -> bool:
+ return (
+ self.kind == other.kind
+ and self.scope == other.scope
+ and self.path == other.path
+ and self.file_format == other.file_format
+ )
+
+ def schema_metadata(self) -> dict[str, Any]:
+ return {
+ "kind": self.kind,
+ "scope": self.scope,
+ "path": self.path,
+ "pattern": self.pattern,
+ "format": self.file_format,
+ "source_format": self.source_format,
+ "csv": {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes},
+ }
+
+ @classmethod
+ def from_schema_metadata(cls, name: str, metadata: Mapping[str, Any] | None) -> "TableDefinition | None":
+ """Rebuild a table from what `schema_metadata` persisted, or None when the keys are missing."""
+ if not metadata:
+ return None
+ kind = metadata.get("kind")
+ scope = metadata.get("scope")
+ path = metadata.get("path")
+ file_format = metadata.get("format")
+ source_format = metadata.get("source_format", file_format)
+ pattern = metadata.get("pattern")
+ csv = metadata.get("csv") or {}
+ if (
+ kind not in ("files", "delta")
+ or (kind == "files" and scope not in ("file", "folder", "all"))
+ or not isinstance(path, str)
+ or file_format not in _FILE_FORMAT_CHOICES
+ or source_format not in _FILE_FORMAT_CHOICES
+ or not (pattern is None or isinstance(pattern, str))
+ or not isinstance(csv, dict)
+ ):
+ return None
+ return cls(
+ name=name,
+ kind=cast(TableKind, kind),
+ path=path,
+ scope=cast(FileScope, scope) if kind == "files" else None,
+ file_format=cast(FileFormatChoice, file_format),
+ csv=CsvOptions(
+ has_header=bool(csv.get("has_header", True)),
+ allow_double_quotes=bool(csv.get("allow_double_quotes", True)),
+ ),
+ pattern=pattern,
+ source_format=cast(FileFormatChoice, source_format),
+ )
+
+
+def sanitize_table_name(name: str) -> str:
+ return re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_").lower()
+
+
+def _strip_compression(path: str) -> str:
+ return path[: -len(COMPRESSED_SUFFIX)] if path.lower().endswith(COMPRESSED_SUFFIX) else path
+
+
+def is_compressed(path: str) -> bool:
+ return path.lower().endswith(COMPRESSED_SUFFIX)
+
+
+def infer_file_format(path: str) -> FileFormat | None:
+ extension = posixpath.splitext(_strip_compression(path).lower())[1]
+ return EXTENSION_FORMATS.get(extension)
+
+
+def resolve_file_format(path: str, choice: FileFormatChoice) -> FileFormat | None:
+ """The format one file is read with, or None when `infer` can't tell from its extension."""
+ if choice in ("csv", "csv_no_header"):
+ return "csv"
+ if choice in ("jsonl", "parquet"):
+ return cast(FileFormat, choice)
+ return infer_file_format(path)
+
+
+def _table_format(path: str, choice: FileFormatChoice) -> FileFormatChoice | None:
+ if choice == "infer":
+ return infer_file_format(path)
+ return choice
+
+
+def _is_hidden(segments: Iterable[str]) -> bool:
+ # `_SUCCESS`, `_committed_*`, `.crc` and `_temporary/` are Spark and Hadoop bookkeeping, not data.
+ return any(segment.startswith(("_", ".")) for segment in segments)
+
+
+def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:
+ roots: set[str] = set()
+ for stored in objects:
+ segments = stored.path.split("/")
+ if DELTA_LOG_DIRECTORY in segments[:-1]:
+ root_segments = segments[: segments.index(DELTA_LOG_DIRECTORY)]
+ if not _is_hidden(root_segments):
+ roots.add("/".join(root_segments))
+ return sorted(roots)
+
+
+def _is_under(path: str, roots: frozenset[str]) -> bool:
+ if not roots:
+ return False
+ if "" in roots:
+ return True
+ segments = path.split("/")[:-1]
+ return any("/".join(segments[:depth]) in roots for depth in range(1, len(segments) + 1))
+
+
+def _table_folder(path: str) -> str | None:
+ """The folder a file's table is named after, or None for a file directly under the root.
+
+ Hive partition folders (`year=2024/month=01`) belong to the table above them, so the folder ends
+ at the first `key=value` segment.
+ """
+ directories = path.split("/")[:-1]
+ if not directories:
+ return None
+ folder: list[str] = []
+ for directory in directories:
+ if "=" in directory:
+ break
+ folder.append(directory)
+ return "/".join(folder)
+
+
+def _format_suffix(file_format: FileFormatChoice) -> str:
+ return "csv" if file_format == "csv_no_header" else file_format
+
+
+def group_tables(
+ objects: Iterable[StoredObject],
+ options: DiscoveryOptions,
+ root_name: str,
+ logger: FilteringBoundLogger | None = None,
+ max_tables: int | None = MAX_TABLES,
+) -> list[TableDefinition]:
+ """Turn a bucket listing into tables.
+
+ - A folder holding `_delta_log/` is one Delta table, and none of its files are offered again.
+ - Other files group by their folder, with hive partition folders folded into the table above.
+ - A file directly under the root is a table of its own.
+ - `combined_table_name` puts every file into one table.
+
+ The same listing and options always give the same tables, which is what lets a sync find its
+ table again from `schema_metadata`.
+ """
+ ordered = sorted(objects, key=lambda stored: stored.path)
+ delta_roots = frozenset(_delta_roots(ordered))
+ choice = options.file_format
+ pattern = options.file_pattern or None
+ combined_name = (options.combined_table_name or "").strip() or None
+
+ def definition(
+ name: str,
+ kind: TableKind,
+ path: str,
+ scope: FileScope | None,
+ file_format: FileFormatChoice,
+ files: list[StoredObject],
+ ) -> TableDefinition:
+ return TableDefinition(
+ name=name,
+ kind=kind,
+ path=path,
+ scope=scope,
+ file_format=file_format,
+ csv=CsvOptions(has_header=file_format != "csv_no_header", allow_double_quotes=options.allow_double_quotes),
+ pattern=pattern,
+ source_format=choice,
+ files=tuple(files),
+ )
+
+ candidates: list[TableDefinition] = []
+ if choice in ("infer", "delta"):
+ for root in sorted(delta_roots):
+ candidates.append(definition(root.replace("/", "_") or root_name, "delta", root, None, "delta", []))
+
+ if choice != "delta":
+ data_files: list[tuple[StoredObject, FileFormatChoice]] = []
+ for stored in ordered:
+ segments = stored.path.split("/")
+ if not segments[-1] or _is_hidden(segments) or _is_under(stored.path, delta_roots):
+ continue
+ if pattern is not None and not fnmatch.fnmatchcase(stored.path, pattern):
+ continue
+ file_format = _table_format(stored.path, choice)
+ if file_format is not None:
+ data_files.append((stored, file_format))
+
+ if combined_name is not None:
+ if data_files:
+ formats = {file_format for _, file_format in data_files}
+ combined_format = formats.pop() if len(formats) == 1 else "infer"
+ candidates.append(
+ definition(combined_name, "files", "", "all", combined_format, [f for f, _ in data_files])
+ )
+ else:
+ folders: dict[tuple[str, FileFormatChoice], list[StoredObject]] = {}
+ for stored, file_format in data_files:
+ folder = _table_folder(stored.path)
+ if folder is None:
+ stem = posixpath.splitext(_strip_compression(stored.path))[0]
+ candidates.append(definition(stem, "files", stored.path, "file", file_format, [stored]))
+ else:
+ folders.setdefault((folder, file_format), []).append(stored)
+ formats_per_folder: dict[str, int] = {}
+ for folder, _ in folders:
+ formats_per_folder[folder] = formats_per_folder.get(folder, 0) + 1
+ for (folder, file_format), files in folders.items():
+ base = folder.replace("/", "_") or root_name
+ name = f"{base}_{_format_suffix(file_format)}" if formats_per_folder[folder] > 1 else base
+ candidates.append(definition(name, "files", folder, "folder", file_format, files))
+
+ tables: list[TableDefinition] = []
+ taken: set[str] = set()
+ next_suffix: dict[str, int] = {}
+ for candidate in candidates:
+ base = sanitize_table_name(candidate.name) or "data"
+ name = base
+ suffix = next_suffix.get(base, 2)
+ while name in taken:
+ name = f"{base}_{suffix}"
+ suffix += 1
+ next_suffix[base] = suffix
+ taken.add(name)
+ tables.append(dataclasses.replace(candidate, name=name))
+
+ if max_tables is not None and len(tables) > max_tables:
+ if logger is not None:
+ logger.warning("Reached the object storage table limit; later tables are ignored", max_tables=max_tables)
+ tables = tables[:max_tables]
+ return tables
+
+
+def resolve_table_files(
+ store: ObjectStore,
+ table: TableDefinition,
+ max_objects: int = MAX_TABLE_OBJECTS,
+ logger: FilteringBoundLogger | None = None,
+) -> TableDefinition:
+ """List a `files` table's folder again and return the table with the files it holds now.
+
+ The listing runs through `group_tables` with the options the table was discovered with, so a
+ subfolder that is a table of its own, or a file another format claims, stays out.
+ """
+ if table.kind != "files":
+ return table
+ listing_prefix = "" if table.scope == "all" else table.path
+ objects = store.list_objects(listing_prefix, max_objects + 1)
+ if len(objects) > max_objects:
+ raise ObjectStorageError(
+ f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{listing_prefix or '/'}'. Point the source at a "
+ "narrower folder, or set a file pattern that matches fewer files."
+ )
+ options = DiscoveryOptions(
+ file_format=table.source_format,
+ file_pattern=table.pattern,
+ allow_double_quotes=table.csv.allow_double_quotes,
+ combined_table_name=table.name if table.scope == "all" else None,
+ )
+ for candidate in group_tables(objects, options, root_name=table.name, logger=logger, max_tables=None):
+ # A combined table keeps its identity when a new file changes which formats it mixes.
+ if candidate.same_location(table) or (table.scope == "all" and candidate.scope == "all"):
+ return dataclasses.replace(candidate, name=table.name)
+ raise ObjectStorageError(
+ f"{NO_TABLE_FILES_ERROR} '{table.name}'. The files it was discovered from may have been renamed, moved, "
+ "or removed from the bucket."
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.py
new file mode 100644
index 000000000000..d2a96f7e3488
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/errors.py
@@ -0,0 +1,48 @@
+# Error message prefixes. `get_non_retryable_errors` matches on these, so they have to stay stable
+# and stay out of the interpolated part of each message.
+INVALID_CONFIG_ERROR = "The storage settings are invalid"
+BLOCKED_LOCATION_ERROR = "PostHog can't import from this storage location"
+ACCESS_DENIED_ERROR = "The storage provider denied access to the bucket"
+NO_SUCH_BUCKET_ERROR = "The bucket doesn't exist"
+WRONG_REGION_ERROR = "The bucket is in a different region"
+NO_IMPORTABLE_FILES_ERROR = "No importable files were found"
+NO_TABLE_FILES_ERROR = "No files are available for table"
+TOO_MANY_FILES_ERROR = "The table has more files than PostHog can import"
+FILE_PARSE_ERROR = "Couldn't parse the file"
+DELTA_UNSUPPORTED_ERROR = "The Delta table uses features PostHog can't read"
+DELTA_READ_ERROR = "Couldn't read the Delta table"
+
+# S3 error codes, and the message S3 sends with a 301, that mean the request went to the wrong
+# regional endpoint for this bucket. s3fs keeps only the message when it translates the error.
+_WRONG_REGION_MARKERS = (
+ "PermanentRedirect",
+ "AuthorizationHeaderMalformed",
+ "IllegalLocationConstraint",
+ "must be addressed using the specified endpoint",
+)
+
+
+class ObjectStorageError(Exception):
+ """A problem the user has to fix: rejected credentials, a missing bucket, an unreadable file."""
+
+
+def translate_storage_error(error: Exception, location: str) -> Exception:
+ """Map a filesystem error to an `ObjectStorageError` with a stable prefix, or return it unchanged.
+
+ s3fs turns S3 error codes into builtin exceptions: `AccessDenied`, `InvalidAccessKeyId` and
+ `SignatureDoesNotMatch` become `PermissionError`, and `NoSuchBucket` becomes `FileNotFoundError`.
+ Anything else stays as it is, so a transient failure keeps its retryable message.
+ """
+ if isinstance(error, ObjectStorageError):
+ return error
+ if isinstance(error, PermissionError):
+ return ObjectStorageError(
+ f"{ACCESS_DENIED_ERROR} '{location}'. Check the access key and secret, and that the key can list "
+ "and read objects in this bucket."
+ )
+ if isinstance(error, FileNotFoundError):
+ return ObjectStorageError(f"{NO_SUCH_BUCKET_ERROR}: '{location}'. Check the bucket name.")
+ message = str(error)
+ if any(marker in message for marker in _WRONG_REGION_MARKERS):
+ return ObjectStorageError(f"{WRONG_REGION_ERROR} from the one configured for '{location}'. Fix the region.")
+ return error
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
new file mode 100644
index 000000000000..087825c3b01c
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
@@ -0,0 +1,205 @@
+import dataclasses
+from datetime import UTC, datetime
+from typing import IO, Any, cast
+
+import s3fs
+import boto3
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import ClientError
+from fsspec import AbstractFileSystem
+from s3fs.errors import translate_boto_error
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ ObjectStoreConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ TOO_MANY_FILES_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ CONNECT_TIMEOUT_SECONDS,
+ MAX_LISTED_KEYS,
+ MAX_REQUEST_ATTEMPTS,
+ READ_TIMEOUT_SECONDS,
+)
+
+S3_MAX_PAGE_SIZE = 1000
+
+
+@frozen
+class StoredObject:
+ # Relative to the store root, '/'-separated.
+ path: str
+ size: int
+ modified_at: datetime
+
+
+@frozen
+class ObjectStore:
+ """A filesystem rooted at the configured bucket folder, plus what deltalake needs to read it.
+
+ Delta tables are read by delta-rs, which does not use the fsspec filesystem, so the store also
+ carries the URI prefix and the storage options for it.
+ """
+
+ fs: AbstractFileSystem
+ # The fsspec path of the folder: `bucket/prefix` for S3.
+ root: str
+ delta_uri_prefix: str
+ delta_storage_options: dict[str, str] = dataclasses.field(repr=False)
+
+ def path(self, relative: str) -> str:
+ return f"{self.root}/{relative}" if relative else self.root
+
+ def delta_uri(self, relative: str) -> str:
+ return f"{self.delta_uri_prefix}{self.path(relative)}"
+
+ def list_objects(self, relative_prefix: str, max_objects: int) -> list[StoredObject]:
+ """List up to `max_objects` files under the prefix, in path order.
+
+ The prefix is a folder, or one file. This generic version walks the whole fsspec tree, so
+ stores with a large remote listing override it with a paginated, capped one.
+ """
+ found = cast(dict[str, dict[str, Any]], self.fs.find(self.path(relative_prefix), detail=True))
+ root_prefix = f"{self.root}/"
+ objects: list[StoredObject] = []
+ for name in sorted(found):
+ info = found[name]
+ if info.get("type") != "file" or not name.startswith(root_prefix):
+ continue
+ objects.append(
+ StoredObject(
+ path=name[len(root_prefix) :], size=int(info.get("size") or 0), modified_at=_modified_at(info)
+ )
+ )
+ if len(objects) >= max_objects:
+ break
+ return objects
+
+ def open(self, relative: str) -> IO[bytes]:
+ return cast(IO[bytes], self.fs.open(self.path(relative), "rb"))
+
+
+def _modified_at(info: dict[str, Any]) -> datetime:
+ last_modified = info.get("LastModified")
+ if isinstance(last_modified, datetime):
+ return last_modified if last_modified.tzinfo else last_modified.replace(tzinfo=UTC)
+ return datetime.fromtimestamp(float(info.get("mtime") or 0), tz=UTC)
+
+
+def build_s3_filesystem(config: ObjectStoreConfig) -> s3fs.S3FileSystem:
+ client_kwargs: dict[str, Any] = {"region_name": config.region}
+ return s3fs.S3FileSystem(
+ key=config.access_key_id,
+ secret=config.secret_access_key,
+ endpoint_url=config.endpoint_url,
+ client_kwargs=client_kwargs,
+ config_kwargs={
+ "connect_timeout": CONNECT_TIMEOUT_SECONDS,
+ "read_timeout": READ_TIMEOUT_SECONDS,
+ "retries": {"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},
+ },
+ # fsspec caches instances by their arguments, which would keep every customer's credentials
+ # alive in the worker. Directory listings are re-read on every run anyway.
+ skip_instance_cache=True,
+ use_listings_cache=False,
+ )
+
+
+def delta_storage_options(config: ObjectStoreConfig) -> dict[str, str]:
+ options = {
+ "AWS_ACCESS_KEY_ID": config.access_key_id,
+ "AWS_SECRET_ACCESS_KEY": config.secret_access_key,
+ "AWS_REGION": config.region,
+ "connect_timeout": f"{CONNECT_TIMEOUT_SECONDS}s",
+ "timeout": f"{READ_TIMEOUT_SECONDS}s",
+ }
+ if config.endpoint_url:
+ options["AWS_ENDPOINT_URL"] = config.endpoint_url
+ return options
+
+
+@frozen
+class S3ObjectStore(ObjectStore):
+ """Lists through a boto3 client and reads through s3fs.
+
+ s3fs caps a listing by asking for a page of twice the cap, which S3-compatible stores reject
+ above their own page limit. The boto3 paginator keeps pages at the S3 maximum and stops at the cap.
+ """
+
+ client: Any = dataclasses.field(repr=False)
+ bucket: str
+ prefix: str
+
+ def list_objects(self, relative_prefix: str, max_objects: int) -> list[StoredObject]:
+ key = "/".join(part for part in (self.prefix, relative_prefix) if part)
+ objects = self._list(f"{key}/" if key else "", max_objects)
+ if not objects and relative_prefix:
+ # The prefix may name one file rather than a folder.
+ objects = [stored for stored in self._list(key, 1) if stored.path == relative_prefix]
+ return objects
+
+ def _list(self, key_prefix: str, max_objects: int) -> list[StoredObject]:
+ root_key = f"{self.prefix}/" if self.prefix else ""
+ paginator = self.client.get_paginator("list_objects_v2")
+ # The file cap counts files, not keys: folder marker keys are skipped, so stopping after a fixed
+ # number of keys could hide that the folder holds more files than the cap. A separate cap on
+ # the keys walked keeps a prefix full of marker keys from running the listing without end.
+ pages = paginator.paginate(
+ Bucket=self.bucket, Prefix=key_prefix, PaginationConfig={"PageSize": S3_MAX_PAGE_SIZE}
+ )
+ objects: list[StoredObject] = []
+ walked_keys = 0
+ try:
+ for page in pages:
+ for item in page.get("Contents", []):
+ walked_keys += 1
+ if walked_keys > MAX_LISTED_KEYS:
+ raise ObjectStorageError(
+ f"{TOO_MANY_FILES_ERROR} ({MAX_LISTED_KEYS:,} keys) under '{key_prefix or '/'}'. Point "
+ "the source at a narrower folder."
+ )
+ object_key: str = item["Key"]
+ if not object_key.startswith(root_key) or object_key.endswith("/"):
+ continue
+ objects.append(
+ StoredObject(
+ path=object_key[len(root_key) :],
+ size=int(item.get("Size") or 0),
+ modified_at=_modified_at(item),
+ )
+ )
+ if len(objects) >= max_objects:
+ return objects
+ except ClientError as e:
+ raise translate_boto_error(e) from e
+ return objects
+
+
+def build_s3_client(config: ObjectStoreConfig) -> Any:
+ return boto3.client(
+ "s3",
+ aws_access_key_id=config.access_key_id,
+ aws_secret_access_key=config.secret_access_key,
+ region_name=config.region,
+ endpoint_url=config.endpoint_url,
+ config=BotoConfig(
+ connect_timeout=CONNECT_TIMEOUT_SECONDS,
+ read_timeout=READ_TIMEOUT_SECONDS,
+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},
+ ),
+ )
+
+
+def open_s3_store(config: ObjectStoreConfig) -> S3ObjectStore:
+ return S3ObjectStore(
+ fs=build_s3_filesystem(config),
+ root=f"{config.bucket}/{config.prefix}" if config.prefix else config.bucket,
+ delta_uri_prefix="s3://",
+ delta_storage_options=delta_storage_options(config),
+ client=build_s3_client(config),
+ bucket=config.bucket,
+ prefix=config.prefix,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
new file mode 100644
index 000000000000..ef63b282aa4b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -0,0 +1,330 @@
+import io
+import csv
+import gzip
+import json
+import zlib
+import tempfile
+from collections.abc import Iterator
+from typing import IO, Any, Literal, cast
+
+import pyarrow as pa
+import pyarrow.csv as pa_csv
+import pyarrow.parquet as pq
+from structlog.types import FilteringBoundLogger
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (
+ FILE_MODIFIED_AT_COLUMN,
+ FILE_PATH_COLUMN,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ CsvOptions,
+ FileFormat,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressed
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ FILE_PARSE_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import StoredObject
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ CSV_BLOCK_SIZE_BYTES,
+ JSONL_CHUNK_BYTES,
+ JSONL_CHUNK_ROWS,
+ MAX_FIRST_CSV_RECORD_BYTES,
+ MAX_RECORD_BYTES,
+ MAX_UNPACKED_PARQUET_BYTES,
+ PARQUET_BATCH_ROWS,
+)
+
+TSV_SUFFIXES = (".tsv", ".tsv.gz")
+UTF8_BOM = b"\xef\xbb\xbf"
+SPOOL_MEMORY_BYTES = 8 * 1024 * 1024
+COPY_CHUNK_BYTES = 1024 * 1024
+
+
+class _PrefixedStream(io.RawIOBase):
+ """Replays bytes already read from a stream before the rest of it.
+
+ The first CSV line is read ahead to name the columns. A headerless file still needs that line
+ parsed as data, so it goes back in front of the stream.
+ """
+
+ def __init__(self, prefix: bytes, rest: IO[bytes]) -> None:
+ self._prefix = prefix
+ self._rest = rest
+
+ def readable(self) -> bool:
+ return True
+
+ def readinto(self, buffer: Any) -> int:
+ view = memoryview(buffer)
+ if self._prefix:
+ count = min(len(view), len(self._prefix))
+ view[:count] = self._prefix[:count]
+ self._prefix = self._prefix[count:]
+ return count
+ data = self._rest.read(len(view))
+ view[: len(data)] = data
+ return len(data)
+
+
+def csv_column_names(cells: list[str], has_header: bool) -> list[str]:
+ """Header names as written, made unique. A headerless file gets ClickHouse's `c1`, `c2`, ... names."""
+ if not has_header:
+ return [f"c{index}" for index in range(1, len(cells) + 1)]
+ names: list[str] = []
+ seen: set[str] = set()
+ next_suffix: dict[str, int] = {}
+ for index, cell in enumerate(cells, start=1):
+ base = cell.strip() or f"c{index}"
+ name = base
+ suffix = next_suffix.get(base, 2)
+ while name in seen:
+ name = f"{base}_{suffix}"
+ suffix += 1
+ next_suffix[base] = suffix
+ seen.add(name)
+ names.append(name)
+ return names
+
+
+def _read_bounded_line(stream: IO[bytes], file_path: str, kind: str) -> bytes:
+ """Read one line, refusing a line longer than `MAX_RECORD_BYTES` rather than holding it in memory."""
+ line = stream.readline(MAX_RECORD_BYTES + 1)
+ if len(line) > MAX_RECORD_BYTES:
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{file_path}' as {kind}: a line is longer than {MAX_RECORD_BYTES:,} bytes."
+ )
+ return line
+
+
+def _read_first_csv_record(
+ stream: IO[bytes], delimiter: str, allow_double_quotes: bool, file_path: str
+) -> tuple[bytes, list[str]]:
+ """The first non-blank record as the bytes it spans and its fields.
+
+ The CSV reader pulls lines until the record is complete, so a quoted field that spans lines is
+ one record, and a stray quote inside an unquoted field is not mistaken for the start of one.
+ """
+ consumed = bytearray()
+
+ def lines() -> Iterator[str]:
+ while True:
+ line = _read_bounded_line(stream, file_path, "CSV")
+ if not line:
+ return
+ if not consumed:
+ if not line.strip():
+ continue
+ line = line.removeprefix(UTF8_BOM)
+ if len(consumed) + len(line) > MAX_FIRST_CSV_RECORD_BYTES:
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{file_path}' as CSV: the first record is longer than "
+ f"{MAX_FIRST_CSV_RECORD_BYTES:,} bytes."
+ )
+ consumed.extend(line)
+ yield line.decode("utf-8", errors="replace")
+
+ if allow_double_quotes:
+ reader = csv.reader(lines(), delimiter=delimiter)
+ else:
+ reader = csv.reader(lines(), delimiter=delimiter, quoting=csv.QUOTE_NONE)
+ try:
+ fields = next(reader, [])
+ except csv.Error as e:
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: the first record is invalid ({e}).") from e
+ return bytes(consumed), fields
+
+
+def iter_csv_tables(
+ stream: IO[bytes],
+ options: CsvOptions,
+ delimiter: str = ",",
+ file_path: str = "",
+ logger: FilteringBoundLogger | None = None,
+) -> Iterator[pa.Table]:
+ """Stream a CSV file as string columns.
+
+ Every column is read as text: Arrow fixes a column's type from the first block it sees, so a
+ later value that does not fit would fail the whole file mid-stream, and two files of one table
+ could disagree on a type. Empty unquoted fields are null.
+ """
+ first_record, first_fields = _read_first_csv_record(stream, delimiter, options.allow_double_quotes, file_path)
+ if not first_record:
+ return
+ names = csv_column_names(first_fields, options.has_header)
+ body: IO[bytes] = stream if options.has_header else cast(IO[bytes], _PrefixedStream(first_record, stream))
+
+ skipped_rows = 0
+
+ def skip_invalid_row(_row: pa_csv.InvalidRow) -> Literal["skip", "error"]:
+ nonlocal skipped_rows
+ skipped_rows += 1
+ return "skip"
+
+ yielded = False
+ try:
+ reader = pa_csv.open_csv(
+ body,
+ read_options=pa_csv.ReadOptions(column_names=names, block_size=CSV_BLOCK_SIZE_BYTES, use_threads=False),
+ parse_options=pa_csv.ParseOptions(
+ delimiter=delimiter,
+ quote_char='"' if options.allow_double_quotes else False,
+ newlines_in_values=options.allow_double_quotes,
+ ignore_empty_lines=True,
+ invalid_row_handler=skip_invalid_row,
+ ),
+ convert_options=pa_csv.ConvertOptions(
+ column_types=dict.fromkeys(names, pa.string()),
+ null_values=[""],
+ strings_can_be_null=True,
+ quoted_strings_can_be_null=False,
+ ),
+ )
+ for batch in reader:
+ if batch.num_rows:
+ yielded = True
+ yield pa.Table.from_batches([batch])
+ except pa.ArrowInvalid as e:
+ if options.has_header and not yielded and "Empty CSV file" in str(e):
+ # The file holds only its header. It is a table with no rows, not a broken file.
+ yield pa.table({name: pa.array([], pa.string()) for name in names})
+ return
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: {e}") from e
+
+ if skipped_rows and logger is not None:
+ # A row whose field count differs from the header is malformed, or the file uses another
+ # delimiter. Skipping it keeps its values out of the wrong columns.
+ logger.warning(
+ "Skipped CSV rows whose field count differs from the header",
+ file=file_path,
+ skipped_rows=skipped_rows,
+ expected=len(names),
+ )
+
+
+def _as_rows(value: Any) -> Iterator[dict[str, Any]]:
+ # A generator, so a line holding a huge array is wrapped into rows a batch at a time.
+ if isinstance(value, dict):
+ yield value
+ elif isinstance(value, list):
+ for item in value:
+ yield item if isinstance(item, dict) else {"value": item}
+ else:
+ yield {"value": value}
+
+
+def iter_jsonl_tables(
+ stream: IO[bytes],
+ file_path: str = "",
+ max_rows: int = JSONL_CHUNK_ROWS,
+ max_bytes: int = JSONL_CHUNK_BYTES,
+) -> Iterator[pa.Table]:
+ rows: list[dict[str, Any]] = []
+ pending_bytes = 0
+ line_number = 0
+ while True:
+ line = _read_bounded_line(stream, file_path, "JSON Lines")
+ if not line:
+ break
+ line_number += 1
+ if not line.strip():
+ continue
+ try:
+ parsed = json.loads(line)
+ except (ValueError, RecursionError) as e:
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{file_path}' as JSON Lines: line {line_number} isn't valid JSON ({e})."
+ ) from e
+ pending_bytes += len(line)
+ for row in _as_rows(parsed):
+ rows.append(row)
+ if len(rows) >= max_rows:
+ yield table_from_py_list(rows)
+ rows = []
+ pending_bytes = 0
+ if rows and pending_bytes >= max_bytes:
+ yield table_from_py_list(rows)
+ rows = []
+ pending_bytes = 0
+ if rows:
+ yield table_from_py_list(rows)
+
+
+def iter_parquet_tables(
+ stream: IO[bytes], file_path: str = "", batch_rows: int = PARQUET_BATCH_ROWS
+) -> Iterator[pa.Table]:
+ try:
+ parquet_file = pq.ParquetFile(stream)
+ yielded = False
+ for batch in parquet_file.iter_batches(batch_size=batch_rows, use_threads=False):
+ if batch.num_rows:
+ yielded = True
+ yield pa.Table.from_batches([batch])
+ if not yielded:
+ # A file with no rows still carries a schema, which the pipeline needs to create the table.
+ yield parquet_file.schema_arrow.empty_table()
+ except pa.ArrowInvalid as e:
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as Parquet: {e}") from e
+
+
+def with_file_columns(table: pa.Table, stored: StoredObject) -> pa.Table:
+ """Tag every row with the file it came from, so a table built from many files stays traceable."""
+ for column in (FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN):
+ if column in table.column_names:
+ table = table.drop_columns([column])
+ rows = table.num_rows
+ table = table.append_column(FILE_PATH_COLUMN, pa.repeat(pa.scalar(stored.path, pa.string()), rows))
+ return table.append_column(
+ FILE_MODIFIED_AT_COLUMN,
+ pa.repeat(pa.scalar(stored.modified_at, pa.timestamp("us", tz="UTC")), rows),
+ )
+
+
+def _iter_gzipped_parquet(stream: IO[bytes], stored: StoredObject) -> Iterator[pa.Table]:
+ """Parquet readers seek to the footer, which a gzip stream can't, so the file is unpacked to disk first."""
+ with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES) as unpacked:
+ try:
+ with gzip.GzipFile(fileobj=stream) as compressed:
+ unpacked_bytes = 0
+ while chunk := compressed.read(COPY_CHUNK_BYTES):
+ unpacked_bytes += len(chunk)
+ if unpacked_bytes > MAX_UNPACKED_PARQUET_BYTES:
+ raise ObjectStorageError(
+ f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: it unpacks to more than "
+ f"{MAX_UNPACKED_PARQUET_BYTES:,} bytes."
+ )
+ unpacked.write(chunk)
+ except (gzip.BadGzipFile, zlib.error, EOFError) as e:
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: {e}") from e
+ unpacked.seek(0)
+ for table in iter_parquet_tables(cast(IO[bytes], unpacked), stored.path):
+ yield with_file_columns(table, stored)
+
+
+def iter_file_tables(
+ stream: IO[bytes],
+ stored: StoredObject,
+ file_format: FileFormat,
+ csv_options: CsvOptions,
+ logger: FilteringBoundLogger | None = None,
+) -> Iterator[pa.Table]:
+ """Read one file as Arrow tables of bounded size, each carrying the `_file_*` columns."""
+ if file_format == "parquet":
+ if is_compressed(stored.path):
+ yield from _iter_gzipped_parquet(stream, stored)
+ return
+ tables = iter_parquet_tables(stream, stored.path)
+ else:
+ binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else stream
+ if file_format == "csv":
+ delimiter = "\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ","
+ tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)
+ else:
+ tables = iter_jsonl_tables(binary, stored.path)
+ try:
+ for table in tables:
+ yield with_file_columns(table, stored)
+ except (gzip.BadGzipFile, zlib.error, EOFError) as e:
+ raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{stored.path}' as gzip: {e}") from e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py
new file mode 100644
index 000000000000..c092feffe954
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/settings.py
@@ -0,0 +1,30 @@
+# A bucket listing is one flat, paginated walk (1,000 keys per request), so these caps bound both the
+# request count and the memory a listing holds.
+MAX_DISCOVERY_OBJECTS = 10_000
+# A sync lists only the table's own folder. The cap is higher than discovery's because a sync that
+# silently skipped files would replace the table with a partial copy, so going past it fails the sync.
+MAX_TABLE_OBJECTS = 50_000
+# A listing also walks folder marker keys (ending in `/`), which are never files, so the walk has its own cap.
+MAX_LISTED_KEYS = 100_000
+# One table per root-level file can turn a large flat folder into thousands of tables.
+MAX_TABLES = 1_000
+
+CONNECT_TIMEOUT_SECONDS = 10
+READ_TIMEOUT_SECONDS = 60
+MAX_REQUEST_ATTEMPTS = 3
+
+CSV_BLOCK_SIZE_BYTES = 4 * 1024 * 1024
+JSONL_CHUNK_ROWS = 5_000
+JSONL_CHUNK_BYTES = 64 * 1024 * 1024
+# One JSON Lines record, or the first line of a CSV file, is held whole in memory to be parsed. A gzip
+# file can expand far past its stored size, so reads stop at this many decompressed bytes.
+MAX_RECORD_BYTES = 32 * 1024 * 1024
+# The first record of a CSV file names the columns, and is read whole before the streaming reader starts.
+MAX_FIRST_CSV_RECORD_BYTES = 1024 * 1024
+# A gzipped Parquet file is unpacked to local disk to be read, so the unpacked size is capped too.
+MAX_UNPACKED_PARQUET_BYTES = 2 * 1024 * 1024 * 1024
+PARQUET_BATCH_ROWS = 10_000
+DELTA_BATCH_ROWS = 16_384
+
+# Arrow batches from wide files can be large, so the pipeline's batcher flushes before this many bytes.
+CHUNK_SIZE_BYTES = 100 * 1024 * 1024
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
new file mode 100644
index 000000000000..b0a377d642a5
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -0,0 +1,205 @@
+from abc import abstractmethod
+from collections.abc import Iterator
+from typing import Generic, NoReturn, Optional
+
+import pyarrow as pa
+from structlog.types import FilteringBoundLogger
+
+from products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, SimpleSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ DiscoveryOptions,
+ ObjectStoreConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import iter_delta_tables
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
+ TableDefinition,
+ group_tables,
+ resolve_file_format,
+ resolve_table_files,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ACCESS_DENIED_ERROR,
+ BLOCKED_LOCATION_ERROR,
+ DELTA_UNSUPPORTED_ERROR,
+ FILE_PARSE_ERROR,
+ INVALID_CONFIG_ERROR,
+ NO_IMPORTABLE_FILES_ERROR,
+ NO_SUCH_BUCKET_ERROR,
+ NO_TABLE_FILES_ERROR,
+ TOO_MANY_FILES_ERROR,
+ WRONG_REGION_ERROR,
+ ObjectStorageError,
+ translate_storage_error,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ ObjectStore,
+ open_s3_store,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (
+ iter_file_tables,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ CHUNK_SIZE_BYTES,
+ MAX_DISCOVERY_OBJECTS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
+
+
+def _raise_translated(error: Exception, location: str) -> NoReturn:
+ translated = translate_storage_error(error, location)
+ if translated is error:
+ raise error
+ raise translated from error
+
+
+def validate_store_location(config: ObjectStoreConfig) -> None:
+ """Apply the checks a self-managed warehouse table URL gets: https, no internal hosts, and no
+ bucket that PostHog itself owns."""
+ is_valid, error = validate_warehouse_table_url_pattern(config.location_url)
+ if not is_valid:
+ raise ObjectStorageError(f"{BLOCKED_LOCATION_ERROR}: {error}")
+
+
+class ObjectStorageSource(SimpleSource[ConfigType], Generic[ConfigType]):
+ """Imports the files in a bucket folder: CSV, JSON Lines and Parquet files, and Delta tables.
+
+ Every sync reads each table in full, so the table matches the bucket at the time of the sync.
+ """
+
+ @abstractmethod
+ def object_store_config(self, config: ConfigType) -> ObjectStoreConfig: ...
+
+ @abstractmethod
+ def discovery_options(self, config: ConfigType) -> DiscoveryOptions: ...
+
+ def open_store(self, store_config: ObjectStoreConfig) -> ObjectStore:
+ """Re-check the location every time the stored credentials are about to be sent to it."""
+ validate_store_location(store_config)
+ return open_s3_store(store_config)
+
+ def get_non_retryable_errors(self) -> dict[str, str | None]:
+ return {
+ INVALID_CONFIG_ERROR: None,
+ BLOCKED_LOCATION_ERROR: None,
+ ACCESS_DENIED_ERROR: (
+ "The storage provider rejected these credentials. Check the access key and secret, and that "
+ "the key can list and read the bucket, then reconnect the source."
+ ),
+ NO_SUCH_BUCKET_ERROR: "The bucket doesn't exist. Check the bucket name, then reconnect the source.",
+ WRONG_REGION_ERROR: "The bucket is in a different region. Fix the region, then reconnect the source.",
+ NO_TABLE_FILES_ERROR: (
+ "The files behind this table are no longer in the bucket. Refresh the source's tables, or "
+ "restore the files."
+ ),
+ TOO_MANY_FILES_ERROR: None,
+ FILE_PARSE_ERROR: None,
+ DELTA_UNSUPPORTED_ERROR: None,
+ }
+
+ def _discover(
+ self, store: ObjectStore, config: ConfigType, logger: FilteringBoundLogger | None = None
+ ) -> list[TableDefinition]:
+ store_config = self.object_store_config(config)
+ try:
+ objects = store.list_objects("", MAX_DISCOVERY_OBJECTS)
+ except Exception as e:
+ _raise_translated(e, store_config.display_location)
+ if len(objects) >= MAX_DISCOVERY_OBJECTS and logger is not None:
+ logger.warning("Reached the object storage listing limit; later objects are ignored")
+ root_name = store_config.prefix.rsplit("/", 1)[-1] if store_config.prefix else store_config.bucket
+ return group_tables(objects, self.discovery_options(config), root_name=root_name, logger=logger)
+
+ def validate_credentials(
+ self,
+ config: ConfigType,
+ team_id: int,
+ schema_name: Optional[str] = None,
+ api_version: str | None = None,
+ ) -> tuple[bool, str | None]:
+ try:
+ store_config = self.object_store_config(config)
+ tables = self._discover(self.open_store(store_config), config)
+ except ObjectStorageError as e:
+ return False, str(e)
+ if not tables:
+ return False, (
+ f"{NO_IMPORTABLE_FILES_ERROR} in '{store_config.display_location}'. PostHog can read the bucket but "
+ "found nothing to import there. Check the folder, the file pattern, and the file format."
+ )
+ return True, None
+
+ def get_schemas(
+ self,
+ config: ConfigType,
+ team_id: int,
+ with_counts: bool = False,
+ names: list[str] | None = None,
+ force_refresh: bool = False,
+ api_version: str | None = None,
+ ) -> list[SourceSchema]:
+ store = self.open_store(self.object_store_config(config))
+ requested = set(names) if names is not None else None
+ return [
+ SourceSchema(
+ name=table.name,
+ supports_incremental=False,
+ supports_append=False,
+ schema_metadata=table.schema_metadata(),
+ )
+ for table in self._discover(store, config)
+ if requested is None or table.name in requested
+ ]
+
+ def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> SourceResponse:
+ store = self.open_store(self.object_store_config(config))
+ table = TableDefinition.from_schema_metadata(inputs.schema_name, inputs.schema_metadata)
+
+ def items() -> Iterator[pa.Table]:
+ resolved = (
+ table if table is not None else self._find_table(store, config, inputs.schema_name, inputs.logger)
+ )
+ yield from iter_table(store, resolved, inputs.logger)
+
+ return SourceResponse(
+ name=inputs.schema_name,
+ items=items,
+ # Files carry no stable row identity, so tables are replaced wholesale on every sync.
+ primary_keys=None,
+ chunk_size_bytes=CHUNK_SIZE_BYTES,
+ )
+
+ def _find_table(
+ self, store: ObjectStore, config: ConfigType, schema_name: str, logger: FilteringBoundLogger
+ ) -> TableDefinition:
+ """Find a table by name when its schema predates `schema_metadata`."""
+ for table in self._discover(store, config, logger):
+ if table.name == schema_name:
+ return table
+ raise ObjectStorageError(
+ f"{NO_TABLE_FILES_ERROR} '{schema_name}'. The files it was discovered from may have been renamed, "
+ "moved, or removed from the bucket."
+ )
+
+
+def iter_table(store: ObjectStore, table: TableDefinition, logger: FilteringBoundLogger) -> Iterator[pa.Table]:
+ if table.kind == "delta":
+ yield from iter_delta_tables(
+ store.delta_uri(table.path), store.delta_storage_options, table.name, logger=logger
+ )
+ return
+
+ try:
+ resolved = resolve_table_files(store, table, logger=logger)
+ except Exception as e:
+ _raise_translated(e, store.root)
+ for stored in resolved.files:
+ file_format = resolve_file_format(stored.path, resolved.file_format)
+ if file_format is None:
+ continue
+ try:
+ with store.open(stored.path) as stream:
+ yield from iter_file_tables(stream, stored, file_format, resolved.csv, logger)
+ except PermissionError as e:
+ _raise_translated(e, store.root)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
new file mode 100644
index 000000000000..6624d8efb173
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -0,0 +1,668 @@
+import io
+import csv
+import gzip
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from django.test import override_settings
+
+import boto3
+import pyarrow as pa
+import structlog
+import pyarrow.parquet as pq
+from botocore.stub import Stubber
+from deltalake import write_deltalake
+from fsspec.implementations.local import LocalFileSystem
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import formats
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ CsvOptions,
+ DiscoveryOptions,
+ ObjectStoreConfig,
+ ObjectStoreProvider,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import iter_delta_tables
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
+ TableDefinition,
+ group_tables,
+ resolve_table_files,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ACCESS_DENIED_ERROR,
+ BLOCKED_LOCATION_ERROR,
+ FILE_PARSE_ERROR,
+ INVALID_CONFIG_ERROR,
+ NO_SUCH_BUCKET_ERROR,
+ NO_TABLE_FILES_ERROR,
+ TOO_MANY_FILES_ERROR,
+ WRONG_REGION_ERROR,
+ ObjectStorageError,
+ translate_storage_error,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ ObjectStore,
+ S3ObjectStore,
+ StoredObject,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (
+ iter_csv_tables,
+ iter_jsonl_tables,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import MAX_TABLES
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
+ iter_table,
+ validate_store_location,
+)
+
+MODIFIED_AT = datetime(2024, 5, 1, 12, 0, tzinfo=UTC)
+LOGGER = structlog.get_logger()
+
+
+def stored(path: str) -> StoredObject:
+ return StoredObject(path=path, size=10, modified_at=MODIFIED_AT)
+
+
+def local_store(root: Path) -> ObjectStore:
+ return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})
+
+
+def summarize(tables: list[TableDefinition]) -> list[tuple[str, str, str | None, str, str, list[str]]]:
+ return [
+ (table.name, table.kind, table.scope, table.path, table.file_format, [f.path for f in table.files])
+ for table in tables
+ ]
+
+
+def read_rows(store: ObjectStore, table: TableDefinition) -> list[dict]:
+ return [row for batch in iter_table(store, table, LOGGER) for row in batch.to_pylist()]
+
+
+class TestGroupTables:
+ @parameterized.expand(
+ [
+ (
+ "delta_folder_is_one_table_and_hides_its_files",
+ [
+ "sales/_delta_log/00000000000000000000.json",
+ "sales/part-0.parquet",
+ "events/a.csv",
+ ],
+ DiscoveryOptions(),
+ [
+ ("sales", "delta", None, "sales", "delta", []),
+ ("events", "files", "folder", "events", "csv", ["events/a.csv"]),
+ ],
+ ),
+ (
+ "root_files_are_one_table_each",
+ ["orders.csv.gz", "users.jsonl", "notes.txt"],
+ DiscoveryOptions(),
+ [
+ ("orders", "files", "file", "orders.csv.gz", "csv", ["orders.csv.gz"]),
+ ("users", "files", "file", "users.jsonl", "jsonl", ["users.jsonl"]),
+ ],
+ ),
+ (
+ "hive_partitions_fold_into_the_table_above",
+ ["events/year=2024/month=01/a.parquet", "events/year=2024/month=02/b.parquet"],
+ DiscoveryOptions(),
+ [
+ (
+ "events",
+ "files",
+ "folder",
+ "events",
+ "parquet",
+ ["events/year=2024/month=01/a.parquet", "events/year=2024/month=02/b.parquet"],
+ ),
+ ],
+ ),
+ (
+ "hive_partitions_under_the_root_name_the_table_after_the_root",
+ ["year=2024/a.parquet"],
+ DiscoveryOptions(),
+ [("exports", "files", "folder", "", "parquet", ["year=2024/a.parquet"])],
+ ),
+ (
+ "combine_puts_every_file_in_one_table_and_keeps_delta_tables",
+ ["a.csv", "daily/b.csv", "sales/_delta_log/0.json"],
+ DiscoveryOptions(combined_table_name="All Orders"),
+ [
+ ("sales", "delta", None, "sales", "delta", []),
+ ("all_orders", "files", "all", "", "csv", ["a.csv", "daily/b.csv"]),
+ ],
+ ),
+ (
+ "combine_with_mixed_formats_reads_each_file_by_extension",
+ ["a.csv", "b.parquet"],
+ DiscoveryOptions(combined_table_name="mixed"),
+ [("mixed", "files", "all", "", "infer", ["a.csv", "b.parquet"])],
+ ),
+ (
+ "delta_format_offers_only_delta_tables",
+ ["a.csv", "sales/_delta_log/0.json", "sales/p.parquet"],
+ DiscoveryOptions(file_format="delta"),
+ [("sales", "delta", None, "sales", "delta", [])],
+ ),
+ (
+ "explicit_format_reads_any_extension_and_skips_delta_tables",
+ ["exports/a.txt", "exports/b", "sales/_delta_log/0.json", "sales/p.parquet"],
+ DiscoveryOptions(file_format="csv_no_header"),
+ [("exports", "files", "folder", "exports", "csv_no_header", ["exports/a.txt", "exports/b"])],
+ ),
+ (
+ "spark_bookkeeping_files_are_skipped",
+ ["out/_SUCCESS", "out/.part-0.crc", "out/_temporary/x.csv", "out/part-0.csv"],
+ DiscoveryOptions(),
+ [("out", "files", "folder", "out", "csv", ["out/part-0.csv"])],
+ ),
+ (
+ "pattern_matches_across_folders",
+ ["a/x.csv", "a/y.json", "b/z.csv"],
+ DiscoveryOptions(file_pattern="*.csv"),
+ [
+ ("a", "files", "folder", "a", "csv", ["a/x.csv"]),
+ ("b", "files", "folder", "b", "csv", ["b/z.csv"]),
+ ],
+ ),
+ (
+ "a_folder_with_two_formats_names_a_table_per_format",
+ ["data/a.csv", "data/b.parquet"],
+ DiscoveryOptions(),
+ [
+ ("data_csv", "files", "folder", "data", "csv", ["data/a.csv"]),
+ ("data_parquet", "files", "folder", "data", "parquet", ["data/b.parquet"]),
+ ],
+ ),
+ (
+ "names_that_sanitize_alike_get_a_suffix",
+ ["a-b.csv", "a_b.csv"],
+ DiscoveryOptions(),
+ [
+ ("a_b", "files", "file", "a-b.csv", "csv", ["a-b.csv"]),
+ ("a_b_2", "files", "file", "a_b.csv", "csv", ["a_b.csv"]),
+ ],
+ ),
+ ]
+ )
+ def test_group_tables(
+ self,
+ _name: str,
+ paths: list[str],
+ options: DiscoveryOptions,
+ expected: list[tuple[str, str, str | None, str, str, list[str]]],
+ ) -> None:
+ tables = group_tables([stored(path) for path in paths], options, root_name="exports")
+
+ assert summarize(tables) == expected
+
+ def test_metadata_round_trip_rebuilds_the_same_table(self) -> None:
+ options = DiscoveryOptions(file_format="csv_no_header", file_pattern="*.txt", allow_double_quotes=False)
+ [table] = group_tables([stored("exports/a.txt")], options, root_name="root")
+
+ rebuilt = TableDefinition.from_schema_metadata(table.name, table.schema_metadata())
+
+ assert rebuilt is not None
+ assert rebuilt.same_location(table)
+ assert (rebuilt.csv, rebuilt.pattern, rebuilt.source_format) == (
+ CsvOptions(has_header=False, allow_double_quotes=False),
+ "*.txt",
+ "csv_no_header",
+ )
+
+ @parameterized.expand(
+ [
+ ("missing", None),
+ ("unknown_kind", {"kind": "folder", "path": "a", "format": "csv"}),
+ ("files_without_scope", {"kind": "files", "path": "a", "format": "csv"}),
+ ("unknown_format", {"kind": "files", "scope": "folder", "path": "a", "format": "xml"}),
+ ]
+ )
+ def test_unusable_metadata_falls_back_to_discovery(self, _name: str, metadata: dict | None) -> None:
+ assert TableDefinition.from_schema_metadata("a", metadata) is None
+
+
+class TestTableLimit:
+ def test_names_that_sanitize_alike_get_increasing_suffixes(self) -> None:
+ objects = [stored(f"{name}.csv") for name in ("a-b", "a_b", "a b", "a.b")]
+
+ names = [table.name for table in group_tables(objects, DiscoveryOptions(), root_name="root")]
+
+ assert sorted(names) == ["a_b", "a_b_2", "a_b_3", "a_b_4"]
+
+ def test_an_unknown_file_format_is_an_invalid_config(self) -> None:
+ with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):
+ DiscoveryOptions(file_format="avro") # type: ignore[arg-type]
+
+ def test_discovery_stops_at_the_table_limit_but_resolving_a_table_does_not(self) -> None:
+ objects = [stored(f"file_{index:05d}.csv") for index in range(MAX_TABLES + 1)]
+
+ assert len(group_tables(objects, DiscoveryOptions(), root_name="root")) == MAX_TABLES
+ assert len(group_tables(objects, DiscoveryOptions(), root_name="root", max_tables=None)) == MAX_TABLES + 1
+
+
+class TestResolveTableFiles:
+ def test_a_folder_table_leaves_out_subfolders_that_are_tables_of_their_own(self, tmp_path: Path) -> None:
+ for path in ("events/a.csv", "events/year=2024/b.csv", "events/clicks/c.csv"):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("id\n1\n")
+ store = local_store(tmp_path)
+ table = TableDefinition.from_schema_metadata(
+ "events",
+ {"kind": "files", "scope": "folder", "path": "events", "format": "csv", "source_format": "infer"},
+ )
+ assert table is not None
+
+ resolved = resolve_table_files(store, table)
+
+ assert [f.path for f in resolved.files] == ["events/a.csv", "events/year=2024/b.csv"]
+
+
+ "paths, max_objects, expected",
+ [
+ (["t/a.csv", "t/b.csv", "t/c.csv"], 2, TOO_MANY_FILES_ERROR),
+ (["other/a.csv"], 10, NO_TABLE_FILES_ERROR),
+ ],
+)
+def test_sync_refuses_a_partial_or_missing_table(
+ tmp_path: Path, paths: list[str], max_objects: int, expected: str
+) -> None:
+ for path in paths:
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("id\n1\n")
+ table = TableDefinition.from_schema_metadata(
+ "t", {"kind": "files", "scope": "folder", "path": "t", "format": "csv", "source_format": "infer"}
+ )
+ assert table is not None
+
+ with pytest.raises(ObjectStorageError, match=expected):
+ resolve_table_files(local_store(tmp_path), table, max_objects=max_objects)
+
+
+class TestCsv:
+ @parameterized.expand(
+ [
+ (
+ "header_with_quoted_delimiter",
+ b'id,name\n1,"Smith, Jo"\n',
+ CsvOptions(),
+ [{"id": "1", "name": "Smith, Jo"}],
+ ),
+ (
+ "no_header_uses_clickhouse_names",
+ b"1,a\n2,b\n",
+ CsvOptions(has_header=False),
+ [
+ {"c1": "1", "c2": "a"},
+ {"c1": "2", "c2": "b"},
+ ],
+ ),
+ (
+ "double_quotes_off_keeps_quotes_as_text",
+ b'id,name\n"1,2"\n',
+ CsvOptions(allow_double_quotes=False),
+ [{"id": '"1', "name": '2"'}],
+ ),
+ (
+ "quoted_value_spans_lines",
+ b'id,note\n1,"line one\nline two"\n',
+ CsvOptions(),
+ [{"id": "1", "note": "line one\nline two"}],
+ ),
+ (
+ "empty_unquoted_is_null_and_empty_quoted_is_text",
+ b'a,b\n,""\n',
+ CsvOptions(),
+ [{"a": None, "b": ""}],
+ ),
+ (
+ "values_stay_text_when_later_rows_change_type",
+ b"n\n1\nnot a number\n",
+ CsvOptions(),
+ [{"n": "1"}, {"n": "not a number"}],
+ ),
+ (
+ "rows_with_the_wrong_field_count_are_skipped",
+ b"a,b\n1,2\n1,2,3\n4,5\n",
+ CsvOptions(),
+ [{"a": "1", "b": "2"}, {"a": "4", "b": "5"}],
+ ),
+ (
+ "byte_order_mark_and_duplicate_headers",
+ b"\xef\xbb\xbfid,id,\n1,2,3\n",
+ CsvOptions(),
+ [{"id": "1", "id_2": "2", "c3": "3"}],
+ ),
+ ("empty_file", b"", CsvOptions(), []),
+ (
+ "headerless_quote_inside_an_unquoted_field_does_not_open_a_record",
+ b'1,12" pipe,3\n4,5,6\n',
+ CsvOptions(has_header=False),
+ [{"c1": "1", "c2": '12" pipe', "c3": "3"}, {"c1": "4", "c2": "5", "c3": "6"}],
+ ),
+ (
+ "repeated_headers_get_increasing_suffixes",
+ b"a,a,a,a_2\n1,2,3,4\n",
+ CsvOptions(),
+ [{"a": "1", "a_2": "2", "a_3": "3", "a_2_2": "4"}],
+ ),
+ (
+ "headerless_first_record_spans_lines",
+ b'1,"line one\nline two",3\n4,5,6\n',
+ CsvOptions(has_header=False),
+ [{"c1": "1", "c2": "line one\nline two", "c3": "3"}, {"c1": "4", "c2": "5", "c3": "6"}],
+ ),
+ ]
+ )
+ def test_csv_rows(self, _name: str, content: bytes, options: CsvOptions, expected: list[dict]) -> None:
+ rows = [row for table in iter_csv_tables(io.BytesIO(content), options) for row in table.to_pylist()]
+
+ assert rows == expected
+
+ def test_an_unterminated_quote_in_the_first_record_stays_within_the_record_limit(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(formats, "MAX_FIRST_CSV_RECORD_BYTES", 64)
+ content = b'1,"open\n' + b"line\n" * 100
+
+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
+ list(iter_csv_tables(io.BytesIO(content), CsvOptions(has_header=False), file_path="x.csv"))
+
+ def test_a_header_only_file_is_a_table_with_no_rows(self) -> None:
+ tables = list(iter_csv_tables(io.BytesIO(b"id,name\n"), CsvOptions()))
+
+ assert [(table.column_names, table.num_rows) for table in tables] == [(["id", "name"], 0)]
+
+ def test_an_oversized_first_row_field_is_a_parse_error(self) -> None:
+ # Other code in the process may have raised the csv module's global field limit.
+ previous_limit = csv.field_size_limit(1024)
+ try:
+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
+ content = io.BytesIO(b"a," + b"x" * 2048 + b"\n")
+ list(iter_csv_tables(content, CsvOptions(has_header=False), file_path="big.csv"))
+ finally:
+ csv.field_size_limit(previous_limit)
+
+
+class TestJsonl:
+ def test_a_line_holding_a_large_array_is_chunked_by_row_count(self) -> None:
+ tables = list(iter_jsonl_tables(io.BytesIO(b"[" + b",".join(b"1" for _ in range(25)) + b"]\n"), max_rows=10))
+
+ assert [table.num_rows for table in tables] == [10, 10, 5]
+
+ def test_chunks_by_row_count_and_wraps_scalars(self) -> None:
+ content = b'{"id": 1}\n\n{"id": 2}\n[{"id": 3}, 4]\n"five"\n'
+
+ tables = list(iter_jsonl_tables(io.BytesIO(content), max_rows=2))
+
+ assert [table.num_rows for table in tables] == [2, 2, 1]
+ assert [row for table in tables for row in table.to_pylist()] == [
+ {"id": 1},
+ {"id": 2},
+ {"id": 3, "value": None},
+ {"id": None, "value": 4},
+ {"value": "five"},
+ ]
+
+ def test_a_too_deeply_nested_record_is_a_parse_error(self) -> None:
+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
+ list(iter_jsonl_tables(io.BytesIO(b"[" * 100_000 + b"]" * 100_000 + b"\n"), "deep.jsonl"))
+
+ def test_invalid_line_names_the_file_and_line(self) -> None:
+ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'x.jsonl' as JSON Lines: line 2"):
+ list(iter_jsonl_tables(io.BytesIO(b'{"id": 1}\n{oops\n'), "x.jsonl"))
+
+ def test_a_line_over_the_record_limit_is_a_parse_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(formats, "MAX_RECORD_BYTES", 16)
+
+ with pytest.raises(ObjectStorageError, match=f"{FILE_PARSE_ERROR} 'x.jsonl' as JSON Lines"):
+ list(iter_jsonl_tables(io.BytesIO(b'{"id": 1}\n{"id": "' + b"x" * 64 + b'"}\n'), "x.jsonl"))
+
+
+class TestReadTable:
+ def test_reads_each_format_and_tags_rows_with_their_file(self, tmp_path: Path) -> None:
+ (tmp_path / "mixed").mkdir()
+ (tmp_path / "mixed" / "a.csv.gz").write_bytes(gzip.compress(b"id\n1\n"))
+ (tmp_path / "mixed" / "b.jsonl").write_bytes(b'{"id": "2"}\n')
+ pq.write_table(pa.table({"id": ["3"]}), tmp_path / "mixed" / "c.parquet")
+ store = local_store(tmp_path)
+ [table] = group_tables(
+ store.list_objects("", 100), DiscoveryOptions(combined_table_name="mixed"), root_name="root"
+ )
+
+ rows = read_rows(store, table)
+
+ assert [(row["id"], row["_file_name"]) for row in rows] == [
+ ("1", "mixed/a.csv.gz"),
+ ("2", "mixed/b.jsonl"),
+ ("3", "mixed/c.parquet"),
+ ]
+ assert all(row["_file_modified_at"].tzinfo is not None for row in rows)
+
+ def test_gzipped_parquet_is_unpacked_before_it_is_read(self, tmp_path: Path) -> None:
+ buffer = io.BytesIO()
+ pq.write_table(pa.table({"id": ["1", "2"]}), buffer)
+ (tmp_path / "orders.parquet.gz").write_bytes(gzip.compress(buffer.getvalue()))
+ store = local_store(tmp_path)
+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")
+
+ assert [row["id"] for row in read_rows(store, table)] == ["1", "2"]
+
+ def test_gzipped_parquet_over_the_unpacked_size_limit_is_a_parse_error(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(formats, "MAX_UNPACKED_PARQUET_BYTES", 16)
+ (tmp_path / "big.parquet.gz").write_bytes(gzip.compress(b"x" * 1024))
+ store = local_store(tmp_path)
+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")
+
+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
+ read_rows(store, table)
+
+ def test_a_parquet_file_with_no_rows_still_yields_its_schema(self, tmp_path: Path) -> None:
+ pq.write_table(pa.table({"id": pa.array([], pa.string())}), tmp_path / "empty.parquet")
+ store = local_store(tmp_path)
+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")
+
+ batches = list(iter_table(store, table, LOGGER))
+
+ assert [(batch.column_names[0], batch.num_rows) for batch in batches] == [("id", 0)]
+
+ def test_corrupt_gzip_is_a_parse_error(self, tmp_path: Path) -> None:
+ (tmp_path / "bad.csv.gz").write_bytes(b"not gzip")
+ store = local_store(tmp_path)
+ [table] = group_tables(store.list_objects("", 10), DiscoveryOptions(), root_name="root")
+
+ with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
+ read_rows(store, table)
+
+ def test_delta_read_stays_on_the_version_it_opened(self, tmp_path: Path) -> None:
+ table_path = tmp_path / "sales"
+ write_deltalake(str(table_path), pa.table({"id": [1]}))
+ write_deltalake(str(table_path), pa.table({"id": [2]}), mode="append")
+ store = local_store(tmp_path)
+ [table] = group_tables(store.list_objects("", 100), DiscoveryOptions(), root_name="root")
+ assert table.kind == "delta"
+
+ batches = iter_delta_tables(store.delta_uri(table.path), {}, table.name, batch_rows=1)
+ first = next(batches)
+ write_deltalake(str(table_path), pa.table({"id": [7, 8, 9]}), mode="overwrite")
+ rest = list(batches)
+
+ assert sorted(row["id"] for batch in [first, *rest] for row in batch.to_pylist()) == [1, 2]
+
+ def test_an_empty_delta_snapshot_still_yields_its_schema(self, tmp_path: Path) -> None:
+ table_path = tmp_path / "sales"
+ write_deltalake(str(table_path), pa.table({"id": [1]}))
+ write_deltalake(str(table_path), pa.table({"id": pa.array([], pa.int64())}), mode="overwrite")
+ store = local_store(tmp_path)
+
+ tables = list(iter_delta_tables(store.delta_uri("sales"), {}, "sales"))
+
+ assert [(table.column_names, table.num_rows) for table in tables] == [(["id"], 0)]
+
+
+class TestLocationAndErrors:
+ @parameterized.expand(
+ [
+ ("bucket_with_a_host", {"bucket": "evil.com/x"}, INVALID_CONFIG_ERROR),
+ ("bucket_with_a_scheme", {"bucket": "s3://data"}, INVALID_CONFIG_ERROR),
+ ("region_with_a_host", {"region": "us-east-1.evil.com"}, INVALID_CONFIG_ERROR),
+ ("missing_secret", {"secret_access_key": " "}, INVALID_CONFIG_ERROR),
+ ("parent_folder", {"prefix": "a/../b"}, INVALID_CONFIG_ERROR),
+ ]
+ )
+ def test_config_rejects_values_that_could_retarget_the_request(
+ self, _name: str, overrides: dict, expected: str
+ ) -> None:
+ values = {
+ "provider": ObjectStoreProvider.S3,
+ "bucket": "customer-data",
+ "region": "us-east-1",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "secret",
+ **overrides,
+ }
+
+ with pytest.raises(ObjectStorageError, match=expected):
+ ObjectStoreConfig(**values)
+
+ def test_posthog_owned_bucket_is_blocked(self) -> None:
+ config = ObjectStoreConfig(
+ provider=ObjectStoreProvider.S3,
+ bucket="posthog-owned",
+ region="us-east-1",
+ access_key_id="AKIAEXAMPLE",
+ secret_access_key="secret",
+ )
+
+ with override_settings(DATAWAREHOUSE_BUCKET="posthog-owned"):
+ with pytest.raises(ObjectStorageError, match=BLOCKED_LOCATION_ERROR):
+ validate_store_location(config)
+
+ @parameterized.expand(
+ [
+ ("access_denied", PermissionError("Access Denied"), ACCESS_DENIED_ERROR),
+ ("no_such_bucket", FileNotFoundError("The specified bucket does not exist"), NO_SUCH_BUCKET_ERROR),
+ (
+ "wrong_region",
+ OSError("An error occurred (PermanentRedirect) when calling the ListObjectsV2 operation"),
+ WRONG_REGION_ERROR,
+ ),
+ ]
+ )
+ def test_storage_errors_map_to_stable_prefixes(self, _name: str, error: Exception, expected: str) -> None:
+ assert str(translate_storage_error(error, "customer-data")).startswith(expected)
+
+ def test_unknown_storage_errors_stay_retryable(self) -> None:
+ error = ConnectionResetError("reset by peer")
+
+ assert translate_storage_error(error, "customer-data") is error
+
+
+def _s3_store(client: Any) -> S3ObjectStore:
+ return S3ObjectStore(
+ fs=LocalFileSystem(),
+ root="customer-data/exports",
+ delta_uri_prefix="s3://",
+ delta_storage_options={},
+ client=client,
+ bucket="customer-data",
+ prefix="exports",
+ )
+
+
+def _listing(*keys: str, next_token: str | None = None) -> dict[str, Any]:
+ response: dict[str, Any] = {
+ "Contents": [{"Key": key, "Size": 5, "LastModified": MODIFIED_AT} for key in keys],
+ "IsTruncated": next_token is not None,
+ }
+ if next_token is not None:
+ response["NextContinuationToken"] = next_token
+ return response
+
+
+class TestS3Listing:
+ @parameterized.expand(
+ [
+ (
+ "pages_until_the_cap",
+ "",
+ 3,
+ [
+ ({"Prefix": "exports/"}, _listing("exports/a.csv", "exports/dir/", next_token="t1")),
+ (
+ {"Prefix": "exports/", "ContinuationToken": "t1"},
+ _listing("exports/b.csv", "exports/c.csv", "exports/d.csv"),
+ ),
+ ],
+ ["a.csv", "b.csv", "c.csv"],
+ ),
+ (
+ "a_prefix_that_names_one_file",
+ "orders.csv",
+ 10,
+ [
+ ({"Prefix": "exports/orders.csv/"}, _listing()),
+ ({"Prefix": "exports/orders.csv", "MaxKeys": 1000}, _listing("exports/orders.csv")),
+ ],
+ ["orders.csv"],
+ ),
+ ]
+ )
+ def test_list_objects(
+ self,
+ _name: str,
+ relative_prefix: str,
+ max_objects: int,
+ pages: list[tuple[dict[str, Any], dict[str, Any]]],
+ expected: list[str],
+ ) -> None:
+ client = boto3.client(
+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"
+ )
+ with Stubber(client) as stubber:
+ for params, response in pages:
+ stubber.add_response(
+ "list_objects_v2", response, {"Bucket": "customer-data", "MaxKeys": 1000, **params}
+ )
+
+ listed = _s3_store(client).list_objects(relative_prefix, max_objects)
+
+ assert [stored.path for stored in listed] == expected
+
+ def test_a_prefix_of_folder_markers_fails_instead_of_walking_every_key(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem."
+ "MAX_LISTED_KEYS",
+ 3,
+ )
+ client = boto3.client(
+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"
+ )
+ with Stubber(client) as stubber:
+ stubber.add_response(
+ "list_objects_v2",
+ _listing("exports/a/", "exports/b/", "exports/c/", "exports/d/"),
+ {"Bucket": "customer-data", "MaxKeys": 1000, "Prefix": "exports/"},
+ )
+
+ with pytest.raises(ObjectStorageError, match=TOO_MANY_FILES_ERROR):
+ _s3_store(client).list_objects("", 10)
+
+ def test_rejected_credentials_become_an_access_error(self) -> None:
+ client = boto3.client(
+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"
+ )
+ with Stubber(client) as stubber:
+ stubber.add_client_error("list_objects_v2", service_error_code="InvalidAccessKeyId", http_status_code=403)
+
+ with pytest.raises(PermissionError):
+ _s3_store(client).list_objects("", 10)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
index 68b0163449da..9fb8f23589dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
@@ -1,9 +1,25 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
+from typing import Literal
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+class AmazonS3CombineFilesConfig(config.Config):
+ table_name: str
+ enabled: bool = config.value(converter=config.str_to_bool, default=False)
+
+
@config.config
class AmazonS3SourceConfig(config.Config):
- pass
+ bucket: str
+ region: str
+ access_key_id: str
+ secret_access_key: str
+ path: str | None = None
+ file_pattern: str | None = None
+ file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")
+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
+ combine_files: AmazonS3CombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index f1ff82120f6a..fe778d40d5f0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -3,6 +3,7 @@ Adyen fail:safe-point
Airbrake fail:rate-limit-budget,stall-budget
AlphaVantage fail:rate-limit-budget,stall-budget
AmazonAds fail:rate-limit-budget,stall-budget
+AmazonS3 not-checkable:schemas
Amplitude fail:rate-limit-budget,stall-budget
Anthropic fail:safe-point
Apollo fail:rate-limit-budget,stall-budget
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
index e30f7d15e0a8..9d5422caaa02 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
@@ -49,6 +49,7 @@ PASS = "pass"
# `config`: `get_schemas` rejects every placeholder config.
# `empty`: `get_schemas` returns no schema for a placeholder config.
SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {
+ "AmazonS3": "network",
"BigQuery": "driver",
"Baserow": "network",
"CheckoutCom": "network",
@@ -94,6 +95,7 @@ SCHEMA_FILTER_NOT_CHECKABLE: dict[str, str] = {
# `request`: the extraction sends a request before it checks the schema, or the vendor supplies the list.
# `webhook`: the schemas are webhook-only, so the extraction accepts any name and yields no rows.
UNKNOWN_SCHEMA_NOT_CHECKABLE: dict[str, str] = {
+ "AmazonS3": "request",
"Attentive": "webhook",
"Baserow": "request",
"BigQuery": "driver",
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index a4ce99e4352f..2ffa77aaaa9f 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -161,6 +161,7 @@ def _configure_source_mock_versioning(mock_get_source) -> None:
mock_get_source.return_value.default_version = "v1"
mock_get_source.return_value.get_version_deprecation.return_value = None
mock_get_source.return_value.max_instances_per_team = None
+ mock_get_source.return_value.requires_team_feature_flag = None
mock_get_source.return_value.connection_host_fields = []
mock_get_source.return_value.detects_primary_keys = False
mock_get_source.return_value.server_managed_job_input_fields.return_value = []
@@ -5972,6 +5973,7 @@ class TestExternalDataSource(APIBaseTest):
]
mock_source.get_endpoint_permissions.return_value = {}
mock_source.detects_primary_keys = False
+ mock_source.requires_team_feature_flag = None
response = self.client.post(
f"/api/environments/{self.team.pk}/external_data_sources/database_schema/",
@@ -9324,6 +9326,41 @@ class TestExternalDataSource(APIBaseTest):
else:
assert response.json()["message"] != limit_message
+ @parameterized.expand(
+ [
+ ("create_without_flag", "", False),
+ ("create_with_flag", "", True),
+ ("database_schema_without_flag", "database_schema/", False),
+ ("database_schema_with_flag", "database_schema/", True),
+ ("setup_without_flag", "setup/", False),
+ ("setup_with_flag", "setup/", True),
+ ("store_credentials_without_flag", "store_credentials/", False),
+ ("store_credentials_with_flag", "store_credentials/", True),
+ ]
+ )
+ def test_source_behind_a_team_feature_flag(self, _name: str, path: str, flag_enabled: bool) -> None:
+ payload = {
+ "bucket": "Not A Bucket",
+ "region": "us-east-1",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "secret",
+ }
+ with patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_source.helpers.feature_enabled_or_false",
+ return_value=flag_enabled,
+ ):
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_sources/{path}",
+ data={"source_type": "AmazonS3", "prefix": "s3_", "payload": payload, **payload},
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ # With the flag the request reaches credential validation, which rejects the bucket name
+ # before any network call.
+ expected = "The storage settings are invalid" if flag_enabled else "isn't available to connect yet"
+ assert expected in response.json()["message"]
+ assert not ExternalDataSource.objects.filter(team_id=self.team.pk, source_type="AmazonS3").exists()
+
def test_revenue_analytics_config_created_automatically(self):
source = self._create_external_data_source()
@@ -12985,10 +13022,10 @@ class TestExternalDataSourceSetup(APIBaseTest):
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.capture_exception")
def test_setup_rejects_source_without_schema_discovery(self, mock_capture_exception):
- # AmazonS3 doesn't implement get_schemas, so the base raises NotImplementedError.
+ # AmazonCloudWatch doesn't implement get_schemas, so the base raises NotImplementedError.
response = self.client.post(
f"/api/environments/{self.team.pk}/external_data_sources/setup/",
- data={"source_type": "AmazonS3", "prefix": "s3_setup_test", "payload": {}},
+ data={"source_type": "AmazonCloudWatch", "prefix": "scaffold_setup_test", "payload": {}},
)
assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
assert "isn't available to connect yet" in response.json()["message"]