:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): migrate self-managed tables into object storage sources (#120353)
Tom Owersac63f82b8ddf
diff --git a/products/access_control/backend/facade/api.py b/products/access_control/backend/facade/api.pyindex d4e2b2f58dba..4ac98417474e 100644--- a/products/access_control/backend/facade/api.py+++ b/products/access_control/backend/facade/api.py@@ -379,6 +379,28 @@ def object_ids_restricted_from_any_member(return {resource_id for resource_id in restricted if resource_id}+def object_ids_with_rules(*, team_id: int, resource: APIScopeObject) -> set[str]:+ """Ids of `resource` objects that carry any object-level rule, whatever its level.++ For a caller that copies an object into a new one that can't carry the same rules. It skips+ every object with a rule rather than judging which rules restrict, and it ignores whether+ the organization enforces rules today, because the rules apply again when it does.+ """+ resource_ids = AccessControl.objects.filter(+ team_id=team_id, resource=resource, resource_id__isnull=False+ ).values_list("resource_id", flat=True)+ return {resource_id for resource_id in resource_ids if resource_id}+++def resource_has_rules(*, team_id: int, resource: APIScopeObject) -> bool:+ """Whether the team has any resource-level rule for `resource`, whatever its level.++ For a caller about to make objects inherit from `resource`, which changes what those rules+ reach. Like `object_ids_with_rules`, it ignores whether the organization enforces rules today.+ """+ return AccessControl.objects.filter(team_id=team_id, resource=resource, resource_id__isnull=True).exists()++def role_belongs_to_organization(*, role_id: str | UUID, organization_id: UUID) -> bool:"""Whether the role is a role of that organization.diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.pynew file mode 100644index 000000000000..6792305ef2bf--- /dev/null+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py@@ -0,0 +1,101 @@+from collections import Counter+from typing import Any++from django.core.management.base import BaseCommand, CommandError, CommandParser++from products.warehouse_sources.backend.self_managed_migration import (+ DEFAULT_ABANDON_REASON,+ MigrationOptions,+ abandon_tables,+ describe_status,+ evaluate_tables,+ migrate_self_managed_tables,+)+++def _split_ids(raw: str | None) -> list[str]:+ seen: dict[str, None] = {}+ for part in (raw or "").split(","):+ if part.strip():+ seen.setdefault(part.strip(), None)+ return list(seen)+++class Command(BaseCommand):+ help = (+ "Link self-managed warehouse tables to object storage sources. Each table gets a turned-off schema "+ "in a source for its bucket, and nothing syncs: the table keeps reading its own bucket until the user "+ "turns syncing on, and the first sync then replaces it with the same id and name. Runs only for teams "+ "with the object storage sources feature flag. Without --live-run the command only reports what it "+ "would do."+ )++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, default=None, help="Only act on this team's tables.")+ parser.add_argument("--table-ids", type=str, default=None, help="Comma-separated warehouse table ids.")+ parser.add_argument(+ "--limit", type=int, default=None, help="Link at most N tables in this run, for a cautious first batch."+ )+ parser.add_argument(+ "--status", action="store_true", help="Report each table's migration state and change nothing."+ )+ parser.add_argument(+ "--abandon",+ type=str,+ default=None,+ help="Comma-separated table ids to leave self-managed. Removes the schema the migration made for them.",+ )+ parser.add_argument(+ "--abandon-reason",+ type=str,+ default=DEFAULT_ABANDON_REASON,+ help="The reason recorded for each abandoned table.",+ )+ parser.add_argument(+ "--live-run", action="store_true", help="Act. Without this the command only reports what it would do."+ )++ def handle(self, *args: Any, **options: Any) -> None:+ if options["limit"] is not None and options["limit"] < 1:+ raise CommandError("--limit must be at least 1.")+ if options["status"] and options["abandon"] is not None:+ raise CommandError("Pass only one of --status and --abandon.")++ team_id: int | None = options["team_id"]+ table_ids = _split_ids(options["table_ids"])+ live_run: bool = options["live_run"]++ if options["status"]:+ self._report_status(team_id, table_ids)+ elif options["abandon"] is not None:+ self._abandon(_split_ids(options["abandon"]), options["abandon_reason"], team_id, live_run)+ else:+ self._migrate(team_id, table_ids, live_run, options["limit"])++ def _report_status(self, team_id: int | None, table_ids: list[str]) -> None:+ statuses = evaluate_tables(team_id=team_id, table_ids=table_ids)+ for status in statuses:+ self.stdout.write(describe_status(status))+ counts = Counter(str(status.state) for status in statuses)+ self.stdout.write(f"\n{len(statuses)} table(s): " + ", ".join(f"{n} {s}" for s, n in sorted(counts.items())))++ def _abandon(self, table_ids: list[str], reason: str, team_id: int | None, live_run: bool) -> None:+ if not table_ids:+ raise CommandError("--abandon needs at least one table id.")+ for outcome in abandon_tables(table_ids, reason=reason, team_id=team_id, live_run=live_run):+ detail = f" ({outcome.reason})" if outcome.reason else ""+ self.stdout.write(f"table={outcome.table_id} {outcome.action}{detail}")+ if not live_run:+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to abandon these tables."))++ def _migrate(self, team_id: int | None, table_ids: list[str], live_run: bool, limit: int | None) -> None:+ outcomes = migrate_self_managed_tables(+ MigrationOptions(team_id=team_id, table_ids=tuple(table_ids), live_run=live_run, limit=limit)+ )+ for outcome in outcomes:+ error = f" error={outcome.error}" if outcome.error else ""+ self.stdout.write(f"{describe_status(outcome.status)} action={outcome.action}{error}")+ actions = Counter(outcome.action for outcome in outcomes)+ self.stdout.write(f"\n{len(outcomes)} table(s): " + ", ".join(f"{n} {a}" for a, n in sorted(actions.items())))+ if not live_run:+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to link these tables."))diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex 78807efa6ae3..238d733f957f 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -2347,6 +2347,56 @@ def get_schemas_for_direct_reconciliation(# offering the new name as a separate one.SCHEMA_RESOURCE_ID_METADATA_KEY = "source_resource_id"+# Set to True in a schema's `schema_metadata` when the schema reads a definition PostHog stored on it,+# not one that discovery reports (a self-managed table moved into an object storage source reads the+# exact files of its old URL pattern). Discovery can't report such a schema, so it must not disable it.+SCHEMA_PINNED_METADATA_KEY = "pinned"+# Set in a schema's `schema_metadata` to the self-managed table that the schema's first sync replaces.+# The schema does not link to that table before the sync, so the table stays as it is until then.+SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY = "self_managed_table"+++@frozen+class SelfManagedTableRef:+ """The self-managed table a schema replaces, and the files the schema was pinned to read."""++ table_id: str+ url_pattern: str+ format: str+ # Set when an operator abandoned the migration of the table.+ abandon_reason: str | None = None++ def to_metadata(self) -> dict[str, Any]:+ metadata = {"table_id": self.table_id, "url_pattern": self.url_pattern, "format": self.format}+ if self.abandon_reason is not None:+ metadata["abandon_reason"] = self.abandon_reason+ return metadata+++def _schema_metadata(schema: "ExternalDataSchema") -> dict[str, Any]:+ metadata = (schema.sync_type_config or {}).get("schema_metadata")+ return metadata if isinstance(metadata, dict) else {}+++def is_pinned_schema(schema: "ExternalDataSchema") -> bool:+ return _schema_metadata(schema).get(SCHEMA_PINNED_METADATA_KEY) is True+++def self_managed_table_ref(schema: "ExternalDataSchema") -> SelfManagedTableRef | None:+ ref = _schema_metadata(schema).get(SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY)+ if not isinstance(ref, dict):+ return None+ table_id, url_pattern, table_format = ref.get("table_id"), ref.get("url_pattern"), ref.get("format")+ if not (isinstance(table_id, str) and isinstance(url_pattern, str) and isinstance(table_format, str)):+ return None+ reason = ref.get("abandon_reason")+ return SelfManagedTableRef(+ table_id=table_id,+ url_pattern=url_pattern,+ format=table_format,+ abandon_reason=reason if isinstance(reason, str) else None,+ )+def _resource_id(metadata: object) -> str | None:if isinstance(metadata, dict) and metadata.get(SCHEMA_RESOURCE_ID_METADATA_KEY) is not None:@@ -2540,10 +2590,16 @@ def sync_old_schemas_with_new_schemas(for schema in schemas_to_create:seeded_metadata = (schema_metadata_by_name or {}).get(schema)- deleted_obj = (- ExternalDataSchema.objects.filter(team_id=team_id, source_id=source_id, name=schema, deleted=True)- .order_by("-updated_at", "-created_at")- .first()+ deleted_obj = next(+ (+ row+ for row in ExternalDataSchema.objects.filter(+ team_id=team_id, source_id=source_id, name=schema, deleted=True+ ).order_by("-updated_at", "-created_at")+ # A deleted migration schema records that its table stays self-managed, so discovery must not revive it.+ if SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY not in _schema_metadata(row)+ ),+ None,)if deleted_obj is not None:deleted_obj.deleted = False@@ -2582,6 +2638,8 @@ def sync_old_schemas_with_new_schemas(team_id=team_id, name=schema, source_id=source_id, deleted=False)for s in schemas_to_check:+ if is_pinned_schema(s):+ continue# Only rows nobody enabled disappear entirely. A user-enabled row survives as visibly# disabled, because soft-deleting it would silently discard the user's selection (e.g.# a scope-gated table the source stopped offering before its first successful sync).diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 8a551d84a1a7..4f8cd4d20cb8 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -76,6 +76,8 @@ if TYPE_CHECKING:from posthog.models import User+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+SERIALIZED_FIELD_TO_CLICKHOUSE_MAPPING: dict[DatabaseSerializedFieldType, str] = {DatabaseSerializedFieldType.INTEGER: "Int64",DatabaseSerializedFieldType.FLOAT: "Float64",@@ -553,6 +555,20 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletdef csv_allow_double_quotes(self) -> bool | None:return self.options.get("csv_allow_double_quotes")+ def adopt_into_source(self, source: "ExternalDataSource") -> list[str]:+ """Make a self-managed table one that `source` syncs. Returns the fields this changed.++ The caller saves these fields together with the new `url_pattern` and `format`: the+ customer's key can't read PostHog's bucket, and PostHog's key can't read the customer's, so+ a row that has only some of these changes can't be read at all. The row id and the name stay,+ so joins, views and saved queries that refer to the table keep working.+ """+ self.external_data_source = source+ self.credential = None+ # A CSV setting for the customer's files. The table now reads Parquet files PostHog wrote.+ self.options = {key: value for key, value in (self.options or {}).items() if key != "csv_allow_double_quotes"}+ return ["external_data_source", "credential", "options"]+def soft_delete(self):from products.data_tools.backend.models.join import DataWarehouseJoindiff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.pynew file mode 100644index 000000000000..ac27e4bda09a--- /dev/null+++ b/products/warehouse_sources/backend/self_managed_migration.py@@ -0,0 +1,664 @@+"""Move self-managed warehouse tables into the object storage sources, without a copy.++A self-managed table is read in place from a customer bucket. Each one gets an object storage source+for its bucket and a turned-off schema that reads exactly the table's files. The schema does not link+to the table: it names the table in its pinned `schema_metadata` (see `SelfManagedTableRef`), and the+table stays as it is, read from the customer bucket and not billed. When the user turns syncing on,+the first sync is a normal billable sync, and its last step points the existing table row at the copy+in PostHog storage (see `find_replaced_self_managed_table` and `DataWarehouseTable.adopt_into_source`),+so the row id and the name stay and existing queries keep working.++The state of each table comes from the rows themselves: a migration source carries a marker in its+`connection_metadata`, and its schemas name their tables. Running the migration again therefore+continues where it stopped.+"""++import re+import uuid+import dataclasses+from collections import defaultdict+from collections.abc import Callable, Iterable, Mapping+from enum import StrEnum+from typing import Any, Literal++from django.db import transaction+from django.db.models import Q++import boto3+import structlog+from botocore.config import Config as BotoConfig+from botocore.exceptions import BotoCoreError, ClientError++from posthog.dataclasses import frozen+from posthog.models.team import Team++from products.access_control.backend.facade.api import object_ids_with_rules, resource_has_rules+from products.data_warehouse.backend.facade.api import delete_external_data_schedule+from products.warehouse_sources.backend.models.credential import DataWarehouseCredential+from products.warehouse_sources.backend.models.external_data_destination import (+ ExternalDataSourceDestination,+ get_or_create_warehouse_destination,+)+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.models.external_data_schema import (+ SCHEMA_PINNED_METADATA_KEY,+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,+ ExternalDataSchema,+ SelfManagedTableRef,+ self_managed_table_ref,+)+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (+ source_feature_flag_missing,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (+ TableDefinition,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+ StorageService,+ UnmappableTableError,+ parse_self_managed_url,+ table_definition_for,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (+ CONNECT_TIMEOUT_SECONDS,+ MAX_REQUEST_ATTEMPTS,+ READ_TIMEOUT_SECONDS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (+ ObjectStorageSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.types import ExternalDataJobStatus++logger = structlog.get_logger(__name__)++# The `connection_metadata` key of a source the migration created.+MIGRATION_MARKER_KEY = "self_managed_migration"+DEFAULT_ABANDON_REASON = "Left self-managed by an operator."+_DELETED_REASON = "its migration schema or source was deleted"++# Resolves the region of an S3 bucket whose URL names none, or returns None when it can't.+BucketRegionResolver = Callable[[str, SelfManagedCredential], str | None]+++class TableMigrationState(StrEnum):+ # Not linked to a source yet.+ PENDING = "pending"+ # Has a migration schema, and the table still reads its own bucket until a sync replaces it.+ AWAITING_SYNC = "awaiting_sync"+ MIGRATED = "migrated"+ ABANDONED = "abandoned"+ SKIPPED = "skipped"+++MigrationAction = Literal["none", "would_link", "linked", "link_failed", "limit_reached"]+AbandonAction = Literal["would_abandon", "abandoned", "refused"]+++@frozen+class TableMigrationStatus:+ table_id: str+ team_id: int+ name: str+ state: TableMigrationState+ reason: str | None = None+ source_id: str | None = None+ schema_id: str | None = None+++@frozen+class TableMigrationOutcome:+ status: TableMigrationStatus+ action: MigrationAction = "none"+ error: str | None = None+++@frozen+class AbandonOutcome:+ table_id: str+ action: AbandonAction+ reason: str | None = None+++@frozen+class MigrationOptions:+ team_id: int | None = None+ table_ids: tuple[str, ...] = ()+ live_run: bool = False+ # How many tables one run may link. None links every one it can.+ limit: int | None = None+++@frozen+class _TablePlan:+ """How one self-managed table maps onto an object storage source."""++ location: SelfManagedLocation+ source: ObjectStorageSource[Any]+ definition: TableDefinition+ job_inputs: dict[str, Any] = dataclasses.field(repr=False)+++@frozen+class _Evaluation:+ table: DataWarehouseTable+ status: TableMigrationStatus+ plan: _TablePlan | None = None+ # The live migration schema of the table.+ schema: ExternalDataSchema | None = None+ job: ExternalDataJob | None = None+++def resolve_s3_bucket_region(bucket: str, credential: SelfManagedCredential) -> str | None:+ """Ask S3 for the bucket's region with the table's own key.++ S3 names the region in the `x-amz-bucket-region` header of a HEAD request, also when it+ answers with a redirect or a 403, so a key that may only read objects still finds it.+ """+ if not credential.access_key.strip() or not credential.access_secret.strip():+ return None+ client = boto3.client(+ "s3",+ region_name="us-east-1",+ aws_access_key_id=credential.access_key.strip(),+ aws_secret_access_key=credential.access_secret.strip(),+ config=BotoConfig(+ connect_timeout=CONNECT_TIMEOUT_SECONDS,+ read_timeout=READ_TIMEOUT_SECONDS,+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},+ ),+ )+ response: Mapping[str, Any]+ try:+ response = client.head_bucket(Bucket=bucket)+ except ClientError as e:+ response = e.response+ except BotoCoreError:+ return None+ headers = response.get("ResponseMetadata", {}).get("HTTPHeaders", {})+ region = headers.get("x-amz-bucket-region")+ return region if isinstance(region, str) and region else None+++def _object_storage_sources() -> dict[StorageService, ObjectStorageSource[Any]]:+ return {+ source.self_managed_service: source+ for source in SourceRegistry.get_all_sources().values()+ if isinstance(source, ObjectStorageSource)+ }+++def _credential_of(table: DataWarehouseTable) -> SelfManagedCredential | None:+ credential = table.credential+ if credential is None:+ return None+ return SelfManagedCredential(access_key=credential.access_key or "", access_secret=credential.access_secret or "")+++class _Planner:+ """Maps tables onto sources, caching the per-team flag check and the per-bucket region lookup."""++ def __init__(self, resolve_region: BucketRegionResolver) -> None:+ self._resolve_region = resolve_region+ self._sources = _object_storage_sources()+ self._flag_missing: dict[tuple[int, StorageService], bool] = {}+ self._regions: dict[tuple[str, str], str | None] = {}+ self._tables_with_rules: dict[int, set[str]] = {}+ self._teams_with_source_rules: dict[int, bool] = {}++ def plan(self, table: DataWarehouseTable) -> _TablePlan | str:+ """The plan for a table, or the reason the table can't be migrated."""+ credential = _credential_of(table)+ if credential is None:+ return "the table has no credential"+ if self._has_access_rules(table):+ # The new source would hold the table's key with default access, and a member the+ # table's rules shut out could read the same files through a schema of that source.+ return "the table has its own access rules, which the source PostHog would create can't carry"+ if self._has_source_rules(table):+ # A table in a source falls back to the rules for sources, which never reached a+ # self-managed table, so moving it could take access away from members who have it.+ # Rules for the whole warehouse don't reach the new source, so a member they shut out+ # could read the table's files through it.+ return "the project has access rules for sources or for the warehouse, which the move would change"+ try:+ location = parse_self_managed_url(table.url_pattern)+ source = self._sources.get(location.service)+ if source is None:+ return f"no import source reads '{location.service}' storage"+ if self._team_lacks_flag(table.team, source):+ return f"the team doesn't have the '{source.requires_team_feature_flag}' feature flag"+ if location.service == "s3" and location.region is None:+ region = self._region(table, location, credential)+ if region is None:+ return "the URL doesn't name the bucket's region, and S3 didn't return it"+ location = dataclasses.replace(location, region=region)+ definition = table_definition_for(+ table.name, location, table.format, table.columns, table.csv_allow_double_quotes+ )+ job_inputs = source.job_inputs_for_self_managed(location, credential)+ except UnmappableTableError as e:+ return str(e)+ return _TablePlan(location=location, source=source, definition=definition, job_inputs=job_inputs)++ def _has_access_rules(self, table: DataWarehouseTable) -> bool:+ if table.team_id not in self._tables_with_rules:+ self._tables_with_rules[table.team_id] = object_ids_with_rules(+ team_id=table.team_id, resource="warehouse_table"+ )+ return str(table.id) in self._tables_with_rules[table.team_id]++ def _has_source_rules(self, table: DataWarehouseTable) -> bool:+ if table.team_id not in self._teams_with_source_rules:+ self._teams_with_source_rules[table.team_id] = resource_has_rules(+ team_id=table.team_id, resource="external_data_source"+ ) or resource_has_rules(team_id=table.team_id, resource="warehouse_objects")+ return self._teams_with_source_rules[table.team_id]++ def _team_lacks_flag(self, team: Team, source: ObjectStorageSource[Any]) -> bool:+ key = (team.pk, source.self_managed_service)+ if key not in self._flag_missing:+ self._flag_missing[key] = source_feature_flag_missing(source, team)+ return self._flag_missing[key]++ def _region(+ self, table: DataWarehouseTable, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> str | None:+ key = (str(table.credential_id), location.bucket)+ if key not in self._regions:+ try:+ self._regions[key] = self._resolve_region(location.bucket, credential)+ except Exception:+ logger.warning("Could not resolve the S3 bucket region", table_id=str(table.id), exc_info=True)+ self._regions[key] = None+ return self._regions[key]+++def _migration_schemas(team_id: int | None, table_ids: Iterable[str]) -> dict[str, list[ExternalDataSchema]]:+ """Every schema, deleted or not, that the migration made for a table, by table id, oldest first."""+ queryset = (+ ExternalDataSchema.objects.filter(+ source__connection_metadata__has_key=MIGRATION_MARKER_KEY,+ sync_type_config__schema_metadata__has_key=SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,+ )+ .select_related("source")+ .order_by("created_at", "id")+ )+ if team_id is not None:+ queryset = queryset.filter(team_id=team_id)+ ids = list(table_ids)+ if ids:+ queryset = queryset.filter(sync_type_config__schema_metadata__self_managed_table__table_id__in=ids)+ by_table: dict[str, list[ExternalDataSchema]] = defaultdict(list)+ for schema in queryset:+ ref = self_managed_table_ref(schema)+ if ref is not None:+ by_table[ref.table_id].append(schema)+ return by_table+++def _is_live(schema: ExternalDataSchema) -> bool:+ return not schema.deleted and not schema.source.deleted+++def _select_tables(+ team_id: int | None, table_ids: Iterable[str], migration_schemas: Mapping[str, list[ExternalDataSchema]]+) -> list[DataWarehouseTable]:+ """Self-managed tables, plus every table the migration made a schema for, oldest first."""+ queryset = (+ DataWarehouseTable.raw_objects.exclude(deleted=True)+ .filter(Q(external_data_source__isnull=True, credential__isnull=False) | Q(id__in=list(migration_schemas)))+ .select_related("credential", "team")+ .order_by("created_at", "id")+ )+ if team_id is not None:+ queryset = queryset.filter(team_id=team_id)+ ids = list(table_ids)+ if ids:+ queryset = queryset.filter(id__in=ids)+ return list(queryset)+++def _latest_jobs(schemas: Iterable[ExternalDataSchema]) -> dict[str, ExternalDataJob]:+ schema_ids = [schema.id for schema in schemas]+ if not schema_ids:+ return {}+ jobs = ExternalDataJob.objects.filter(schema_id__in=schema_ids).order_by("schema_id", "-created_at")+ latest: dict[str, ExternalDataJob] = {}+ for job in jobs:+ latest.setdefault(str(job.schema_id), job)+ return latest+++def _awaiting_reason(table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None) -> str:+ ref = self_managed_table_ref(schema)+ if schema.table_id is not None:+ return "the table changed after the migration, so its schema syncs a separate table"+ if ref is None or (table.url_pattern, table.format) != (ref.url_pattern, ref.format):+ return "the table changed after the migration, so a sync makes a separate table"+ if job is None:+ return "syncing is on; waiting for the first sync" if schema.should_sync else "syncing is off"+ if job.status == ExternalDataJobStatus.RUNNING:+ return "the first sync is running"+ detail = job.latest_error or f"the last sync ended with status {job.status}"+ return f"{detail}; the table still reads its own bucket"+++def _evaluate(+ table: DataWarehouseTable,+ schemas: list[ExternalDataSchema],+ jobs: Mapping[str, ExternalDataJob],+ planner: _Planner,+) -> _Evaluation:+ live = next((schema for schema in schemas if _is_live(schema)), None)+ job = jobs.get(str(live.id)) if live is not None else None++ def status(state: TableMigrationState, reason: str | None = None) -> TableMigrationStatus:+ named = live or (schemas[-1] if schemas else None)+ return TableMigrationStatus(+ table_id=str(table.id),+ team_id=table.team_id,+ name=table.name,+ state=state,+ reason=reason,+ source_id=str(named.source_id) if named is not None else None,+ schema_id=str(named.id) if named is not None else None,+ )++ if table.external_data_source_id is not None:+ return _Evaluation(table=table, status=status(TableMigrationState.MIGRATED), schema=live, job=job)+ if live is not None:+ reason = _awaiting_reason(table, live, job)+ return _Evaluation(table=table, status=status(TableMigrationState.AWAITING_SYNC, reason), schema=live, job=job)+ if schemas:+ refs = [ref for ref in (self_managed_table_ref(schema) for schema in schemas) if ref is not None]+ reason = next((ref.abandon_reason for ref in reversed(refs) if ref.abandon_reason), _DELETED_REASON)+ return _Evaluation(table=table, status=status(TableMigrationState.ABANDONED, reason))++ planned = planner.plan(table)+ if isinstance(planned, _TablePlan):+ return _Evaluation(table=table, status=status(TableMigrationState.PENDING, "not linked yet"), plan=planned)+ return _Evaluation(table=table, status=status(TableMigrationState.SKIPPED, planned))+++def evaluate_tables(+ team_id: int | None = None,+ table_ids: Iterable[str] = (),+ resolve_region: BucketRegionResolver = resolve_s3_bucket_region,+) -> list[TableMigrationStatus]:+ return [evaluation.status for evaluation in _evaluate_all(team_id, table_ids, _Planner(resolve_region))]+++def _evaluate_all(team_id: int | None, table_ids: Iterable[str], planner: _Planner) -> list[_Evaluation]:+ ids = list(table_ids)+ schemas = _migration_schemas(team_id, ids)+ tables = _select_tables(team_id, ids, schemas)+ jobs = _latest_jobs(schema for table_schemas in schemas.values() for schema in table_schemas if _is_live(schema))+ return [_evaluate(table, schemas.get(str(table.id), []), jobs, planner) for table in tables]+++def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:+ """A table prefix no other source of the type uses, so the new source's table names can't clash.++ Like a source a user creates, the first source of a type needs none.+ """+ sources = ExternalDataSource.objects.filter(team_id=team_id, source_type=plan.source.source_type).exclude(+ deleted=True+ )+ taken = {(prefix or "").strip() for prefix in sources.values_list("prefix", flat=True)}+ if "" not in taken:+ return None+ base = re.sub(r"[^A-Za-z0-9_]", "_", plan.location.bucket).strip("_").lower() or "bucket"+ if not base[0].isalpha():+ base = f"bucket_{base}"+ candidate, suffix = base, 2+ while candidate in taken or ExternalDataSource.is_system_managed_prefix(candidate):+ candidate, suffix = f"{base}_{suffix}", suffix + 1+ return candidate+++def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:+ """The group's source for the table's team, credential and bucket. Runs under the credential lock."""+ existing = (+ ExternalDataSource.objects.filter(+ team_id=table.team_id,+ source_type=plan.source.source_type,+ connection_metadata__self_managed_migration__group_key=plan.location.group_key,+ connection_metadata__self_managed_migration__credential_id=str(table.credential_id),+ )+ .exclude(deleted=True)+ .order_by("created_at")+ .first()+ )+ if existing is not None:+ return existing+ source = ExternalDataSource.objects.create(+ prefix=_free_prefix(table.team_id, plan),+ source_id=str(uuid.uuid4()),+ connection_id=str(uuid.uuid4()),+ destination_id=str(uuid.uuid4()),+ team_id=table.team_id,+ status=ExternalDataSource.Status.RUNNING,+ source_type=plan.source.source_type,+ api_version=plan.source.default_version,+ job_inputs=plan.job_inputs,+ description=f"Self-managed tables in {plan.location.bucket}",+ access_method=ExternalDataSource.AccessMethod.WAREHOUSE,+ connection_metadata={+ MIGRATION_MARKER_KEY: {+ "group_key": plan.location.group_key,+ "credential_id": str(table.credential_id),+ }+ },+ )+ ExternalDataSourceDestination.objects.for_team(table.team_id).create(+ team_id=table.team_id, source_id=source.pk, destination=get_or_create_warehouse_destination(table.team_id)+ )+ return source+++def _credential_id(table: DataWarehouseTable) -> uuid.UUID:+ if table.credential_id is None:+ raise ValueError(f"Table {table.id} has no credential to migrate with")+ return table.credential_id+++def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:+ """The table's migration schema, created with its source when there is none yet.++ The credential row is the group lock: every table of one source shares it, and nothing else+ locks it, so two runs can't create two sources or two schemas for the same table.+ """+ with transaction.atomic():+ DataWarehouseCredential.objects.select_for_update().get(team_id=table.team_id, pk=_credential_id(table))+ schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])+ live = next((schema for schema in schemas if _is_live(schema)), None)+ if live is not None:+ return live+ if schemas:+ raise UnmappableTableError("the table's migration was abandoned")+ source = _find_or_create_source(table, plan)+ if (+ ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk, name=table.name)+ .exclude(deleted=True)+ .exists()+ ):+ raise UnmappableTableError(f"another table in this bucket already uses the name '{table.name}'")+ ref = SelfManagedTableRef(table_id=str(table.id), url_pattern=table.url_pattern, format=table.format)+ return ExternalDataSchema.objects.create(+ name=table.name,+ team_id=table.team_id,+ source=source,+ should_sync=False,+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,+ sync_type_config={+ "schema_metadata": {+ **plan.definition.schema_metadata(),+ SCHEMA_PINNED_METADATA_KEY: True,+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref.to_metadata(),+ }+ },+ )+++def migrate_self_managed_tables(+ options: MigrationOptions, resolve_region: BucketRegionResolver = resolve_s3_bucket_region+) -> list[TableMigrationOutcome]:+ """Plan, and on a live run create, the source and the turned-off schema of every selected table.++ Nothing syncs and no table changes. A dry run reads the same rows and reports the same+ decisions, but writes nothing.+ """+ planner = _Planner(resolve_region)+ outcomes: list[TableMigrationOutcome] = []+ linked = 0+ for evaluation in _evaluate_all(options.team_id, options.table_ids, planner):+ if evaluation.plan is None or evaluation.status.state != TableMigrationState.PENDING:+ outcomes.append(TableMigrationOutcome(status=evaluation.status))+ continue+ if options.limit is not None and linked >= options.limit:+ outcomes.append(TableMigrationOutcome(status=evaluation.status, action="limit_reached"))+ continue+ linked += 1+ if not options.live_run:+ outcomes.append(TableMigrationOutcome(status=evaluation.status, action="would_link"))+ continue+ outcomes.append(_link(evaluation.table, evaluation.plan, evaluation.status))+ return outcomes+++def _link(table: DataWarehouseTable, plan: _TablePlan, status: TableMigrationStatus) -> TableMigrationOutcome:+ try:+ schema = _ensure_schema(table, plan)+ except UnmappableTableError as e:+ return TableMigrationOutcome(status=status, action="link_failed", error=str(e))+ except Exception as e:+ # A database error on one table must not stop the batch or hide its outcomes.+ logger.warning("Could not link a self-managed table", table_id=str(table.id), exc_info=True)+ return TableMigrationOutcome(status=status, action="link_failed", error=f"{type(e).__name__}: {e}")+ logger.info(+ "Linked a self-managed table to an object storage source",+ table_id=str(table.id),+ team_id=table.team_id,+ schema_id=str(schema.id),+ )+ return TableMigrationOutcome(+ status=dataclasses.replace(+ status,+ state=TableMigrationState.AWAITING_SYNC,+ reason="syncing is off",+ schema_id=str(schema.id),+ source_id=str(schema.source_id),+ ),+ action="linked",+ )+++def _abandon_refusal(evaluation: _Evaluation) -> str | None:+ state = evaluation.status.state+ if state == TableMigrationState.MIGRATED:+ return "the table is already migrated"+ if state == TableMigrationState.ABANDONED:+ return "the table's migration is already abandoned"+ if state == TableMigrationState.SKIPPED:+ return f"the table can't be migrated, so there is nothing to abandon ({evaluation.status.reason})"+ schema = evaluation.schema+ if schema is None:+ return None+ if schema.table_id is not None:+ return "the table's schema syncs a separate table"+ if schema.should_sync:+ return "the user turned syncing on for the table's schema"+ if evaluation.job is not None and evaluation.job.status == ExternalDataJobStatus.RUNNING:+ return "the table's first sync is running"+ return None+++def abandon_tables(+ table_ids: Iterable[str],+ reason: str = DEFAULT_ABANDON_REASON,+ team_id: int | None = None,+ live_run: bool = False,+ resolve_region: BucketRegionResolver = resolve_s3_bucket_region,+) -> list[AbandonOutcome]:+ """Leave tables self-managed for good, and remove the schema the migration made for them.++ The schema stays as a deleted row that names the table and the reason, so later runs leave the+ table alone. A table with no schema yet gets one first, for that record. A table's source goes+ too when the table was the last one it held. The table itself never changes.+ """+ ids = list(table_ids)+ evaluations = {+ evaluation.status.table_id: evaluation for evaluation in _evaluate_all(team_id, ids, _Planner(resolve_region))+ }+ outcomes: list[AbandonOutcome] = []+ for table_id in ids:+ evaluation = evaluations.get(table_id)+ if evaluation is None:+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason="no such self-managed table"))+ continue+ refusal = _abandon_refusal(evaluation)+ if refusal is not None:+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason=refusal))+ elif not live_run:+ outcomes.append(AbandonOutcome(table_id=table_id, action="would_abandon"))+ else:+ try:+ _abandon(evaluation, reason)+ except UnmappableTableError as e:+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason=str(e)))+ continue+ outcomes.append(AbandonOutcome(table_id=table_id, action="abandoned", reason=reason))+ return outcomes+++def _abandon(evaluation: _Evaluation, reason: str) -> None:+ table = evaluation.table+ removed_source_id: str | None = None+ with transaction.atomic():+ if evaluation.schema is not None:+ schema = evaluation.schema+ else:+ assert evaluation.plan is not None+ schema = _ensure_schema(table, evaluation.plan)+ ref = self_managed_table_ref(schema)+ assert ref is not None+ metadata = (schema.sync_type_config or {})["schema_metadata"]+ metadata[SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY] = dataclasses.replace(ref, abandon_reason=reason).to_metadata()+ schema.save(update_fields=["sync_type_config", "updated_at"])+ schema.soft_delete()+ source = schema.source+ if (+ not ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk)+ .exclude(deleted=True)+ .exists()+ ):+ source.soft_delete()+ removed_source_id = str(source.pk)+ if evaluation.schema is not None:+ # A schedule exists only when the user turned syncing on before, so it is paused now.+ try:+ delete_external_data_schedule(str(schema.id))+ except Exception:+ # The schema is deleted, so its schedule can't start a sync.+ logger.warning(+ "Could not delete the schedule of an abandoned migration schema",+ table_id=str(table.id),+ schema_id=str(schema.id),+ exc_info=True,+ )+ logger.info(+ "Abandoned a self-managed table migration",+ table_id=str(table.id),+ team_id=table.team_id,+ removed_source_id=removed_source_id,+ )+++def describe_status(status: TableMigrationStatus) -> str:+ reason = f" ({status.reason})" if status.reason else ""+ return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}"diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.pyindex 62d7c0bcf713..2c71e846636d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py@@ -7,7 +7,7 @@ from datetime import date, datetimefrom typing import Any, Optionalfrom django.db import transaction-from django.db.models import Prefetch+from django.db.models import Prefetch, Qfrom django.utils import timezoneimport dlt@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.models.external_data_job import Externalfrom products.warehouse_sources.backend.models.external_data_schema import (ExternalDataSchema,mark_initial_sync_complete,+ self_managed_table_ref,update_sync_type_config_keys,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource@@ -322,6 +323,48 @@ def find_orphan_table(schema: ExternalDataSchema, source: ExternalDataSource) ->).first()+def find_replaced_self_managed_table(+ schema: ExternalDataSchema, source: ExternalDataSource+) -> DataWarehouseTable | None:+ """The self-managed table the schema's first sync replaces, when it still reads the files the schema copies.++ A table the user edited after the migration reads other files now, so the sync must not replace it,+ and gets a table of its own. A table this source already took over but that no schema owns is+ the result of a publish that stopped before the link, so the next publish finishes it.+ """+ ref = self_managed_table_ref(schema)+ if ref is None:+ return None+ return (+ DataWarehouseTable.objects.filter(team_id=schema.team_id, id=ref.table_id, deleted=False)+ .filter(externaldataschema__isnull=True)+ .filter(+ Q(external_data_source__isnull=True, url_pattern=ref.url_pattern, format=ref.format)+ | Q(external_data_source_id=source.id)+ )+ .first()+ )+++def find_adopted_self_managed_table(+ schema: ExternalDataSchema, source: ExternalDataSource+) -> DataWarehouseTable | None:+ """The self-managed table an earlier publish of the schema already moved into the source, but didn't link.++ The table keeps its own name, so the name lookup in `find_orphan_table` can't find it.+ """+ ref = self_managed_table_ref(schema)+ if ref is None:+ return None+ return DataWarehouseTable.objects.filter(+ team_id=schema.team_id,+ id=ref.table_id,+ external_data_source_id=source.id,+ deleted=False,+ externaldataschema__isnull=True,+ ).first()++def relink_orphan_table(schema: ExternalDataSchema, source: ExternalDataSource, logger: FilteringBoundLogger) -> DataWarehouseTable | None:@@ -334,7 +377,7 @@ def relink_orphan_table(if schema.table_id is not None or (schema.is_cdc and schema.cdc_table_mode == "cdc_only"):return schema.table- orphan = find_orphan_table(schema, source)+ orphan = find_orphan_table(schema, source) or find_adopted_self_managed_table(schema, source)if orphan is None:return None@@ -436,6 +479,9 @@ async def validate_schema_and_update_table(if table_created is not None:logger.debug(f"Found existing table {table_created.id} - reusing it for schema {_schema_id}")+ if table_created is None:+ table_created = find_replaced_self_managed_table(external_data_schema, job.pipeline)+# A reported row_count of 0 does not always mean the run wrote nothing: the v3 consumer# can read 0 on a redelivered final batch, and a resumed run counts only its own attempt.# A publish step with nothing to make queryable is what an empty first sync looks like.@@ -466,6 +512,10 @@ async def validate_schema_and_update_table(# new_url_pattern above is derived from the job's own destination folder, not from# request input, so this sync is a trusted writer of a credential-less table's URL.update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]+ if table.external_data_source_id is None:+ # A self-managed table this schema replaces reads its own bucket until now.+ update_fields.extend(table.adopt_into_source(job.pipeline))+ logger.info(f"Adopted self-managed table {table.id} into source {job.pipeline.id}")if live_size_mib is not None:table.size_in_s3_mib = live_size_mibupdate_fields.append("size_in_s3_mib")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex c6d8db6c601c..a336b90553e0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -1050,7 +1050,8 @@ class PipelineV3(Generic[ResumableData]):async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:pa_table = _append_debug_column_to_pyarrows_table(pa_table, self._load_id)- pa_table = normalize_table_column_names(pa_table)+ if not self._resource.preserve_column_names:+ pa_table = normalize_table_column_names(pa_table)if self._uses_delta_write_column_selection:pa_table = await observe_and_project_table(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 4fbee007e583..fe5537f59fd2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1320,6 +1320,37 @@ class TestSingleTableRunIsUntouched:assert set(base) - set(laned) == set()+class TestColumnNamesReachTheWrite:+ @pytest.mark.parametrize(+ "preserve, expected",+ [+ (False, ["order_id", "total_amount", "_ph_debug"]),+ (True, ["Order ID", "totalAmount", "_ph_debug"]),+ ],+ )+ async def test_preserving_names_skips_the_snake_case_rename(self, preserve: bool, expected: list[str]) -> None:+ # A self-managed table moved into a source keeps the column names its queries use.+ pipeline = _make_pipeline()+ pipeline._resource = SourceResponse(+ name="orders", items=lambda: [], primary_keys=None, preserve_column_names=preserve+ )+ cast(MagicMock, pipeline._schema).configure_mock(incremental_field=None, enabled_columns=None)+ pipeline._last_incremental_field_value = None+ pipeline._earliest_incremental_field_value = None+ table = pa.table({"Order ID": pa.array([1], pa.int64()), "totalAmount": pa.array([2.5], pa.float64())})++ with (+ patch.object(PipelineV3, "_stage_batch", AsyncMock(return_value=1)),+ patch.object(PipelineV3, "_advance_handoff_checkpoint", AsyncMock()),+ patch(f"{_PIPELINE}.update_incremental_field_values", AsyncMock(return_value=MagicMock())),+ patch(f"{_PIPELINE}.update_row_tracking_after_batch", AsyncMock()),+ ):+ await pipeline._process_batch(pa_table=table, batch_index=0, row_count=1)++ staged = cast(AsyncMock, pipeline._sinks.stage_chunk).call_args.args[1]+ assert staged.column_names == expected++class TestFinalizeStagesTheWatermarkFirst:@pytest.mark.asyncioasync def test_desc_watermark_is_staged_before_the_final_batch_notification(self) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.pyindex 4078ef2e0565..64ade129ac45 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py@@ -14,8 +14,13 @@ from parameterized import parameterizedfrom posthog.models import Team+from products.warehouse_sources.backend.models.credential import DataWarehouseCredentialfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_schema import (+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,+ ExternalDataSchema,+ SelfManagedTableRef,+)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention@@ -28,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_refresh_cumulative_row_count,merge_columns,register_cdc_companion_table,+ relink_orphan_table,update_last_synced_at,validate_schema_and_update_table,)@@ -44,6 +50,7 @@ from products.warehouse_sources.backend.types import (_PIPELINE_SYNC_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync"_DB_RETRY_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry"_TABLE_MODULE = "products.warehouse_sources.backend.models.table"+_SELF_MANAGED_URL = "https://exports.s3.us-east-1.amazonaws.com/orders/*.csv"class TestResolveTableAndFolderNames:@@ -597,6 +604,178 @@ class TestValidateSchemaAndUpdateTable:assert schema.table.size_in_s3_mib == expected_size_mibassert job.storage_delta_mib == expected_delta_mib+ def _self_managed_table(self, team: Team, schema: ExternalDataSchema, **fields: object) -> DataWarehouseTable:+ credential = DataWarehouseCredential.objects.create(access_key="AKIAEXAMPLE", access_secret="secret", team=team)+ table = DataWarehouseTable.objects.create(+ name="Raw_Orders",+ format=DataWarehouseTableFormat.CSVWithNames,+ url_pattern=_SELF_MANAGED_URL,+ team=team,+ credential=credential,+ options={"csv_allow_double_quotes": False, "kept": 1},+ columns={"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField"}},+ )+ ref = SelfManagedTableRef(table_id=str(table.id), url_pattern=table.url_pattern, format=table.format)+ schema.sync_type_config = {"schema_metadata": {SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref.to_metadata()}}+ schema.save()+ if fields:+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(**fields)+ table.refresh_from_db()+ return table++ def _publish(self, team: Team, schema: ExternalDataSchema, job: ExternalDataJob, row_count: int = 10) -> None:+ async_to_sync(validate_schema_and_update_table)(+ run_id=str(job.id),+ team_id=team.pk,+ schema_id=schema.id,+ row_count=row_count,+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,+ queryable_folder="raw_orders__query_a",+ live_size_mib=3.0,+ )++ @pytest.mark.parametrize("row_count", [10, 0])+ def test_the_first_publish_adopts_the_self_managed_table_the_schema_names(self, team: Team, row_count: int) -> None:+ schema, job = self._schema_and_job(team)+ table = self._self_managed_table(team, schema)+ credential_seen_by_introspection: list[object] = []++ def get_columns(table_self: DataWarehouseTable, safe_expose_ch_error: bool = True) -> dict:+ credential_seen_by_introspection.append(table_self.credential_id)+ return {"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField"}}++ with (+ patch.object(DataWarehouseTable, "get_columns", get_columns),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ self._publish(team, schema, job, row_count=row_count)++ table.refresh_from_db()+ schema.refresh_from_db()+ assert DataWarehouseTable.objects.filter(team=team, deleted=False).count() == 1+ assert schema.table_id == table.id+ assert (table.name, table.external_data_source_id, table.credential_id) == ("Raw_Orders", job.pipeline_id, None)+ assert (table.format, table.queryable_folder, table.size_in_s3_mib) == (+ DataWarehouseTableFormat.DeltaS3Wrapper,+ "raw_orders__query_a",+ 3.0,+ )+ names = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)+ assert table.url_pattern == job.url_pattern_by_schema(names.folder_name)+ assert table.options == {"kept": 1}+ # The customer's key can't read PostHog's bucket, so introspection must see the adopted row.+ assert credential_seen_by_introspection == [None]+ job.refresh_from_db()+ assert job.storage_delta_mib == 3.0++ @pytest.mark.parametrize(+ "change",+ [+ {"url_pattern": "https://exports.s3.us-east-1.amazonaws.com/other/*.csv"},+ {"format": DataWarehouseTableFormat.Parquet},+ {"deleted": True},+ ],+ )+ def test_a_self_managed_table_that_changed_is_not_replaced(self, team: Team, change: dict) -> None:+ schema, job = self._schema_and_job(team)+ table = self._self_managed_table(team, schema, **change)+ before = (table.url_pattern, table.format, table.credential_id, table.deleted)++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ self._publish(team, schema, job)++ table.refresh_from_db()+ schema.refresh_from_db()+ assert (table.url_pattern, table.format, table.credential_id, table.deleted) == before+ assert table.external_data_source_id is None+ assert schema.table is not None and schema.table_id != table.id+ assert schema.table.name == build_table_name(job.pipeline, "orders")++ def test_a_table_of_another_team_is_not_replaced(self, team: Team) -> None:+ schema, job = self._schema_and_job(team)+ other_team = Team.objects.create(organization=team.organization, name="Other team")+ table = self._self_managed_table(team, schema, team_id=other_team.pk)++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ self._publish(team, schema, job)++ table.refresh_from_db()+ schema.refresh_from_db()+ assert table.external_data_source_id is None+ assert schema.table_id is not None and schema.table_id != table.id++ def test_an_adopted_table_left_unlinked_is_linked_by_the_next_publish(self, team: Team) -> None:+ schema, job = self._schema_and_job(team)+ table = self._self_managed_table(team, schema)+ no_files_yet = ServerException("No files in log segment", code=742)++ with patch.object(DataWarehouseTable, "get_columns", side_effect=no_files_yet):+ self._publish(team, schema, job)+ schema.refresh_from_db()+ table.refresh_from_db()+ assert (schema.table_id, table.external_data_source_id) == (None, job.pipeline_id)++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ self._publish(team, schema, job)++ schema.refresh_from_db()+ assert schema.table_id == table.id+ assert DataWarehouseTable.objects.filter(team=team, deleted=False).count() == 1++ def test_an_adopted_table_left_unlinked_is_relinked_before_the_next_extraction(self, team: Team) -> None:+ schema, job = self._schema_and_job(team)+ table = self._self_managed_table(team, schema)+ no_files_yet = ServerException("No files in log segment", code=742)+ with patch.object(DataWarehouseTable, "get_columns", side_effect=no_files_yet):+ self._publish(team, schema, job)+ schema.refresh_from_db()+ assert schema.table_id is None++ relinked = relink_orphan_table(schema, job.pipeline, MagicMock())++ schema.refresh_from_db()+ assert relinked is not None and relinked.id == table.id+ assert schema.table_id == table.id++ def test_a_self_managed_table_not_adopted_yet_is_not_relinked(self, team: Team) -> None:+ schema, job = self._schema_and_job(team)+ self._self_managed_table(team, schema)++ assert relink_orphan_table(schema, job.pipeline, MagicMock()) is None++ schema.refresh_from_db()+ assert schema.table_id is None++ def test_a_synced_table_keeps_its_options(self, team: Team) -> None:+ schema, job = self._schema_and_job(team)+ table = self._linked_table(team, schema, job, queryable_folder="orders__query_a")+ DataWarehouseTable.objects.filter(id=table.id).update(options={"csv_allow_double_quotes": True})++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=150),+ ):+ async_to_sync(validate_schema_and_update_table)(+ run_id=str(job.id),+ team_id=team.pk,+ schema_id=schema.id,+ row_count=10,+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,+ queryable_folder="orders__query_b",+ )++ table.refresh_from_db()+ assert table.options == {"csv_allow_double_quotes": True}+def test_a_schema_linked_to_its_cdc_companion_gets_its_own_table(self, team: Team) -> None:schema, job = self._schema_and_job(team)companion = self._linked_table(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.pyindex a5ef6fb5112b..5ae8d3a1512e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py@@ -1,4 +1,4 @@-from typing import cast+from typing import Any, castfrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -26,6 +26,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objS3_EXAMPLE_URL,parse_s3_url,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+ UnmappableTableError,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (ObjectStorageSource,)@@ -48,6 +53,7 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):api_docs_url = "https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html"requires_team_feature_flag = FEATURE_FLAG+ self_managed_service = "s3"@propertydef source_type(self) -> ExternalDataSourceType:@@ -69,6 +75,17 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):def discovery_options(self, config: AmazonS3SourceConfig) -> DiscoveryOptions:return discovery_options_from(config)+ def self_managed_connection_inputs(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ if location.region is None:+ raise UnmappableTableError("the URL doesn't name the bucket's region")+ return {+ "url_pattern": f"https://s3.{location.region}.amazonaws.com/{location.bucket}/",+ "access_key_id": credential.access_key,+ "secret_access_key": credential.access_secret,+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.pyindex 81a35048a36c..c2c8c8fddee5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py@@ -1,4 +1,4 @@-from typing import cast+from typing import Any, castfrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -24,6 +24,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objAZURE_EXAMPLE_URL,parse_azure_url,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+ UnmappableTableError,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (ObjectStorageSource,)@@ -46,6 +51,7 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):api_docs_url = "https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api"requires_team_feature_flag = FEATURE_FLAG+ self_managed_service = "azure"@propertydef source_type(self) -> ExternalDataSourceType:@@ -62,6 +68,18 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):def discovery_options(self, config: AzureBlobSourceConfig) -> DiscoveryOptions:return discovery_options_from(config)+ def self_managed_connection_inputs(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ # ClickHouse reads Azure with the storage account name as the access key. A key for another+ # account than the URL's never read these files, so it isn't moved to this one.+ if credential.access_key.strip().lower() != location.account:+ raise UnmappableTableError("the access key isn't the storage account name in the URL")+ return {+ "url_pattern": f"https://{location.account}.blob.core.windows.net/{location.bucket}/",+ "account_key": credential.access_secret,+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.pyindex 3c9f2cf9c353..5606d5c57452 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py@@ -1,4 +1,4 @@-from typing import cast+from typing import Any, castfrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objR2_EXAMPLE_URL,parse_r2_url,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (ObjectStorageSource,)@@ -43,6 +47,7 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):api_docs_url = "https://developers.cloudflare.com/r2/api/s3/api/"requires_team_feature_flag = FEATURE_FLAG+ self_managed_service = "r2"@propertydef source_type(self) -> ExternalDataSourceType:@@ -64,6 +69,15 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):def discovery_options(self, config: CloudflareR2SourceConfig) -> DiscoveryOptions:return discovery_options_from(config)+ def self_managed_connection_inputs(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ return {+ "url_pattern": f"https://{location.account or ''}.r2.cloudflarestorage.com/{location.bucket}/",+ "access_key_id": credential.access_key,+ "secret_access_key": credential.access_secret,+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.pyindex 27ccd852b797..a1beb2d2fd3e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py@@ -5,6 +5,7 @@ from typing import Literal, get_argsfrom posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import CsvColumnTypefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (INVALID_CONFIG_ERROR,ObjectStorageError,@@ -220,6 +221,12 @@ class CsvOptions:# Mirrors ClickHouse's `format_csv_allow_double_quotes`: off means a double quote is an ordinary# character and never encloses a field.allow_double_quotes: bool = True+ # Columns cast from text to a typed column. Every other column stays a string.+ column_types: tuple[CsvColumnType, ...] = ()+ # Read an unquoted `\N` as null, as ClickHouse does. Only tables moved from a self-managed table set it.+ null_marker: bool = False+ # The field delimiter for every file. None picks it from the file name: tabs for `.tsv`, else commas.+ delimiter: str | None = None@frozendiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.pynew file mode 100644index 000000000000..2a58328db1ff--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py@@ -0,0 +1,352 @@+import re+from collections.abc import Callable, Mapping+from typing import Any, Literal, cast, get_args+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError++import pyarrow as pa+import pyarrow.compute as pc++from posthog.dataclasses import frozen++# The types a CSV column can be cast to. Anything else stays a string column.+CsvColumnKind = Literal["integer", "unsigned", "float", "decimal", "boolean", "date", "datetime"]+CSV_COLUMN_KINDS: frozenset[str] = frozenset(get_args(CsvColumnKind))++_WRAPPER = re.compile(r"^(?:Nullable|LowCardinality)\((?P<inner>.*)\)$")+_TIMEZONE_ARGUMENT = re.compile(r"'(?P<timezone>[^']+)'")+_DECIMAL_ARGUMENTS = re.compile(r"^\s*(?P<first>\d+)\s*(?:,\s*(?P<second>\d+)\s*)?$")+_CLICKHOUSE_KINDS: dict[str, CsvColumnKind] = {+ "Int8": "integer",+ "Int16": "integer",+ "Int32": "integer",+ "Int64": "integer",+ "UInt8": "integer",+ "UInt16": "integer",+ "UInt32": "integer",+ "UInt64": "unsigned",+ "Float32": "float",+ "Float64": "float",+ "Bool": "boolean",+ "Date": "date",+ "Date32": "date",+ "DateTime": "datetime",+ "DateTime64": "datetime",+}+# `Decimal32(S)` and its siblings fix the precision and take only the scale.+_DECIMAL_PRECISIONS: dict[str, int | None] = {+ "Decimal": None,+ "Decimal32": 9,+ "Decimal64": 18,+ "Decimal128": 38,+ "Decimal256": 76,+}+# Arrow's 128-bit decimal holds 38 digits, which is also the most a Delta table can store.+_MAX_DECIMAL_PRECISION = 38+# Numbers no Arrow integer or 38-digit decimal holds, so a CSV column of them can't keep its type.+_WIDE_INTEGERS = frozenset({"Int128", "Int256", "UInt128", "UInt256"})+# A Delta table has no unsigned integers, and an Int64 can't hold the top half of UInt64, so a+# UInt64 is stored as a 20-digit decimal, which holds every UInt64 and stays a number.+_UNSIGNED_STORAGE = pa.decimal128(20, 0)+_FIXED_ARROW_TYPES: dict[CsvColumnKind, pa.DataType] = {+ "integer": pa.int64(),+ "unsigned": _UNSIGNED_STORAGE,+ "float": pa.float64(),+ "boolean": pa.bool_(),+ "date": pa.date32(),+ "datetime": pa.timestamp("us", tz="UTC"),+}+# The spellings ClickHouse accepts for a Bool in text formats.+_TRUE_VALUES = ["true", "1", "yes", "y", "on", "t", "enable"]+_FALSE_VALUES = ["false", "0", "no", "n", "off", "f", "disable"]+_UTC = "UTC"+# A failed cast is split in halves until a slice is this small, and only then cast value by value.+_PER_VALUE_SLICE = 32+_CAST_ERRORS = (pa.ArrowInvalid, pa.ArrowNotImplementedError, OverflowError)+++class UnsupportedCsvColumnError(ValueError):+ """A stored column type that a CSV import can't keep. The message names the column."""+++@frozen+class CsvColumnType:+ name: str+ kind: CsvColumnKind+ precision: int | None = None+ scale: int | None = None+ # The zone ClickHouse read naive values of a `datetime` column in. None reads them as UTC.+ timezone: str | None = None++ @property+ def arrow_type(self) -> pa.DataType:+ if self.kind == "decimal":+ return pa.decimal128(self.precision or _MAX_DECIMAL_PRECISION, self.scale or 0)+ return _FIXED_ARROW_TYPES[self.kind]++ def to_metadata(self) -> str | dict[str, Any]:+ if self.kind == "decimal":+ return {"kind": self.kind, "precision": self.precision, "scale": self.scale}+ if self.kind == "datetime" and self.timezone is not None:+ return {"kind": self.kind, "timezone": self.timezone}+ return self.kind+++@frozen+class CsvCastResult:+ table: pa.Table+ # Per column, how many non-empty values could not be read as the column's type and became null.+ nulled_values: dict[str, int]+++@frozen+class _ClickHouseType:+ base: str+ arguments: str+++def _unwrap(clickhouse_type: str) -> _ClickHouseType:+ """The base type name and its arguments, without `Nullable(...)` or `LowCardinality(...)`."""+ inner = clickhouse_type.strip()+ while (wrapped := _WRAPPER.match(inner)) is not None:+ inner = wrapped.group("inner").strip()+ base, _, arguments = inner.partition("(")+ return _ClickHouseType(base=base.strip(), arguments=arguments.removesuffix(")"))+++def clickhouse_base_type(clickhouse_type: str) -> str:+ return _unwrap(clickhouse_type).base+++def _datetime_timezone(name: str, arguments: str) -> str | None:+ match = _TIMEZONE_ARGUMENT.search(arguments)+ if match is None:+ return None+ timezone = match.group("timezone")+ try:+ ZoneInfo(timezone)+ except (ZoneInfoNotFoundError, ValueError) as e:+ raise UnsupportedCsvColumnError(f"the column '{name}' has a time zone '{timezone}' PostHog can't read") from e+ return None if timezone == _UTC else timezone+++def _decimal_type(name: str, base: str, arguments: str) -> CsvColumnType:+ match = _DECIMAL_ARGUMENTS.match(arguments)+ if match is None:+ raise UnsupportedCsvColumnError(f"the column '{name}' has a decimal type PostHog can't read")+ fixed_precision = _DECIMAL_PRECISIONS[base]+ if fixed_precision is None:+ precision, scale = int(match.group("first")), int(match.group("second") or 0)+ else:+ precision, scale = fixed_precision, int(match.group("first"))+ if precision > _MAX_DECIMAL_PRECISION:+ raise UnsupportedCsvColumnError(+ f"the column '{name}' is a decimal with more than {_MAX_DECIMAL_PRECISION} digits"+ )+ return CsvColumnType(name=name, kind="decimal", precision=precision, scale=scale)+++def csv_column_type(name: str, clickhouse_type: str) -> CsvColumnType | None:+ """The cast for a stored ClickHouse type, or None when the column stays a string.++ Raises `UnsupportedCsvColumnError` for a number that neither a cast nor a string column keeps.+ """+ parsed = _unwrap(clickhouse_type)+ base, arguments = parsed.base, parsed.arguments+ if base in _WIDE_INTEGERS:+ raise UnsupportedCsvColumnError(f"the column '{name}' is a {base}, which PostHog can't store")+ if base in _DECIMAL_PRECISIONS:+ return _decimal_type(name, base, arguments)+ kind = _CLICKHOUSE_KINDS.get(base)+ if kind is None:+ return None+ if kind == "datetime":+ return CsvColumnType(name=name, kind=kind, timezone=_datetime_timezone(name, arguments))+ return CsvColumnType(name=name, kind=kind)+++def stored_clickhouse_types(columns: Mapping[str, Any] | None) -> dict[str, str]:+ """The ClickHouse type of each stored column, in stored order.++ A column ClickHouse marked as invalid never read as its type, so it is left out.+ """+ types: dict[str, str] = {}+ for name, definition in (columns or {}).items():+ if isinstance(definition, str):+ types[name] = definition+ elif isinstance(definition, Mapping) and definition.get("valid") is not False:+ types[name] = str(definition.get("clickhouse") or "")+ return types+++def csv_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:+ """The casts for a warehouse table's stored columns, in their stored order."""+ column_types: list[CsvColumnType] = []+ for name, clickhouse_type in stored_clickhouse_types(columns).items():+ column_type = csv_column_type(name, clickhouse_type)+ if column_type is not None:+ column_types.append(column_type)+ return tuple(column_types)+++def _column_type_from_metadata(name: str, raw: object) -> CsvColumnType | None:+ if isinstance(raw, str):+ return CsvColumnType(name=name, kind=cast(CsvColumnKind, raw)) if raw in CSV_COLUMN_KINDS else None+ if not isinstance(raw, Mapping) or raw.get("kind") not in CSV_COLUMN_KINDS:+ return None+ precision, scale, timezone = raw.get("precision"), raw.get("scale"), raw.get("timezone")+ return CsvColumnType(+ name=name,+ kind=cast(CsvColumnKind, raw["kind"]),+ precision=precision if isinstance(precision, int) else None,+ scale=scale if isinstance(scale, int) else None,+ timezone=timezone if isinstance(timezone, str) else None,+ )+++def csv_column_types_from_metadata(raw: object) -> tuple[CsvColumnType, ...]:+ if not isinstance(raw, Mapping):+ return ()+ column_types = (_column_type_from_metadata(name, value) for name, value in raw.items() if isinstance(name, str))+ return tuple(column_type for column_type in column_types if column_type is not None)+++def _cast_each(values: pa.Array, parse: Callable[[pa.Array], pa.Array], target: pa.DataType) -> pa.Array:+ parsed: list[Any] = []+ for value in values.to_pylist():+ if value is None:+ parsed.append(None)+ continue+ try:+ parsed.append(parse(pa.array([value], pa.string()))[0].as_py())+ except _CAST_ERRORS:+ parsed.append(None)+ return pa.array(parsed, target)+++def _cast_isolating_failures(values: pa.Array, parse: Callable[[pa.Array], pa.Array], target: pa.DataType) -> pa.Array:+ """Cast a slice that failed as a whole by halves, so only small slices are cast value by value.++ The cost then grows with the number of bad values times the log of the slice length, not with+ the length of every slice that holds one bad value.+ """+ if len(values) <= _PER_VALUE_SLICE:+ return _cast_each(values, parse, target)+ middle = len(values) // 2+ halves: list[pa.Array] = []+ for half in (values.slice(0, middle), values.slice(middle)):+ try:+ halves.append(parse(half))+ except _CAST_ERRORS:+ halves.append(_cast_isolating_failures(half, parse, target))+ return pa.concat_arrays(halves)+++def _without_plus_sign(values: pa.Array) -> pa.Array:+ # Arrow refuses an explicit plus sign, which ClickHouse reads as part of the number.+ return cast(pa.Array, pc.replace_substring_regex(values, pattern=r"^\+", replacement=""))+++def _parse_number(target: pa.DataType) -> Callable[[pa.Array], pa.Array]:+ return lambda values: pc.cast(_without_plus_sign(values), target)+++def _parse_decimal(target: pa.DataType, scale: int) -> Callable[[pa.Array], pa.Array]:+ # ClickHouse drops the fractional digits a decimal's scale can't hold, rather than refusing the value.+ excess_digits = rf"^([+-]?\d*\.\d{{{scale}}})\d+$" if scale else r"^([+-]?\d+)\.\d*$"++ def parse(values: pa.Array) -> pa.Array:+ truncated = cast(+ pa.Array, pc.replace_substring_regex(_without_plus_sign(values), pattern=excess_digits, replacement=r"\1")+ )+ return pc.cast(truncated, target)++ return parse+++def _parse_unsigned(values: pa.Array) -> pa.Array:+ # Read as UInt64 first, so a negative or too large value fails like it does in ClickHouse.+ return pc.cast(pc.cast(_without_plus_sign(values), pa.uint64()), _UNSIGNED_STORAGE)+++def _parse_boolean(values: pa.Array) -> pa.Array:+ lowered = pc.utf8_lower(values)+ is_true = pc.is_in(lowered, value_set=pa.array(_TRUE_VALUES))+ is_false = pc.is_in(lowered, value_set=pa.array(_FALSE_VALUES))+ if not pc.all(pc.or_(pc.or_(is_true, is_false), pc.is_null(values))).as_py():+ raise pa.ArrowInvalid("not a boolean")+ return pc.if_else(pc.is_null(values), pa.scalar(None, pa.bool_()), is_true)+++def _parse_timestamp(values: pa.Array, tz: str | None) -> pa.Array:+ """Parse text into a microsecond timestamp, truncating finer fractions of a second.++ A `DateTime64(9)` column, the type ClickHouse infers for a timestamp, can hold nanoseconds,+ which Arrow refuses to read as microseconds. Those values are read as nanoseconds first and+ then truncated to the microseconds a Delta table stores.+ """+ micros = pa.timestamp("us") if tz is None else pa.timestamp("us", tz=tz)+ nanos = pa.timestamp("ns") if tz is None else pa.timestamp("ns", tz=tz)+ try:+ return pc.cast(values, micros)+ except pa.ArrowInvalid:+ return pc.cast(pc.cast(values, nanos), micros, safe=False)+++def _parse_datetime(timezone: str | None) -> Callable[[pa.Array], pa.Array]:+ target = _FIXED_ARROW_TYPES["datetime"]++ def parse(values: pa.Array) -> pa.Array:+ # ClickHouse reads a value without an offset in the column's time zone, or in UTC.+ try:+ naive = _parse_timestamp(values, None)+ except pa.ArrowInvalid:+ return _parse_timestamp(values, _UTC)+ if timezone is None:+ return pc.cast(naive, target)+ # A local time that a clock change skips or repeats takes the earlier instant.+ localized = pc.assume_timezone(+ cast(Any, naive), timezone=timezone, ambiguous="earliest", nonexistent="earliest"+ )+ return pc.cast(localized, target)++ return parse+++def _parser(column_type: CsvColumnType) -> Callable[[pa.Array], pa.Array]:+ if column_type.kind == "boolean":+ return _parse_boolean+ if column_type.kind == "datetime":+ return _parse_datetime(column_type.timezone)+ if column_type.kind == "unsigned":+ return _parse_unsigned+ if column_type.kind == "decimal":+ return _parse_decimal(column_type.arrow_type, column_type.scale or 0)+ return _parse_number(column_type.arrow_type)+++def cast_csv_columns(table: pa.Table, column_types: tuple[CsvColumnType, ...]) -> CsvCastResult:+ """Cast the string columns of a CSV batch to the stored types.++ A value that can't be read as its column's type becomes null rather than failing the file,+ and the caller reports how many values that happened to. The batch is cast as a whole first,+ so only the slices that hold such a value pay for a slower pass.+ """+ nulled: dict[str, int] = {}+ for column_type in column_types:+ if column_type.name not in table.column_names:+ continue+ index = table.schema.get_field_index(column_type.name)+ values = pc.utf8_trim_whitespace(table.column(index).combine_chunks())+ # A quoted empty field is an empty string, which no typed column can hold.+ values = pc.if_else(pc.equal(values, pa.scalar("", pa.string())), pa.scalar(None, pa.string()), values)+ target = column_type.arrow_type+ parse = _parser(column_type)+ try:+ cast_values = parse(values)+ except _CAST_ERRORS:+ cast_values = _cast_isolating_failures(values, parse, target)+ failed = cast_values.null_count - values.null_count+ if failed:+ nulled[column_type.name] = failed+ table = table.set_column(index, pa.field(column_type.name, target), cast_values)+ return CsvCastResult(table=table, nulled_values=nulled)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.pyindex 5c0c2e103659..1d0c9bcb2b19 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py@@ -14,6 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objFileFormat,FileFormatChoice,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ csv_column_types_from_metadata,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (NO_TABLE_FILES_ERROR,TOO_MANY_FILES_ERROR,@@ -26,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (ClickHouseGlob,compile_clickhouse_glob,+ glob_key_prefix,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (MAX_TABLE_OBJECTS,@@ -33,8 +37,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj)TableKind = Literal["files", "delta"]-# Which files a `files` table reads: one file, a folder and its hive partitions, or every file.-FileScope = Literal["file", "folder", "all"]+# Which files a `files` table reads: one file, a folder and its hive partitions, every file, or the+# files a ClickHouse-style glob in `pattern` matches. Discovery never makes a `glob` table: it keeps+# the file set of a table that ClickHouse read from a URL pattern before it was imported.+FileScope = Literal["file", "folder", "all", "glob"]+_FILE_SCOPES: frozenset[str] = frozenset(get_args(FileScope))DELTA_LOG_DIRECTORY = "_delta_log"COMPRESSED_SUFFIX = ".gz"@@ -71,6 +78,8 @@ class TableDefinition:# The source's configured format at discovery, replayed at sync so the same files group together.source_format: FileFormatChoicefiles: tuple[StoredObject, ...] = ()+ # Keep the column names as the files write them, so queries written against them keep working.+ preserve_column_names: bool = Falsedef same_location(self, other: "TableDefinition") -> bool:return (@@ -81,15 +90,25 @@ class TableDefinition:)def schema_metadata(self) -> dict[str, Any]:- return {+ csv: dict[str, Any] = {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes}+ if self.csv.column_types:+ csv["column_types"] = {column.name: column.to_metadata() for column in self.csv.column_types}+ if self.csv.null_marker:+ csv["null_marker"] = True+ if self.csv.delimiter is not None:+ csv["delimiter"] = self.csv.delimiter+ metadata: dict[str, Any] = {"kind": self.kind,"scope": self.scope,"path": self.path,"pattern": self.pattern,"format": self.file_format,"source_format": self.source_format,- "csv": {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes},+ "csv": csv,}+ if self.preserve_column_names:+ metadata["preserve_column_names"] = True+ return metadata@classmethoddef from_schema_metadata(cls, name: str, metadata: Mapping[str, Any] | None) -> "TableDefinition | None":@@ -103,9 +122,11 @@ class TableDefinition:source_format = metadata.get("source_format", file_format)pattern = metadata.get("pattern")csv = metadata.get("csv") or {}+ delimiter = csv.get("delimiter") if isinstance(csv, dict) else Noneif (kind not in ("files", "delta")- or (kind == "files" and scope not in ("file", "folder", "all"))+ or (kind == "files" and scope not in _FILE_SCOPES)+ or (kind == "files" and scope == "glob" and not isinstance(pattern, str))or not isinstance(path, str)or file_format not in _FILE_FORMAT_CHOICESor source_format not in _FILE_FORMAT_CHOICES@@ -122,9 +143,13 @@ class TableDefinition:csv=CsvOptions(has_header=bool(csv.get("has_header", True)),allow_double_quotes=bool(csv.get("allow_double_quotes", True)),+ column_types=csv_column_types_from_metadata(csv.get("column_types")),+ null_marker=csv.get("null_marker") is True,+ delimiter=delimiter if isinstance(delimiter, str) and len(delimiter) == 1 else None,),pattern=pattern,source_format=cast(FileFormatChoice, source_format),+ preserve_column_names=metadata.get("preserve_column_names") is True,)@@ -360,12 +385,19 @@ def resolve_table_files("""if table.kind != "files":return table- objects = store.list_objects(table.path, max_objects + 1)+ if table.scope == "glob":+ listing_prefix = glob_key_prefix(table.pattern or "")+ objects = store.list_key_prefix(listing_prefix, max_objects + 1)+ else:+ listing_prefix = table.path+ objects = store.list_objects(listing_prefix, max_objects + 1)if len(objects) > max_objects:raise ObjectStorageError(- f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{table.path or '/'}'. Point the URL at a "+ f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{listing_prefix or '/'}'. Point the URL at a ""narrower folder, or use a pattern that matches fewer files.")+ if table.scope == "glob":+ return _resolve_glob_files(objects, table)options = DiscoveryOptions(file_format=table.source_format,allow_double_quotes=table.csv.allow_double_quotes,@@ -387,3 +419,18 @@ def resolve_table_files(f"{NO_TABLE_FILES_ERROR} '{table.name}'. The files it was discovered from may have been renamed, moved, ""or removed from the bucket.")+++def _resolve_glob_files(objects: list[StoredObject], table: TableDefinition) -> TableDefinition:+ """Every listed file the table's glob matches, with no other filter.++ Hidden files, Delta folders and file extensions are not checked, because the files ClickHouse+ read for the same pattern were not filtered either.+ """+ glob = compile_clickhouse_glob(table.pattern or "")+ files = tuple(stored for stored in objects if glob.matches(stored.path))+ if not files:+ raise ObjectStorageError(+ f"{NO_TABLE_FILES_ERROR} '{table.name}'. No file in the bucket matches '{table.pattern}'."+ )+ return dataclasses.replace(table, files=files)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.pyindex 50720c7b6cda..b21457ad007c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py@@ -84,6 +84,24 @@ class ObjectStore:breakreturn objects+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:+ """List files whose relative paths start with the key prefix, in path order."""+ folder = key_prefix.rpartition("/")[0]+ found = cast(dict[str, dict[str, Any]], self.fs.find(self.path(folder), detail=True))+ root_prefix = f"{self.root}/"+ objects: list[StoredObject] = []+ for name in sorted(found):+ info = found[name]+ if info.get("type") != "file" or not name.startswith(root_prefix):+ continue+ relative = name[len(root_prefix) :]+ if not relative.startswith(key_prefix):+ continue+ objects.append(StoredObject(path=relative, size=int(info.get("size") or 0), modified_at=_modified_at(info)))+ if len(objects) >= max_objects:+ break+ return objects+def open(self, relative: str) -> IO[bytes]:return cast(IO[bytes], self.fs.open(self.path(relative), "rb"))@@ -157,6 +175,12 @@ class S3ObjectStore(ObjectStore):objects = [stored for stored in self._list(key, 1) if stored.path == relative_prefix]return objects+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:+ key = "/".join(part for part in (self.prefix, key_prefix) if part)+ if self.prefix and not key_prefix:+ key = f"{self.prefix}/"+ return self._list(key, max_objects)+def _list(self, key_prefix: str, max_objects: int) -> list[StoredObject]:root_key = f"{self.prefix}/" if self.prefix else ""paginator = self.client.get_paginator("list_objects_v2")@@ -236,6 +260,12 @@ class AzureObjectStore(ObjectStore):objects = [stored for stored in self._list(name, 1) if stored.path == relative_prefix]return objects+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:+ key = "/".join(part for part in (self.prefix, key_prefix) if part)+ if self.prefix and not key_prefix:+ key = f"{self.prefix}/"+ return self._list(key, max_objects)+def _list(self, name_prefix: str, max_objects: int) -> list[StoredObject]:root_name = f"{self.prefix}/" if self.prefix else ""objects: list[StoredObject] = []diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pyindex ffca2e509438..782e4ff6c079 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -21,6 +21,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objCsvOptions,FileFormat,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ cast_csv_columns,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (FILE_PARSE_ERROR,@@ -72,6 +75,10 @@ class _PrefixedStream(io.RawIOBase):return len(data)+# How ClickHouse writes a null in a CSV file, and reads one back when it is unquoted.+CLICKHOUSE_CSV_NULL = "\\N"++def csv_column_names(cells: list[str], has_header: bool) -> list[str]:"""Header names as written, made unique. A headerless file gets ClickHouse's `c1`, `c2`, ... names."""if not has_header:@@ -147,11 +154,11 @@ def iter_csv_tables(file_path: str = "",logger: FilteringBoundLogger | None = None,) -> Iterator[pa.Table]:- """Stream a CSV file as string columns.+ """Stream a CSV file as string columns, except the columns `options.column_types` casts.- Every column is read as text: Arrow fixes a column's type from the first block it sees, so a- later value that does not fit would fail the whole file mid-stream, and two files of one table- could disagree on a type. Empty unquoted fields are null.+ Every other column is read as text: Arrow fixes a column's type from the first block it sees,+ so a later value that does not fit would fail the whole file mid-stream, and two files of one+ table could disagree on a type. Empty unquoted fields are null."""first_record, first_fields = _read_first_csv_record(stream, delimiter, options.allow_double_quotes, file_path)if not first_record:@@ -160,12 +167,21 @@ def iter_csv_tables(body: IO[bytes] = stream if options.has_header else cast(IO[bytes], _PrefixedStream(first_record, stream))skipped_rows = 0+ nulled_values: dict[str, int] = {}def skip_invalid_row(_row: pa_csv.InvalidRow) -> Literal["skip", "error"]:nonlocal skipped_rowsskipped_rows += 1return "skip"+ def typed(table: pa.Table) -> pa.Table:+ if not options.column_types:+ return table+ result = cast_csv_columns(table, options.column_types)+ for column, count in result.nulled_values.items():+ nulled_values[column] = nulled_values.get(column, 0) + count+ return result.table+yielded = Falsetry:reader = pa_csv.open_csv(@@ -180,7 +196,7 @@ def iter_csv_tables(),convert_options=pa_csv.ConvertOptions(column_types=dict.fromkeys(names, pa.string()),- null_values=[""],+ null_values=["", CLICKHOUSE_CSV_NULL] if options.null_marker else [""],strings_can_be_null=True,quoted_strings_can_be_null=False,),@@ -188,14 +204,23 @@ def iter_csv_tables(for batch in reader:if batch.num_rows:yielded = True- yield pa.Table.from_batches([batch])+ yield typed(pa.Table.from_batches([batch]))except pa.ArrowInvalid as e:if options.has_header and not yielded and "Empty CSV file" in str(e):# The file holds only its header. It is a table with no rows, not a broken file.- yield pa.table({name: pa.array([], pa.string()) for name in names})+ yield typed(pa.table({name: pa.array([], pa.string()) for name in names}))returnraise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: {e}") from e+ if nulled_values and logger is not None:+ # The stored column types come from a sample of the files, so a later value can fail to+ # read as its type. The value becomes null so the rest of the file still imports.+ logger.warning(+ "Read CSV values that don't match their column type as null",+ file=file_path,+ nulled_values=nulled_values,+ )+if skipped_rows and logger is not None:# A row whose field count differs from the header is malformed, or the file uses another# delimiter. Skipping it keeps its values out of the wrong columns.@@ -328,7 +353,7 @@ def iter_file_tables(else:binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else streamif file_format == "csv":- delimiter = "\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ","+ delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)else:tables = iter_jsonl_tables(binary, stored.path)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.pynew file mode 100644index 000000000000..6058611df09d--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py@@ -0,0 +1,269 @@+"""Map a self-managed warehouse table onto the object storage sources that import the same files.++A self-managed table is a URL pattern that ClickHouse reads in place with the table's own access+key. Every mapping here is exact or refused: a table this module can't map is skipped with a reason,+never imported from files that differ from the ones ClickHouse read.+"""++import re+import dataclasses+from collections.abc import Iterable, Mapping+from typing import Any, Literal+from urllib.parse import unquote, urlparse++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import normalize_column_name+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (+ FILE_MODIFIED_AT_COLUMN,+ FILE_PATH_COLUMN,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (+ CsvOptions,+ FileFormatChoice,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ UnsupportedCsvColumnError,+ clickhouse_base_type,+ csv_column_types_from_columns,+ stored_clickhouse_types,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (+ TableDefinition,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (+ compile_clickhouse_glob,+ glob_listing_prefix,+ has_glob,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.incremental import (+ FILE_ROW_NUMBER_COLUMN,+)++StorageService = Literal["s3", "gcs", "r2", "azure"]++_S3_HOST = re.compile(r"(?:(?P<bucket>.+)\.)?s3(?:[.-](?P<endpoint>[a-z0-9.-]+?))?\.amazonaws\.com(?:\.cn)?")+_S3_ENDPOINT = re.compile(r"(?:(?:dualstack|fips)\.)?(?P<region>[a-z]{2}(?:-[a-z0-9]+)+-\d+)")+# The legacy name of the us-east-1 endpoint.+_S3_EXTERNAL_ENDPOINT = "external-1"+_S3_EXTERNAL_REGION = "us-east-1"+_GCS_HOST = "storage.googleapis.com"+_R2_HOST = re.compile(r"(?P<account>[0-9a-f]{32})\.r2\.cloudflarestorage\.com")+_AZURE_HOST = re.compile(r"(?P<account>[a-z0-9]{3,24})\.blob\.core\.windows\.net")++# ClickHouse formats a self-managed table can have, and the source format that reads the same files.+_FORMATS: dict[str, FileFormatChoice] = {+ "CSV": "csv_no_header",+ "CSVWithNames": "csv",+ "Parquet": "parquet",+ "JSONEachRow": "jsonl",+ "Delta": "delta",+ "DeltaS3Wrapper": "delta",+}+# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts.+_NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})+# Types ClickHouse reads from JSON strings, which the JSON import keeps as strings.+_JSON_TEXT_TYPES = frozenset({"Date", "Date32", "DateTime", "DateTime64"})+# Columns the import adds to every row. A stored column of the same name would lose its values to them.+_FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})+++class UnmappableTableError(Exception):+ """The table can't be imported from exactly the files ClickHouse reads. The message says why."""+++@frozen+class SelfManagedLocation:+ """Where a self-managed table's files are, split into the parts an object storage source takes."""++ service: StorageService+ # The bucket, or the container for Azure.+ bucket: str+ # The decoded object key after the bucket. It can hold ClickHouse glob syntax.+ key: str+ # The S3 region, or None when the URL names no region.+ region: str | None = None+ # The Cloudflare account ID for R2, or the storage account name for Azure.+ account: str | None = None++ @property+ def group_key(self) -> str:+ return "/".join((self.service, self.account or self.region or "", self.bucket))+++@frozen+class SelfManagedCredential:+ """A self-managed table's key pair. For Azure the access key is the storage account name."""++ access_key: str = dataclasses.field(repr=False)+ access_secret: str = dataclasses.field(repr=False)+++def _s3_location(host: str, path: str) -> SelfManagedLocation:+ match = _S3_HOST.fullmatch(host)+ if match is None:+ raise UnmappableTableError(f"'{host}' isn't an S3 endpoint PostHog can import from")+ endpoint = match.group("endpoint")+ region: str | None = None+ if endpoint == _S3_EXTERNAL_ENDPOINT:+ region = _S3_EXTERNAL_REGION+ elif endpoint is not None:+ region_match = _S3_ENDPOINT.fullmatch(endpoint)+ if region_match is None:+ raise UnmappableTableError(f"'{host}' isn't an S3 API endpoint, such as a website or access point endpoint")+ region = region_match.group("region")+ if match.group("bucket") is not None:+ return SelfManagedLocation(service="s3", bucket=match.group("bucket"), key=path, region=region)+ bucket, _, key = path.partition("/")+ return SelfManagedLocation(service="s3", bucket=bucket, key=key, region=region)+++def parse_self_managed_url(url: str) -> SelfManagedLocation:+ """Split a self-managed table's URL pattern into its service, bucket, account and key."""+ parsed = urlparse(url.strip())+ if parsed.scheme not in ("https", "http"):+ raise UnmappableTableError("the URL isn't an http or https URL")+ if parsed.query or parsed.fragment or "?" in url or "#" in url:+ raise UnmappableTableError("the URL has a '?' or '#', which PostHog can't map to a file pattern")+ if parsed.username or parsed.password or parsed.port:+ raise UnmappableTableError("the URL names a user or a port")+ host = (parsed.hostname or "").lower()+ # Only the separator after the host goes: a further slash is part of the object key.+ path = unquote(parsed.path).removeprefix("/")+ if host.endswith(".amazonaws.com") or host.endswith(".amazonaws.com.cn"):+ location = _s3_location(host, path)+ elif host == _GCS_HOST:+ bucket, _, key = path.partition("/")+ location = SelfManagedLocation(service="gcs", bucket=bucket, key=key)+ elif host.endswith(f".{_GCS_HOST}"):+ location = SelfManagedLocation(service="gcs", bucket=host[: -len(_GCS_HOST) - 1], key=path)+ elif (r2 := _R2_HOST.fullmatch(host)) is not None:+ bucket, _, key = path.partition("/")+ location = SelfManagedLocation(service="r2", bucket=bucket, key=key, account=r2.group("account"))+ elif (azure := _AZURE_HOST.fullmatch(host)) is not None:+ container, _, key = path.partition("/")+ location = SelfManagedLocation(service="azure", bucket=container, key=key, account=azure.group("account"))+ else:+ raise UnmappableTableError(f"'{host}' isn't a storage host PostHog can import from")+ if not location.bucket or has_glob(location.bucket) or has_glob(location.account or ""):+ raise UnmappableTableError("the URL doesn't name one bucket")+ if location.key.startswith("/") or "//" in location.key:+ raise UnmappableTableError("the URL path has an empty segment, which the import can't read")+ if any(segment in (".", "..") for segment in location.key.split("/")):+ raise UnmappableTableError("the URL path has a '.' or '..' segment")+ return location+++def source_format_for(table_format: str) -> FileFormatChoice:+ choice = _FORMATS.get(table_format)+ if choice is None:+ raise UnmappableTableError(f"the '{table_format}' format has no import equivalent")+ return choice+++def needs_preserved_column_names(column_names: Iterable[str]) -> bool:+ """Whether the import pipeline would rename a column that existing queries refer to."""+ return any(normalize_column_name(name) != name for name in column_names)+++def _refuse_nested_columns(columns: Mapping[str, Any] | None) -> None:+ for name, clickhouse_type in stored_clickhouse_types(columns).items():+ base = clickhouse_base_type(clickhouse_type)+ if base in _NESTED_TYPES:+ raise UnmappableTableError(+ f"the column '{name}' is a {base}, and the import would store it as text, not as a {base}"+ )+++def _refuse_file_column_names(columns: Mapping[str, Any] | None) -> None:+ for name in (columns or {}).keys():+ if name in _FILE_COLUMNS:+ raise UnmappableTableError(+ f"the column '{name}' has the name of a column the import adds, which would replace its values"+ )+++def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:+ # The CSV reader trims spaces around header names, so a column ClickHouse read as ' id ' would be renamed.+ for name in (columns or {}).keys():+ if name != name.strip():+ raise UnmappableTableError(+ f"the column '{name}' has spaces around its name, which the CSV import doesn't keep"+ )+++def _refuse_json_text_types(columns: Mapping[str, Any] | None) -> None:+ # A JSON file holds dates and times as strings, and the JSON import keeps them as strings.+ for name, clickhouse_type in stored_clickhouse_types(columns).items():+ base = clickhouse_base_type(clickhouse_type)+ if base in _JSON_TEXT_TYPES:+ raise UnmappableTableError(+ f"the column '{name}' is a {base}, and the JSON import would store it as text, not as a {base}"+ )+++def table_definition_for(+ name: str,+ location: SelfManagedLocation,+ table_format: str,+ columns: Mapping[str, Any] | None,+ csv_allow_double_quotes: bool | None,+) -> TableDefinition:+ """The schema contract that imports exactly the files the table's URL pattern reads."""+ file_format = source_format_for(table_format)+ _refuse_nested_columns(columns)+ _refuse_file_column_names(columns)+ preserve = needs_preserved_column_names((columns or {}).keys())+ if file_format == "delta":+ path = location.key.strip("/")+ if has_glob(path):+ raise UnmappableTableError("a Delta table URL can't hold a file pattern")+ return TableDefinition(+ name=name,+ kind="delta",+ path=path,+ scope=None,+ file_format="delta",+ csv=CsvOptions(),+ pattern=None,+ source_format="delta",+ preserve_column_names=preserve,+ )+ pattern = location.key+ if not pattern or pattern.endswith("/"):+ raise UnmappableTableError("the URL names a folder, not files")+ try:+ compile_clickhouse_glob(pattern)+ except ObjectStorageError as e:+ raise UnmappableTableError(str(e)) from e+ try:+ column_types = csv_column_types_from_columns(columns) if file_format in ("csv", "csv_no_header") else ()+ except UnsupportedCsvColumnError as e:+ raise UnmappableTableError(str(e)) from e+ if file_format == "csv":+ _refuse_padded_header_names(columns)+ if file_format == "jsonl":+ _refuse_json_text_types(columns)+ return TableDefinition(+ name=name,+ kind="files",+ path=glob_listing_prefix(pattern),+ scope="glob",+ file_format=file_format,+ csv=CsvOptions(+ has_header=file_format == "csv",+ # ClickHouse reads quotes as enclosing a field unless the table turned that off.+ allow_double_quotes=csv_allow_double_quotes is not False,+ column_types=column_types,+ # ClickHouse reads an unquoted `\N` as null in a CSV file.+ null_marker=True,+ # ClickHouse reads the declared format whatever the file is called, so a `.tsv` file is still CSV.+ delimiter=",",+ ),+ pattern=pattern,+ source_format=file_format,+ preserve_column_names=preserve,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex d58802c78a0f..565bdf28d8a4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -1,7 +1,7 @@from abc import abstractmethodfrom collections.abc import Iteratorfrom datetime import UTC, datetime-from typing import Generic, NoReturn, Optional+from typing import Any, ClassVar, Generic, NoReturn, Optionalimport pyarrow as pafrom botocore.exceptions import ClientError@@ -61,6 +61,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objFileSelection,plan_mirror,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+ StorageService,+ UnmappableTableError,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (CHUNK_SIZE_BYTES,MAX_DISCOVERY_OBJECTS,@@ -119,6 +125,35 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe@abstractmethoddef discovery_options(self, config: ConfigType) -> DiscoveryOptions: ...+ # The service whose self-managed warehouse tables this source can take over.+ self_managed_service: ClassVar[StorageService]++ @abstractmethod+ def self_managed_connection_inputs(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ """The job inputs that reach the table's bucket with the table's own credential."""++ def job_inputs_for_self_managed(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ """Job inputs for a source rooted at the bucket of a self-managed table.++ The inputs go through the same parsing and location checks as a source a user creates, so+ a credential or bucket the source would refuse is refused here too.+ """+ raw = {+ **self.self_managed_connection_inputs(location, credential),+ "file_format": "infer",+ "csv_allow_double_quotes": "true",+ }+ try:+ config = self.parse_config(raw)+ validate_store_location(self.object_store_config(config))+ except (ObjectStorageError, TypeError, ValueError) as e:+ raise UnmappableTableError(str(e)) from e+ return self.serialize_config(config)+def cursor_class(self) -> type[ObjectStorageDeltaCursor]:return ObjectStorageDeltaCursor@@ -233,6 +268,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe# Only a merge needs a key. Without one, a full refresh replaces the table and an append adds to it.primary_keys=FILE_PRIMARY_KEYS if selection.merges else None,chunk_size_bytes=CHUNK_SIZE_BYTES,+ preserve_column_names=table.preserve_column_names,)def _mirrored_files_response(@@ -291,6 +327,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDeprimary_keys=None,chunk_size_bytes=CHUNK_SIZE_BYTES,destination_reset_required=read.reset_required,+ preserve_column_names=table.preserve_column_names,)def _find_table(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 4ef4d5f00e87..ffab85de6375 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -1040,6 +1040,47 @@ class TestS3Listing:with pytest.raises(ObjectStorageError, match=TOO_MANY_FILES_ERROR):_s3_store(client).list_objects("", 10)+ @parameterized.expand(+ [+ ("bucket_root", "", "exports/orders-", "exports/orders-", 10),+ ("store_root", "root", "exports/orders-", "root/exports/orders-", 10),+ ("empty_key_prefix", "root", "", "root/", 10),+ ("empty_bucket_prefix", "", "", "", 10),+ ("matching_file_limit", "root", "exports/orders-", "root/exports/orders-", 1),+ ]+ )+ def test_list_key_prefix(+ self, _name: str, prefix: str, key_prefix: str, expected_prefix: str, max_objects: int+ ) -> None:+ client = boto3.client(+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"+ )+ root_key = f"{prefix}/" if prefix else ""+ store = S3ObjectStore(+ fs=LocalFileSystem(),+ root=f"customer-data/{prefix}" if prefix else "customer-data",+ delta_uri_prefix="s3://",+ delta_storage_options={},+ client=client,+ bucket="customer-data",+ prefix=prefix,+ )+ with Stubber(client) as stubber:+ stubber.add_response(+ "list_objects_v2",+ _listing(+ f"{expected_prefix}folder/",+ f"{root_key}exports/orders-1.csv",+ f"{root_key}exports/orders-2.csv",+ ),+ {"Bucket": "customer-data", "MaxKeys": 1000, "Prefix": expected_prefix},+ )++ listed = store.list_key_prefix(key_prefix, max_objects)++ stubber.assert_no_pending_responses()+ assert [stored.path for stored in listed] == ["exports/orders-1.csv", "exports/orders-2.csv"][:max_objects]+def test_rejected_credentials_become_an_access_error(self) -> None:client = boto3.client("s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.pynew file mode 100644index 000000000000..3a255bffd138--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py@@ -0,0 +1,1015 @@+import io+import time+import uuid+from datetime import UTC, date, datetime+from decimal import Decimal+from pathlib import Path+from tempfile import TemporaryDirectory+from typing import Any++import pytest+from unittest.mock import MagicMock++import pyarrow as pa+import structlog+from fsspec.implementations.local import LocalFileSystem+from parameterized import parameterized++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_blob.source import AzureBlobSource+from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare_r2.source import CloudflareR2Source+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import CsvOptions+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ CsvColumnType,+ UnsupportedCsvColumnError,+ cast_csv_columns,+ csv_column_type,+ csv_column_types_from_columns,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (+ TableDefinition,+ resolve_table_files,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (+ NO_TABLE_FILES_ERROR,+ TOO_MANY_FILES_ERROR,+ ObjectStorageError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (+ ObjectStore,+ StoredObject,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (+ iter_csv_tables,+ iter_file_tables,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (+ compile_clickhouse_glob,+ glob_key_prefix,+ glob_listing_prefix,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+ UnmappableTableError,+ needs_preserved_column_names,+ parse_self_managed_url,+ source_format_for,+ table_definition_for,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs+from products.warehouse_sources.backend.temporal.data_imports.sources.google_cloud_storage.source import (+ GoogleCloudStorageSource,+)++R2_ACCOUNT = "0123456789abcdef0123456789abcdef"+LOGGER = structlog.get_logger()+++def local_store(root: Path) -> ObjectStore:+ return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})+++def write_files(root: Path, files: dict[str, str]) -> None:+ for path, content in files.items():+ (root / path).parent.mkdir(parents=True, exist_ok=True)+ (root / path).write_text(content)+++class TestParseSelfManagedUrl:+ @parameterized.expand(+ [+ (+ "s3_virtual_hosted_regional",+ "https://exports.s3.eu-west-1.amazonaws.com/events/*.parquet",+ SelfManagedLocation(service="s3", bucket="exports", key="events/*.parquet", region="eu-west-1"),+ ),+ (+ "s3_virtual_hosted_dotted_bucket",+ "https://my.exports.s3.us-east-2.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="my.exports", key="a.csv", region="us-east-2"),+ ),+ (+ "s3_virtual_hosted_legacy_dash_region",+ "https://exports.s3-us-west-2.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-west-2"),+ ),+ (+ "s3_virtual_hosted_global_has_no_region",+ "https://exports.s3.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region=None),+ ),+ (+ "s3_virtual_hosted_dualstack",+ "https://exports.s3.dualstack.ap-south-1.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="ap-south-1"),+ ),+ (+ "s3_external_endpoint_is_us_east_1",+ "https://exports.s3-external-1.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-east-1"),+ ),+ (+ "s3_path_style_regional",+ "https://s3.eu-central-1.amazonaws.com/exports/2024/*.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="2024/*.csv", region="eu-central-1"),+ ),+ (+ "s3_path_style_global",+ "https://s3.amazonaws.com/exports/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region=None),+ ),+ (+ "s3_china",+ "https://exports.s3.cn-north-1.amazonaws.com.cn/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="cn-north-1"),+ ),+ (+ "s3_percent_encoded_key",+ "https://exports.s3.us-east-1.amazonaws.com/daily%20export/a%2Bb.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="daily export/a+b.csv", region="us-east-1"),+ ),+ (+ "s3_http_scheme",+ "http://exports.s3.us-east-1.amazonaws.com/a.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-east-1"),+ ),+ (+ "gcs_path_style",+ "https://storage.googleapis.com/analytics-dump/tables/**.parquet",+ SelfManagedLocation(service="gcs", bucket="analytics-dump", key="tables/**.parquet"),+ ),+ (+ "gcs_virtual_hosted",+ "https://analytics-dump.storage.googleapis.com/a.jsonl",+ SelfManagedLocation(service="gcs", bucket="analytics-dump", key="a.jsonl"),+ ),+ (+ "azure",+ "https://examplestore.blob.core.windows.net/exports/orders/*.csv",+ SelfManagedLocation(service="azure", bucket="exports", key="orders/*.csv", account="examplestore"),+ ),+ (+ "r2",+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/orders/{{1..3}}.csv",+ SelfManagedLocation(service="r2", bucket="exports", key="orders/{1..3}.csv", account=R2_ACCOUNT),+ ),+ (+ "upper_case_host",+ "https://EXPORTS.S3.US-EAST-1.AMAZONAWS.COM/A.csv",+ SelfManagedLocation(service="s3", bucket="exports", key="A.csv", region="us-east-1"),+ ),+ ]+ )+ def test_parses(self, _name: str, url: str, expected: SelfManagedLocation) -> None:+ assert parse_self_managed_url(url) == expected++ @parameterized.expand(+ [+ ("not_http", "s3://exports/a.csv"),+ ("unknown_host", "https://minio.example.com/exports/a.csv"),+ ("s3_website_endpoint", "https://exports.s3-website-us-east-1.amazonaws.com/a.csv"),+ ("s3_access_point", "https://ap-123456789012.s3-accesspoint.us-east-1.amazonaws.com/a.csv"),+ ("s3_accelerate", "https://exports.s3-accelerate.amazonaws.com/a.csv"),+ ("query_string", "https://exports.s3.us-east-1.amazonaws.com/a.csv?versionId=1"),+ ("question_mark_glob", "https://exports.s3.us-east-1.amazonaws.com/a?.csv"),+ ("fragment", "https://exports.s3.us-east-1.amazonaws.com/a.csv#x"),+ ("port", "https://exports.s3.us-east-1.amazonaws.com:8443/a.csv"),+ ("userinfo", "https://user:[email protected]/a.csv"),+ ("path_style_without_bucket", "https://s3.us-east-1.amazonaws.com/"),+ ("glob_in_bucket", "https://s3.us-east-1.amazonaws.com/exports-*/a.csv"),+ ("dot_dot_segment", "https://exports.s3.us-east-1.amazonaws.com/a/../b.csv"),+ ("key_with_a_leading_slash", "https://exports.s3.us-east-1.amazonaws.com//orders.csv"),+ ("key_with_an_empty_segment", "https://storage.googleapis.com/lake/orders//a.csv"),+ ("azure_data_lake_endpoint", "https://examplestore.dfs.core.windows.net/exports/a.csv"),+ ("azure_other_cloud", "https://examplestore.blob.core.chinacloudapi.cn/exports/a.csv"),+ ("azure_without_container", "https://examplestore.blob.core.windows.net/"),+ ("r2_jurisdiction_endpoint", f"https://{R2_ACCOUNT}.eu.r2.cloudflarestorage.com/exports/a.csv"),+ ("r2_short_account", "https://abc.r2.cloudflarestorage.com/exports/a.csv"),+ ("gcs_browser_url", "https://storage.cloud.google.com/exports/a.csv"),+ ]+ )+ def test_refuses_urls_it_cant_map_exactly(self, _name: str, url: str) -> None:+ with pytest.raises(UnmappableTableError):+ parse_self_managed_url(url)+++class TestClickHouseGlob:+ @parameterized.expand(+ [+ ("star_stays_in_its_folder", "data/*.csv", "data/a.csv", True),+ ("star_matches_empty", "data/*.csv", "data/.csv", True),+ ("star_does_not_cross_folders", "data/*.csv", "data/2024/a.csv", False),+ ("double_star_crosses_folders", "data/**.csv", "data/2024/01/a.csv", True),+ ("double_star_folder", "data/**", "data/x/y", True),+ ("question_mark_is_one_character", "part-?.csv", "part-1.csv", True),+ ("question_mark_is_not_two", "part-?.csv", "part-10.csv", False),+ ("question_mark_does_not_match_slash", "a?b", "a/b", False),+ ("alternatives", "{orders,refunds}.csv", "refunds.csv", True),+ ("alternatives_are_literal", "{a.b,c}.csv", "aXb.csv", False),+ ("range", "part-{1..12}.csv", "part-12.csv", True),+ ("range_upper_bound", "part-{1..12}.csv", "part-13.csv", False),+ ("range_reversed_bounds", "part-{12..1}.csv", "part-5.csv", True),+ ("range_without_padding_refuses_leading_zero", "part-{1..12}.csv", "part-05.csv", False),+ ("padded_range", "part-{01..12}.csv", "part-05.csv", True),+ ("padded_range_refuses_unpadded", "part-{01..12}.csv", "part-5.csv", False),+ ("literal_regex_characters", "a+b(1).csv", "a+b(1).csv", True),+ ("literal_dot_is_not_any_character", "a.csv", "aXcsv", False),+ ("literal_path_is_exact", "data/a.csv", "data/a.csv.bak", False),+ ("hidden_files_match_like_clickhouse", "data/*", "data/_SUCCESS", True),+ ("range_leaves_digits_for_star", "part-{1..12}*.csv", "part-123.csv", True),+ ("range_does_not_take_extra_digits", "part-{1..12}.csv", "part-123.csv", False),+ ("empty_alternative", "{a,}x.csv", "x.csv", True),+ ("padded_range_in_middle", "d={01..31}/h-*.csv", "d=07/h-1.csv", True),+ ("padded_range_in_middle_refuses_unpadded", "d={01..31}/h-*.csv", "d=7/h-1.csv", False),+ ("zero_range", "{0..0}", "0", True),+ ("zero_range_refuses_extra_zero", "{0..0}", "00", False),+ ("double_star_after_range", "{1..12}**.csv", "123/nested/a.csv", True),+ ("alternatives_before_star", "{a,ab}*.csv", "abc.csv", True),+ ("double_star_matches_empty", "**", "", True),+ ("star_does_not_match_slash", "*", "/", False),+ ("question_mark_does_not_match_only_slash", "?", "/", False),+ ("triple_star_crosses_folders", "***", "a/b", True),+ ("four_stars_match_empty", "****", "", True),+ ("adjacent_empty_alternatives", "*{a,}{b,}x", "x", True),+ ("range_leaves_digits_for_literal", "{1..12}3", "123", True),+ ("padded_reversed_range", "{12..01}", "07", True),+ ("padded_range_uses_widest_bound", "{0..0012}", "0007", True),+ ]+ )+ def test_matches(self, _name: str, pattern: str, path: str, expected: bool) -> None:+ assert compile_clickhouse_glob(pattern).matches(path) is expected++ @parameterized.expand(+ [+ ("unclosed_brace", "a{b,c.csv"),+ ("empty_braces", "a{}.csv"),+ ("nested_glob_in_braces", "{a*,b}.csv"),+ ("square_brackets", "part-[0-9].csv"),+ ("stray_closing_brace", "a}.csv"),+ ("nested_question_mark", "{a?,b}.csv"),+ ("nested_brace", "{{a,b},c}.csv"),+ ("bracket_in_braces", "{a[,b}.csv"),+ ("closing_bracket_in_braces", "{a],b}.csv"),+ ("stray_closing_bracket", "a].csv"),+ ]+ )+ def test_refuses_patterns_it_cant_read(self, _name: str, pattern: str) -> None:+ with pytest.raises(ObjectStorageError):+ compile_clickhouse_glob(pattern)++ @parameterized.expand(+ [+ ("glob_in_file_name", "events/2024/*.csv", "events/2024"),+ ("glob_in_folder", "events/*/a.csv", "events"),+ ("glob_at_root", "*.csv", ""),+ ("range_folder", "y={2023..2024}/a.parquet", ""),+ ("literal_file", "events/a.csv", "events/a.csv"),+ ]+ )+ def test_listing_prefix(self, _name: str, pattern: str, expected: str) -> None:+ assert glob_listing_prefix(pattern) == expected++ @parameterized.expand(+ [+ ("glob_in_file_name", "exports/orders-*.csv", "exports/orders-"),+ ("glob_at_root", "*.csv", ""),+ ("range_folder", "y={2023..2024}/a.parquet", "y="),+ ("literal_file", "events/a.csv", "events/a.csv"),+ ("question_mark", "a?b", "a"),+ ]+ )+ def test_key_prefix(self, _name: str, pattern: str, expected: str) -> None:+ assert glob_key_prefix(pattern) == expected++ @parameterized.expand(+ [+ ("twenty_stars", "*a" * 20 + "*b.csv"),+ ("eight_stars", "*a*a*a*a*a*a*a*a*b.csv"),+ ]+ )+ def test_adversarial_match_time(self, _name: str, pattern: str) -> None:+ glob = compile_clickhouse_glob(pattern)+ path = "a" * 200 + ".csv"++ started = time.perf_counter()+ for _ in range(1000):+ assert glob.matches(path) is False+ assert time.perf_counter() - started < 2+ assert glob.matches("a" * 200 + "b.csv") is True++ @parameterized.expand(+ [+ ("few_matching_keys", 3, False),+ ("matching_key_limit", 2, True),+ ]+ )+ def test_resolves_only_the_key_prefix(self, _name: str, max_objects: int, exceeds_limit: bool) -> None:+ with TemporaryDirectory() as directory:+ root = Path(directory)+ write_files(+ root,+ {+ "exports/other-1.csv": "id\n1\n",+ "exports/other-2.csv": "id\n2\n",+ "exports/other-3.csv": "id\n3\n",+ "exports/orders-2.csv": "id\n2\n",+ "exports/orders-1.csv": "id\n1\n",+ "exports/orders-3.txt": "id\n3\n",+ },+ )+ table = TableDefinition(+ name="orders",+ kind="files",+ path="exports",+ scope="glob",+ file_format="csv",+ csv=CsvOptions(),+ pattern="exports/orders-*.csv",+ source_format="csv",+ )+ if exceeds_limit:+ with pytest.raises(ObjectStorageError, match=f"{TOO_MANY_FILES_ERROR}.*exports/orders-"):+ resolve_table_files(local_store(root), table, max_objects=max_objects)+ else:+ resolved = resolve_table_files(local_store(root), table, max_objects=max_objects)+ assert [stored.path for stored in resolved.files] == ["exports/orders-1.csv", "exports/orders-2.csv"]+++class TestTableDefinition:+ @parameterized.expand(+ [+ ("CSV", "csv_no_header"),+ ("CSVWithNames", "csv"),+ ("Parquet", "parquet"),+ ("JSONEachRow", "jsonl"),+ ("Delta", "delta"),+ ("DeltaS3Wrapper", "delta"),+ ]+ )+ def test_format_mapping(self, table_format: str, expected: str) -> None:+ assert source_format_for(table_format) == expected++ def test_an_unknown_format_is_refused(self) -> None:+ with pytest.raises(UnmappableTableError):+ source_format_for("TabSeparated")++ @parameterized.expand(+ [+ ("all_snake_case", ["id", "created_at", "amount_usd"], False),+ ("no_columns", [], False),+ ("camel_case", ["id", "createdAt"], True),+ ("spaces", ["Order ID"], True),+ ("upper_case", ["ID"], True),+ ]+ )+ def test_preserve_names_only_when_the_pipeline_would_rename(+ self, _name: str, columns: list[str], expected: bool+ ) -> None:+ assert needs_preserved_column_names(columns) is expected++ def test_csv_with_names_keeps_the_stored_types_and_quote_setting(self) -> None:+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/2024-*.csv")+ columns = {+ "id": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True},+ "Order Total": {"clickhouse": "Nullable(Float64)", "hogql": "FloatDatabaseField", "valid": True},+ "note": {"clickhouse": "Nullable(String)", "hogql": "StringDatabaseField", "valid": True},+ }++ definition = table_definition_for("orders", location, "CSVWithNames", columns, csv_allow_double_quotes=False)++ assert definition.schema_metadata() == {+ "kind": "files",+ "scope": "glob",+ "path": "orders",+ "pattern": "orders/2024-*.csv",+ "format": "csv",+ "source_format": "csv",+ "csv": {+ "has_header": True,+ "allow_double_quotes": False,+ "column_types": {"id": "integer", "Order Total": "float"},+ "null_marker": True,+ "delimiter": ",",+ },+ "preserve_column_names": True,+ }++ @parameterized.expand(+ [+ ("csv_without_header", "CSV", "csv_no_header", False, True),+ ("parquet_has_no_csv_types", "Parquet", "parquet", True, False),+ ("jsonl_has_no_csv_types", "JSONEachRow", "jsonl", True, False),+ ]+ )+ def test_csv_header_and_types_follow_the_format(+ self, _name: str, table_format: str, file_format: str, has_header: bool, has_types: bool+ ) -> None:+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/a.file")+ columns = {"c1": {"clickhouse": "Int64", "hogql": "IntegerDatabaseField"}}++ definition = table_definition_for("orders", location, table_format, columns, None)++ assert (definition.scope, definition.path, definition.file_format) == ("glob", "orders/a.file", file_format)+ assert definition.csv.allow_double_quotes is True+ assert definition.csv.has_header is (has_header if file_format.startswith("csv") else False)+ assert bool(definition.csv.column_types) is has_types++ @parameterized.expand([("Delta",), ("DeltaS3Wrapper",)])+ def test_delta_reads_the_table_folder(self, table_format: str) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/tables/orders/")++ definition = table_definition_for("orders", location, table_format, {"OrderId": "Int64"}, None)++ assert (definition.kind, definition.path, definition.pattern) == ("delta", "tables/orders", None)+ assert definition.preserve_column_names is True++ @parameterized.expand(+ [+ ("delta_with_glob", "https://storage.googleapis.com/lake/tables/*", "Delta"),+ ("files_url_is_a_folder", "https://storage.googleapis.com/lake/tables/", "Parquet"),+ ("files_url_is_the_bucket", "https://storage.googleapis.com/lake", "CSVWithNames"),+ ("unreadable_glob", "https://storage.googleapis.com/lake/part-[0-9].parquet", "Parquet"),+ ]+ )+ def test_refuses_tables_it_cant_read_exactly(self, _name: str, url: str, table_format: str) -> None:+ with pytest.raises(UnmappableTableError):+ table_definition_for("orders", parse_self_managed_url(url), table_format, {}, None)++ def test_metadata_round_trip_keeps_the_glob_types_and_names(self) -> None:+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/{1..3}.csv")+ definition = table_definition_for(+ "orders", location, "CSVWithNames", {"OrderId": "Nullable(Int64)", "paid": "Bool"}, None+ )++ rebuilt = TableDefinition.from_schema_metadata("orders", definition.schema_metadata())++ assert rebuilt == definition++ def test_metadata_round_trip_keeps_decimal_unsigned_and_time_zone_types(self) -> None:+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")+ columns = {+ "total": "Nullable(Decimal(12, 3))",+ "big": "UInt64",+ "at": "DateTime64(3, 'Europe/London')",+ "utc": "DateTime('UTC')",+ }+ definition = table_definition_for("orders", location, "CSVWithNames", columns, None)++ metadata = definition.schema_metadata()+ rebuilt = TableDefinition.from_schema_metadata("orders", metadata)++ assert metadata["csv"]["column_types"] == {+ "total": {"kind": "decimal", "precision": 12, "scale": 3},+ "big": "unsigned",+ "at": {"kind": "datetime", "timezone": "Europe/London"},+ "utc": "datetime",+ }+ assert rebuilt == definition++ @parameterized.expand(+ [+ (f"{table_format}_{column_type}", table_format, column_type, url)+ for table_format, url in (+ ("Parquet", "https://storage.googleapis.com/lake/orders/*.parquet"),+ ("JSONEachRow", "https://storage.googleapis.com/lake/orders/*.jsonl"),+ ("CSVWithNames", "https://storage.googleapis.com/lake/orders/*.csv"),+ ("Delta", "https://storage.googleapis.com/lake/orders/"),+ )+ for column_type in (+ "Array(Nullable(Int64))",+ "Map(String, String)",+ "Tuple(a Nullable(Int64), b Nullable(String))",+ "Nullable(JSON)",+ "Object('json')",+ "Nested(a Int64)",+ )+ ]+ )+ def test_nested_columns_are_refused(self, _name: str, table_format: str, column_type: str, url: str) -> None:+ columns = {"id": "Int64", "items": {"clickhouse": column_type, "hogql": "StringJSONDatabaseField"}}++ with pytest.raises(UnmappableTableError, match="'items'"):+ table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)++ def test_an_invalid_nested_column_does_not_block_the_table(self) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.parquet")+ columns = {"items": {"clickhouse": "Array(Int64)", "hogql": "StringArrayDatabaseField", "valid": False}}++ assert table_definition_for("orders", location, "Parquet", columns, None).kind == "files"++ def test_a_csv_column_no_import_keeps_is_refused(self) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")++ with pytest.raises(UnmappableTableError, match="'id'"):+ table_definition_for("orders", location, "CSVWithNames", {"id": "Int128"}, None)++ @parameterized.expand(+ [+ (f"{table_format}_{column}", table_format, column, url)+ for table_format, url in (+ ("Parquet", "https://storage.googleapis.com/lake/orders/*.parquet"),+ ("CSVWithNames", "https://storage.googleapis.com/lake/orders/*.csv"),+ ("Delta", "https://storage.googleapis.com/lake/orders/"),+ )+ for column in ("_file_name", "_file_modified_at", "_file_row_number")+ ]+ )+ def test_columns_named_like_the_file_columns_are_refused(+ self, _name: str, table_format: str, column: str, url: str+ ) -> None:+ columns = {"id": "Int64", column: "Nullable(String)"}++ with pytest.raises(UnmappableTableError, match=f"'{column}'"):+ table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)++ @parameterized.expand([("Date",), ("Nullable(Date32)",), ("DateTime",), ("Nullable(DateTime64(9))",)])+ def test_json_dates_and_times_are_refused(self, column_type: str) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")++ with pytest.raises(UnmappableTableError, match="'created'"):+ table_definition_for("orders", location, "JSONEachRow", {"id": "Int64", "created": column_type}, None)++ def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")++ with pytest.raises(UnmappableTableError, match="' Order ID '"):+ table_definition_for("orders", location, "CSVWithNames", {" Order ID ": "Nullable(String)"}, None)++ def test_headerless_csv_names_are_not_checked_for_spaces(self) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")++ assert table_definition_for("orders", location, "CSV", {"c1": "Nullable(String)"}, None).kind == "files"++ @parameterized.expand(+ [+ ("glob_without_pattern", {"kind": "files", "scope": "glob", "path": "", "format": "csv"}),+ ("unknown_scope", {"kind": "files", "scope": "everything", "path": "", "format": "csv"}),+ ]+ )+ def test_unusable_glob_metadata_is_ignored(self, _name: str, metadata: dict[str, Any]) -> None:+ assert TableDefinition.from_schema_metadata("orders", metadata) is None++ def test_unknown_column_kinds_in_metadata_are_dropped(self) -> None:+ rebuilt = TableDefinition.from_schema_metadata(+ "orders",+ {+ "kind": "files",+ "scope": "glob",+ "path": "",+ "pattern": "*.csv",+ "format": "csv",+ "csv": {"column_types": {"id": "integer", "blob": "binary", "n": 3}},+ },+ )++ assert rebuilt is not None+ assert rebuilt.csv.column_types == (CsvColumnType(name="id", kind="integer"),)+ assert rebuilt.preserve_column_names is False+++class TestResolveGlobFiles:+ def test_reads_exactly_the_files_the_glob_matches(self, tmp_path: Path) -> None:+ write_files(+ tmp_path,+ {+ "orders/part-1.csv": "id\n1\n",+ "orders/part-2.csv": "id\n2\n",+ "orders/part-3.csv": "id\n3\n",+ "orders/_SUCCESS": "",+ "orders/nested/part-1.csv": "id\n4\n",+ "orders/part-1.csv.bak": "id\n5\n",+ "refunds/part-1.csv": "id\n6\n",+ },+ )+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/part-{1..2}.csv")+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)++ resolved = resolve_table_files(local_store(tmp_path), table)++ assert [stored.path for stored in resolved.files] == ["orders/part-1.csv", "orders/part-2.csv"]++ def test_a_literal_file_resolves_to_that_file(self, tmp_path: Path) -> None:+ write_files(tmp_path, {"orders.csv": "id\n1\n", "orders.csv.old": "id\n2\n"})+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders.csv")+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)++ resolved = resolve_table_files(local_store(tmp_path), table)++ assert [stored.path for stored in resolved.files] == ["orders.csv"]++ def test_no_matching_file_is_a_missing_table(self, tmp_path: Path) -> None:+ write_files(tmp_path, {"orders/a.parquet": "x"})+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)++ with pytest.raises(ObjectStorageError, match=NO_TABLE_FILES_ERROR):+ resolve_table_files(local_store(tmp_path), table)+++class TestCsvColumnTypes:+ @parameterized.expand(+ [+ ("Int64", CsvColumnType(name="c", kind="integer")),+ ("Nullable(Int32)", CsvColumnType(name="c", kind="integer")),+ ("UInt8", CsvColumnType(name="c", kind="integer")),+ ("LowCardinality(Nullable(Int16))", CsvColumnType(name="c", kind="integer")),+ ("UInt64", CsvColumnType(name="c", kind="unsigned")),+ ("Nullable(UInt64)", CsvColumnType(name="c", kind="unsigned")),+ ("Float32", CsvColumnType(name="c", kind="float")),+ ("Nullable(Float64)", CsvColumnType(name="c", kind="float")),+ ("Decimal(10, 2)", CsvColumnType(name="c", kind="decimal", precision=10, scale=2)),+ ("Nullable(Decimal(38,0))", CsvColumnType(name="c", kind="decimal", precision=38, scale=0)),+ ("Decimal32(4)", CsvColumnType(name="c", kind="decimal", precision=9, scale=4)),+ ("Decimal64(6)", CsvColumnType(name="c", kind="decimal", precision=18, scale=6)),+ ("Decimal128(10)", CsvColumnType(name="c", kind="decimal", precision=38, scale=10)),+ ("Bool", CsvColumnType(name="c", kind="boolean")),+ ("Date", CsvColumnType(name="c", kind="date")),+ ("Date32", CsvColumnType(name="c", kind="date")),+ ("DateTime", CsvColumnType(name="c", kind="datetime")),+ ("Nullable(DateTime64(6, 'UTC'))", CsvColumnType(name="c", kind="datetime")),+ ("DateTime('Europe/London')", CsvColumnType(name="c", kind="datetime", timezone="Europe/London")),+ (+ "Nullable(DateTime64(3, 'America/New_York'))",+ CsvColumnType(name="c", kind="datetime", timezone="America/New_York"),+ ),+ ("String", None),+ ("Array(Int64)", None),+ ("Nullable(UUID)", None),+ ("", None),+ ]+ )+ def test_type_for_stored_type(self, clickhouse_type: str, expected: CsvColumnType | None) -> None:+ assert csv_column_type("c", clickhouse_type) == expected++ @parameterized.expand(+ [+ ("int128", "Int128"),+ ("uint256", "Nullable(UInt256)"),+ ("decimal_wider_than_38_digits", "Decimal(50, 2)"),+ ("decimal256", "Decimal256(4)"),+ ("decimal_without_arguments", "Decimal"),+ ("unknown_time_zone", "DateTime('Nowhere/Land')"),+ ]+ )+ def test_types_no_import_keeps_are_refused(self, _name: str, clickhouse_type: str) -> None:+ with pytest.raises(UnsupportedCsvColumnError, match="'c'"):+ csv_column_type("c", clickhouse_type)++ def test_types_from_stored_columns_skip_invalid_and_string_columns(self) -> None:+ columns: dict[str, Any] = {+ "id": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True},+ "broken": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": False},+ "legacy": "Nullable(Float64)",+ "name": {"clickhouse": "Nullable(String)", "hogql": "StringDatabaseField"},+ "odd": 5,+ }++ assert csv_column_types_from_columns(columns) == (+ CsvColumnType(name="id", kind="integer"),+ CsvColumnType(name="legacy", kind="float"),+ )++ @parameterized.expand(+ [+ ("integers", "integer", ["1", " -2 ", "+3", None, ""], [1, -2, 3, None, None], 0),+ (+ "bad_integers_become_null",+ "integer",+ ["1", "1.5", "x", "99999999999999999999"],+ [1, None, None, None],+ 3,+ ),+ ("floats", "float", ["1.5", "1e3", " 2 ", "-0.25", "+4"], [1.5, 1000.0, 2.0, -0.25, 4.0], 0),+ ("bad_floats_become_null", "float", ["1.5", "one"], [1.5, None], 1),+ (+ "booleans",+ "boolean",+ ["true", "FALSE", "1", "0", "yes", "No", None],+ [True, False, True, False, True, False, None],+ 0,+ ),+ ("bad_booleans_become_null", "boolean", ["true", "maybe"], [True, None], 1),+ ("dates", "date", ["2024-01-15", "1999-12-31"], [date(2024, 1, 15), date(1999, 12, 31)], 0),+ (+ "bad_dates_become_null",+ "date",+ ["2024-01-15", "2024-13-01", "15/01/2024"],+ [date(2024, 1, 15), None, None],+ 2,+ ),+ (+ "naive_datetimes_are_utc",+ "datetime",+ ["2024-01-15 10:30:00", "2024-01-15T10:30:00.123456"],+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 10, 30, 0, 123456, tzinfo=UTC)],+ 0,+ ),+ (+ "datetimes_with_offsets",+ "datetime",+ ["2024-01-15T10:30:00Z", "2024-01-15T10:30:00+02:00"],+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 8, 30, tzinfo=UTC)],+ 0,+ ),+ (+ "mixed_datetimes_are_read_one_by_one",+ "datetime",+ ["2024-01-15 10:30:00", "2024-01-15T10:30:00Z", "soon"],+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 10, 30, tzinfo=UTC), None],+ 1,+ ),+ (+ "unsigned_keeps_values_above_int64",+ CsvColumnType(name="value", kind="unsigned"),+ ["18446744073709551615", "0", "-1", "+7", "18446744073709551616"],+ [Decimal(18446744073709551615), Decimal(0), None, Decimal(7), None],+ 2,+ ),+ (+ "decimals_keep_their_scale",+ CsvColumnType(name="value", kind="decimal", precision=10, scale=2),+ ["1.25", "-3.1", "7", "1.255", "x", "123456789012.5"],+ [Decimal("1.25"), Decimal("-3.10"), Decimal("7.00"), Decimal("1.25"), None, None],+ 2,+ ),+ (+ "naive_datetimes_take_the_column_time_zone",+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),+ ["2024-07-01 10:00:00", "2024-01-15 10:00:00"],+ [datetime(2024, 7, 1, 9, 0, tzinfo=UTC), datetime(2024, 1, 15, 10, 0, tzinfo=UTC)],+ 0,+ ),+ (+ "offsets_win_over_the_column_time_zone",+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),+ ["2024-07-01T10:00:00Z", "2024-07-01T10:00:00+02:00"],+ [datetime(2024, 7, 1, 10, 0, tzinfo=UTC), datetime(2024, 7, 1, 8, 0, tzinfo=UTC)],+ 0,+ ),+ (+ "mixed_values_in_a_time_zone_are_read_one_by_one",+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),+ ["2024-07-01 10:00:00", "2024-07-01T10:00:00Z", "soon"],+ [datetime(2024, 7, 1, 9, 0, tzinfo=UTC), datetime(2024, 7, 1, 10, 0, tzinfo=UTC), None],+ 1,+ ),+ (+ "nanosecond_datetimes_are_truncated_to_microseconds",+ "datetime",+ ["2024-07-01 10:00:00.123456789", "2024-07-01 10:00:00", "2024-07-01T10:00:00.123456789+02:00"],+ [+ datetime(2024, 7, 1, 10, 0, 0, 123456, tzinfo=UTC),+ datetime(2024, 7, 1, 10, 0, tzinfo=UTC),+ datetime(2024, 7, 1, 8, 0, 0, 123456, tzinfo=UTC),+ ],+ 0,+ ),+ (+ "nanosecond_datetimes_take_the_column_time_zone",+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),+ ["2024-07-01 10:00:00.123456789"],+ [datetime(2024, 7, 1, 9, 0, 0, 123456, tzinfo=UTC)],+ 0,+ ),+ (+ "repeated_local_time_takes_the_earlier_instant",+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),+ ["2024-10-27 01:30:00"],+ [datetime(2024, 10, 27, 0, 30, tzinfo=UTC)],+ 0,+ ),+ ]+ )+ def test_cast(+ self, _name: str, kind: Any, values: list[str | None], expected: list[Any], expected_nulled: int+ ) -> None:+ table = pa.table({"value": pa.array(values, pa.string()), "other": pa.array(["x"] * len(values))})+ column_type = kind if isinstance(kind, CsvColumnType) else CsvColumnType(name="value", kind=kind)++ result = cast_csv_columns(table, (column_type,))++ assert result.table.column("value").to_pylist() == expected+ assert result.table.column("other").type == pa.string()+ assert result.nulled_values == ({"value": expected_nulled} if expected_nulled else {})++ def test_a_column_missing_from_the_file_is_ignored(self) -> None:+ table = pa.table({"id": pa.array(["1"])})++ result = cast_csv_columns(table, (CsvColumnType(name="gone", kind="integer"),))++ assert result.table.equals(table)++ @parameterized.expand(+ [+ ("no_bad_values", 1000, ()),+ ("one_bad_value", 1000, (517,)),+ ("bad_first_and_last", 1000, (0, 999)),+ ("bad_values_in_one_run", 1000, tuple(range(300, 340))),+ ("scattered_bad_values", 5000, tuple(range(0, 5000, 97))),+ ("every_value_bad", 200, tuple(range(200))),+ ("shorter_than_one_slice", 5, (2,)),+ ]+ )+ def test_split_cast_equals_a_value_by_value_cast(self, _name: str, rows: int, bad_rows: tuple[int, ...]) -> None:+ bad = set(bad_rows)+ values = [None if row % 11 == 0 else ("x" if row in bad else str(row)) for row in range(rows)]+ table = pa.table({"value": pa.array(values, pa.string())})++ result = cast_csv_columns(table, (CsvColumnType(name="value", kind="integer"),))++ expected = [None if value is None or value == "x" else int(value) for value in values]+ assert result.table.column("value").to_pylist() == expected+ bad_non_null = sum(1 for value in values if value == "x")+ assert result.nulled_values == ({"value": bad_non_null} if bad_non_null else {})++ def test_split_cast_parses_far_fewer_slices_than_values(self) -> None:+ values = [str(row) for row in range(100_000)]+ values[54_321] = "x"+ table = pa.table({"value": pa.array(values, pa.string())})++ started = time.perf_counter()+ result = cast_csv_columns(table, (CsvColumnType(name="value", kind="integer"),))++ assert time.perf_counter() - started < 1+ assert result.nulled_values == {"value": 1}++ def test_the_reader_casts_typed_columns_and_reports_bad_values(self) -> None:+ logger = MagicMock()+ content = b"id,amount,created,label\n1,2.5,2024-01-01,a\nx,3,2024-01-02,b\n"+ options = CsvOptions(+ column_types=(+ CsvColumnType(name="id", kind="integer"),+ CsvColumnType(name="amount", kind="float"),+ CsvColumnType(name="created", kind="date"),+ )+ )++ [table] = list(iter_csv_tables(io.BytesIO(content), options, file_path="orders.csv", logger=logger))++ assert table.schema.types == [pa.int64(), pa.float64(), pa.date32(), pa.string()]+ assert table.column("id").to_pylist() == [1, None]+ logger.warning.assert_called_once()+ assert logger.warning.call_args.kwargs["nulled_values"] == {"id": 1}++ def test_a_header_only_file_gets_typed_empty_columns(self) -> None:+ options = CsvOptions(column_types=(CsvColumnType(name="id", kind="integer"),))++ [table] = list(iter_csv_tables(io.BytesIO(b"id,label\n"), options))++ assert table.num_rows == 0+ assert table.schema.types == [pa.int64(), pa.string()]++ @parameterized.expand(+ [+ ("migrated_table", True, [None, "\\N", "a"]),+ ("new_source", False, ["\\N", "\\N", "a"]),+ ]+ )+ def test_an_unquoted_null_marker_is_null_only_for_migrated_tables(+ self, _name: str, null_marker: bool, expected: list[str | None]+ ) -> None:+ content = b'label\n\\N\n"\\N"\na\n'++ [table] = list(iter_csv_tables(io.BytesIO(content), CsvOptions(null_marker=null_marker)))++ assert table.column("label").to_pylist() == expected++ def test_a_fixed_delimiter_wins_over_the_file_name(self) -> None:+ content = io.BytesIO(b"id,label\n1,a\n")+ stored = StoredObject(path="orders.tsv", size=0, modified_at=datetime(2024, 1, 1, tzinfo=UTC))++ [table] = list(iter_file_tables(content, stored, "csv", CsvOptions(delimiter=",")))++ assert table.column("label").to_pylist() == ["a"]++ def test_new_csv_sources_still_read_strings(self) -> None:+ [table] = list(iter_csv_tables(io.BytesIO(b"id\n1\n"), CsvOptions()))++ assert table.schema.types == [pa.string()]+++def _credential(access_key: str = "AKIAEXAMPLE", access_secret: str = "c2VjcmV0") -> SelfManagedCredential:+ return SelfManagedCredential(access_key=access_key, access_secret=access_secret)+++def resolvable_hosts(monkeypatch: pytest.MonkeyPatch) -> None:+ # The location check resolves the host, and the example hosts here don't exist.+ monkeypatch.setattr(source_base, "validate_warehouse_table_url_pattern", lambda _url: (True, ""))++[email protected]("resolvable_hosts")+class TestJobInputsForSelfManaged:+ @parameterized.expand(+ [+ (+ "s3",+ AmazonS3Source,+ "https://exports.s3.eu-west-1.amazonaws.com/a.csv",+ {"url_pattern": "https://s3.eu-west-1.amazonaws.com/exports/", "access_key_id": "AKIAEXAMPLE"},+ ),+ (+ "gcs",+ GoogleCloudStorageSource,+ "https://storage.googleapis.com/exports/a.csv",+ {"url_pattern": "https://storage.googleapis.com/exports/", "access_key_id": "AKIAEXAMPLE"},+ ),+ (+ "r2",+ CloudflareR2Source,+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/a.csv",+ {+ "url_pattern": f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/",+ "access_key_id": "AKIAEXAMPLE",+ },+ ),+ ]+ )+ def test_maps_the_bucket_and_credential(+ self, _name: str, source_class: type, url: str, expected: dict[str, str]+ ) -> None:+ job_inputs = source_class().job_inputs_for_self_managed(parse_self_managed_url(url), _credential())++ assert {key: job_inputs[key] for key in expected} == expected+ assert job_inputs["secret_access_key"] == "c2VjcmV0"+ file_format = job_inputs["file_format"]+ selection = file_format["selection"] if isinstance(file_format, dict) else file_format+ assert selection == "infer"++ def test_azure_takes_the_account_from_the_url_and_the_key_from_the_secret(self) -> None:+ location = parse_self_managed_url("https://examplestore.blob.core.windows.net/exports/a.csv")++ job_inputs = AzureBlobSource().job_inputs_for_self_managed(location, _credential(access_key="ExampleStore"))++ assert (job_inputs["url_pattern"], job_inputs["account_key"]) == (+ "https://examplestore.blob.core.windows.net/exports/",+ "c2VjcmV0",+ )++ @parameterized.expand(+ [+ ("s3_without_region", AmazonS3Source, "https://exports.s3.amazonaws.com/a.csv", _credential()),+ ("s3_bucket_name_too_short", AmazonS3Source, "https://s3.us-east-1.amazonaws.com/ab/a.csv", _credential()),+ (+ "s3_empty_secret",+ AmazonS3Source,+ "https://exports.s3.us-east-1.amazonaws.com/a.csv",+ _credential(access_secret=" "),+ ),+ (+ "azure_key_for_another_account",+ AzureBlobSource,+ "https://examplestore.blob.core.windows.net/exports/a.csv",+ _credential(access_key="otherstore"),+ ),+ (+ "azure_key_that_is_not_base64",+ AzureBlobSource,+ "https://examplestore.blob.core.windows.net/exports/a.csv",+ _credential(access_key="examplestore", access_secret="a;EndpointSuffix=example.com"),+ ),+ ]+ )+ def test_refuses_what_the_source_would_refuse(+ self, _name: str, source_class: type, url: str, credential: SelfManagedCredential+ ) -> None:+ with pytest.raises(UnmappableTableError):+ source_class().job_inputs_for_self_managed(parse_self_managed_url(url), credential)+++class TestSourceForPipeline:+ @parameterized.expand([("preserved", True), ("normalized", False)])+ def test_the_response_carries_the_column_name_choice(self, _name: str, preserve: bool) -> None:+ source = AmazonS3Source()+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")+ definition = table_definition_for("orders", location, "CSVWithNames", {}, None)+ metadata = {**definition.schema_metadata(), "preserve_column_names": preserve}+ config = source.parse_config(+ {+ "url_pattern": "https://s3.us-east-1.amazonaws.com/exports/",+ "access_key_id": "AKIAEXAMPLE",+ "secret_access_key": "c2VjcmV0",+ }+ )+ inputs = SourceInputs(+ schema_name="orders",+ schema_id=str(uuid.uuid4()),+ source_id=str(uuid.uuid4()),+ team_id=1,+ should_use_incremental_field=False,+ db_incremental_field_last_value=None,+ db_incremental_field_earliest_value=None,+ incremental_field=None,+ incremental_field_type=None,+ job_id=str(uuid.uuid4()),+ logger=LOGGER,+ reset_pipeline=False,+ schema_metadata=metadata,+ )+ source.open_store = MagicMock(return_value=MagicMock()) # type: ignore[method-assign]++ response = source.source_for_pipeline(config, inputs)++ assert response.preserve_column_names is preservediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.pyindex 9dbcfeb38ccd..51b41a7ca431 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py@@ -158,6 +158,9 @@ class SourceResponse:"""Set by an incremental source that replaces rows by origin instead of merging on a primary key(see `ReplaceScope`). The run loads as appends and needs no primary key. A first sync or a resetignores it, because the table starts empty."""+ preserve_column_names: bool = False+ """Write the column names as the source yields them instead of normalizing them to snake_case.+ Set for a table whose existing queries name its columns as they were before the import."""# Not frozen: nothing mutates it in place today, so freezing it is plausible, but every sourcediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.pyindex db5cf46edca5..b7ac2bc3a608 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py@@ -1,4 +1,4 @@-from typing import cast+from typing import Any, castfrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objGCS_EXAMPLE_URL,parse_gcs_url,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ SelfManagedLocation,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (ObjectStorageSource,)@@ -47,6 +51,7 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfiapi_docs_url = "https://cloud.google.com/storage/docs/interoperability"requires_team_feature_flag = FEATURE_FLAG+ self_managed_service = "gcs"@propertydef source_type(self) -> ExternalDataSourceType:@@ -68,6 +73,15 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfidef discovery_options(self, config: GoogleCloudStorageSourceConfig) -> DiscoveryOptions:return discovery_options_from(config)+ def self_managed_connection_inputs(+ self, location: SelfManagedLocation, credential: SelfManagedCredential+ ) -> dict[str, Any]:+ return {+ "url_pattern": f"https://storage.googleapis.com/{location.bucket}/",+ "access_key_id": credential.access_key,+ "secret_access_key": credential.access_secret,+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(diff --git a/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py b/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.pyindex dda0a8478442..a775fb005ba9 100644--- a/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py+++ b/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py@@ -8,7 +8,9 @@ from django.test import SimpleTestCasefrom parameterized import parameterizedfrom products.warehouse_sources.backend.models.external_data_schema import (+ SCHEMA_PINNED_METADATA_KEY,SCHEMA_RESOURCE_ID_METADATA_KEY,+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,ExternalDataSchema,auto_enable_new_schemas,schema_name_matches_auto_sync_patterns,@@ -237,6 +239,61 @@ class TestSchemaDiscoveryReconcile(BaseTest):assert enabled_unsynced.status == ExternalDataSchema.Status.COMPLETEDmock_pause.assert_called_once_with(str(enabled_unsynced.id))+ @parameterized.expand(+ [+ ("pinned_enabled_keeps_syncing", True, True, False),+ ("pinned_off_with_table_stays", True, False, False),+ ("unpinned_enabled_is_disabled", False, True, True),+ ("non_boolean_value_is_not_pinned", "yes", True, True),+ ]+ )+ def test_discovery_leaves_pinned_schemas_alone(+ self, _name: str, pinned: object, should_sync: bool, expect_disabled: bool+ ) -> None:+ source = self._make_source()+ schema = self._make_synced_schema(source, "orders_raw")+ metadata = {"kind": "files", "scope": "glob", "pattern": "orders/*.csv", SCHEMA_PINNED_METADATA_KEY: pinned}+ schema.should_sync = should_sync+ schema.sync_type_config = {"schema_metadata": metadata}+ schema.save()++ with patch(_PAUSE_FN) as mock_pause, self.captureOnCommitCallbacks(execute=True):+ sync_result = sync_old_schemas_with_new_schemas(+ {"orders": None},+ source_id=str(source.pk),+ team_id=self.team.pk,+ )++ schema.refresh_from_db()+ assert schema.deleted is False+ assert sync_result.deleted == []+ assert schema.should_sync is (should_sync and not expect_disabled)+ assert mock_pause.called is expect_disabled++ def test_discovery_does_not_revive_a_deleted_migration_schema(self) -> None:+ source = self._make_source()+ ref = {"table_id": str(uuid.uuid4()), "url_pattern": "https://b.s3.amazonaws.com/o/*.csv", "format": "CSV"}+ tombstone = ExternalDataSchema.objects.create(+ team_id=self.team.pk,+ source_id=source.pk,+ name="orders",+ should_sync=False,+ deleted=True,+ sync_type_config={+ "schema_metadata": {SCHEMA_PINNED_METADATA_KEY: True, SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref}+ },+ )++ sync_old_schemas_with_new_schemas({"orders": None}, source_id=str(source.pk), team_id=self.team.pk)++ tombstone.refresh_from_db()+ assert tombstone.deleted is True+ revived = ExternalDataSchema.objects.get(+ team_id=self.team.pk, source_id=source.pk, name="orders", deleted=False+ )+ assert revived.pk != tombstone.pk+ assert SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY not in (revived.sync_type_config or {}).get("schema_metadata", {})+def test_failed_pause_leaves_the_table_on_and_still_pauses_the_others(self) -> None:# A row written off while its schedule still runs keeps billing, and the next discovery run# would skip it as already off, so a failed pause has to leave the row on for that retry.diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.pynew file mode 100644index 000000000000..491ea7f074ac--- /dev/null+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py@@ -0,0 +1,666 @@+from datetime import UTC, datetime+from io import StringIO+from typing import Any++from posthog.test.base import APIBaseTest, BaseTest+from unittest.mock import MagicMock, patch++from django.core.management import call_command+from django.core.management.base import CommandError++from parameterized import parameterized+from temporalio.service import RPCError, RPCStatusCode++from posthog.hogql.database.database import Database++from posthog.models.organization import OrganizationMembership+from posthog.models.team import Team++from products.access_control.backend.models.access_control import AccessControl+from products.warehouse_sources.backend.models.credential import DataWarehouseCredential+from products.warehouse_sources.backend.models.external_data_destination import ExternalDataSourceDestination+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.models.external_data_schema import (+ SCHEMA_PINNED_METADATA_KEY,+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,+ ExternalDataSchema,+ self_managed_table_ref,+)+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.self_managed_migration import (+ MIGRATION_MARKER_KEY,+ MigrationOptions,+ TableMigrationState,+ abandon_tables,+ evaluate_tables,+ migrate_self_managed_tables,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (+ SelfManagedCredential,+ parse_self_managed_url,+ table_definition_for,+)+from products.warehouse_sources.backend.types import DataWarehouseTableFormat, ExternalDataJobStatus++MODULE = "products.warehouse_sources.backend.self_managed_migration"+SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source"+SCHEMA_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_schema"+EXPORTS_URL = "https://exports.s3.eu-west-1.amazonaws.com/orders/*.csv"+++def _no_region(_bucket: str, _credential: SelfManagedCredential) -> str | None:+ raise AssertionError("every URL in this test names its region")+++def _schemas_for(table: DataWarehouseTable) -> list[ExternalDataSchema]:+ return [+ schema+ for schema in ExternalDataSchema.objects.select_related("source").filter(team_id=table.team_id)+ if (ref := self_managed_table_ref(schema)) is not None and ref.table_id == str(table.id)+ ]+++def _schema_for(table: DataWarehouseTable) -> ExternalDataSchema:+ [schema] = [schema for schema in _schemas_for(table) if not schema.deleted]+ return schema+++class _MigrationTest(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ patches = [+ patch(f"{MODULE}.source_feature_flag_missing", return_value=False),+ patch(f"{MODULE}.delete_external_data_schedule"),+ # The location check resolves the bucket host, and these example buckets don't exist.+ patch.object(source_base, "validate_warehouse_table_url_pattern", return_value=(True, "")),+ ]+ mocks = [p.start() for p in patches]+ for p in patches:+ self.addCleanup(p.stop)+ self.flag_missing, self.delete_schedule, _ = mocks+ self.credential = self._credential("AKIAFIRST")++ def _credential(self, access_key: str) -> DataWarehouseCredential:+ return DataWarehouseCredential.objects.create(access_key=access_key, access_secret="c2VjcmV0", team=self.team)++ def _table(+ self,+ name: str = "orders",+ url: str = EXPORTS_URL,+ credential: DataWarehouseCredential | None = None,+ table_format: str = DataWarehouseTableFormat.CSVWithNames,+ created_at: datetime | None = None,+ **fields: Any,+ ) -> DataWarehouseTable:+ table = DataWarehouseTable.objects.create(+ name=name,+ format=table_format,+ url_pattern=url,+ team=self.team,+ credential=credential or self.credential,+ columns={"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True}},+ options={"csv_allow_double_quotes": True},+ **fields,+ )+ if created_at is not None:+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(created_at=created_at)+ table.refresh_from_db()+ return table++ def _migrate(self, **options: Any) -> list[Any]:+ resolve_region = options.pop("resolve_region", _no_region)+ live_run = options.pop("live_run", True)+ return migrate_self_managed_tables(+ MigrationOptions(team_id=self.team.pk, live_run=live_run, **options), resolve_region=resolve_region+ )++ def _abandon(self, table: DataWarehouseTable, **options: Any) -> Any:+ options.setdefault("live_run", True)+ [outcome] = abandon_tables([str(table.id)], resolve_region=_no_region, **options)+ return outcome++ def _status(self, table: DataWarehouseTable) -> Any:+ [status] = evaluate_tables(team_id=self.team.pk, table_ids=[str(table.id)], resolve_region=_no_region)+ return status++ def _job(self, table: DataWarehouseTable, status: str, latest_error: str | None = None) -> ExternalDataJob:+ schema = _schema_for(table)+ return ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=status, latest_error=latest_error+ )++ def _adopt(self, table: DataWarehouseTable) -> None:+ schema = _schema_for(table)+ table.refresh_from_db()+ table.adopt_into_source(schema.source)+ table.url_pattern = f"https://bucket.s3.amazonaws.com/team_{self.team.pk}/{table.name}"+ table.format = DataWarehouseTableFormat.DeltaS3Wrapper+ table.save(internally_computed_url_pattern=True)+ schema.table = table+ schema.save()+++class TestSelfManagedMigration(_MigrationTest):+ def test_tables_group_into_one_source_per_credential_and_bucket(self) -> None:+ other_credential = self._credential("AKIASECOND")+ virtual_hosted = self._table("orders")+ path_style = self._table("refunds", url="https://s3.eu-west-1.amazonaws.com/exports/refunds/*.csv")+ other_bucket = self._table("archive", url="https://archive.s3.eu-west-1.amazonaws.com/a.parquet")+ other_key = self._table("mirror", credential=other_credential)++ self._migrate()++ def source_of(table: DataWarehouseTable) -> ExternalDataSource:+ return _schema_for(table).source++ assert source_of(virtual_hosted).pk == source_of(path_style).pk+ assert len({source_of(t).pk for t in (virtual_hosted, other_bucket, other_key)}) == 3+ assert ExternalDataSource.objects.filter(team=self.team).count() == 3+ # Two sources of one type can't both go without a prefix, or their table names would clash.+ prefixes = [source_of(t).prefix or "" for t in (virtual_hosted, other_bucket, other_key)]+ assert prefixes.count("") == 1 and len(set(prefixes)) == 3+ assert set(prefixes) - {""} <= {"exports", "archive"}+ assert source_of(other_bucket).job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/archive/"+ assert source_of(other_key).job_inputs["access_key_id"] == "AKIASECOND"+ assert (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY] == {+ "group_key": "s3/eu-west-1/exports",+ "credential_id": str(self.credential.pk),+ }++ def test_a_live_run_creates_a_turned_off_schema_and_changes_nothing_else(self) -> None:+ created_at = datetime(2024, 3, 1, tzinfo=UTC)+ table = self._table(created_at=created_at)+ before = DataWarehouseTable.raw_objects.values().get(pk=table.pk)++ [outcome] = self._migrate()++ schema = _schema_for(table)+ expected_contract = table_definition_for(+ "orders", parse_self_managed_url(EXPORTS_URL), "CSVWithNames", table.columns, True+ ).schema_metadata()+ assert (outcome.action, outcome.status.state) == ("linked", TableMigrationState.AWAITING_SYNC)+ assert (schema.name, schema.should_sync, schema.sync_type, schema.table_id) == (+ "orders",+ False,+ "full_refresh",+ None,+ )+ assert schema.sync_type_config == {+ "schema_metadata": {+ **expected_contract,+ SCHEMA_PINNED_METADATA_KEY: True,+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: {+ "table_id": str(table.id),+ "url_pattern": EXPORTS_URL,+ "format": "CSVWithNames",+ },+ }+ }+ assert expected_contract["preserve_column_names"] is True+ assert schema.source.source_type == "AmazonS3"+ # A normal new source: its date is not moved to the table's.+ assert schema.source.created_at > created_at+ assert schema.source.job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/exports/"+ assert ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=schema.source).exists()+ assert not ExternalDataJob.objects.filter(team=self.team).exists()+ assert DataWarehouseTable.raw_objects.values().get(pk=table.pk) == before++ def test_a_dry_run_writes_nothing(self) -> None:+ self._table()++ [outcome] = self._migrate(live_run=False)++ assert (outcome.action, outcome.status.state) == ("would_link", TableMigrationState.PENDING)+ assert not ExternalDataSource.objects.filter(team=self.team).exists()+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()++ def test_running_again_changes_nothing(self) -> None:+ table = self._table()+ self._migrate()++ [again] = self._migrate()++ assert (again.action, again.status.state) == ("none", TableMigrationState.AWAITING_SYNC)+ assert again.status.schema_id == str(_schema_for(table).id)+ assert ExternalDataSchema.objects.filter(team=self.team).count() == 1+ assert ExternalDataSource.objects.filter(team=self.team).count() == 1++ @parameterized.expand(+ [+ ("syncing_off", False, None, None, {}, "syncing is off"),+ ("syncing_on", True, None, None, {}, "syncing is on; waiting for the first sync"),+ ("first_sync_running", True, ExternalDataJobStatus.RUNNING, None, {}, "the first sync is running"),+ (+ "first_sync_failed",+ True,+ ExternalDataJobStatus.FAILED,+ "The bucket doesn't exist",+ {},+ "The bucket doesn't exist; the table still reads its own bucket",+ ),+ (+ "table_edited",+ False,+ None,+ None,+ {"url_pattern": "https://exports.s3.eu-west-1.amazonaws.com/new/*.csv"},+ "the table changed after the migration, so a sync makes a separate table",+ ),+ ]+ )+ def test_a_linked_table_reports_why_it_still_reads_its_own_bucket(+ self,+ _name: str,+ should_sync: bool,+ job_status: str | None,+ latest_error: str | None,+ table_change: dict[str, Any],+ reason: str,+ ) -> None:+ table = self._table()+ self._migrate()+ ExternalDataSchema.objects.filter(pk=_schema_for(table).pk).update(should_sync=should_sync)+ if job_status is not None:+ self._job(table, job_status, latest_error=latest_error)+ if table_change:+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(**table_change)++ status = self._status(table)++ assert (status.state, status.reason) == (TableMigrationState.AWAITING_SYNC, reason)+ table.refresh_from_db()+ assert (table.external_data_source_id, table.credential_id) == (None, self.credential.pk)++ def test_a_schema_that_synced_a_separate_table_is_reported(self) -> None:+ table = self._table()+ self._migrate()+ schema = _schema_for(table)+ schema.table = DataWarehouseTable.objects.create(+ name="amazons3_orders", format="Parquet", team=self.team, external_data_source=schema.source+ )+ schema.save()++ status = self._status(table)++ assert (status.state, status.reason) == (+ TableMigrationState.AWAITING_SYNC,+ "the table changed after the migration, so its schema syncs a separate table",+ )++ def test_a_migrated_table_is_never_linked_again(self) -> None:+ table = self._table()+ self._migrate()+ self._adopt(table)++ [outcome] = self._migrate()++ assert (outcome.action, outcome.status.state) == ("none", TableMigrationState.MIGRATED)+ assert ExternalDataSchema.objects.filter(team=self.team).count() == 1++ @parameterized.expand(+ [+ ("unknown_host", {"url": "https://minio.example.com/exports/a.csv"}, "isn't a storage host"),+ ("unsupported_format", {"table_format": "TabSeparated"}, "has no import equivalent"),+ ("delta_with_glob", {"table_format": "Delta", "url": EXPORTS_URL}, "can't hold a file pattern"),+ (+ "azure_key_for_another_account",+ {"url": "https://examplestore.blob.core.windows.net/exports/a.csv"},+ "isn't the storage account name",+ ),+ ]+ )+ def test_a_table_that_cant_be_mapped_is_skipped_with_its_reason(+ self, _name: str, fields: dict[str, Any], reason: str+ ) -> None:+ self._table(**fields)++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.SKIPPED+ assert reason in (outcome.status.reason or "")+ assert outcome.action == "none"+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()++ def test_teams_without_the_flag_are_skipped(self) -> None:+ self.flag_missing.return_value = True+ self._table()++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.SKIPPED+ assert "dwh-object-storage-sources" in (outcome.status.reason or "")+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()++ @parameterized.expand(+ [+ ("object_rule_restricts", True, "none", TableMigrationState.SKIPPED),+ ("object_rule_grants", True, "editor", TableMigrationState.SKIPPED),+ ("rule_on_another_table", False, "none", TableMigrationState.AWAITING_SYNC),+ ]+ )+ def test_a_table_with_its_own_access_rules_is_skipped(+ self, _name: str, on_this_table: bool, access_level: str, expected: TableMigrationState+ ) -> None:+ table = self._table()+ other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")+ AccessControl.objects.create(+ team=self.team,+ resource="warehouse_table",+ resource_id=str(table.id if on_this_table else other.id),+ access_level=access_level,+ organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),+ )++ outcome = next(o for o in self._migrate() if o.status.table_id == str(table.id))++ assert outcome.status.state == expected+ if expected == TableMigrationState.SKIPPED:+ assert "own access rules" in (outcome.status.reason or "")+ assert _schemas_for(table) == []++ @parameterized.expand(+ [+ ("member_rule_on_sources", "external_data_source", True),+ ("default_rule_on_sources", "external_data_source", False),+ ("member_rule_on_the_warehouse", "warehouse_objects", True),+ ("default_rule_on_the_warehouse", "warehouse_objects", False),+ ]+ )+ def test_a_project_with_access_rules_for_sources_is_skipped(+ self, _name: str, resource: str, for_member: bool+ ) -> None:+ table = self._table()+ AccessControl.objects.create(+ team=self.team,+ resource=resource,+ resource_id=None,+ access_level="none",+ organization_member=(+ OrganizationMembership.objects.get(user=self.user, organization=self.organization)+ if for_member+ else None+ ),+ )++ [outcome] = self._migrate()++ assert outcome.status.table_id == str(table.id)+ assert outcome.status.state == TableMigrationState.SKIPPED+ assert "access rules for sources" in (outcome.status.reason or "")+ assert _schemas_for(table) == []++ def test_status_reports_every_state(self) -> None:+ pending = self._table("pending_table")+ self._table("skipped_table", url="https://minio.example.com/exports/a.csv")+ awaiting = self._table("awaiting_table", url="https://exports.s3.eu-west-1.amazonaws.com/w/*.csv")+ migrated = self._table("migrated_table", url="https://exports.s3.eu-west-1.amazonaws.com/m/*.csv")+ abandoned = self._table("abandoned_table", url="https://exports.s3.eu-west-1.amazonaws.com/a/*.csv")+ self._migrate(table_ids=tuple(str(t.id) for t in (awaiting, migrated, abandoned)))+ self._adopt(migrated)+ self._abandon(abandoned, reason="customer asked to keep it")++ statuses = evaluate_tables(team_id=self.team.pk, resolve_region=_no_region)++ assert {status.name: status.state for status in statuses} == {+ pending.name: TableMigrationState.PENDING,+ "skipped_table": TableMigrationState.SKIPPED,+ awaiting.name: TableMigrationState.AWAITING_SYNC,+ migrated.name: TableMigrationState.MIGRATED,+ abandoned.name: TableMigrationState.ABANDONED,+ }+ assert next(s for s in statuses if s.name == abandoned.name).reason == "customer asked to keep it"++ def test_selection_ignores_synced_deleted_and_other_teams_tables(self) -> None:+ self._table("deleted_table", deleted=True)+ self._table("no_credential", credential=None)+ DataWarehouseTable.raw_objects.filter(name="no_credential").update(credential=None)+ source = ExternalDataSource.objects.create(team=self.team, source_type="Postgres", job_inputs={})+ self._table("synced_table", external_data_source=source)+ other_team = Team.objects.create(organization=self.organization, name="Other team")+ DataWarehouseTable.objects.create(+ name="other_team_table", format="CSVWithNames", url_pattern=EXPORTS_URL, team=other_team+ )+ kept = self._table("kept")++ assert [status.table_id for status in evaluate_tables(team_id=self.team.pk)] == [str(kept.id)]++ def test_limit_and_table_ids_bound_a_run(self) -> None:+ first = self._table("first", created_at=datetime(2024, 1, 1, tzinfo=UTC))+ second = self._table(+ "second",+ url="https://exports.s3.eu-west-1.amazonaws.com/b/*.csv",+ created_at=datetime(2024, 1, 2, tzinfo=UTC),+ )+ third = self._table("third", url="https://exports.s3.eu-west-1.amazonaws.com/c/*.csv")++ outcomes = self._migrate(limit=1, table_ids=(str(second.id), str(third.id)))++ assert [(o.status.name, o.action) for o in outcomes] == [("second", "linked"), ("third", "limit_reached")]+ assert _schemas_for(first) == []++ def test_a_name_already_taken_in_the_bucket_fails_the_link_of_that_table_only(self) -> None:+ self._table("orders")+ self._migrate()+ source = ExternalDataSource.objects.get(team=self.team)+ ExternalDataSchema.objects.create(name="dupe", team=self.team, source=source)+ self._table("dupe", url="https://exports.s3.eu-west-1.amazonaws.com/d/*.csv")+ self._table("fine", url="https://exports.s3.eu-west-1.amazonaws.com/f/*.csv")++ outcomes = self._migrate()++ assert [(o.status.name, o.action) for o in outcomes][1:] == [("dupe", "link_failed"), ("fine", "linked")]+ assert "already uses the name 'dupe'" in (outcomes[1].error or "")++ @parameterized.expand([("named_region", "eu-west-1", "linked"), ("unknown_region", None, "none")])+ def test_a_bucket_url_without_a_region_asks_s3_once_per_bucket(+ self, _name: str, region: str | None, expected_action: str+ ) -> None:+ self._table("a", url="https://exports.s3.amazonaws.com/a/*.csv")+ self._table("b", url="https://s3.amazonaws.com/exports/b/*.csv")+ resolver = MagicMock(return_value=region)++ outcomes = self._migrate(resolve_region=resolver)++ assert [o.action for o in outcomes] == [expected_action] * 2+ resolver.assert_called_once()+ if region is not None:+ job_inputs = ExternalDataSource.objects.get(team=self.team).job_inputs+ assert job_inputs["url_pattern"] == f"https://s3.{region}.amazonaws.com/exports/"++ def test_an_adopted_table_is_queried_by_its_bare_name_before_and_after(self) -> None:+ table = self._table("orders_raw")+ self._migrate()++ before = Database.create_for(team=self.team)+ assert before.has_table("orders_raw")+ assert not before.has_table("amazons3.orders_raw")++ self._adopt(table)++ after = Database.create_for(team=self.team)+ assert after.has_table("orders_raw")+ assert after.has_table("amazons3.orders_raw")+++class TestAbandon(_MigrationTest):+ def test_abandon_removes_the_schema_and_keeps_the_table(self) -> None:+ table = self._table()+ self._migrate()+ self._job(table, ExternalDataJobStatus.FAILED)+ schema = _schema_for(table)++ outcome = self._abandon(table, reason="customer asked to keep it")++ assert outcome.action == "abandoned"+ schema.refresh_from_db()+ assert schema.deleted is True+ ref = self_managed_table_ref(schema)+ assert ref is not None and ref.abandon_reason == "customer asked to keep it"+ assert ExternalDataSource.objects.get(pk=schema.source_id).deleted is True+ self.delete_schedule.assert_called_once_with(str(schema.id))+ table.refresh_from_db()+ assert (table.deleted, table.credential_id, table.url_pattern) == (False, self.credential.pk, EXPORTS_URL)+ [again] = self._migrate()+ assert (again.action, again.status.state) == ("none", TableMigrationState.ABANDONED)+ assert ExternalDataSchema.objects.filter(team=self.team, deleted=False).count() == 0++ def test_abandon_is_kept_when_the_schedule_delete_fails(self) -> None:+ table = self._table()+ self._migrate()+ self.delete_schedule.side_effect = RPCError("unavailable", RPCStatusCode.UNAVAILABLE, b"")++ assert self._abandon(table).action == "abandoned"+ assert self._status(table).state == TableMigrationState.ABANDONED++ def test_abandon_keeps_a_source_that_holds_other_tables(self) -> None:+ table = self._table()+ other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")+ self._migrate()++ self._abandon(table)++ assert _schema_for(other).source.deleted is False++ def test_abandon_of_a_table_that_was_never_linked_records_it_and_removes_the_new_source(self) -> None:+ table = self._table()++ outcome = self._abandon(table, reason="not wanted")++ assert outcome.action == "abandoned"+ [schema] = _schemas_for(table)+ assert (schema.deleted, schema.source.deleted) == (True, True)+ self.delete_schedule.assert_not_called()+ assert (self._status(table).state, self._status(table).reason) == (TableMigrationState.ABANDONED, "not wanted")+ assert self._migrate()[0].action == "none"++ @parameterized.expand(+ [+ ("migrated", "the table is already migrated"),+ ("abandoned", "the table's migration is already abandoned"),+ ("syncing_on", "the user turned syncing on for the table's schema"),+ ("running", "the table's first sync is running"),+ ("separate_table", "the table's schema syncs a separate table"),+ ]+ )+ def test_abandon_refuses_a_table_it_must_not_touch(self, case: str, reason: str) -> None:+ table = self._table()+ self._migrate()+ schema = _schema_for(table)+ if case == "migrated":+ self._adopt(table)+ elif case == "abandoned":+ self._abandon(table)+ elif case == "syncing_on":+ ExternalDataSchema.objects.filter(pk=schema.pk).update(should_sync=True)+ elif case == "running":+ self._job(table, ExternalDataJobStatus.RUNNING)+ else:+ schema.table = DataWarehouseTable.objects.create(+ name="amazons3_orders", format="Parquet", team=self.team, external_data_source=schema.source+ )+ schema.save()+ deleted_before = ExternalDataSchema.objects.get(pk=schema.pk).deleted++ outcome = self._abandon(table)++ assert (outcome.action, outcome.reason) == ("refused", reason)+ assert ExternalDataSchema.objects.get(pk=schema.pk).deleted is deleted_before++ def test_abandon_refuses_a_table_that_cant_be_migrated(self) -> None:+ table = self._table(url="https://minio.example.com/exports/a.csv")++ outcome = self._abandon(table)++ assert outcome.action == "refused"+ assert "can't be migrated" in (outcome.reason or "")+ assert not ExternalDataSource.objects.filter(team=self.team).exists()++ def test_a_dry_run_abandon_writes_nothing(self) -> None:+ table = self._table()+ self._migrate()++ outcomes = abandon_tables(+ [str(table.id), "00000000-0000-0000-0000-000000000000"], resolve_region=_no_region, live_run=False+ )++ assert [o.action for o in outcomes] == ["would_abandon", "refused"]+ assert _schema_for(table).deleted is False+ self.delete_schedule.assert_not_called()+++class TestMigrateSelfManagedTablesCommand(BaseTest):+ @parameterized.expand(+ [+ ("zero_limit", {"limit": 0}, "--limit"),+ ("status_and_abandon", {"status": True, "abandon": "x"}, "only one"),+ ("empty_abandon", {"abandon": " , "}, "at least one table id"),+ ]+ )+ def test_rejects_invalid_options(self, _name: str, options: dict[str, Any], message: str) -> None:+ with self.assertRaisesRegex(CommandError, message):+ call_command("migrate_self_managed_tables", **options)++ def test_status_and_dry_run_report_each_table_and_change_nothing(self) -> None:+ DataWarehouseTable.objects.create(+ name="orders",+ format="CSVWithNames",+ url_pattern="https://minio.example.com/exports/a.csv",+ team=self.team,+ credential=DataWarehouseCredential.objects.create(access_key="k", access_secret="s", team=self.team),+ )++ for options in ({"status": True}, {}):+ out = StringIO()+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=out, **options)+ assert "name=orders state=skipped" in out.getvalue()++ assert not ExternalDataSource.objects.filter(team=self.team).exists()+++class TestDeletingAMigrationSchema(APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ patches = [+ patch(f"{MODULE}.source_feature_flag_missing", return_value=False),+ patch.object(source_base, "validate_warehouse_table_url_pattern", return_value=(True, "")),+ patch(f"{SOURCE_VIEWS}.base.bulk_delete_external_data_schedules", return_value=[]),+ patch(f"{SOURCE_VIEWS}.base.delete_external_data_schedule"),+ patch(f"{SOURCE_VIEWS}.base.delete_discover_schemas_schedule"),+ patch(f"{SCHEMA_VIEWS}.external_data_workflow_exists", return_value=False),+ patch("products.data_warehouse.backend.s3.get_s3_client"),+ ]+ for p in patches:+ p.start()+ self.addCleanup(p.stop)+ self.credential = DataWarehouseCredential.objects.create(+ access_key="AKIAFIRST", access_secret="c2VjcmV0", team=self.team+ )++ @parameterized.expand([("source",), ("schema",), ("schema_data",)])+ def test_deleting_before_the_first_sync_keeps_the_table_and_ends_its_migration(self, path: str) -> None:+ table = DataWarehouseTable.objects.create(+ name="orders",+ format=DataWarehouseTableFormat.CSVWithNames,+ url_pattern=EXPORTS_URL,+ team=self.team,+ credential=self.credential,+ )+ migrate_self_managed_tables(MigrationOptions(team_id=self.team.pk, live_run=True), resolve_region=_no_region)+ schema = _schema_for(table)+ urls = {+ "source": f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}",+ "schema": f"/api/environments/{self.team.pk}/external_data_schemas/{schema.pk}",+ "schema_data": f"/api/environments/{self.team.pk}/external_data_schemas/{schema.pk}/delete_data",+ }++ response = self.client.delete(urls[path])++ assert response.status_code in (200, 204), response.content+ table.refresh_from_db()+ assert (table.deleted, table.credential_id, table.url_pattern) == (False, self.credential.pk, EXPORTS_URL)+ [status] = evaluate_tables(team_id=self.team.pk, resolve_region=_no_region)+ expected = TableMigrationState.AWAITING_SYNC if path == "schema_data" else TableMigrationState.ABANDONED+ assert status.state == expected+ assert Database.create_for(team=self.team).has_table("orders")