Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): migrate self-managed tables into object storage sources (#120353)

Tom Owersac63f82b8ddf
diff --git a/products/access_control/backend/facade/api.py b/products/access_control/backend/facade/api.py
index d4e2b2f58dba..4ac98417474e 100644
--- a/products/access_control/backend/facade/api.py
+++ b/products/access_control/backend/facade/api.py
@@ -379,6 +379,28 @@ def object_ids_restricted_from_any_member(
return {resource_id for resource_id in restricted if resource_id}
+def object_ids_with_rules(*, team_id: int, resource: APIScopeObject) -> set[str]:
+ """Ids of `resource` objects that carry any object-level rule, whatever its level.
+
+ For a caller that copies an object into a new one that can't carry the same rules. It skips
+ every object with a rule rather than judging which rules restrict, and it ignores whether
+ the organization enforces rules today, because the rules apply again when it does.
+ """
+ resource_ids = AccessControl.objects.filter(
+ team_id=team_id, resource=resource, resource_id__isnull=False
+ ).values_list("resource_id", flat=True)
+ return {resource_id for resource_id in resource_ids if resource_id}
+
+
+def resource_has_rules(*, team_id: int, resource: APIScopeObject) -> bool:
+ """Whether the team has any resource-level rule for `resource`, whatever its level.
+
+ For a caller about to make objects inherit from `resource`, which changes what those rules
+ reach. Like `object_ids_with_rules`, it ignores whether the organization enforces rules today.
+ """
+ return AccessControl.objects.filter(team_id=team_id, resource=resource, resource_id__isnull=True).exists()
+
+
def role_belongs_to_organization(*, role_id: str | UUID, organization_id: UUID) -> bool:
"""Whether the role is a role of that organization.
diff --git a/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
new file mode 100644
index 000000000000..6792305ef2bf
--- /dev/null
+++ b/products/warehouse_sources/backend/management/commands/migrate_self_managed_tables.py
@@ -0,0 +1,101 @@
+from collections import Counter
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+
+from products.warehouse_sources.backend.self_managed_migration import (
+ DEFAULT_ABANDON_REASON,
+ MigrationOptions,
+ abandon_tables,
+ describe_status,
+ evaluate_tables,
+ migrate_self_managed_tables,
+)
+
+
+def _split_ids(raw: str | None) -> list[str]:
+ seen: dict[str, None] = {}
+ for part in (raw or "").split(","):
+ if part.strip():
+ seen.setdefault(part.strip(), None)
+ return list(seen)
+
+
+class Command(BaseCommand):
+ help = (
+ "Link self-managed warehouse tables to object storage sources. Each table gets a turned-off schema "
+ "in a source for its bucket, and nothing syncs: the table keeps reading its own bucket until the user "
+ "turns syncing on, and the first sync then replaces it with the same id and name. Runs only for teams "
+ "with the object storage sources feature flag. Without --live-run the command only reports what it "
+ "would do."
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, default=None, help="Only act on this team's tables.")
+ parser.add_argument("--table-ids", type=str, default=None, help="Comma-separated warehouse table ids.")
+ parser.add_argument(
+ "--limit", type=int, default=None, help="Link at most N tables in this run, for a cautious first batch."
+ )
+ parser.add_argument(
+ "--status", action="store_true", help="Report each table's migration state and change nothing."
+ )
+ parser.add_argument(
+ "--abandon",
+ type=str,
+ default=None,
+ help="Comma-separated table ids to leave self-managed. Removes the schema the migration made for them.",
+ )
+ parser.add_argument(
+ "--abandon-reason",
+ type=str,
+ default=DEFAULT_ABANDON_REASON,
+ help="The reason recorded for each abandoned table.",
+ )
+ parser.add_argument(
+ "--live-run", action="store_true", help="Act. Without this the command only reports what it would do."
+ )
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ if options["limit"] is not None and options["limit"] < 1:
+ raise CommandError("--limit must be at least 1.")
+ if options["status"] and options["abandon"] is not None:
+ raise CommandError("Pass only one of --status and --abandon.")
+
+ team_id: int | None = options["team_id"]
+ table_ids = _split_ids(options["table_ids"])
+ live_run: bool = options["live_run"]
+
+ if options["status"]:
+ self._report_status(team_id, table_ids)
+ elif options["abandon"] is not None:
+ self._abandon(_split_ids(options["abandon"]), options["abandon_reason"], team_id, live_run)
+ else:
+ self._migrate(team_id, table_ids, live_run, options["limit"])
+
+ def _report_status(self, team_id: int | None, table_ids: list[str]) -> None:
+ statuses = evaluate_tables(team_id=team_id, table_ids=table_ids)
+ for status in statuses:
+ self.stdout.write(describe_status(status))
+ counts = Counter(str(status.state) for status in statuses)
+ self.stdout.write(f"\n{len(statuses)} table(s): " + ", ".join(f"{n} {s}" for s, n in sorted(counts.items())))
+
+ def _abandon(self, table_ids: list[str], reason: str, team_id: int | None, live_run: bool) -> None:
+ if not table_ids:
+ raise CommandError("--abandon needs at least one table id.")
+ for outcome in abandon_tables(table_ids, reason=reason, team_id=team_id, live_run=live_run):
+ detail = f" ({outcome.reason})" if outcome.reason else ""
+ self.stdout.write(f"table={outcome.table_id} {outcome.action}{detail}")
+ if not live_run:
+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to abandon these tables."))
+
+ def _migrate(self, team_id: int | None, table_ids: list[str], live_run: bool, limit: int | None) -> None:
+ outcomes = migrate_self_managed_tables(
+ MigrationOptions(team_id=team_id, table_ids=tuple(table_ids), live_run=live_run, limit=limit)
+ )
+ for outcome in outcomes:
+ error = f" error={outcome.error}" if outcome.error else ""
+ self.stdout.write(f"{describe_status(outcome.status)} action={outcome.action}{error}")
+ actions = Counter(outcome.action for outcome in outcomes)
+ self.stdout.write(f"\n{len(outcomes)} table(s): " + ", ".join(f"{n} {a}" for a, n in sorted(actions.items())))
+ if not live_run:
+ self.stdout.write(self.style.WARNING("\nDry run. Pass --live-run to link these tables."))
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.py
index 78807efa6ae3..238d733f957f 100644
--- a/products/warehouse_sources/backend/models/external_data_schema.py
+++ b/products/warehouse_sources/backend/models/external_data_schema.py
@@ -2347,6 +2347,56 @@ def get_schemas_for_direct_reconciliation(
# offering the new name as a separate one.
SCHEMA_RESOURCE_ID_METADATA_KEY = "source_resource_id"
+# Set to True in a schema's `schema_metadata` when the schema reads a definition PostHog stored on it,
+# not one that discovery reports (a self-managed table moved into an object storage source reads the
+# exact files of its old URL pattern). Discovery can't report such a schema, so it must not disable it.
+SCHEMA_PINNED_METADATA_KEY = "pinned"
+# Set in a schema's `schema_metadata` to the self-managed table that the schema's first sync replaces.
+# The schema does not link to that table before the sync, so the table stays as it is until then.
+SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY = "self_managed_table"
+
+
+@frozen
+class SelfManagedTableRef:
+ """The self-managed table a schema replaces, and the files the schema was pinned to read."""
+
+ table_id: str
+ url_pattern: str
+ format: str
+ # Set when an operator abandoned the migration of the table.
+ abandon_reason: str | None = None
+
+ def to_metadata(self) -> dict[str, Any]:
+ metadata = {"table_id": self.table_id, "url_pattern": self.url_pattern, "format": self.format}
+ if self.abandon_reason is not None:
+ metadata["abandon_reason"] = self.abandon_reason
+ return metadata
+
+
+def _schema_metadata(schema: "ExternalDataSchema") -> dict[str, Any]:
+ metadata = (schema.sync_type_config or {}).get("schema_metadata")
+ return metadata if isinstance(metadata, dict) else {}
+
+
+def is_pinned_schema(schema: "ExternalDataSchema") -> bool:
+ return _schema_metadata(schema).get(SCHEMA_PINNED_METADATA_KEY) is True
+
+
+def self_managed_table_ref(schema: "ExternalDataSchema") -> SelfManagedTableRef | None:
+ ref = _schema_metadata(schema).get(SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY)
+ if not isinstance(ref, dict):
+ return None
+ table_id, url_pattern, table_format = ref.get("table_id"), ref.get("url_pattern"), ref.get("format")
+ if not (isinstance(table_id, str) and isinstance(url_pattern, str) and isinstance(table_format, str)):
+ return None
+ reason = ref.get("abandon_reason")
+ return SelfManagedTableRef(
+ table_id=table_id,
+ url_pattern=url_pattern,
+ format=table_format,
+ abandon_reason=reason if isinstance(reason, str) else None,
+ )
+
def _resource_id(metadata: object) -> str | None:
if isinstance(metadata, dict) and metadata.get(SCHEMA_RESOURCE_ID_METADATA_KEY) is not None:
@@ -2540,10 +2590,16 @@ def sync_old_schemas_with_new_schemas(
for schema in schemas_to_create:
seeded_metadata = (schema_metadata_by_name or {}).get(schema)
- deleted_obj = (
- ExternalDataSchema.objects.filter(team_id=team_id, source_id=source_id, name=schema, deleted=True)
- .order_by("-updated_at", "-created_at")
- .first()
+ deleted_obj = next(
+ (
+ row
+ for row in ExternalDataSchema.objects.filter(
+ team_id=team_id, source_id=source_id, name=schema, deleted=True
+ ).order_by("-updated_at", "-created_at")
+ # A deleted migration schema records that its table stays self-managed, so discovery must not revive it.
+ if SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY not in _schema_metadata(row)
+ ),
+ None,
)
if deleted_obj is not None:
deleted_obj.deleted = False
@@ -2582,6 +2638,8 @@ def sync_old_schemas_with_new_schemas(
team_id=team_id, name=schema, source_id=source_id, deleted=False
)
for s in schemas_to_check:
+ if is_pinned_schema(s):
+ continue
# Only rows nobody enabled disappear entirely. A user-enabled row survives as visibly
# disabled, because soft-deleting it would silently discard the user's selection (e.g.
# a scope-gated table the source stopped offering before its first successful sync).
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 8a551d84a1a7..4f8cd4d20cb8 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -76,6 +76,8 @@ if TYPE_CHECKING:
from posthog.models import User
+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+
SERIALIZED_FIELD_TO_CLICKHOUSE_MAPPING: dict[DatabaseSerializedFieldType, str] = {
DatabaseSerializedFieldType.INTEGER: "Int64",
DatabaseSerializedFieldType.FLOAT: "Float64",
@@ -553,6 +555,20 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
def csv_allow_double_quotes(self) -> bool | None:
return self.options.get("csv_allow_double_quotes")
+ def adopt_into_source(self, source: "ExternalDataSource") -> list[str]:
+ """Make a self-managed table one that `source` syncs. Returns the fields this changed.
+
+ The caller saves these fields together with the new `url_pattern` and `format`: the
+ customer's key can't read PostHog's bucket, and PostHog's key can't read the customer's, so
+ a row that has only some of these changes can't be read at all. The row id and the name stay,
+ so joins, views and saved queries that refer to the table keep working.
+ """
+ self.external_data_source = source
+ self.credential = None
+ # A CSV setting for the customer's files. The table now reads Parquet files PostHog wrote.
+ self.options = {key: value for key, value in (self.options or {}).items() if key != "csv_allow_double_quotes"}
+ return ["external_data_source", "credential", "options"]
+
def soft_delete(self):
from products.data_tools.backend.models.join import DataWarehouseJoin
diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.py
new file mode 100644
index 000000000000..ac27e4bda09a
--- /dev/null
+++ b/products/warehouse_sources/backend/self_managed_migration.py
@@ -0,0 +1,664 @@
+"""Move self-managed warehouse tables into the object storage sources, without a copy.
+
+A self-managed table is read in place from a customer bucket. Each one gets an object storage source
+for its bucket and a turned-off schema that reads exactly the table's files. The schema does not link
+to the table: it names the table in its pinned `schema_metadata` (see `SelfManagedTableRef`), and the
+table stays as it is, read from the customer bucket and not billed. When the user turns syncing on,
+the first sync is a normal billable sync, and its last step points the existing table row at the copy
+in PostHog storage (see `find_replaced_self_managed_table` and `DataWarehouseTable.adopt_into_source`),
+so the row id and the name stay and existing queries keep working.
+
+The state of each table comes from the rows themselves: a migration source carries a marker in its
+`connection_metadata`, and its schemas name their tables. Running the migration again therefore
+continues where it stopped.
+"""
+
+import re
+import uuid
+import dataclasses
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Mapping
+from enum import StrEnum
+from typing import Any, Literal
+
+from django.db import transaction
+from django.db.models import Q
+
+import boto3
+import structlog
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import BotoCoreError, ClientError
+
+from posthog.dataclasses import frozen
+from posthog.models.team import Team
+
+from products.access_control.backend.facade.api import object_ids_with_rules, resource_has_rules
+from products.data_warehouse.backend.facade.api import delete_external_data_schedule
+from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
+from products.warehouse_sources.backend.models.external_data_destination import (
+ ExternalDataSourceDestination,
+ get_or_create_warehouse_destination,
+)
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import (
+ SCHEMA_PINNED_METADATA_KEY,
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
+ ExternalDataSchema,
+ SelfManagedTableRef,
+ self_managed_table_ref,
+)
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
+ source_feature_flag_missing,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
+ TableDefinition,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+ StorageService,
+ UnmappableTableError,
+ parse_self_managed_url,
+ table_definition_for,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
+ CONNECT_TIMEOUT_SECONDS,
+ MAX_REQUEST_ATTEMPTS,
+ READ_TIMEOUT_SECONDS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
+ ObjectStorageSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.types import ExternalDataJobStatus
+
+logger = structlog.get_logger(__name__)
+
+# The `connection_metadata` key of a source the migration created.
+MIGRATION_MARKER_KEY = "self_managed_migration"
+DEFAULT_ABANDON_REASON = "Left self-managed by an operator."
+_DELETED_REASON = "its migration schema or source was deleted"
+
+# Resolves the region of an S3 bucket whose URL names none, or returns None when it can't.
+BucketRegionResolver = Callable[[str, SelfManagedCredential], str | None]
+
+
+class TableMigrationState(StrEnum):
+ # Not linked to a source yet.
+ PENDING = "pending"
+ # Has a migration schema, and the table still reads its own bucket until a sync replaces it.
+ AWAITING_SYNC = "awaiting_sync"
+ MIGRATED = "migrated"
+ ABANDONED = "abandoned"
+ SKIPPED = "skipped"
+
+
+MigrationAction = Literal["none", "would_link", "linked", "link_failed", "limit_reached"]
+AbandonAction = Literal["would_abandon", "abandoned", "refused"]
+
+
+@frozen
+class TableMigrationStatus:
+ table_id: str
+ team_id: int
+ name: str
+ state: TableMigrationState
+ reason: str | None = None
+ source_id: str | None = None
+ schema_id: str | None = None
+
+
+@frozen
+class TableMigrationOutcome:
+ status: TableMigrationStatus
+ action: MigrationAction = "none"
+ error: str | None = None
+
+
+@frozen
+class AbandonOutcome:
+ table_id: str
+ action: AbandonAction
+ reason: str | None = None
+
+
+@frozen
+class MigrationOptions:
+ team_id: int | None = None
+ table_ids: tuple[str, ...] = ()
+ live_run: bool = False
+ # How many tables one run may link. None links every one it can.
+ limit: int | None = None
+
+
+@frozen
+class _TablePlan:
+ """How one self-managed table maps onto an object storage source."""
+
+ location: SelfManagedLocation
+ source: ObjectStorageSource[Any]
+ definition: TableDefinition
+ job_inputs: dict[str, Any] = dataclasses.field(repr=False)
+
+
+@frozen
+class _Evaluation:
+ table: DataWarehouseTable
+ status: TableMigrationStatus
+ plan: _TablePlan | None = None
+ # The live migration schema of the table.
+ schema: ExternalDataSchema | None = None
+ job: ExternalDataJob | None = None
+
+
+def resolve_s3_bucket_region(bucket: str, credential: SelfManagedCredential) -> str | None:
+ """Ask S3 for the bucket's region with the table's own key.
+
+ S3 names the region in the `x-amz-bucket-region` header of a HEAD request, also when it
+ answers with a redirect or a 403, so a key that may only read objects still finds it.
+ """
+ if not credential.access_key.strip() or not credential.access_secret.strip():
+ return None
+ client = boto3.client(
+ "s3",
+ region_name="us-east-1",
+ aws_access_key_id=credential.access_key.strip(),
+ aws_secret_access_key=credential.access_secret.strip(),
+ config=BotoConfig(
+ connect_timeout=CONNECT_TIMEOUT_SECONDS,
+ read_timeout=READ_TIMEOUT_SECONDS,
+ retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": "standard"},
+ ),
+ )
+ response: Mapping[str, Any]
+ try:
+ response = client.head_bucket(Bucket=bucket)
+ except ClientError as e:
+ response = e.response
+ except BotoCoreError:
+ return None
+ headers = response.get("ResponseMetadata", {}).get("HTTPHeaders", {})
+ region = headers.get("x-amz-bucket-region")
+ return region if isinstance(region, str) and region else None
+
+
+def _object_storage_sources() -> dict[StorageService, ObjectStorageSource[Any]]:
+ return {
+ source.self_managed_service: source
+ for source in SourceRegistry.get_all_sources().values()
+ if isinstance(source, ObjectStorageSource)
+ }
+
+
+def _credential_of(table: DataWarehouseTable) -> SelfManagedCredential | None:
+ credential = table.credential
+ if credential is None:
+ return None
+ return SelfManagedCredential(access_key=credential.access_key or "", access_secret=credential.access_secret or "")
+
+
+class _Planner:
+ """Maps tables onto sources, caching the per-team flag check and the per-bucket region lookup."""
+
+ def __init__(self, resolve_region: BucketRegionResolver) -> None:
+ self._resolve_region = resolve_region
+ self._sources = _object_storage_sources()
+ self._flag_missing: dict[tuple[int, StorageService], bool] = {}
+ self._regions: dict[tuple[str, str], str | None] = {}
+ self._tables_with_rules: dict[int, set[str]] = {}
+ self._teams_with_source_rules: dict[int, bool] = {}
+
+ def plan(self, table: DataWarehouseTable) -> _TablePlan | str:
+ """The plan for a table, or the reason the table can't be migrated."""
+ credential = _credential_of(table)
+ if credential is None:
+ return "the table has no credential"
+ if self._has_access_rules(table):
+ # The new source would hold the table's key with default access, and a member the
+ # table's rules shut out could read the same files through a schema of that source.
+ return "the table has its own access rules, which the source PostHog would create can't carry"
+ if self._has_source_rules(table):
+ # A table in a source falls back to the rules for sources, which never reached a
+ # self-managed table, so moving it could take access away from members who have it.
+ # Rules for the whole warehouse don't reach the new source, so a member they shut out
+ # could read the table's files through it.
+ return "the project has access rules for sources or for the warehouse, which the move would change"
+ try:
+ location = parse_self_managed_url(table.url_pattern)
+ source = self._sources.get(location.service)
+ if source is None:
+ return f"no import source reads '{location.service}' storage"
+ if self._team_lacks_flag(table.team, source):
+ return f"the team doesn't have the '{source.requires_team_feature_flag}' feature flag"
+ if location.service == "s3" and location.region is None:
+ region = self._region(table, location, credential)
+ if region is None:
+ return "the URL doesn't name the bucket's region, and S3 didn't return it"
+ location = dataclasses.replace(location, region=region)
+ definition = table_definition_for(
+ table.name, location, table.format, table.columns, table.csv_allow_double_quotes
+ )
+ job_inputs = source.job_inputs_for_self_managed(location, credential)
+ except UnmappableTableError as e:
+ return str(e)
+ return _TablePlan(location=location, source=source, definition=definition, job_inputs=job_inputs)
+
+ def _has_access_rules(self, table: DataWarehouseTable) -> bool:
+ if table.team_id not in self._tables_with_rules:
+ self._tables_with_rules[table.team_id] = object_ids_with_rules(
+ team_id=table.team_id, resource="warehouse_table"
+ )
+ return str(table.id) in self._tables_with_rules[table.team_id]
+
+ def _has_source_rules(self, table: DataWarehouseTable) -> bool:
+ if table.team_id not in self._teams_with_source_rules:
+ self._teams_with_source_rules[table.team_id] = resource_has_rules(
+ team_id=table.team_id, resource="external_data_source"
+ ) or resource_has_rules(team_id=table.team_id, resource="warehouse_objects")
+ return self._teams_with_source_rules[table.team_id]
+
+ def _team_lacks_flag(self, team: Team, source: ObjectStorageSource[Any]) -> bool:
+ key = (team.pk, source.self_managed_service)
+ if key not in self._flag_missing:
+ self._flag_missing[key] = source_feature_flag_missing(source, team)
+ return self._flag_missing[key]
+
+ def _region(
+ self, table: DataWarehouseTable, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> str | None:
+ key = (str(table.credential_id), location.bucket)
+ if key not in self._regions:
+ try:
+ self._regions[key] = self._resolve_region(location.bucket, credential)
+ except Exception:
+ logger.warning("Could not resolve the S3 bucket region", table_id=str(table.id), exc_info=True)
+ self._regions[key] = None
+ return self._regions[key]
+
+
+def _migration_schemas(team_id: int | None, table_ids: Iterable[str]) -> dict[str, list[ExternalDataSchema]]:
+ """Every schema, deleted or not, that the migration made for a table, by table id, oldest first."""
+ queryset = (
+ ExternalDataSchema.objects.filter(
+ source__connection_metadata__has_key=MIGRATION_MARKER_KEY,
+ sync_type_config__schema_metadata__has_key=SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
+ )
+ .select_related("source")
+ .order_by("created_at", "id")
+ )
+ if team_id is not None:
+ queryset = queryset.filter(team_id=team_id)
+ ids = list(table_ids)
+ if ids:
+ queryset = queryset.filter(sync_type_config__schema_metadata__self_managed_table__table_id__in=ids)
+ by_table: dict[str, list[ExternalDataSchema]] = defaultdict(list)
+ for schema in queryset:
+ ref = self_managed_table_ref(schema)
+ if ref is not None:
+ by_table[ref.table_id].append(schema)
+ return by_table
+
+
+def _is_live(schema: ExternalDataSchema) -> bool:
+ return not schema.deleted and not schema.source.deleted
+
+
+def _select_tables(
+ team_id: int | None, table_ids: Iterable[str], migration_schemas: Mapping[str, list[ExternalDataSchema]]
+) -> list[DataWarehouseTable]:
+ """Self-managed tables, plus every table the migration made a schema for, oldest first."""
+ queryset = (
+ DataWarehouseTable.raw_objects.exclude(deleted=True)
+ .filter(Q(external_data_source__isnull=True, credential__isnull=False) | Q(id__in=list(migration_schemas)))
+ .select_related("credential", "team")
+ .order_by("created_at", "id")
+ )
+ if team_id is not None:
+ queryset = queryset.filter(team_id=team_id)
+ ids = list(table_ids)
+ if ids:
+ queryset = queryset.filter(id__in=ids)
+ return list(queryset)
+
+
+def _latest_jobs(schemas: Iterable[ExternalDataSchema]) -> dict[str, ExternalDataJob]:
+ schema_ids = [schema.id for schema in schemas]
+ if not schema_ids:
+ return {}
+ jobs = ExternalDataJob.objects.filter(schema_id__in=schema_ids).order_by("schema_id", "-created_at")
+ latest: dict[str, ExternalDataJob] = {}
+ for job in jobs:
+ latest.setdefault(str(job.schema_id), job)
+ return latest
+
+
+def _awaiting_reason(table: DataWarehouseTable, schema: ExternalDataSchema, job: ExternalDataJob | None) -> str:
+ ref = self_managed_table_ref(schema)
+ if schema.table_id is not None:
+ return "the table changed after the migration, so its schema syncs a separate table"
+ if ref is None or (table.url_pattern, table.format) != (ref.url_pattern, ref.format):
+ return "the table changed after the migration, so a sync makes a separate table"
+ if job is None:
+ return "syncing is on; waiting for the first sync" if schema.should_sync else "syncing is off"
+ if job.status == ExternalDataJobStatus.RUNNING:
+ return "the first sync is running"
+ detail = job.latest_error or f"the last sync ended with status {job.status}"
+ return f"{detail}; the table still reads its own bucket"
+
+
+def _evaluate(
+ table: DataWarehouseTable,
+ schemas: list[ExternalDataSchema],
+ jobs: Mapping[str, ExternalDataJob],
+ planner: _Planner,
+) -> _Evaluation:
+ live = next((schema for schema in schemas if _is_live(schema)), None)
+ job = jobs.get(str(live.id)) if live is not None else None
+
+ def status(state: TableMigrationState, reason: str | None = None) -> TableMigrationStatus:
+ named = live or (schemas[-1] if schemas else None)
+ return TableMigrationStatus(
+ table_id=str(table.id),
+ team_id=table.team_id,
+ name=table.name,
+ state=state,
+ reason=reason,
+ source_id=str(named.source_id) if named is not None else None,
+ schema_id=str(named.id) if named is not None else None,
+ )
+
+ if table.external_data_source_id is not None:
+ return _Evaluation(table=table, status=status(TableMigrationState.MIGRATED), schema=live, job=job)
+ if live is not None:
+ reason = _awaiting_reason(table, live, job)
+ return _Evaluation(table=table, status=status(TableMigrationState.AWAITING_SYNC, reason), schema=live, job=job)
+ if schemas:
+ refs = [ref for ref in (self_managed_table_ref(schema) for schema in schemas) if ref is not None]
+ reason = next((ref.abandon_reason for ref in reversed(refs) if ref.abandon_reason), _DELETED_REASON)
+ return _Evaluation(table=table, status=status(TableMigrationState.ABANDONED, reason))
+
+ planned = planner.plan(table)
+ if isinstance(planned, _TablePlan):
+ return _Evaluation(table=table, status=status(TableMigrationState.PENDING, "not linked yet"), plan=planned)
+ return _Evaluation(table=table, status=status(TableMigrationState.SKIPPED, planned))
+
+
+def evaluate_tables(
+ team_id: int | None = None,
+ table_ids: Iterable[str] = (),
+ resolve_region: BucketRegionResolver = resolve_s3_bucket_region,
+) -> list[TableMigrationStatus]:
+ return [evaluation.status for evaluation in _evaluate_all(team_id, table_ids, _Planner(resolve_region))]
+
+
+def _evaluate_all(team_id: int | None, table_ids: Iterable[str], planner: _Planner) -> list[_Evaluation]:
+ ids = list(table_ids)
+ schemas = _migration_schemas(team_id, ids)
+ tables = _select_tables(team_id, ids, schemas)
+ jobs = _latest_jobs(schema for table_schemas in schemas.values() for schema in table_schemas if _is_live(schema))
+ return [_evaluate(table, schemas.get(str(table.id), []), jobs, planner) for table in tables]
+
+
+def _free_prefix(team_id: int, plan: _TablePlan) -> str | None:
+ """A table prefix no other source of the type uses, so the new source's table names can't clash.
+
+ Like a source a user creates, the first source of a type needs none.
+ """
+ sources = ExternalDataSource.objects.filter(team_id=team_id, source_type=plan.source.source_type).exclude(
+ deleted=True
+ )
+ taken = {(prefix or "").strip() for prefix in sources.values_list("prefix", flat=True)}
+ if "" not in taken:
+ return None
+ base = re.sub(r"[^A-Za-z0-9_]", "_", plan.location.bucket).strip("_").lower() or "bucket"
+ if not base[0].isalpha():
+ base = f"bucket_{base}"
+ candidate, suffix = base, 2
+ while candidate in taken or ExternalDataSource.is_system_managed_prefix(candidate):
+ candidate, suffix = f"{base}_{suffix}", suffix + 1
+ return candidate
+
+
+def _find_or_create_source(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSource:
+ """The group's source for the table's team, credential and bucket. Runs under the credential lock."""
+ existing = (
+ ExternalDataSource.objects.filter(
+ team_id=table.team_id,
+ source_type=plan.source.source_type,
+ connection_metadata__self_managed_migration__group_key=plan.location.group_key,
+ connection_metadata__self_managed_migration__credential_id=str(table.credential_id),
+ )
+ .exclude(deleted=True)
+ .order_by("created_at")
+ .first()
+ )
+ if existing is not None:
+ return existing
+ source = ExternalDataSource.objects.create(
+ prefix=_free_prefix(table.team_id, plan),
+ source_id=str(uuid.uuid4()),
+ connection_id=str(uuid.uuid4()),
+ destination_id=str(uuid.uuid4()),
+ team_id=table.team_id,
+ status=ExternalDataSource.Status.RUNNING,
+ source_type=plan.source.source_type,
+ api_version=plan.source.default_version,
+ job_inputs=plan.job_inputs,
+ description=f"Self-managed tables in {plan.location.bucket}",
+ access_method=ExternalDataSource.AccessMethod.WAREHOUSE,
+ connection_metadata={
+ MIGRATION_MARKER_KEY: {
+ "group_key": plan.location.group_key,
+ "credential_id": str(table.credential_id),
+ }
+ },
+ )
+ ExternalDataSourceDestination.objects.for_team(table.team_id).create(
+ team_id=table.team_id, source_id=source.pk, destination=get_or_create_warehouse_destination(table.team_id)
+ )
+ return source
+
+
+def _credential_id(table: DataWarehouseTable) -> uuid.UUID:
+ if table.credential_id is None:
+ raise ValueError(f"Table {table.id} has no credential to migrate with")
+ return table.credential_id
+
+
+def _ensure_schema(table: DataWarehouseTable, plan: _TablePlan) -> ExternalDataSchema:
+ """The table's migration schema, created with its source when there is none yet.
+
+ The credential row is the group lock: every table of one source shares it, and nothing else
+ locks it, so two runs can't create two sources or two schemas for the same table.
+ """
+ with transaction.atomic():
+ DataWarehouseCredential.objects.select_for_update().get(team_id=table.team_id, pk=_credential_id(table))
+ schemas = _migration_schemas(table.team_id, [str(table.id)]).get(str(table.id), [])
+ live = next((schema for schema in schemas if _is_live(schema)), None)
+ if live is not None:
+ return live
+ if schemas:
+ raise UnmappableTableError("the table's migration was abandoned")
+ source = _find_or_create_source(table, plan)
+ if (
+ ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk, name=table.name)
+ .exclude(deleted=True)
+ .exists()
+ ):
+ raise UnmappableTableError(f"another table in this bucket already uses the name '{table.name}'")
+ ref = SelfManagedTableRef(table_id=str(table.id), url_pattern=table.url_pattern, format=table.format)
+ return ExternalDataSchema.objects.create(
+ name=table.name,
+ team_id=table.team_id,
+ source=source,
+ should_sync=False,
+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,
+ sync_type_config={
+ "schema_metadata": {
+ **plan.definition.schema_metadata(),
+ SCHEMA_PINNED_METADATA_KEY: True,
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref.to_metadata(),
+ }
+ },
+ )
+
+
+def migrate_self_managed_tables(
+ options: MigrationOptions, resolve_region: BucketRegionResolver = resolve_s3_bucket_region
+) -> list[TableMigrationOutcome]:
+ """Plan, and on a live run create, the source and the turned-off schema of every selected table.
+
+ Nothing syncs and no table changes. A dry run reads the same rows and reports the same
+ decisions, but writes nothing.
+ """
+ planner = _Planner(resolve_region)
+ outcomes: list[TableMigrationOutcome] = []
+ linked = 0
+ for evaluation in _evaluate_all(options.team_id, options.table_ids, planner):
+ if evaluation.plan is None or evaluation.status.state != TableMigrationState.PENDING:
+ outcomes.append(TableMigrationOutcome(status=evaluation.status))
+ continue
+ if options.limit is not None and linked >= options.limit:
+ outcomes.append(TableMigrationOutcome(status=evaluation.status, action="limit_reached"))
+ continue
+ linked += 1
+ if not options.live_run:
+ outcomes.append(TableMigrationOutcome(status=evaluation.status, action="would_link"))
+ continue
+ outcomes.append(_link(evaluation.table, evaluation.plan, evaluation.status))
+ return outcomes
+
+
+def _link(table: DataWarehouseTable, plan: _TablePlan, status: TableMigrationStatus) -> TableMigrationOutcome:
+ try:
+ schema = _ensure_schema(table, plan)
+ except UnmappableTableError as e:
+ return TableMigrationOutcome(status=status, action="link_failed", error=str(e))
+ except Exception as e:
+ # A database error on one table must not stop the batch or hide its outcomes.
+ logger.warning("Could not link a self-managed table", table_id=str(table.id), exc_info=True)
+ return TableMigrationOutcome(status=status, action="link_failed", error=f"{type(e).__name__}: {e}")
+ logger.info(
+ "Linked a self-managed table to an object storage source",
+ table_id=str(table.id),
+ team_id=table.team_id,
+ schema_id=str(schema.id),
+ )
+ return TableMigrationOutcome(
+ status=dataclasses.replace(
+ status,
+ state=TableMigrationState.AWAITING_SYNC,
+ reason="syncing is off",
+ schema_id=str(schema.id),
+ source_id=str(schema.source_id),
+ ),
+ action="linked",
+ )
+
+
+def _abandon_refusal(evaluation: _Evaluation) -> str | None:
+ state = evaluation.status.state
+ if state == TableMigrationState.MIGRATED:
+ return "the table is already migrated"
+ if state == TableMigrationState.ABANDONED:
+ return "the table's migration is already abandoned"
+ if state == TableMigrationState.SKIPPED:
+ return f"the table can't be migrated, so there is nothing to abandon ({evaluation.status.reason})"
+ schema = evaluation.schema
+ if schema is None:
+ return None
+ if schema.table_id is not None:
+ return "the table's schema syncs a separate table"
+ if schema.should_sync:
+ return "the user turned syncing on for the table's schema"
+ if evaluation.job is not None and evaluation.job.status == ExternalDataJobStatus.RUNNING:
+ return "the table's first sync is running"
+ return None
+
+
+def abandon_tables(
+ table_ids: Iterable[str],
+ reason: str = DEFAULT_ABANDON_REASON,
+ team_id: int | None = None,
+ live_run: bool = False,
+ resolve_region: BucketRegionResolver = resolve_s3_bucket_region,
+) -> list[AbandonOutcome]:
+ """Leave tables self-managed for good, and remove the schema the migration made for them.
+
+ The schema stays as a deleted row that names the table and the reason, so later runs leave the
+ table alone. A table with no schema yet gets one first, for that record. A table's source goes
+ too when the table was the last one it held. The table itself never changes.
+ """
+ ids = list(table_ids)
+ evaluations = {
+ evaluation.status.table_id: evaluation for evaluation in _evaluate_all(team_id, ids, _Planner(resolve_region))
+ }
+ outcomes: list[AbandonOutcome] = []
+ for table_id in ids:
+ evaluation = evaluations.get(table_id)
+ if evaluation is None:
+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason="no such self-managed table"))
+ continue
+ refusal = _abandon_refusal(evaluation)
+ if refusal is not None:
+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason=refusal))
+ elif not live_run:
+ outcomes.append(AbandonOutcome(table_id=table_id, action="would_abandon"))
+ else:
+ try:
+ _abandon(evaluation, reason)
+ except UnmappableTableError as e:
+ outcomes.append(AbandonOutcome(table_id=table_id, action="refused", reason=str(e)))
+ continue
+ outcomes.append(AbandonOutcome(table_id=table_id, action="abandoned", reason=reason))
+ return outcomes
+
+
+def _abandon(evaluation: _Evaluation, reason: str) -> None:
+ table = evaluation.table
+ removed_source_id: str | None = None
+ with transaction.atomic():
+ if evaluation.schema is not None:
+ schema = evaluation.schema
+ else:
+ assert evaluation.plan is not None
+ schema = _ensure_schema(table, evaluation.plan)
+ ref = self_managed_table_ref(schema)
+ assert ref is not None
+ metadata = (schema.sync_type_config or {})["schema_metadata"]
+ metadata[SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY] = dataclasses.replace(ref, abandon_reason=reason).to_metadata()
+ schema.save(update_fields=["sync_type_config", "updated_at"])
+ schema.soft_delete()
+ source = schema.source
+ if (
+ not ExternalDataSchema.objects.filter(team_id=table.team_id, source_id=source.pk)
+ .exclude(deleted=True)
+ .exists()
+ ):
+ source.soft_delete()
+ removed_source_id = str(source.pk)
+ if evaluation.schema is not None:
+ # A schedule exists only when the user turned syncing on before, so it is paused now.
+ try:
+ delete_external_data_schedule(str(schema.id))
+ except Exception:
+ # The schema is deleted, so its schedule can't start a sync.
+ logger.warning(
+ "Could not delete the schedule of an abandoned migration schema",
+ table_id=str(table.id),
+ schema_id=str(schema.id),
+ exc_info=True,
+ )
+ logger.info(
+ "Abandoned a self-managed table migration",
+ table_id=str(table.id),
+ team_id=table.team_id,
+ removed_source_id=removed_source_id,
+ )
+
+
+def describe_status(status: TableMigrationStatus) -> str:
+ reason = f" ({status.reason})" if status.reason else ""
+ return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
index 62d7c0bcf713..2c71e846636d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
@@ -7,7 +7,7 @@ from datetime import date, datetime
from typing import Any, Optional
from django.db import transaction
-from django.db.models import Prefetch
+from django.db.models import Prefetch, Q
from django.utils import timezone
import dlt
@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.models.external_data_job import External
from products.warehouse_sources.backend.models.external_data_schema import (
ExternalDataSchema,
mark_initial_sync_complete,
+ self_managed_table_ref,
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
@@ -322,6 +323,48 @@ def find_orphan_table(schema: ExternalDataSchema, source: ExternalDataSource) ->
).first()
+def find_replaced_self_managed_table(
+ schema: ExternalDataSchema, source: ExternalDataSource
+) -> DataWarehouseTable | None:
+ """The self-managed table the schema's first sync replaces, when it still reads the files the schema copies.
+
+ A table the user edited after the migration reads other files now, so the sync must not replace it,
+ and gets a table of its own. A table this source already took over but that no schema owns is
+ the result of a publish that stopped before the link, so the next publish finishes it.
+ """
+ ref = self_managed_table_ref(schema)
+ if ref is None:
+ return None
+ return (
+ DataWarehouseTable.objects.filter(team_id=schema.team_id, id=ref.table_id, deleted=False)
+ .filter(externaldataschema__isnull=True)
+ .filter(
+ Q(external_data_source__isnull=True, url_pattern=ref.url_pattern, format=ref.format)
+ | Q(external_data_source_id=source.id)
+ )
+ .first()
+ )
+
+
+def find_adopted_self_managed_table(
+ schema: ExternalDataSchema, source: ExternalDataSource
+) -> DataWarehouseTable | None:
+ """The self-managed table an earlier publish of the schema already moved into the source, but didn't link.
+
+ The table keeps its own name, so the name lookup in `find_orphan_table` can't find it.
+ """
+ ref = self_managed_table_ref(schema)
+ if ref is None:
+ return None
+ return DataWarehouseTable.objects.filter(
+ team_id=schema.team_id,
+ id=ref.table_id,
+ external_data_source_id=source.id,
+ deleted=False,
+ externaldataschema__isnull=True,
+ ).first()
+
+
def relink_orphan_table(
schema: ExternalDataSchema, source: ExternalDataSource, logger: FilteringBoundLogger
) -> DataWarehouseTable | None:
@@ -334,7 +377,7 @@ def relink_orphan_table(
if schema.table_id is not None or (schema.is_cdc and schema.cdc_table_mode == "cdc_only"):
return schema.table
- orphan = find_orphan_table(schema, source)
+ orphan = find_orphan_table(schema, source) or find_adopted_self_managed_table(schema, source)
if orphan is None:
return None
@@ -436,6 +479,9 @@ async def validate_schema_and_update_table(
if table_created is not None:
logger.debug(f"Found existing table {table_created.id} - reusing it for schema {_schema_id}")
+ if table_created is None:
+ table_created = find_replaced_self_managed_table(external_data_schema, job.pipeline)
+
# A reported row_count of 0 does not always mean the run wrote nothing: the v3 consumer
# can read 0 on a redelivered final batch, and a resumed run counts only its own attempt.
# A publish step with nothing to make queryable is what an empty first sync looks like.
@@ -466,6 +512,10 @@ async def validate_schema_and_update_table(
# new_url_pattern above is derived from the job's own destination folder, not from
# request input, so this sync is a trusted writer of a credential-less table's URL.
update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]
+ if table.external_data_source_id is None:
+ # A self-managed table this schema replaces reads its own bucket until now.
+ update_fields.extend(table.adopt_into_source(job.pipeline))
+ logger.info(f"Adopted self-managed table {table.id} into source {job.pipeline.id}")
if live_size_mib is not None:
table.size_in_s3_mib = live_size_mib
update_fields.append("size_in_s3_mib")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index c6d8db6c601c..a336b90553e0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -1050,7 +1050,8 @@ class PipelineV3(Generic[ResumableData]):
async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:
pa_table = _append_debug_column_to_pyarrows_table(pa_table, self._load_id)
- pa_table = normalize_table_column_names(pa_table)
+ if not self._resource.preserve_column_names:
+ pa_table = normalize_table_column_names(pa_table)
if self._uses_delta_write_column_selection:
pa_table = await observe_and_project_table(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 4fbee007e583..fe5537f59fd2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1320,6 +1320,37 @@ class TestSingleTableRunIsUntouched:
assert set(base) - set(laned) == set()
+class TestColumnNamesReachTheWrite:
+ @pytest.mark.parametrize(
+ "preserve, expected",
+ [
+ (False, ["order_id", "total_amount", "_ph_debug"]),
+ (True, ["Order ID", "totalAmount", "_ph_debug"]),
+ ],
+ )
+ async def test_preserving_names_skips_the_snake_case_rename(self, preserve: bool, expected: list[str]) -> None:
+ # A self-managed table moved into a source keeps the column names its queries use.
+ pipeline = _make_pipeline()
+ pipeline._resource = SourceResponse(
+ name="orders", items=lambda: [], primary_keys=None, preserve_column_names=preserve
+ )
+ cast(MagicMock, pipeline._schema).configure_mock(incremental_field=None, enabled_columns=None)
+ pipeline._last_incremental_field_value = None
+ pipeline._earliest_incremental_field_value = None
+ table = pa.table({"Order ID": pa.array([1], pa.int64()), "totalAmount": pa.array([2.5], pa.float64())})
+
+ with (
+ patch.object(PipelineV3, "_stage_batch", AsyncMock(return_value=1)),
+ patch.object(PipelineV3, "_advance_handoff_checkpoint", AsyncMock()),
+ patch(f"{_PIPELINE}.update_incremental_field_values", AsyncMock(return_value=MagicMock())),
+ patch(f"{_PIPELINE}.update_row_tracking_after_batch", AsyncMock()),
+ ):
+ await pipeline._process_batch(pa_table=table, batch_index=0, row_count=1)
+
+ staged = cast(AsyncMock, pipeline._sinks.stage_chunk).call_args.args[1]
+ assert staged.column_names == expected
+
+
class TestFinalizeStagesTheWatermarkFirst:
@pytest.mark.asyncio
async def test_desc_watermark_is_staged_before_the_final_batch_notification(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
index 4078ef2e0565..64ade129ac45 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
@@ -14,8 +14,13 @@ from parameterized import parameterized
from posthog.models import Team
+from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_schema import (
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
+ ExternalDataSchema,
+ SelfManagedTableRef,
+)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
@@ -28,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
_refresh_cumulative_row_count,
merge_columns,
register_cdc_companion_table,
+ relink_orphan_table,
update_last_synced_at,
validate_schema_and_update_table,
)
@@ -44,6 +50,7 @@ from products.warehouse_sources.backend.types import (
_PIPELINE_SYNC_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync"
_DB_RETRY_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry"
_TABLE_MODULE = "products.warehouse_sources.backend.models.table"
+_SELF_MANAGED_URL = "https://exports.s3.us-east-1.amazonaws.com/orders/*.csv"
class TestResolveTableAndFolderNames:
@@ -597,6 +604,178 @@ class TestValidateSchemaAndUpdateTable:
assert schema.table.size_in_s3_mib == expected_size_mib
assert job.storage_delta_mib == expected_delta_mib
+ def _self_managed_table(self, team: Team, schema: ExternalDataSchema, **fields: object) -> DataWarehouseTable:
+ credential = DataWarehouseCredential.objects.create(access_key="AKIAEXAMPLE", access_secret="secret", team=team)
+ table = DataWarehouseTable.objects.create(
+ name="Raw_Orders",
+ format=DataWarehouseTableFormat.CSVWithNames,
+ url_pattern=_SELF_MANAGED_URL,
+ team=team,
+ credential=credential,
+ options={"csv_allow_double_quotes": False, "kept": 1},
+ columns={"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField"}},
+ )
+ ref = SelfManagedTableRef(table_id=str(table.id), url_pattern=table.url_pattern, format=table.format)
+ schema.sync_type_config = {"schema_metadata": {SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref.to_metadata()}}
+ schema.save()
+ if fields:
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(**fields)
+ table.refresh_from_db()
+ return table
+
+ def _publish(self, team: Team, schema: ExternalDataSchema, job: ExternalDataJob, row_count: int = 10) -> None:
+ async_to_sync(validate_schema_and_update_table)(
+ run_id=str(job.id),
+ team_id=team.pk,
+ schema_id=schema.id,
+ row_count=row_count,
+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,
+ queryable_folder="raw_orders__query_a",
+ live_size_mib=3.0,
+ )
+
+ @pytest.mark.parametrize("row_count", [10, 0])
+ def test_the_first_publish_adopts_the_self_managed_table_the_schema_names(self, team: Team, row_count: int) -> None:
+ schema, job = self._schema_and_job(team)
+ table = self._self_managed_table(team, schema)
+ credential_seen_by_introspection: list[object] = []
+
+ def get_columns(table_self: DataWarehouseTable, safe_expose_ch_error: bool = True) -> dict:
+ credential_seen_by_introspection.append(table_self.credential_id)
+ return {"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField"}}
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", get_columns),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ self._publish(team, schema, job, row_count=row_count)
+
+ table.refresh_from_db()
+ schema.refresh_from_db()
+ assert DataWarehouseTable.objects.filter(team=team, deleted=False).count() == 1
+ assert schema.table_id == table.id
+ assert (table.name, table.external_data_source_id, table.credential_id) == ("Raw_Orders", job.pipeline_id, None)
+ assert (table.format, table.queryable_folder, table.size_in_s3_mib) == (
+ DataWarehouseTableFormat.DeltaS3Wrapper,
+ "raw_orders__query_a",
+ 3.0,
+ )
+ names = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)
+ assert table.url_pattern == job.url_pattern_by_schema(names.folder_name)
+ assert table.options == {"kept": 1}
+ # The customer's key can't read PostHog's bucket, so introspection must see the adopted row.
+ assert credential_seen_by_introspection == [None]
+ job.refresh_from_db()
+ assert job.storage_delta_mib == 3.0
+
+ @pytest.mark.parametrize(
+ "change",
+ [
+ {"url_pattern": "https://exports.s3.us-east-1.amazonaws.com/other/*.csv"},
+ {"format": DataWarehouseTableFormat.Parquet},
+ {"deleted": True},
+ ],
+ )
+ def test_a_self_managed_table_that_changed_is_not_replaced(self, team: Team, change: dict) -> None:
+ schema, job = self._schema_and_job(team)
+ table = self._self_managed_table(team, schema, **change)
+ before = (table.url_pattern, table.format, table.credential_id, table.deleted)
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ self._publish(team, schema, job)
+
+ table.refresh_from_db()
+ schema.refresh_from_db()
+ assert (table.url_pattern, table.format, table.credential_id, table.deleted) == before
+ assert table.external_data_source_id is None
+ assert schema.table is not None and schema.table_id != table.id
+ assert schema.table.name == build_table_name(job.pipeline, "orders")
+
+ def test_a_table_of_another_team_is_not_replaced(self, team: Team) -> None:
+ schema, job = self._schema_and_job(team)
+ other_team = Team.objects.create(organization=team.organization, name="Other team")
+ table = self._self_managed_table(team, schema, team_id=other_team.pk)
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ self._publish(team, schema, job)
+
+ table.refresh_from_db()
+ schema.refresh_from_db()
+ assert table.external_data_source_id is None
+ assert schema.table_id is not None and schema.table_id != table.id
+
+ def test_an_adopted_table_left_unlinked_is_linked_by_the_next_publish(self, team: Team) -> None:
+ schema, job = self._schema_and_job(team)
+ table = self._self_managed_table(team, schema)
+ no_files_yet = ServerException("No files in log segment", code=742)
+
+ with patch.object(DataWarehouseTable, "get_columns", side_effect=no_files_yet):
+ self._publish(team, schema, job)
+ schema.refresh_from_db()
+ table.refresh_from_db()
+ assert (schema.table_id, table.external_data_source_id) == (None, job.pipeline_id)
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ self._publish(team, schema, job)
+
+ schema.refresh_from_db()
+ assert schema.table_id == table.id
+ assert DataWarehouseTable.objects.filter(team=team, deleted=False).count() == 1
+
+ def test_an_adopted_table_left_unlinked_is_relinked_before_the_next_extraction(self, team: Team) -> None:
+ schema, job = self._schema_and_job(team)
+ table = self._self_managed_table(team, schema)
+ no_files_yet = ServerException("No files in log segment", code=742)
+ with patch.object(DataWarehouseTable, "get_columns", side_effect=no_files_yet):
+ self._publish(team, schema, job)
+ schema.refresh_from_db()
+ assert schema.table_id is None
+
+ relinked = relink_orphan_table(schema, job.pipeline, MagicMock())
+
+ schema.refresh_from_db()
+ assert relinked is not None and relinked.id == table.id
+ assert schema.table_id == table.id
+
+ def test_a_self_managed_table_not_adopted_yet_is_not_relinked(self, team: Team) -> None:
+ schema, job = self._schema_and_job(team)
+ self._self_managed_table(team, schema)
+
+ assert relink_orphan_table(schema, job.pipeline, MagicMock()) is None
+
+ schema.refresh_from_db()
+ assert schema.table_id is None
+
+ def test_a_synced_table_keeps_its_options(self, team: Team) -> None:
+ schema, job = self._schema_and_job(team)
+ table = self._linked_table(team, schema, job, queryable_folder="orders__query_a")
+ DataWarehouseTable.objects.filter(id=table.id).update(options={"csv_allow_double_quotes": True})
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=150),
+ ):
+ async_to_sync(validate_schema_and_update_table)(
+ run_id=str(job.id),
+ team_id=team.pk,
+ schema_id=schema.id,
+ row_count=10,
+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,
+ queryable_folder="orders__query_b",
+ )
+
+ table.refresh_from_db()
+ assert table.options == {"csv_allow_double_quotes": True}
+
def test_a_schema_linked_to_its_cdc_companion_gets_its_own_table(self, team: Team) -> None:
schema, job = self._schema_and_job(team)
companion = self._linked_table(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
index a5ef6fb5112b..5ae8d3a1512e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -26,6 +26,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
S3_EXAMPLE_URL,
parse_s3_url,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+ UnmappableTableError,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
)
@@ -48,6 +53,7 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
api_docs_url = "https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html"
requires_team_feature_flag = FEATURE_FLAG
+ self_managed_service = "s3"
@property
def source_type(self) -> ExternalDataSourceType:
@@ -69,6 +75,17 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
def discovery_options(self, config: AmazonS3SourceConfig) -> DiscoveryOptions:
return discovery_options_from(config)
+ def self_managed_connection_inputs(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ if location.region is None:
+ raise UnmappableTableError("the URL doesn't name the bucket's region")
+ return {
+ "url_pattern": f"https://s3.{location.region}.amazonaws.com/{location.bucket}/",
+ "access_key_id": credential.access_key,
+ "secret_access_key": credential.access_secret,
+ }
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
index 81a35048a36c..c2c8c8fddee5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -24,6 +24,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
AZURE_EXAMPLE_URL,
parse_azure_url,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+ UnmappableTableError,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
)
@@ -46,6 +51,7 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):
api_docs_url = "https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api"
requires_team_feature_flag = FEATURE_FLAG
+ self_managed_service = "azure"
@property
def source_type(self) -> ExternalDataSourceType:
@@ -62,6 +68,18 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):
def discovery_options(self, config: AzureBlobSourceConfig) -> DiscoveryOptions:
return discovery_options_from(config)
+ def self_managed_connection_inputs(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ # ClickHouse reads Azure with the storage account name as the access key. A key for another
+ # account than the URL's never read these files, so it isn't moved to this one.
+ if credential.access_key.strip().lower() != location.account:
+ raise UnmappableTableError("the access key isn't the storage account name in the URL")
+ return {
+ "url_pattern": f"https://{location.account}.blob.core.windows.net/{location.bucket}/",
+ "account_key": credential.access_secret,
+ }
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
index 3c9f2cf9c353..5606d5c57452 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
R2_EXAMPLE_URL,
parse_r2_url,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
)
@@ -43,6 +47,7 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):
api_docs_url = "https://developers.cloudflare.com/r2/api/s3/api/"
requires_team_feature_flag = FEATURE_FLAG
+ self_managed_service = "r2"
@property
def source_type(self) -> ExternalDataSourceType:
@@ -64,6 +69,15 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):
def discovery_options(self, config: CloudflareR2SourceConfig) -> DiscoveryOptions:
return discovery_options_from(config)
+ def self_managed_connection_inputs(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ return {
+ "url_pattern": f"https://{location.account or ''}.r2.cloudflarestorage.com/{location.bucket}/",
+ "access_key_id": credential.access_key,
+ "secret_access_key": credential.access_secret,
+ }
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
index 27ccd852b797..a1beb2d2fd3e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
@@ -5,6 +5,7 @@ from typing import Literal, get_args
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import CsvColumnType
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
INVALID_CONFIG_ERROR,
ObjectStorageError,
@@ -220,6 +221,12 @@ class CsvOptions:
# Mirrors ClickHouse's `format_csv_allow_double_quotes`: off means a double quote is an ordinary
# character and never encloses a field.
allow_double_quotes: bool = True
+ # Columns cast from text to a typed column. Every other column stays a string.
+ column_types: tuple[CsvColumnType, ...] = ()
+ # Read an unquoted `\N` as null, as ClickHouse does. Only tables moved from a self-managed table set it.
+ null_marker: bool = False
+ # The field delimiter for every file. None picks it from the file name: tabs for `.tsv`, else commas.
+ delimiter: str | None = None
@frozen
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py
new file mode 100644
index 000000000000..2a58328db1ff
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py
@@ -0,0 +1,352 @@
+import re
+from collections.abc import Callable, Mapping
+from typing import Any, Literal, cast, get_args
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from posthog.dataclasses import frozen
+
+# The types a CSV column can be cast to. Anything else stays a string column.
+CsvColumnKind = Literal["integer", "unsigned", "float", "decimal", "boolean", "date", "datetime"]
+CSV_COLUMN_KINDS: frozenset[str] = frozenset(get_args(CsvColumnKind))
+
+_WRAPPER = re.compile(r"^(?:Nullable|LowCardinality)\((?P<inner>.*)\)$")
+_TIMEZONE_ARGUMENT = re.compile(r"'(?P<timezone>[^']+)'")
+_DECIMAL_ARGUMENTS = re.compile(r"^\s*(?P<first>\d+)\s*(?:,\s*(?P<second>\d+)\s*)?$")
+_CLICKHOUSE_KINDS: dict[str, CsvColumnKind] = {
+ "Int8": "integer",
+ "Int16": "integer",
+ "Int32": "integer",
+ "Int64": "integer",
+ "UInt8": "integer",
+ "UInt16": "integer",
+ "UInt32": "integer",
+ "UInt64": "unsigned",
+ "Float32": "float",
+ "Float64": "float",
+ "Bool": "boolean",
+ "Date": "date",
+ "Date32": "date",
+ "DateTime": "datetime",
+ "DateTime64": "datetime",
+}
+# `Decimal32(S)` and its siblings fix the precision and take only the scale.
+_DECIMAL_PRECISIONS: dict[str, int | None] = {
+ "Decimal": None,
+ "Decimal32": 9,
+ "Decimal64": 18,
+ "Decimal128": 38,
+ "Decimal256": 76,
+}
+# Arrow's 128-bit decimal holds 38 digits, which is also the most a Delta table can store.
+_MAX_DECIMAL_PRECISION = 38
+# Numbers no Arrow integer or 38-digit decimal holds, so a CSV column of them can't keep its type.
+_WIDE_INTEGERS = frozenset({"Int128", "Int256", "UInt128", "UInt256"})
+# A Delta table has no unsigned integers, and an Int64 can't hold the top half of UInt64, so a
+# UInt64 is stored as a 20-digit decimal, which holds every UInt64 and stays a number.
+_UNSIGNED_STORAGE = pa.decimal128(20, 0)
+_FIXED_ARROW_TYPES: dict[CsvColumnKind, pa.DataType] = {
+ "integer": pa.int64(),
+ "unsigned": _UNSIGNED_STORAGE,
+ "float": pa.float64(),
+ "boolean": pa.bool_(),
+ "date": pa.date32(),
+ "datetime": pa.timestamp("us", tz="UTC"),
+}
+# The spellings ClickHouse accepts for a Bool in text formats.
+_TRUE_VALUES = ["true", "1", "yes", "y", "on", "t", "enable"]
+_FALSE_VALUES = ["false", "0", "no", "n", "off", "f", "disable"]
+_UTC = "UTC"
+# A failed cast is split in halves until a slice is this small, and only then cast value by value.
+_PER_VALUE_SLICE = 32
+_CAST_ERRORS = (pa.ArrowInvalid, pa.ArrowNotImplementedError, OverflowError)
+
+
+class UnsupportedCsvColumnError(ValueError):
+ """A stored column type that a CSV import can't keep. The message names the column."""
+
+
+@frozen
+class CsvColumnType:
+ name: str
+ kind: CsvColumnKind
+ precision: int | None = None
+ scale: int | None = None
+ # The zone ClickHouse read naive values of a `datetime` column in. None reads them as UTC.
+ timezone: str | None = None
+
+ @property
+ def arrow_type(self) -> pa.DataType:
+ if self.kind == "decimal":
+ return pa.decimal128(self.precision or _MAX_DECIMAL_PRECISION, self.scale or 0)
+ return _FIXED_ARROW_TYPES[self.kind]
+
+ def to_metadata(self) -> str | dict[str, Any]:
+ if self.kind == "decimal":
+ return {"kind": self.kind, "precision": self.precision, "scale": self.scale}
+ if self.kind == "datetime" and self.timezone is not None:
+ return {"kind": self.kind, "timezone": self.timezone}
+ return self.kind
+
+
+@frozen
+class CsvCastResult:
+ table: pa.Table
+ # Per column, how many non-empty values could not be read as the column's type and became null.
+ nulled_values: dict[str, int]
+
+
+@frozen
+class _ClickHouseType:
+ base: str
+ arguments: str
+
+
+def _unwrap(clickhouse_type: str) -> _ClickHouseType:
+ """The base type name and its arguments, without `Nullable(...)` or `LowCardinality(...)`."""
+ inner = clickhouse_type.strip()
+ while (wrapped := _WRAPPER.match(inner)) is not None:
+ inner = wrapped.group("inner").strip()
+ base, _, arguments = inner.partition("(")
+ return _ClickHouseType(base=base.strip(), arguments=arguments.removesuffix(")"))
+
+
+def clickhouse_base_type(clickhouse_type: str) -> str:
+ return _unwrap(clickhouse_type).base
+
+
+def _datetime_timezone(name: str, arguments: str) -> str | None:
+ match = _TIMEZONE_ARGUMENT.search(arguments)
+ if match is None:
+ return None
+ timezone = match.group("timezone")
+ try:
+ ZoneInfo(timezone)
+ except (ZoneInfoNotFoundError, ValueError) as e:
+ raise UnsupportedCsvColumnError(f"the column '{name}' has a time zone '{timezone}' PostHog can't read") from e
+ return None if timezone == _UTC else timezone
+
+
+def _decimal_type(name: str, base: str, arguments: str) -> CsvColumnType:
+ match = _DECIMAL_ARGUMENTS.match(arguments)
+ if match is None:
+ raise UnsupportedCsvColumnError(f"the column '{name}' has a decimal type PostHog can't read")
+ fixed_precision = _DECIMAL_PRECISIONS[base]
+ if fixed_precision is None:
+ precision, scale = int(match.group("first")), int(match.group("second") or 0)
+ else:
+ precision, scale = fixed_precision, int(match.group("first"))
+ if precision > _MAX_DECIMAL_PRECISION:
+ raise UnsupportedCsvColumnError(
+ f"the column '{name}' is a decimal with more than {_MAX_DECIMAL_PRECISION} digits"
+ )
+ return CsvColumnType(name=name, kind="decimal", precision=precision, scale=scale)
+
+
+def csv_column_type(name: str, clickhouse_type: str) -> CsvColumnType | None:
+ """The cast for a stored ClickHouse type, or None when the column stays a string.
+
+ Raises `UnsupportedCsvColumnError` for a number that neither a cast nor a string column keeps.
+ """
+ parsed = _unwrap(clickhouse_type)
+ base, arguments = parsed.base, parsed.arguments
+ if base in _WIDE_INTEGERS:
+ raise UnsupportedCsvColumnError(f"the column '{name}' is a {base}, which PostHog can't store")
+ if base in _DECIMAL_PRECISIONS:
+ return _decimal_type(name, base, arguments)
+ kind = _CLICKHOUSE_KINDS.get(base)
+ if kind is None:
+ return None
+ if kind == "datetime":
+ return CsvColumnType(name=name, kind=kind, timezone=_datetime_timezone(name, arguments))
+ return CsvColumnType(name=name, kind=kind)
+
+
+def stored_clickhouse_types(columns: Mapping[str, Any] | None) -> dict[str, str]:
+ """The ClickHouse type of each stored column, in stored order.
+
+ A column ClickHouse marked as invalid never read as its type, so it is left out.
+ """
+ types: dict[str, str] = {}
+ for name, definition in (columns or {}).items():
+ if isinstance(definition, str):
+ types[name] = definition
+ elif isinstance(definition, Mapping) and definition.get("valid") is not False:
+ types[name] = str(definition.get("clickhouse") or "")
+ return types
+
+
+def csv_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:
+ """The casts for a warehouse table's stored columns, in their stored order."""
+ column_types: list[CsvColumnType] = []
+ for name, clickhouse_type in stored_clickhouse_types(columns).items():
+ column_type = csv_column_type(name, clickhouse_type)
+ if column_type is not None:
+ column_types.append(column_type)
+ return tuple(column_types)
+
+
+def _column_type_from_metadata(name: str, raw: object) -> CsvColumnType | None:
+ if isinstance(raw, str):
+ return CsvColumnType(name=name, kind=cast(CsvColumnKind, raw)) if raw in CSV_COLUMN_KINDS else None
+ if not isinstance(raw, Mapping) or raw.get("kind") not in CSV_COLUMN_KINDS:
+ return None
+ precision, scale, timezone = raw.get("precision"), raw.get("scale"), raw.get("timezone")
+ return CsvColumnType(
+ name=name,
+ kind=cast(CsvColumnKind, raw["kind"]),
+ precision=precision if isinstance(precision, int) else None,
+ scale=scale if isinstance(scale, int) else None,
+ timezone=timezone if isinstance(timezone, str) else None,
+ )
+
+
+def csv_column_types_from_metadata(raw: object) -> tuple[CsvColumnType, ...]:
+ if not isinstance(raw, Mapping):
+ return ()
+ column_types = (_column_type_from_metadata(name, value) for name, value in raw.items() if isinstance(name, str))
+ return tuple(column_type for column_type in column_types if column_type is not None)
+
+
+def _cast_each(values: pa.Array, parse: Callable[[pa.Array], pa.Array], target: pa.DataType) -> pa.Array:
+ parsed: list[Any] = []
+ for value in values.to_pylist():
+ if value is None:
+ parsed.append(None)
+ continue
+ try:
+ parsed.append(parse(pa.array([value], pa.string()))[0].as_py())
+ except _CAST_ERRORS:
+ parsed.append(None)
+ return pa.array(parsed, target)
+
+
+def _cast_isolating_failures(values: pa.Array, parse: Callable[[pa.Array], pa.Array], target: pa.DataType) -> pa.Array:
+ """Cast a slice that failed as a whole by halves, so only small slices are cast value by value.
+
+ The cost then grows with the number of bad values times the log of the slice length, not with
+ the length of every slice that holds one bad value.
+ """
+ if len(values) <= _PER_VALUE_SLICE:
+ return _cast_each(values, parse, target)
+ middle = len(values) // 2
+ halves: list[pa.Array] = []
+ for half in (values.slice(0, middle), values.slice(middle)):
+ try:
+ halves.append(parse(half))
+ except _CAST_ERRORS:
+ halves.append(_cast_isolating_failures(half, parse, target))
+ return pa.concat_arrays(halves)
+
+
+def _without_plus_sign(values: pa.Array) -> pa.Array:
+ # Arrow refuses an explicit plus sign, which ClickHouse reads as part of the number.
+ return cast(pa.Array, pc.replace_substring_regex(values, pattern=r"^\+", replacement=""))
+
+
+def _parse_number(target: pa.DataType) -> Callable[[pa.Array], pa.Array]:
+ return lambda values: pc.cast(_without_plus_sign(values), target)
+
+
+def _parse_decimal(target: pa.DataType, scale: int) -> Callable[[pa.Array], pa.Array]:
+ # ClickHouse drops the fractional digits a decimal's scale can't hold, rather than refusing the value.
+ excess_digits = rf"^([+-]?\d*\.\d{{{scale}}})\d+$" if scale else r"^([+-]?\d+)\.\d*$"
+
+ def parse(values: pa.Array) -> pa.Array:
+ truncated = cast(
+ pa.Array, pc.replace_substring_regex(_without_plus_sign(values), pattern=excess_digits, replacement=r"\1")
+ )
+ return pc.cast(truncated, target)
+
+ return parse
+
+
+def _parse_unsigned(values: pa.Array) -> pa.Array:
+ # Read as UInt64 first, so a negative or too large value fails like it does in ClickHouse.
+ return pc.cast(pc.cast(_without_plus_sign(values), pa.uint64()), _UNSIGNED_STORAGE)
+
+
+def _parse_boolean(values: pa.Array) -> pa.Array:
+ lowered = pc.utf8_lower(values)
+ is_true = pc.is_in(lowered, value_set=pa.array(_TRUE_VALUES))
+ is_false = pc.is_in(lowered, value_set=pa.array(_FALSE_VALUES))
+ if not pc.all(pc.or_(pc.or_(is_true, is_false), pc.is_null(values))).as_py():
+ raise pa.ArrowInvalid("not a boolean")
+ return pc.if_else(pc.is_null(values), pa.scalar(None, pa.bool_()), is_true)
+
+
+def _parse_timestamp(values: pa.Array, tz: str | None) -> pa.Array:
+ """Parse text into a microsecond timestamp, truncating finer fractions of a second.
+
+ A `DateTime64(9)` column, the type ClickHouse infers for a timestamp, can hold nanoseconds,
+ which Arrow refuses to read as microseconds. Those values are read as nanoseconds first and
+ then truncated to the microseconds a Delta table stores.
+ """
+ micros = pa.timestamp("us") if tz is None else pa.timestamp("us", tz=tz)
+ nanos = pa.timestamp("ns") if tz is None else pa.timestamp("ns", tz=tz)
+ try:
+ return pc.cast(values, micros)
+ except pa.ArrowInvalid:
+ return pc.cast(pc.cast(values, nanos), micros, safe=False)
+
+
+def _parse_datetime(timezone: str | None) -> Callable[[pa.Array], pa.Array]:
+ target = _FIXED_ARROW_TYPES["datetime"]
+
+ def parse(values: pa.Array) -> pa.Array:
+ # ClickHouse reads a value without an offset in the column's time zone, or in UTC.
+ try:
+ naive = _parse_timestamp(values, None)
+ except pa.ArrowInvalid:
+ return _parse_timestamp(values, _UTC)
+ if timezone is None:
+ return pc.cast(naive, target)
+ # A local time that a clock change skips or repeats takes the earlier instant.
+ localized = pc.assume_timezone(
+ cast(Any, naive), timezone=timezone, ambiguous="earliest", nonexistent="earliest"
+ )
+ return pc.cast(localized, target)
+
+ return parse
+
+
+def _parser(column_type: CsvColumnType) -> Callable[[pa.Array], pa.Array]:
+ if column_type.kind == "boolean":
+ return _parse_boolean
+ if column_type.kind == "datetime":
+ return _parse_datetime(column_type.timezone)
+ if column_type.kind == "unsigned":
+ return _parse_unsigned
+ if column_type.kind == "decimal":
+ return _parse_decimal(column_type.arrow_type, column_type.scale or 0)
+ return _parse_number(column_type.arrow_type)
+
+
+def cast_csv_columns(table: pa.Table, column_types: tuple[CsvColumnType, ...]) -> CsvCastResult:
+ """Cast the string columns of a CSV batch to the stored types.
+
+ A value that can't be read as its column's type becomes null rather than failing the file,
+ and the caller reports how many values that happened to. The batch is cast as a whole first,
+ so only the slices that hold such a value pay for a slower pass.
+ """
+ nulled: dict[str, int] = {}
+ for column_type in column_types:
+ if column_type.name not in table.column_names:
+ continue
+ index = table.schema.get_field_index(column_type.name)
+ values = pc.utf8_trim_whitespace(table.column(index).combine_chunks())
+ # A quoted empty field is an empty string, which no typed column can hold.
+ values = pc.if_else(pc.equal(values, pa.scalar("", pa.string())), pa.scalar(None, pa.string()), values)
+ target = column_type.arrow_type
+ parse = _parser(column_type)
+ try:
+ cast_values = parse(values)
+ except _CAST_ERRORS:
+ cast_values = _cast_isolating_failures(values, parse, target)
+ failed = cast_values.null_count - values.null_count
+ if failed:
+ nulled[column_type.name] = failed
+ table = table.set_column(index, pa.field(column_type.name, target), cast_values)
+ return CsvCastResult(table=table, nulled_values=nulled)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
index 5c0c2e103659..1d0c9bcb2b19 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
@@ -14,6 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
FileFormat,
FileFormatChoice,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ csv_column_types_from_metadata,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
NO_TABLE_FILES_ERROR,
TOO_MANY_FILES_ERROR,
@@ -26,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
ClickHouseGlob,
compile_clickhouse_glob,
+ glob_key_prefix,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
MAX_TABLE_OBJECTS,
@@ -33,8 +37,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
)
TableKind = Literal["files", "delta"]
-# Which files a `files` table reads: one file, a folder and its hive partitions, or every file.
-FileScope = Literal["file", "folder", "all"]
+# Which files a `files` table reads: one file, a folder and its hive partitions, every file, or the
+# files a ClickHouse-style glob in `pattern` matches. Discovery never makes a `glob` table: it keeps
+# the file set of a table that ClickHouse read from a URL pattern before it was imported.
+FileScope = Literal["file", "folder", "all", "glob"]
+_FILE_SCOPES: frozenset[str] = frozenset(get_args(FileScope))
DELTA_LOG_DIRECTORY = "_delta_log"
COMPRESSED_SUFFIX = ".gz"
@@ -71,6 +78,8 @@ class TableDefinition:
# The source's configured format at discovery, replayed at sync so the same files group together.
source_format: FileFormatChoice
files: tuple[StoredObject, ...] = ()
+ # Keep the column names as the files write them, so queries written against them keep working.
+ preserve_column_names: bool = False
def same_location(self, other: "TableDefinition") -> bool:
return (
@@ -81,15 +90,25 @@ class TableDefinition:
)
def schema_metadata(self) -> dict[str, Any]:
- return {
+ csv: dict[str, Any] = {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes}
+ if self.csv.column_types:
+ csv["column_types"] = {column.name: column.to_metadata() for column in self.csv.column_types}
+ if self.csv.null_marker:
+ csv["null_marker"] = True
+ if self.csv.delimiter is not None:
+ csv["delimiter"] = self.csv.delimiter
+ metadata: dict[str, Any] = {
"kind": self.kind,
"scope": self.scope,
"path": self.path,
"pattern": self.pattern,
"format": self.file_format,
"source_format": self.source_format,
- "csv": {"has_header": self.csv.has_header, "allow_double_quotes": self.csv.allow_double_quotes},
+ "csv": csv,
}
+ if self.preserve_column_names:
+ metadata["preserve_column_names"] = True
+ return metadata
@classmethod
def from_schema_metadata(cls, name: str, metadata: Mapping[str, Any] | None) -> "TableDefinition | None":
@@ -103,9 +122,11 @@ class TableDefinition:
source_format = metadata.get("source_format", file_format)
pattern = metadata.get("pattern")
csv = metadata.get("csv") or {}
+ delimiter = csv.get("delimiter") if isinstance(csv, dict) else None
if (
kind not in ("files", "delta")
- or (kind == "files" and scope not in ("file", "folder", "all"))
+ or (kind == "files" and scope not in _FILE_SCOPES)
+ or (kind == "files" and scope == "glob" and not isinstance(pattern, str))
or not isinstance(path, str)
or file_format not in _FILE_FORMAT_CHOICES
or source_format not in _FILE_FORMAT_CHOICES
@@ -122,9 +143,13 @@ class TableDefinition:
csv=CsvOptions(
has_header=bool(csv.get("has_header", True)),
allow_double_quotes=bool(csv.get("allow_double_quotes", True)),
+ column_types=csv_column_types_from_metadata(csv.get("column_types")),
+ null_marker=csv.get("null_marker") is True,
+ delimiter=delimiter if isinstance(delimiter, str) and len(delimiter) == 1 else None,
),
pattern=pattern,
source_format=cast(FileFormatChoice, source_format),
+ preserve_column_names=metadata.get("preserve_column_names") is True,
)
@@ -360,12 +385,19 @@ def resolve_table_files(
"""
if table.kind != "files":
return table
- objects = store.list_objects(table.path, max_objects + 1)
+ if table.scope == "glob":
+ listing_prefix = glob_key_prefix(table.pattern or "")
+ objects = store.list_key_prefix(listing_prefix, max_objects + 1)
+ else:
+ listing_prefix = table.path
+ objects = store.list_objects(listing_prefix, max_objects + 1)
if len(objects) > max_objects:
raise ObjectStorageError(
- f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{table.path or '/'}'. Point the URL at a "
+ f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{listing_prefix or '/'}'. Point the URL at a "
"narrower folder, or use a pattern that matches fewer files."
)
+ if table.scope == "glob":
+ return _resolve_glob_files(objects, table)
options = DiscoveryOptions(
file_format=table.source_format,
allow_double_quotes=table.csv.allow_double_quotes,
@@ -387,3 +419,18 @@ def resolve_table_files(
f"{NO_TABLE_FILES_ERROR} '{table.name}'. The files it was discovered from may have been renamed, moved, "
"or removed from the bucket."
)
+
+
+def _resolve_glob_files(objects: list[StoredObject], table: TableDefinition) -> TableDefinition:
+ """Every listed file the table's glob matches, with no other filter.
+
+ Hidden files, Delta folders and file extensions are not checked, because the files ClickHouse
+ read for the same pattern were not filtered either.
+ """
+ glob = compile_clickhouse_glob(table.pattern or "")
+ files = tuple(stored for stored in objects if glob.matches(stored.path))
+ if not files:
+ raise ObjectStorageError(
+ f"{NO_TABLE_FILES_ERROR} '{table.name}'. No file in the bucket matches '{table.pattern}'."
+ )
+ return dataclasses.replace(table, files=files)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
index 50720c7b6cda..b21457ad007c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/filesystem.py
@@ -84,6 +84,24 @@ class ObjectStore:
break
return objects
+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:
+ """List files whose relative paths start with the key prefix, in path order."""
+ folder = key_prefix.rpartition("/")[0]
+ found = cast(dict[str, dict[str, Any]], self.fs.find(self.path(folder), detail=True))
+ root_prefix = f"{self.root}/"
+ objects: list[StoredObject] = []
+ for name in sorted(found):
+ info = found[name]
+ if info.get("type") != "file" or not name.startswith(root_prefix):
+ continue
+ relative = name[len(root_prefix) :]
+ if not relative.startswith(key_prefix):
+ continue
+ objects.append(StoredObject(path=relative, size=int(info.get("size") or 0), modified_at=_modified_at(info)))
+ if len(objects) >= max_objects:
+ break
+ return objects
+
def open(self, relative: str) -> IO[bytes]:
return cast(IO[bytes], self.fs.open(self.path(relative), "rb"))
@@ -157,6 +175,12 @@ class S3ObjectStore(ObjectStore):
objects = [stored for stored in self._list(key, 1) if stored.path == relative_prefix]
return objects
+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:
+ key = "/".join(part for part in (self.prefix, key_prefix) if part)
+ if self.prefix and not key_prefix:
+ key = f"{self.prefix}/"
+ return self._list(key, max_objects)
+
def _list(self, key_prefix: str, max_objects: int) -> list[StoredObject]:
root_key = f"{self.prefix}/" if self.prefix else ""
paginator = self.client.get_paginator("list_objects_v2")
@@ -236,6 +260,12 @@ class AzureObjectStore(ObjectStore):
objects = [stored for stored in self._list(name, 1) if stored.path == relative_prefix]
return objects
+ def list_key_prefix(self, key_prefix: str, max_objects: int) -> list[StoredObject]:
+ key = "/".join(part for part in (self.prefix, key_prefix) if part)
+ if self.prefix and not key_prefix:
+ key = f"{self.prefix}/"
+ return self._list(key, max_objects)
+
def _list(self, name_prefix: str, max_objects: int) -> list[StoredObject]:
root_name = f"{self.prefix}/" if self.prefix else ""
objects: list[StoredObject] = []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
index ffca2e509438..782e4ff6c079 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -21,6 +21,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
CsvOptions,
FileFormat,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ cast_csv_columns,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressed
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
FILE_PARSE_ERROR,
@@ -72,6 +75,10 @@ class _PrefixedStream(io.RawIOBase):
return len(data)
+# How ClickHouse writes a null in a CSV file, and reads one back when it is unquoted.
+CLICKHOUSE_CSV_NULL = "\\N"
+
+
def csv_column_names(cells: list[str], has_header: bool) -> list[str]:
"""Header names as written, made unique. A headerless file gets ClickHouse's `c1`, `c2`, ... names."""
if not has_header:
@@ -147,11 +154,11 @@ def iter_csv_tables(
file_path: str = "",
logger: FilteringBoundLogger | None = None,
) -> Iterator[pa.Table]:
- """Stream a CSV file as string columns.
+ """Stream a CSV file as string columns, except the columns `options.column_types` casts.
- Every column is read as text: Arrow fixes a column's type from the first block it sees, so a
- later value that does not fit would fail the whole file mid-stream, and two files of one table
- could disagree on a type. Empty unquoted fields are null.
+ Every other column is read as text: Arrow fixes a column's type from the first block it sees,
+ so a later value that does not fit would fail the whole file mid-stream, and two files of one
+ table could disagree on a type. Empty unquoted fields are null.
"""
first_record, first_fields = _read_first_csv_record(stream, delimiter, options.allow_double_quotes, file_path)
if not first_record:
@@ -160,12 +167,21 @@ def iter_csv_tables(
body: IO[bytes] = stream if options.has_header else cast(IO[bytes], _PrefixedStream(first_record, stream))
skipped_rows = 0
+ nulled_values: dict[str, int] = {}
def skip_invalid_row(_row: pa_csv.InvalidRow) -> Literal["skip", "error"]:
nonlocal skipped_rows
skipped_rows += 1
return "skip"
+ def typed(table: pa.Table) -> pa.Table:
+ if not options.column_types:
+ return table
+ result = cast_csv_columns(table, options.column_types)
+ for column, count in result.nulled_values.items():
+ nulled_values[column] = nulled_values.get(column, 0) + count
+ return result.table
+
yielded = False
try:
reader = pa_csv.open_csv(
@@ -180,7 +196,7 @@ def iter_csv_tables(
),
convert_options=pa_csv.ConvertOptions(
column_types=dict.fromkeys(names, pa.string()),
- null_values=[""],
+ null_values=["", CLICKHOUSE_CSV_NULL] if options.null_marker else [""],
strings_can_be_null=True,
quoted_strings_can_be_null=False,
),
@@ -188,14 +204,23 @@ def iter_csv_tables(
for batch in reader:
if batch.num_rows:
yielded = True
- yield pa.Table.from_batches([batch])
+ yield typed(pa.Table.from_batches([batch]))
except pa.ArrowInvalid as e:
if options.has_header and not yielded and "Empty CSV file" in str(e):
# The file holds only its header. It is a table with no rows, not a broken file.
- yield pa.table({name: pa.array([], pa.string()) for name in names})
+ yield typed(pa.table({name: pa.array([], pa.string()) for name in names}))
return
raise ObjectStorageError(f"{FILE_PARSE_ERROR} '{file_path}' as CSV: {e}") from e
+ if nulled_values and logger is not None:
+ # The stored column types come from a sample of the files, so a later value can fail to
+ # read as its type. The value becomes null so the rest of the file still imports.
+ logger.warning(
+ "Read CSV values that don't match their column type as null",
+ file=file_path,
+ nulled_values=nulled_values,
+ )
+
if skipped_rows and logger is not None:
# A row whose field count differs from the header is malformed, or the file uses another
# delimiter. Skipping it keeps its values out of the wrong columns.
@@ -328,7 +353,7 @@ def iter_file_tables(
else:
binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else stream
if file_format == "csv":
- delimiter = "\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ","
+ delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")
tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)
else:
tables = iter_jsonl_tables(binary, stored.path)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
new file mode 100644
index 000000000000..6058611df09d
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
@@ -0,0 +1,269 @@
+"""Map a self-managed warehouse table onto the object storage sources that import the same files.
+
+A self-managed table is a URL pattern that ClickHouse reads in place with the table's own access
+key. Every mapping here is exact or refused: a table this module can't map is skipped with a reason,
+never imported from files that differ from the ones ClickHouse read.
+"""
+
+import re
+import dataclasses
+from collections.abc import Iterable, Mapping
+from typing import Any, Literal
+from urllib.parse import unquote, urlparse
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import normalize_column_name
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import (
+ FILE_MODIFIED_AT_COLUMN,
+ FILE_PATH_COLUMN,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ CsvOptions,
+ FileFormatChoice,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ UnsupportedCsvColumnError,
+ clickhouse_base_type,
+ csv_column_types_from_columns,
+ stored_clickhouse_types,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
+ TableDefinition,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ compile_clickhouse_glob,
+ glob_listing_prefix,
+ has_glob,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.incremental import (
+ FILE_ROW_NUMBER_COLUMN,
+)
+
+StorageService = Literal["s3", "gcs", "r2", "azure"]
+
+_S3_HOST = re.compile(r"(?:(?P<bucket>.+)\.)?s3(?:[.-](?P<endpoint>[a-z0-9.-]+?))?\.amazonaws\.com(?:\.cn)?")
+_S3_ENDPOINT = re.compile(r"(?:(?:dualstack|fips)\.)?(?P<region>[a-z]{2}(?:-[a-z0-9]+)+-\d+)")
+# The legacy name of the us-east-1 endpoint.
+_S3_EXTERNAL_ENDPOINT = "external-1"
+_S3_EXTERNAL_REGION = "us-east-1"
+_GCS_HOST = "storage.googleapis.com"
+_R2_HOST = re.compile(r"(?P<account>[0-9a-f]{32})\.r2\.cloudflarestorage\.com")
+_AZURE_HOST = re.compile(r"(?P<account>[a-z0-9]{3,24})\.blob\.core\.windows\.net")
+
+# ClickHouse formats a self-managed table can have, and the source format that reads the same files.
+_FORMATS: dict[str, FileFormatChoice] = {
+ "CSV": "csv_no_header",
+ "CSVWithNames": "csv",
+ "Parquet": "parquet",
+ "JSONEachRow": "jsonl",
+ "Delta": "delta",
+ "DeltaS3Wrapper": "delta",
+}
+# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts.
+_NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})
+# Types ClickHouse reads from JSON strings, which the JSON import keeps as strings.
+_JSON_TEXT_TYPES = frozenset({"Date", "Date32", "DateTime", "DateTime64"})
+# Columns the import adds to every row. A stored column of the same name would lose its values to them.
+_FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})
+
+
+class UnmappableTableError(Exception):
+ """The table can't be imported from exactly the files ClickHouse reads. The message says why."""
+
+
+@frozen
+class SelfManagedLocation:
+ """Where a self-managed table's files are, split into the parts an object storage source takes."""
+
+ service: StorageService
+ # The bucket, or the container for Azure.
+ bucket: str
+ # The decoded object key after the bucket. It can hold ClickHouse glob syntax.
+ key: str
+ # The S3 region, or None when the URL names no region.
+ region: str | None = None
+ # The Cloudflare account ID for R2, or the storage account name for Azure.
+ account: str | None = None
+
+ @property
+ def group_key(self) -> str:
+ return "/".join((self.service, self.account or self.region or "", self.bucket))
+
+
+@frozen
+class SelfManagedCredential:
+ """A self-managed table's key pair. For Azure the access key is the storage account name."""
+
+ access_key: str = dataclasses.field(repr=False)
+ access_secret: str = dataclasses.field(repr=False)
+
+
+def _s3_location(host: str, path: str) -> SelfManagedLocation:
+ match = _S3_HOST.fullmatch(host)
+ if match is None:
+ raise UnmappableTableError(f"'{host}' isn't an S3 endpoint PostHog can import from")
+ endpoint = match.group("endpoint")
+ region: str | None = None
+ if endpoint == _S3_EXTERNAL_ENDPOINT:
+ region = _S3_EXTERNAL_REGION
+ elif endpoint is not None:
+ region_match = _S3_ENDPOINT.fullmatch(endpoint)
+ if region_match is None:
+ raise UnmappableTableError(f"'{host}' isn't an S3 API endpoint, such as a website or access point endpoint")
+ region = region_match.group("region")
+ if match.group("bucket") is not None:
+ return SelfManagedLocation(service="s3", bucket=match.group("bucket"), key=path, region=region)
+ bucket, _, key = path.partition("/")
+ return SelfManagedLocation(service="s3", bucket=bucket, key=key, region=region)
+
+
+def parse_self_managed_url(url: str) -> SelfManagedLocation:
+ """Split a self-managed table's URL pattern into its service, bucket, account and key."""
+ parsed = urlparse(url.strip())
+ if parsed.scheme not in ("https", "http"):
+ raise UnmappableTableError("the URL isn't an http or https URL")
+ if parsed.query or parsed.fragment or "?" in url or "#" in url:
+ raise UnmappableTableError("the URL has a '?' or '#', which PostHog can't map to a file pattern")
+ if parsed.username or parsed.password or parsed.port:
+ raise UnmappableTableError("the URL names a user or a port")
+ host = (parsed.hostname or "").lower()
+ # Only the separator after the host goes: a further slash is part of the object key.
+ path = unquote(parsed.path).removeprefix("/")
+ if host.endswith(".amazonaws.com") or host.endswith(".amazonaws.com.cn"):
+ location = _s3_location(host, path)
+ elif host == _GCS_HOST:
+ bucket, _, key = path.partition("/")
+ location = SelfManagedLocation(service="gcs", bucket=bucket, key=key)
+ elif host.endswith(f".{_GCS_HOST}"):
+ location = SelfManagedLocation(service="gcs", bucket=host[: -len(_GCS_HOST) - 1], key=path)
+ elif (r2 := _R2_HOST.fullmatch(host)) is not None:
+ bucket, _, key = path.partition("/")
+ location = SelfManagedLocation(service="r2", bucket=bucket, key=key, account=r2.group("account"))
+ elif (azure := _AZURE_HOST.fullmatch(host)) is not None:
+ container, _, key = path.partition("/")
+ location = SelfManagedLocation(service="azure", bucket=container, key=key, account=azure.group("account"))
+ else:
+ raise UnmappableTableError(f"'{host}' isn't a storage host PostHog can import from")
+ if not location.bucket or has_glob(location.bucket) or has_glob(location.account or ""):
+ raise UnmappableTableError("the URL doesn't name one bucket")
+ if location.key.startswith("/") or "//" in location.key:
+ raise UnmappableTableError("the URL path has an empty segment, which the import can't read")
+ if any(segment in (".", "..") for segment in location.key.split("/")):
+ raise UnmappableTableError("the URL path has a '.' or '..' segment")
+ return location
+
+
+def source_format_for(table_format: str) -> FileFormatChoice:
+ choice = _FORMATS.get(table_format)
+ if choice is None:
+ raise UnmappableTableError(f"the '{table_format}' format has no import equivalent")
+ return choice
+
+
+def needs_preserved_column_names(column_names: Iterable[str]) -> bool:
+ """Whether the import pipeline would rename a column that existing queries refer to."""
+ return any(normalize_column_name(name) != name for name in column_names)
+
+
+def _refuse_nested_columns(columns: Mapping[str, Any] | None) -> None:
+ for name, clickhouse_type in stored_clickhouse_types(columns).items():
+ base = clickhouse_base_type(clickhouse_type)
+ if base in _NESTED_TYPES:
+ raise UnmappableTableError(
+ f"the column '{name}' is a {base}, and the import would store it as text, not as a {base}"
+ )
+
+
+def _refuse_file_column_names(columns: Mapping[str, Any] | None) -> None:
+ for name in (columns or {}).keys():
+ if name in _FILE_COLUMNS:
+ raise UnmappableTableError(
+ f"the column '{name}' has the name of a column the import adds, which would replace its values"
+ )
+
+
+def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:
+ # The CSV reader trims spaces around header names, so a column ClickHouse read as ' id ' would be renamed.
+ for name in (columns or {}).keys():
+ if name != name.strip():
+ raise UnmappableTableError(
+ f"the column '{name}' has spaces around its name, which the CSV import doesn't keep"
+ )
+
+
+def _refuse_json_text_types(columns: Mapping[str, Any] | None) -> None:
+ # A JSON file holds dates and times as strings, and the JSON import keeps them as strings.
+ for name, clickhouse_type in stored_clickhouse_types(columns).items():
+ base = clickhouse_base_type(clickhouse_type)
+ if base in _JSON_TEXT_TYPES:
+ raise UnmappableTableError(
+ f"the column '{name}' is a {base}, and the JSON import would store it as text, not as a {base}"
+ )
+
+
+def table_definition_for(
+ name: str,
+ location: SelfManagedLocation,
+ table_format: str,
+ columns: Mapping[str, Any] | None,
+ csv_allow_double_quotes: bool | None,
+) -> TableDefinition:
+ """The schema contract that imports exactly the files the table's URL pattern reads."""
+ file_format = source_format_for(table_format)
+ _refuse_nested_columns(columns)
+ _refuse_file_column_names(columns)
+ preserve = needs_preserved_column_names((columns or {}).keys())
+ if file_format == "delta":
+ path = location.key.strip("/")
+ if has_glob(path):
+ raise UnmappableTableError("a Delta table URL can't hold a file pattern")
+ return TableDefinition(
+ name=name,
+ kind="delta",
+ path=path,
+ scope=None,
+ file_format="delta",
+ csv=CsvOptions(),
+ pattern=None,
+ source_format="delta",
+ preserve_column_names=preserve,
+ )
+ pattern = location.key
+ if not pattern or pattern.endswith("/"):
+ raise UnmappableTableError("the URL names a folder, not files")
+ try:
+ compile_clickhouse_glob(pattern)
+ except ObjectStorageError as e:
+ raise UnmappableTableError(str(e)) from e
+ try:
+ column_types = csv_column_types_from_columns(columns) if file_format in ("csv", "csv_no_header") else ()
+ except UnsupportedCsvColumnError as e:
+ raise UnmappableTableError(str(e)) from e
+ if file_format == "csv":
+ _refuse_padded_header_names(columns)
+ if file_format == "jsonl":
+ _refuse_json_text_types(columns)
+ return TableDefinition(
+ name=name,
+ kind="files",
+ path=glob_listing_prefix(pattern),
+ scope="glob",
+ file_format=file_format,
+ csv=CsvOptions(
+ has_header=file_format == "csv",
+ # ClickHouse reads quotes as enclosing a field unless the table turned that off.
+ allow_double_quotes=csv_allow_double_quotes is not False,
+ column_types=column_types,
+ # ClickHouse reads an unquoted `\N` as null in a CSV file.
+ null_marker=True,
+ # ClickHouse reads the declared format whatever the file is called, so a `.tsv` file is still CSV.
+ delimiter=",",
+ ),
+ pattern=pattern,
+ source_format=file_format,
+ preserve_column_names=preserve,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index d58802c78a0f..565bdf28d8a4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -1,7 +1,7 @@
from abc import abstractmethod
from collections.abc import Iterator
from datetime import UTC, datetime
-from typing import Generic, NoReturn, Optional
+from typing import Any, ClassVar, Generic, NoReturn, Optional
import pyarrow as pa
from botocore.exceptions import ClientError
@@ -61,6 +61,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
FileSelection,
plan_mirror,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+ StorageService,
+ UnmappableTableError,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
CHUNK_SIZE_BYTES,
MAX_DISCOVERY_OBJECTS,
@@ -119,6 +125,35 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
@abstractmethod
def discovery_options(self, config: ConfigType) -> DiscoveryOptions: ...
+ # The service whose self-managed warehouse tables this source can take over.
+ self_managed_service: ClassVar[StorageService]
+
+ @abstractmethod
+ def self_managed_connection_inputs(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ """The job inputs that reach the table's bucket with the table's own credential."""
+
+ def job_inputs_for_self_managed(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ """Job inputs for a source rooted at the bucket of a self-managed table.
+
+ The inputs go through the same parsing and location checks as a source a user creates, so
+ a credential or bucket the source would refuse is refused here too.
+ """
+ raw = {
+ **self.self_managed_connection_inputs(location, credential),
+ "file_format": "infer",
+ "csv_allow_double_quotes": "true",
+ }
+ try:
+ config = self.parse_config(raw)
+ validate_store_location(self.object_store_config(config))
+ except (ObjectStorageError, TypeError, ValueError) as e:
+ raise UnmappableTableError(str(e)) from e
+ return self.serialize_config(config)
+
def cursor_class(self) -> type[ObjectStorageDeltaCursor]:
return ObjectStorageDeltaCursor
@@ -233,6 +268,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
# Only a merge needs a key. Without one, a full refresh replaces the table and an append adds to it.
primary_keys=FILE_PRIMARY_KEYS if selection.merges else None,
chunk_size_bytes=CHUNK_SIZE_BYTES,
+ preserve_column_names=table.preserve_column_names,
)
def _mirrored_files_response(
@@ -291,6 +327,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
primary_keys=None,
chunk_size_bytes=CHUNK_SIZE_BYTES,
destination_reset_required=read.reset_required,
+ preserve_column_names=table.preserve_column_names,
)
def _find_table(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 4ef4d5f00e87..ffab85de6375 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -1040,6 +1040,47 @@ class TestS3Listing:
with pytest.raises(ObjectStorageError, match=TOO_MANY_FILES_ERROR):
_s3_store(client).list_objects("", 10)
+ @parameterized.expand(
+ [
+ ("bucket_root", "", "exports/orders-", "exports/orders-", 10),
+ ("store_root", "root", "exports/orders-", "root/exports/orders-", 10),
+ ("empty_key_prefix", "root", "", "root/", 10),
+ ("empty_bucket_prefix", "", "", "", 10),
+ ("matching_file_limit", "root", "exports/orders-", "root/exports/orders-", 1),
+ ]
+ )
+ def test_list_key_prefix(
+ self, _name: str, prefix: str, key_prefix: str, expected_prefix: str, max_objects: int
+ ) -> None:
+ client = boto3.client(
+ "s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"
+ )
+ root_key = f"{prefix}/" if prefix else ""
+ store = S3ObjectStore(
+ fs=LocalFileSystem(),
+ root=f"customer-data/{prefix}" if prefix else "customer-data",
+ delta_uri_prefix="s3://",
+ delta_storage_options={},
+ client=client,
+ bucket="customer-data",
+ prefix=prefix,
+ )
+ with Stubber(client) as stubber:
+ stubber.add_response(
+ "list_objects_v2",
+ _listing(
+ f"{expected_prefix}folder/",
+ f"{root_key}exports/orders-1.csv",
+ f"{root_key}exports/orders-2.csv",
+ ),
+ {"Bucket": "customer-data", "MaxKeys": 1000, "Prefix": expected_prefix},
+ )
+
+ listed = store.list_key_prefix(key_prefix, max_objects)
+
+ stubber.assert_no_pending_responses()
+ assert [stored.path for stored in listed] == ["exports/orders-1.csv", "exports/orders-2.csv"][:max_objects]
+
def test_rejected_credentials_become_an_access_error(self) -> None:
client = boto3.client(
"s3", region_name="us-east-1", aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="secret"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
new file mode 100644
index 000000000000..3a255bffd138
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
@@ -0,0 +1,1015 @@
+import io
+import time
+import uuid
+from datetime import UTC, date, datetime
+from decimal import Decimal
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+import pytest
+from unittest.mock import MagicMock
+
+import pyarrow as pa
+import structlog
+from fsspec.implementations.local import LocalFileSystem
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_blob.source import AzureBlobSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare_r2.source import CloudflareR2Source
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import CsvOptions
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ CsvColumnType,
+ UnsupportedCsvColumnError,
+ cast_csv_columns,
+ csv_column_type,
+ csv_column_types_from_columns,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
+ TableDefinition,
+ resolve_table_files,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ NO_TABLE_FILES_ERROR,
+ TOO_MANY_FILES_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.filesystem import (
+ ObjectStore,
+ StoredObject,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.formats import (
+ iter_csv_tables,
+ iter_file_tables,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ compile_clickhouse_glob,
+ glob_key_prefix,
+ glob_listing_prefix,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+ UnmappableTableError,
+ needs_preserved_column_names,
+ parse_self_managed_url,
+ source_format_for,
+ table_definition_for,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
+from products.warehouse_sources.backend.temporal.data_imports.sources.google_cloud_storage.source import (
+ GoogleCloudStorageSource,
+)
+
+R2_ACCOUNT = "0123456789abcdef0123456789abcdef"
+LOGGER = structlog.get_logger()
+
+
+def local_store(root: Path) -> ObjectStore:
+ return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})
+
+
+def write_files(root: Path, files: dict[str, str]) -> None:
+ for path, content in files.items():
+ (root / path).parent.mkdir(parents=True, exist_ok=True)
+ (root / path).write_text(content)
+
+
+class TestParseSelfManagedUrl:
+ @parameterized.expand(
+ [
+ (
+ "s3_virtual_hosted_regional",
+ "https://exports.s3.eu-west-1.amazonaws.com/events/*.parquet",
+ SelfManagedLocation(service="s3", bucket="exports", key="events/*.parquet", region="eu-west-1"),
+ ),
+ (
+ "s3_virtual_hosted_dotted_bucket",
+ "https://my.exports.s3.us-east-2.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="my.exports", key="a.csv", region="us-east-2"),
+ ),
+ (
+ "s3_virtual_hosted_legacy_dash_region",
+ "https://exports.s3-us-west-2.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-west-2"),
+ ),
+ (
+ "s3_virtual_hosted_global_has_no_region",
+ "https://exports.s3.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region=None),
+ ),
+ (
+ "s3_virtual_hosted_dualstack",
+ "https://exports.s3.dualstack.ap-south-1.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="ap-south-1"),
+ ),
+ (
+ "s3_external_endpoint_is_us_east_1",
+ "https://exports.s3-external-1.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-east-1"),
+ ),
+ (
+ "s3_path_style_regional",
+ "https://s3.eu-central-1.amazonaws.com/exports/2024/*.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="2024/*.csv", region="eu-central-1"),
+ ),
+ (
+ "s3_path_style_global",
+ "https://s3.amazonaws.com/exports/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region=None),
+ ),
+ (
+ "s3_china",
+ "https://exports.s3.cn-north-1.amazonaws.com.cn/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="cn-north-1"),
+ ),
+ (
+ "s3_percent_encoded_key",
+ "https://exports.s3.us-east-1.amazonaws.com/daily%20export/a%2Bb.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="daily export/a+b.csv", region="us-east-1"),
+ ),
+ (
+ "s3_http_scheme",
+ "http://exports.s3.us-east-1.amazonaws.com/a.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="a.csv", region="us-east-1"),
+ ),
+ (
+ "gcs_path_style",
+ "https://storage.googleapis.com/analytics-dump/tables/**.parquet",
+ SelfManagedLocation(service="gcs", bucket="analytics-dump", key="tables/**.parquet"),
+ ),
+ (
+ "gcs_virtual_hosted",
+ "https://analytics-dump.storage.googleapis.com/a.jsonl",
+ SelfManagedLocation(service="gcs", bucket="analytics-dump", key="a.jsonl"),
+ ),
+ (
+ "azure",
+ "https://examplestore.blob.core.windows.net/exports/orders/*.csv",
+ SelfManagedLocation(service="azure", bucket="exports", key="orders/*.csv", account="examplestore"),
+ ),
+ (
+ "r2",
+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/orders/{{1..3}}.csv",
+ SelfManagedLocation(service="r2", bucket="exports", key="orders/{1..3}.csv", account=R2_ACCOUNT),
+ ),
+ (
+ "upper_case_host",
+ "https://EXPORTS.S3.US-EAST-1.AMAZONAWS.COM/A.csv",
+ SelfManagedLocation(service="s3", bucket="exports", key="A.csv", region="us-east-1"),
+ ),
+ ]
+ )
+ def test_parses(self, _name: str, url: str, expected: SelfManagedLocation) -> None:
+ assert parse_self_managed_url(url) == expected
+
+ @parameterized.expand(
+ [
+ ("not_http", "s3://exports/a.csv"),
+ ("unknown_host", "https://minio.example.com/exports/a.csv"),
+ ("s3_website_endpoint", "https://exports.s3-website-us-east-1.amazonaws.com/a.csv"),
+ ("s3_access_point", "https://ap-123456789012.s3-accesspoint.us-east-1.amazonaws.com/a.csv"),
+ ("s3_accelerate", "https://exports.s3-accelerate.amazonaws.com/a.csv"),
+ ("query_string", "https://exports.s3.us-east-1.amazonaws.com/a.csv?versionId=1"),
+ ("question_mark_glob", "https://exports.s3.us-east-1.amazonaws.com/a?.csv"),
+ ("fragment", "https://exports.s3.us-east-1.amazonaws.com/a.csv#x"),
+ ("port", "https://exports.s3.us-east-1.amazonaws.com:8443/a.csv"),
+ ("userinfo", "https://user:[email protected]/a.csv"),
+ ("path_style_without_bucket", "https://s3.us-east-1.amazonaws.com/"),
+ ("glob_in_bucket", "https://s3.us-east-1.amazonaws.com/exports-*/a.csv"),
+ ("dot_dot_segment", "https://exports.s3.us-east-1.amazonaws.com/a/../b.csv"),
+ ("key_with_a_leading_slash", "https://exports.s3.us-east-1.amazonaws.com//orders.csv"),
+ ("key_with_an_empty_segment", "https://storage.googleapis.com/lake/orders//a.csv"),
+ ("azure_data_lake_endpoint", "https://examplestore.dfs.core.windows.net/exports/a.csv"),
+ ("azure_other_cloud", "https://examplestore.blob.core.chinacloudapi.cn/exports/a.csv"),
+ ("azure_without_container", "https://examplestore.blob.core.windows.net/"),
+ ("r2_jurisdiction_endpoint", f"https://{R2_ACCOUNT}.eu.r2.cloudflarestorage.com/exports/a.csv"),
+ ("r2_short_account", "https://abc.r2.cloudflarestorage.com/exports/a.csv"),
+ ("gcs_browser_url", "https://storage.cloud.google.com/exports/a.csv"),
+ ]
+ )
+ def test_refuses_urls_it_cant_map_exactly(self, _name: str, url: str) -> None:
+ with pytest.raises(UnmappableTableError):
+ parse_self_managed_url(url)
+
+
+class TestClickHouseGlob:
+ @parameterized.expand(
+ [
+ ("star_stays_in_its_folder", "data/*.csv", "data/a.csv", True),
+ ("star_matches_empty", "data/*.csv", "data/.csv", True),
+ ("star_does_not_cross_folders", "data/*.csv", "data/2024/a.csv", False),
+ ("double_star_crosses_folders", "data/**.csv", "data/2024/01/a.csv", True),
+ ("double_star_folder", "data/**", "data/x/y", True),
+ ("question_mark_is_one_character", "part-?.csv", "part-1.csv", True),
+ ("question_mark_is_not_two", "part-?.csv", "part-10.csv", False),
+ ("question_mark_does_not_match_slash", "a?b", "a/b", False),
+ ("alternatives", "{orders,refunds}.csv", "refunds.csv", True),
+ ("alternatives_are_literal", "{a.b,c}.csv", "aXb.csv", False),
+ ("range", "part-{1..12}.csv", "part-12.csv", True),
+ ("range_upper_bound", "part-{1..12}.csv", "part-13.csv", False),
+ ("range_reversed_bounds", "part-{12..1}.csv", "part-5.csv", True),
+ ("range_without_padding_refuses_leading_zero", "part-{1..12}.csv", "part-05.csv", False),
+ ("padded_range", "part-{01..12}.csv", "part-05.csv", True),
+ ("padded_range_refuses_unpadded", "part-{01..12}.csv", "part-5.csv", False),
+ ("literal_regex_characters", "a+b(1).csv", "a+b(1).csv", True),
+ ("literal_dot_is_not_any_character", "a.csv", "aXcsv", False),
+ ("literal_path_is_exact", "data/a.csv", "data/a.csv.bak", False),
+ ("hidden_files_match_like_clickhouse", "data/*", "data/_SUCCESS", True),
+ ("range_leaves_digits_for_star", "part-{1..12}*.csv", "part-123.csv", True),
+ ("range_does_not_take_extra_digits", "part-{1..12}.csv", "part-123.csv", False),
+ ("empty_alternative", "{a,}x.csv", "x.csv", True),
+ ("padded_range_in_middle", "d={01..31}/h-*.csv", "d=07/h-1.csv", True),
+ ("padded_range_in_middle_refuses_unpadded", "d={01..31}/h-*.csv", "d=7/h-1.csv", False),
+ ("zero_range", "{0..0}", "0", True),
+ ("zero_range_refuses_extra_zero", "{0..0}", "00", False),
+ ("double_star_after_range", "{1..12}**.csv", "123/nested/a.csv", True),
+ ("alternatives_before_star", "{a,ab}*.csv", "abc.csv", True),
+ ("double_star_matches_empty", "**", "", True),
+ ("star_does_not_match_slash", "*", "/", False),
+ ("question_mark_does_not_match_only_slash", "?", "/", False),
+ ("triple_star_crosses_folders", "***", "a/b", True),
+ ("four_stars_match_empty", "****", "", True),
+ ("adjacent_empty_alternatives", "*{a,}{b,}x", "x", True),
+ ("range_leaves_digits_for_literal", "{1..12}3", "123", True),
+ ("padded_reversed_range", "{12..01}", "07", True),
+ ("padded_range_uses_widest_bound", "{0..0012}", "0007", True),
+ ]
+ )
+ def test_matches(self, _name: str, pattern: str, path: str, expected: bool) -> None:
+ assert compile_clickhouse_glob(pattern).matches(path) is expected
+
+ @parameterized.expand(
+ [
+ ("unclosed_brace", "a{b,c.csv"),
+ ("empty_braces", "a{}.csv"),
+ ("nested_glob_in_braces", "{a*,b}.csv"),
+ ("square_brackets", "part-[0-9].csv"),
+ ("stray_closing_brace", "a}.csv"),
+ ("nested_question_mark", "{a?,b}.csv"),
+ ("nested_brace", "{{a,b},c}.csv"),
+ ("bracket_in_braces", "{a[,b}.csv"),
+ ("closing_bracket_in_braces", "{a],b}.csv"),
+ ("stray_closing_bracket", "a].csv"),
+ ]
+ )
+ def test_refuses_patterns_it_cant_read(self, _name: str, pattern: str) -> None:
+ with pytest.raises(ObjectStorageError):
+ compile_clickhouse_glob(pattern)
+
+ @parameterized.expand(
+ [
+ ("glob_in_file_name", "events/2024/*.csv", "events/2024"),
+ ("glob_in_folder", "events/*/a.csv", "events"),
+ ("glob_at_root", "*.csv", ""),
+ ("range_folder", "y={2023..2024}/a.parquet", ""),
+ ("literal_file", "events/a.csv", "events/a.csv"),
+ ]
+ )
+ def test_listing_prefix(self, _name: str, pattern: str, expected: str) -> None:
+ assert glob_listing_prefix(pattern) == expected
+
+ @parameterized.expand(
+ [
+ ("glob_in_file_name", "exports/orders-*.csv", "exports/orders-"),
+ ("glob_at_root", "*.csv", ""),
+ ("range_folder", "y={2023..2024}/a.parquet", "y="),
+ ("literal_file", "events/a.csv", "events/a.csv"),
+ ("question_mark", "a?b", "a"),
+ ]
+ )
+ def test_key_prefix(self, _name: str, pattern: str, expected: str) -> None:
+ assert glob_key_prefix(pattern) == expected
+
+ @parameterized.expand(
+ [
+ ("twenty_stars", "*a" * 20 + "*b.csv"),
+ ("eight_stars", "*a*a*a*a*a*a*a*a*b.csv"),
+ ]
+ )
+ def test_adversarial_match_time(self, _name: str, pattern: str) -> None:
+ glob = compile_clickhouse_glob(pattern)
+ path = "a" * 200 + ".csv"
+
+ started = time.perf_counter()
+ for _ in range(1000):
+ assert glob.matches(path) is False
+ assert time.perf_counter() - started < 2
+ assert glob.matches("a" * 200 + "b.csv") is True
+
+ @parameterized.expand(
+ [
+ ("few_matching_keys", 3, False),
+ ("matching_key_limit", 2, True),
+ ]
+ )
+ def test_resolves_only_the_key_prefix(self, _name: str, max_objects: int, exceeds_limit: bool) -> None:
+ with TemporaryDirectory() as directory:
+ root = Path(directory)
+ write_files(
+ root,
+ {
+ "exports/other-1.csv": "id\n1\n",
+ "exports/other-2.csv": "id\n2\n",
+ "exports/other-3.csv": "id\n3\n",
+ "exports/orders-2.csv": "id\n2\n",
+ "exports/orders-1.csv": "id\n1\n",
+ "exports/orders-3.txt": "id\n3\n",
+ },
+ )
+ table = TableDefinition(
+ name="orders",
+ kind="files",
+ path="exports",
+ scope="glob",
+ file_format="csv",
+ csv=CsvOptions(),
+ pattern="exports/orders-*.csv",
+ source_format="csv",
+ )
+ if exceeds_limit:
+ with pytest.raises(ObjectStorageError, match=f"{TOO_MANY_FILES_ERROR}.*exports/orders-"):
+ resolve_table_files(local_store(root), table, max_objects=max_objects)
+ else:
+ resolved = resolve_table_files(local_store(root), table, max_objects=max_objects)
+ assert [stored.path for stored in resolved.files] == ["exports/orders-1.csv", "exports/orders-2.csv"]
+
+
+class TestTableDefinition:
+ @parameterized.expand(
+ [
+ ("CSV", "csv_no_header"),
+ ("CSVWithNames", "csv"),
+ ("Parquet", "parquet"),
+ ("JSONEachRow", "jsonl"),
+ ("Delta", "delta"),
+ ("DeltaS3Wrapper", "delta"),
+ ]
+ )
+ def test_format_mapping(self, table_format: str, expected: str) -> None:
+ assert source_format_for(table_format) == expected
+
+ def test_an_unknown_format_is_refused(self) -> None:
+ with pytest.raises(UnmappableTableError):
+ source_format_for("TabSeparated")
+
+ @parameterized.expand(
+ [
+ ("all_snake_case", ["id", "created_at", "amount_usd"], False),
+ ("no_columns", [], False),
+ ("camel_case", ["id", "createdAt"], True),
+ ("spaces", ["Order ID"], True),
+ ("upper_case", ["ID"], True),
+ ]
+ )
+ def test_preserve_names_only_when_the_pipeline_would_rename(
+ self, _name: str, columns: list[str], expected: bool
+ ) -> None:
+ assert needs_preserved_column_names(columns) is expected
+
+ def test_csv_with_names_keeps_the_stored_types_and_quote_setting(self) -> None:
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/2024-*.csv")
+ columns = {
+ "id": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True},
+ "Order Total": {"clickhouse": "Nullable(Float64)", "hogql": "FloatDatabaseField", "valid": True},
+ "note": {"clickhouse": "Nullable(String)", "hogql": "StringDatabaseField", "valid": True},
+ }
+
+ definition = table_definition_for("orders", location, "CSVWithNames", columns, csv_allow_double_quotes=False)
+
+ assert definition.schema_metadata() == {
+ "kind": "files",
+ "scope": "glob",
+ "path": "orders",
+ "pattern": "orders/2024-*.csv",
+ "format": "csv",
+ "source_format": "csv",
+ "csv": {
+ "has_header": True,
+ "allow_double_quotes": False,
+ "column_types": {"id": "integer", "Order Total": "float"},
+ "null_marker": True,
+ "delimiter": ",",
+ },
+ "preserve_column_names": True,
+ }
+
+ @parameterized.expand(
+ [
+ ("csv_without_header", "CSV", "csv_no_header", False, True),
+ ("parquet_has_no_csv_types", "Parquet", "parquet", True, False),
+ ("jsonl_has_no_csv_types", "JSONEachRow", "jsonl", True, False),
+ ]
+ )
+ def test_csv_header_and_types_follow_the_format(
+ self, _name: str, table_format: str, file_format: str, has_header: bool, has_types: bool
+ ) -> None:
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/a.file")
+ columns = {"c1": {"clickhouse": "Int64", "hogql": "IntegerDatabaseField"}}
+
+ definition = table_definition_for("orders", location, table_format, columns, None)
+
+ assert (definition.scope, definition.path, definition.file_format) == ("glob", "orders/a.file", file_format)
+ assert definition.csv.allow_double_quotes is True
+ assert definition.csv.has_header is (has_header if file_format.startswith("csv") else False)
+ assert bool(definition.csv.column_types) is has_types
+
+ @parameterized.expand([("Delta",), ("DeltaS3Wrapper",)])
+ def test_delta_reads_the_table_folder(self, table_format: str) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/tables/orders/")
+
+ definition = table_definition_for("orders", location, table_format, {"OrderId": "Int64"}, None)
+
+ assert (definition.kind, definition.path, definition.pattern) == ("delta", "tables/orders", None)
+ assert definition.preserve_column_names is True
+
+ @parameterized.expand(
+ [
+ ("delta_with_glob", "https://storage.googleapis.com/lake/tables/*", "Delta"),
+ ("files_url_is_a_folder", "https://storage.googleapis.com/lake/tables/", "Parquet"),
+ ("files_url_is_the_bucket", "https://storage.googleapis.com/lake", "CSVWithNames"),
+ ("unreadable_glob", "https://storage.googleapis.com/lake/part-[0-9].parquet", "Parquet"),
+ ]
+ )
+ def test_refuses_tables_it_cant_read_exactly(self, _name: str, url: str, table_format: str) -> None:
+ with pytest.raises(UnmappableTableError):
+ table_definition_for("orders", parse_self_managed_url(url), table_format, {}, None)
+
+ def test_metadata_round_trip_keeps_the_glob_types_and_names(self) -> None:
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/{1..3}.csv")
+ definition = table_definition_for(
+ "orders", location, "CSVWithNames", {"OrderId": "Nullable(Int64)", "paid": "Bool"}, None
+ )
+
+ rebuilt = TableDefinition.from_schema_metadata("orders", definition.schema_metadata())
+
+ assert rebuilt == definition
+
+ def test_metadata_round_trip_keeps_decimal_unsigned_and_time_zone_types(self) -> None:
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")
+ columns = {
+ "total": "Nullable(Decimal(12, 3))",
+ "big": "UInt64",
+ "at": "DateTime64(3, 'Europe/London')",
+ "utc": "DateTime('UTC')",
+ }
+ definition = table_definition_for("orders", location, "CSVWithNames", columns, None)
+
+ metadata = definition.schema_metadata()
+ rebuilt = TableDefinition.from_schema_metadata("orders", metadata)
+
+ assert metadata["csv"]["column_types"] == {
+ "total": {"kind": "decimal", "precision": 12, "scale": 3},
+ "big": "unsigned",
+ "at": {"kind": "datetime", "timezone": "Europe/London"},
+ "utc": "datetime",
+ }
+ assert rebuilt == definition
+
+ @parameterized.expand(
+ [
+ (f"{table_format}_{column_type}", table_format, column_type, url)
+ for table_format, url in (
+ ("Parquet", "https://storage.googleapis.com/lake/orders/*.parquet"),
+ ("JSONEachRow", "https://storage.googleapis.com/lake/orders/*.jsonl"),
+ ("CSVWithNames", "https://storage.googleapis.com/lake/orders/*.csv"),
+ ("Delta", "https://storage.googleapis.com/lake/orders/"),
+ )
+ for column_type in (
+ "Array(Nullable(Int64))",
+ "Map(String, String)",
+ "Tuple(a Nullable(Int64), b Nullable(String))",
+ "Nullable(JSON)",
+ "Object('json')",
+ "Nested(a Int64)",
+ )
+ ]
+ )
+ def test_nested_columns_are_refused(self, _name: str, table_format: str, column_type: str, url: str) -> None:
+ columns = {"id": "Int64", "items": {"clickhouse": column_type, "hogql": "StringJSONDatabaseField"}}
+
+ with pytest.raises(UnmappableTableError, match="'items'"):
+ table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)
+
+ def test_an_invalid_nested_column_does_not_block_the_table(self) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.parquet")
+ columns = {"items": {"clickhouse": "Array(Int64)", "hogql": "StringArrayDatabaseField", "valid": False}}
+
+ assert table_definition_for("orders", location, "Parquet", columns, None).kind == "files"
+
+ def test_a_csv_column_no_import_keeps_is_refused(self) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")
+
+ with pytest.raises(UnmappableTableError, match="'id'"):
+ table_definition_for("orders", location, "CSVWithNames", {"id": "Int128"}, None)
+
+ @parameterized.expand(
+ [
+ (f"{table_format}_{column}", table_format, column, url)
+ for table_format, url in (
+ ("Parquet", "https://storage.googleapis.com/lake/orders/*.parquet"),
+ ("CSVWithNames", "https://storage.googleapis.com/lake/orders/*.csv"),
+ ("Delta", "https://storage.googleapis.com/lake/orders/"),
+ )
+ for column in ("_file_name", "_file_modified_at", "_file_row_number")
+ ]
+ )
+ def test_columns_named_like_the_file_columns_are_refused(
+ self, _name: str, table_format: str, column: str, url: str
+ ) -> None:
+ columns = {"id": "Int64", column: "Nullable(String)"}
+
+ with pytest.raises(UnmappableTableError, match=f"'{column}'"):
+ table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)
+
+ @parameterized.expand([("Date",), ("Nullable(Date32)",), ("DateTime",), ("Nullable(DateTime64(9))",)])
+ def test_json_dates_and_times_are_refused(self, column_type: str) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")
+
+ with pytest.raises(UnmappableTableError, match="'created'"):
+ table_definition_for("orders", location, "JSONEachRow", {"id": "Int64", "created": column_type}, None)
+
+ def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")
+
+ with pytest.raises(UnmappableTableError, match="' Order ID '"):
+ table_definition_for("orders", location, "CSVWithNames", {" Order ID ": "Nullable(String)"}, None)
+
+ def test_headerless_csv_names_are_not_checked_for_spaces(self) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")
+
+ assert table_definition_for("orders", location, "CSV", {"c1": "Nullable(String)"}, None).kind == "files"
+
+ @parameterized.expand(
+ [
+ ("glob_without_pattern", {"kind": "files", "scope": "glob", "path": "", "format": "csv"}),
+ ("unknown_scope", {"kind": "files", "scope": "everything", "path": "", "format": "csv"}),
+ ]
+ )
+ def test_unusable_glob_metadata_is_ignored(self, _name: str, metadata: dict[str, Any]) -> None:
+ assert TableDefinition.from_schema_metadata("orders", metadata) is None
+
+ def test_unknown_column_kinds_in_metadata_are_dropped(self) -> None:
+ rebuilt = TableDefinition.from_schema_metadata(
+ "orders",
+ {
+ "kind": "files",
+ "scope": "glob",
+ "path": "",
+ "pattern": "*.csv",
+ "format": "csv",
+ "csv": {"column_types": {"id": "integer", "blob": "binary", "n": 3}},
+ },
+ )
+
+ assert rebuilt is not None
+ assert rebuilt.csv.column_types == (CsvColumnType(name="id", kind="integer"),)
+ assert rebuilt.preserve_column_names is False
+
+
+class TestResolveGlobFiles:
+ def test_reads_exactly_the_files_the_glob_matches(self, tmp_path: Path) -> None:
+ write_files(
+ tmp_path,
+ {
+ "orders/part-1.csv": "id\n1\n",
+ "orders/part-2.csv": "id\n2\n",
+ "orders/part-3.csv": "id\n3\n",
+ "orders/_SUCCESS": "",
+ "orders/nested/part-1.csv": "id\n4\n",
+ "orders/part-1.csv.bak": "id\n5\n",
+ "refunds/part-1.csv": "id\n6\n",
+ },
+ )
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/part-{1..2}.csv")
+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)
+
+ resolved = resolve_table_files(local_store(tmp_path), table)
+
+ assert [stored.path for stored in resolved.files] == ["orders/part-1.csv", "orders/part-2.csv"]
+
+ def test_a_literal_file_resolves_to_that_file(self, tmp_path: Path) -> None:
+ write_files(tmp_path, {"orders.csv": "id\n1\n", "orders.csv.old": "id\n2\n"})
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders.csv")
+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)
+
+ resolved = resolve_table_files(local_store(tmp_path), table)
+
+ assert [stored.path for stored in resolved.files] == ["orders.csv"]
+
+ def test_no_matching_file_is_a_missing_table(self, tmp_path: Path) -> None:
+ write_files(tmp_path, {"orders/a.parquet": "x"})
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")
+ table = table_definition_for("orders", location, "CSVWithNames", {}, None)
+
+ with pytest.raises(ObjectStorageError, match=NO_TABLE_FILES_ERROR):
+ resolve_table_files(local_store(tmp_path), table)
+
+
+class TestCsvColumnTypes:
+ @parameterized.expand(
+ [
+ ("Int64", CsvColumnType(name="c", kind="integer")),
+ ("Nullable(Int32)", CsvColumnType(name="c", kind="integer")),
+ ("UInt8", CsvColumnType(name="c", kind="integer")),
+ ("LowCardinality(Nullable(Int16))", CsvColumnType(name="c", kind="integer")),
+ ("UInt64", CsvColumnType(name="c", kind="unsigned")),
+ ("Nullable(UInt64)", CsvColumnType(name="c", kind="unsigned")),
+ ("Float32", CsvColumnType(name="c", kind="float")),
+ ("Nullable(Float64)", CsvColumnType(name="c", kind="float")),
+ ("Decimal(10, 2)", CsvColumnType(name="c", kind="decimal", precision=10, scale=2)),
+ ("Nullable(Decimal(38,0))", CsvColumnType(name="c", kind="decimal", precision=38, scale=0)),
+ ("Decimal32(4)", CsvColumnType(name="c", kind="decimal", precision=9, scale=4)),
+ ("Decimal64(6)", CsvColumnType(name="c", kind="decimal", precision=18, scale=6)),
+ ("Decimal128(10)", CsvColumnType(name="c", kind="decimal", precision=38, scale=10)),
+ ("Bool", CsvColumnType(name="c", kind="boolean")),
+ ("Date", CsvColumnType(name="c", kind="date")),
+ ("Date32", CsvColumnType(name="c", kind="date")),
+ ("DateTime", CsvColumnType(name="c", kind="datetime")),
+ ("Nullable(DateTime64(6, 'UTC'))", CsvColumnType(name="c", kind="datetime")),
+ ("DateTime('Europe/London')", CsvColumnType(name="c", kind="datetime", timezone="Europe/London")),
+ (
+ "Nullable(DateTime64(3, 'America/New_York'))",
+ CsvColumnType(name="c", kind="datetime", timezone="America/New_York"),
+ ),
+ ("String", None),
+ ("Array(Int64)", None),
+ ("Nullable(UUID)", None),
+ ("", None),
+ ]
+ )
+ def test_type_for_stored_type(self, clickhouse_type: str, expected: CsvColumnType | None) -> None:
+ assert csv_column_type("c", clickhouse_type) == expected
+
+ @parameterized.expand(
+ [
+ ("int128", "Int128"),
+ ("uint256", "Nullable(UInt256)"),
+ ("decimal_wider_than_38_digits", "Decimal(50, 2)"),
+ ("decimal256", "Decimal256(4)"),
+ ("decimal_without_arguments", "Decimal"),
+ ("unknown_time_zone", "DateTime('Nowhere/Land')"),
+ ]
+ )
+ def test_types_no_import_keeps_are_refused(self, _name: str, clickhouse_type: str) -> None:
+ with pytest.raises(UnsupportedCsvColumnError, match="'c'"):
+ csv_column_type("c", clickhouse_type)
+
+ def test_types_from_stored_columns_skip_invalid_and_string_columns(self) -> None:
+ columns: dict[str, Any] = {
+ "id": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True},
+ "broken": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": False},
+ "legacy": "Nullable(Float64)",
+ "name": {"clickhouse": "Nullable(String)", "hogql": "StringDatabaseField"},
+ "odd": 5,
+ }
+
+ assert csv_column_types_from_columns(columns) == (
+ CsvColumnType(name="id", kind="integer"),
+ CsvColumnType(name="legacy", kind="float"),
+ )
+
+ @parameterized.expand(
+ [
+ ("integers", "integer", ["1", " -2 ", "+3", None, ""], [1, -2, 3, None, None], 0),
+ (
+ "bad_integers_become_null",
+ "integer",
+ ["1", "1.5", "x", "99999999999999999999"],
+ [1, None, None, None],
+ 3,
+ ),
+ ("floats", "float", ["1.5", "1e3", " 2 ", "-0.25", "+4"], [1.5, 1000.0, 2.0, -0.25, 4.0], 0),
+ ("bad_floats_become_null", "float", ["1.5", "one"], [1.5, None], 1),
+ (
+ "booleans",
+ "boolean",
+ ["true", "FALSE", "1", "0", "yes", "No", None],
+ [True, False, True, False, True, False, None],
+ 0,
+ ),
+ ("bad_booleans_become_null", "boolean", ["true", "maybe"], [True, None], 1),
+ ("dates", "date", ["2024-01-15", "1999-12-31"], [date(2024, 1, 15), date(1999, 12, 31)], 0),
+ (
+ "bad_dates_become_null",
+ "date",
+ ["2024-01-15", "2024-13-01", "15/01/2024"],
+ [date(2024, 1, 15), None, None],
+ 2,
+ ),
+ (
+ "naive_datetimes_are_utc",
+ "datetime",
+ ["2024-01-15 10:30:00", "2024-01-15T10:30:00.123456"],
+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 10, 30, 0, 123456, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "datetimes_with_offsets",
+ "datetime",
+ ["2024-01-15T10:30:00Z", "2024-01-15T10:30:00+02:00"],
+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 8, 30, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "mixed_datetimes_are_read_one_by_one",
+ "datetime",
+ ["2024-01-15 10:30:00", "2024-01-15T10:30:00Z", "soon"],
+ [datetime(2024, 1, 15, 10, 30, tzinfo=UTC), datetime(2024, 1, 15, 10, 30, tzinfo=UTC), None],
+ 1,
+ ),
+ (
+ "unsigned_keeps_values_above_int64",
+ CsvColumnType(name="value", kind="unsigned"),
+ ["18446744073709551615", "0", "-1", "+7", "18446744073709551616"],
+ [Decimal(18446744073709551615), Decimal(0), None, Decimal(7), None],
+ 2,
+ ),
+ (
+ "decimals_keep_their_scale",
+ CsvColumnType(name="value", kind="decimal", precision=10, scale=2),
+ ["1.25", "-3.1", "7", "1.255", "x", "123456789012.5"],
+ [Decimal("1.25"), Decimal("-3.10"), Decimal("7.00"), Decimal("1.25"), None, None],
+ 2,
+ ),
+ (
+ "naive_datetimes_take_the_column_time_zone",
+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),
+ ["2024-07-01 10:00:00", "2024-01-15 10:00:00"],
+ [datetime(2024, 7, 1, 9, 0, tzinfo=UTC), datetime(2024, 1, 15, 10, 0, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "offsets_win_over_the_column_time_zone",
+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),
+ ["2024-07-01T10:00:00Z", "2024-07-01T10:00:00+02:00"],
+ [datetime(2024, 7, 1, 10, 0, tzinfo=UTC), datetime(2024, 7, 1, 8, 0, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "mixed_values_in_a_time_zone_are_read_one_by_one",
+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),
+ ["2024-07-01 10:00:00", "2024-07-01T10:00:00Z", "soon"],
+ [datetime(2024, 7, 1, 9, 0, tzinfo=UTC), datetime(2024, 7, 1, 10, 0, tzinfo=UTC), None],
+ 1,
+ ),
+ (
+ "nanosecond_datetimes_are_truncated_to_microseconds",
+ "datetime",
+ ["2024-07-01 10:00:00.123456789", "2024-07-01 10:00:00", "2024-07-01T10:00:00.123456789+02:00"],
+ [
+ datetime(2024, 7, 1, 10, 0, 0, 123456, tzinfo=UTC),
+ datetime(2024, 7, 1, 10, 0, tzinfo=UTC),
+ datetime(2024, 7, 1, 8, 0, 0, 123456, tzinfo=UTC),
+ ],
+ 0,
+ ),
+ (
+ "nanosecond_datetimes_take_the_column_time_zone",
+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),
+ ["2024-07-01 10:00:00.123456789"],
+ [datetime(2024, 7, 1, 9, 0, 0, 123456, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "repeated_local_time_takes_the_earlier_instant",
+ CsvColumnType(name="value", kind="datetime", timezone="Europe/London"),
+ ["2024-10-27 01:30:00"],
+ [datetime(2024, 10, 27, 0, 30, tzinfo=UTC)],
+ 0,
+ ),
+ ]
+ )
+ def test_cast(
+ self, _name: str, kind: Any, values: list[str | None], expected: list[Any], expected_nulled: int
+ ) -> None:
+ table = pa.table({"value": pa.array(values, pa.string()), "other": pa.array(["x"] * len(values))})
+ column_type = kind if isinstance(kind, CsvColumnType) else CsvColumnType(name="value", kind=kind)
+
+ result = cast_csv_columns(table, (column_type,))
+
+ assert result.table.column("value").to_pylist() == expected
+ assert result.table.column("other").type == pa.string()
+ assert result.nulled_values == ({"value": expected_nulled} if expected_nulled else {})
+
+ def test_a_column_missing_from_the_file_is_ignored(self) -> None:
+ table = pa.table({"id": pa.array(["1"])})
+
+ result = cast_csv_columns(table, (CsvColumnType(name="gone", kind="integer"),))
+
+ assert result.table.equals(table)
+
+ @parameterized.expand(
+ [
+ ("no_bad_values", 1000, ()),
+ ("one_bad_value", 1000, (517,)),
+ ("bad_first_and_last", 1000, (0, 999)),
+ ("bad_values_in_one_run", 1000, tuple(range(300, 340))),
+ ("scattered_bad_values", 5000, tuple(range(0, 5000, 97))),
+ ("every_value_bad", 200, tuple(range(200))),
+ ("shorter_than_one_slice", 5, (2,)),
+ ]
+ )
+ def test_split_cast_equals_a_value_by_value_cast(self, _name: str, rows: int, bad_rows: tuple[int, ...]) -> None:
+ bad = set(bad_rows)
+ values = [None if row % 11 == 0 else ("x" if row in bad else str(row)) for row in range(rows)]
+ table = pa.table({"value": pa.array(values, pa.string())})
+
+ result = cast_csv_columns(table, (CsvColumnType(name="value", kind="integer"),))
+
+ expected = [None if value is None or value == "x" else int(value) for value in values]
+ assert result.table.column("value").to_pylist() == expected
+ bad_non_null = sum(1 for value in values if value == "x")
+ assert result.nulled_values == ({"value": bad_non_null} if bad_non_null else {})
+
+ def test_split_cast_parses_far_fewer_slices_than_values(self) -> None:
+ values = [str(row) for row in range(100_000)]
+ values[54_321] = "x"
+ table = pa.table({"value": pa.array(values, pa.string())})
+
+ started = time.perf_counter()
+ result = cast_csv_columns(table, (CsvColumnType(name="value", kind="integer"),))
+
+ assert time.perf_counter() - started < 1
+ assert result.nulled_values == {"value": 1}
+
+ def test_the_reader_casts_typed_columns_and_reports_bad_values(self) -> None:
+ logger = MagicMock()
+ content = b"id,amount,created,label\n1,2.5,2024-01-01,a\nx,3,2024-01-02,b\n"
+ options = CsvOptions(
+ column_types=(
+ CsvColumnType(name="id", kind="integer"),
+ CsvColumnType(name="amount", kind="float"),
+ CsvColumnType(name="created", kind="date"),
+ )
+ )
+
+ [table] = list(iter_csv_tables(io.BytesIO(content), options, file_path="orders.csv", logger=logger))
+
+ assert table.schema.types == [pa.int64(), pa.float64(), pa.date32(), pa.string()]
+ assert table.column("id").to_pylist() == [1, None]
+ logger.warning.assert_called_once()
+ assert logger.warning.call_args.kwargs["nulled_values"] == {"id": 1}
+
+ def test_a_header_only_file_gets_typed_empty_columns(self) -> None:
+ options = CsvOptions(column_types=(CsvColumnType(name="id", kind="integer"),))
+
+ [table] = list(iter_csv_tables(io.BytesIO(b"id,label\n"), options))
+
+ assert table.num_rows == 0
+ assert table.schema.types == [pa.int64(), pa.string()]
+
+ @parameterized.expand(
+ [
+ ("migrated_table", True, [None, "\\N", "a"]),
+ ("new_source", False, ["\\N", "\\N", "a"]),
+ ]
+ )
+ def test_an_unquoted_null_marker_is_null_only_for_migrated_tables(
+ self, _name: str, null_marker: bool, expected: list[str | None]
+ ) -> None:
+ content = b'label\n\\N\n"\\N"\na\n'
+
+ [table] = list(iter_csv_tables(io.BytesIO(content), CsvOptions(null_marker=null_marker)))
+
+ assert table.column("label").to_pylist() == expected
+
+ def test_a_fixed_delimiter_wins_over_the_file_name(self) -> None:
+ content = io.BytesIO(b"id,label\n1,a\n")
+ stored = StoredObject(path="orders.tsv", size=0, modified_at=datetime(2024, 1, 1, tzinfo=UTC))
+
+ [table] = list(iter_file_tables(content, stored, "csv", CsvOptions(delimiter=",")))
+
+ assert table.column("label").to_pylist() == ["a"]
+
+ def test_new_csv_sources_still_read_strings(self) -> None:
+ [table] = list(iter_csv_tables(io.BytesIO(b"id\n1\n"), CsvOptions()))
+
+ assert table.schema.types == [pa.string()]
+
+
+def _credential(access_key: str = "AKIAEXAMPLE", access_secret: str = "c2VjcmV0") -> SelfManagedCredential:
+ return SelfManagedCredential(access_key=access_key, access_secret=access_secret)
+
+
+def resolvable_hosts(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The location check resolves the host, and the example hosts here don't exist.
+ monkeypatch.setattr(source_base, "validate_warehouse_table_url_pattern", lambda _url: (True, ""))
+
+
[email protected]("resolvable_hosts")
+class TestJobInputsForSelfManaged:
+ @parameterized.expand(
+ [
+ (
+ "s3",
+ AmazonS3Source,
+ "https://exports.s3.eu-west-1.amazonaws.com/a.csv",
+ {"url_pattern": "https://s3.eu-west-1.amazonaws.com/exports/", "access_key_id": "AKIAEXAMPLE"},
+ ),
+ (
+ "gcs",
+ GoogleCloudStorageSource,
+ "https://storage.googleapis.com/exports/a.csv",
+ {"url_pattern": "https://storage.googleapis.com/exports/", "access_key_id": "AKIAEXAMPLE"},
+ ),
+ (
+ "r2",
+ CloudflareR2Source,
+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/a.csv",
+ {
+ "url_pattern": f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/exports/",
+ "access_key_id": "AKIAEXAMPLE",
+ },
+ ),
+ ]
+ )
+ def test_maps_the_bucket_and_credential(
+ self, _name: str, source_class: type, url: str, expected: dict[str, str]
+ ) -> None:
+ job_inputs = source_class().job_inputs_for_self_managed(parse_self_managed_url(url), _credential())
+
+ assert {key: job_inputs[key] for key in expected} == expected
+ assert job_inputs["secret_access_key"] == "c2VjcmV0"
+ file_format = job_inputs["file_format"]
+ selection = file_format["selection"] if isinstance(file_format, dict) else file_format
+ assert selection == "infer"
+
+ def test_azure_takes_the_account_from_the_url_and_the_key_from_the_secret(self) -> None:
+ location = parse_self_managed_url("https://examplestore.blob.core.windows.net/exports/a.csv")
+
+ job_inputs = AzureBlobSource().job_inputs_for_self_managed(location, _credential(access_key="ExampleStore"))
+
+ assert (job_inputs["url_pattern"], job_inputs["account_key"]) == (
+ "https://examplestore.blob.core.windows.net/exports/",
+ "c2VjcmV0",
+ )
+
+ @parameterized.expand(
+ [
+ ("s3_without_region", AmazonS3Source, "https://exports.s3.amazonaws.com/a.csv", _credential()),
+ ("s3_bucket_name_too_short", AmazonS3Source, "https://s3.us-east-1.amazonaws.com/ab/a.csv", _credential()),
+ (
+ "s3_empty_secret",
+ AmazonS3Source,
+ "https://exports.s3.us-east-1.amazonaws.com/a.csv",
+ _credential(access_secret=" "),
+ ),
+ (
+ "azure_key_for_another_account",
+ AzureBlobSource,
+ "https://examplestore.blob.core.windows.net/exports/a.csv",
+ _credential(access_key="otherstore"),
+ ),
+ (
+ "azure_key_that_is_not_base64",
+ AzureBlobSource,
+ "https://examplestore.blob.core.windows.net/exports/a.csv",
+ _credential(access_key="examplestore", access_secret="a;EndpointSuffix=example.com"),
+ ),
+ ]
+ )
+ def test_refuses_what_the_source_would_refuse(
+ self, _name: str, source_class: type, url: str, credential: SelfManagedCredential
+ ) -> None:
+ with pytest.raises(UnmappableTableError):
+ source_class().job_inputs_for_self_managed(parse_self_managed_url(url), credential)
+
+
+class TestSourceForPipeline:
+ @parameterized.expand([("preserved", True), ("normalized", False)])
+ def test_the_response_carries_the_column_name_choice(self, _name: str, preserve: bool) -> None:
+ source = AmazonS3Source()
+ location = parse_self_managed_url("https://exports.s3.us-east-1.amazonaws.com/orders/*.csv")
+ definition = table_definition_for("orders", location, "CSVWithNames", {}, None)
+ metadata = {**definition.schema_metadata(), "preserve_column_names": preserve}
+ config = source.parse_config(
+ {
+ "url_pattern": "https://s3.us-east-1.amazonaws.com/exports/",
+ "access_key_id": "AKIAEXAMPLE",
+ "secret_access_key": "c2VjcmV0",
+ }
+ )
+ inputs = SourceInputs(
+ schema_name="orders",
+ schema_id=str(uuid.uuid4()),
+ source_id=str(uuid.uuid4()),
+ team_id=1,
+ should_use_incremental_field=False,
+ db_incremental_field_last_value=None,
+ db_incremental_field_earliest_value=None,
+ incremental_field=None,
+ incremental_field_type=None,
+ job_id=str(uuid.uuid4()),
+ logger=LOGGER,
+ reset_pipeline=False,
+ schema_metadata=metadata,
+ )
+ source.open_store = MagicMock(return_value=MagicMock()) # type: ignore[method-assign]
+
+ response = source.source_for_pipeline(config, inputs)
+
+ assert response.preserve_column_names is preserve
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py
index 9dbcfeb38ccd..51b41a7ca431 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py
@@ -158,6 +158,9 @@ class SourceResponse:
"""Set by an incremental source that replaces rows by origin instead of merging on a primary key
(see `ReplaceScope`). The run loads as appends and needs no primary key. A first sync or a reset
ignores it, because the table starts empty."""
+ preserve_column_names: bool = False
+ """Write the column names as the source yields them instead of normalizing them to snake_case.
+ Set for a table whose existing queries name its columns as they were before the import."""
# Not frozen: nothing mutates it in place today, so freezing it is plausible, but every source
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
index db5cf46edca5..b7ac2bc3a608 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -26,6 +26,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
GCS_EXAMPLE_URL,
parse_gcs_url,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ SelfManagedLocation,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
)
@@ -47,6 +51,7 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfi
api_docs_url = "https://cloud.google.com/storage/docs/interoperability"
requires_team_feature_flag = FEATURE_FLAG
+ self_managed_service = "gcs"
@property
def source_type(self) -> ExternalDataSourceType:
@@ -68,6 +73,15 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfi
def discovery_options(self, config: GoogleCloudStorageSourceConfig) -> DiscoveryOptions:
return discovery_options_from(config)
+ def self_managed_connection_inputs(
+ self, location: SelfManagedLocation, credential: SelfManagedCredential
+ ) -> dict[str, Any]:
+ return {
+ "url_pattern": f"https://storage.googleapis.com/{location.bucket}/",
+ "access_key_id": credential.access_key,
+ "secret_access_key": credential.access_secret,
+ }
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
diff --git a/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py b/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py
index dda0a8478442..a775fb005ba9 100644
--- a/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py
+++ b/products/warehouse_sources/backend/tests/test_schema_discovery_reconcile.py
@@ -8,7 +8,9 @@ from django.test import SimpleTestCase
from parameterized import parameterized
from products.warehouse_sources.backend.models.external_data_schema import (
+ SCHEMA_PINNED_METADATA_KEY,
SCHEMA_RESOURCE_ID_METADATA_KEY,
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
ExternalDataSchema,
auto_enable_new_schemas,
schema_name_matches_auto_sync_patterns,
@@ -237,6 +239,61 @@ class TestSchemaDiscoveryReconcile(BaseTest):
assert enabled_unsynced.status == ExternalDataSchema.Status.COMPLETED
mock_pause.assert_called_once_with(str(enabled_unsynced.id))
+ @parameterized.expand(
+ [
+ ("pinned_enabled_keeps_syncing", True, True, False),
+ ("pinned_off_with_table_stays", True, False, False),
+ ("unpinned_enabled_is_disabled", False, True, True),
+ ("non_boolean_value_is_not_pinned", "yes", True, True),
+ ]
+ )
+ def test_discovery_leaves_pinned_schemas_alone(
+ self, _name: str, pinned: object, should_sync: bool, expect_disabled: bool
+ ) -> None:
+ source = self._make_source()
+ schema = self._make_synced_schema(source, "orders_raw")
+ metadata = {"kind": "files", "scope": "glob", "pattern": "orders/*.csv", SCHEMA_PINNED_METADATA_KEY: pinned}
+ schema.should_sync = should_sync
+ schema.sync_type_config = {"schema_metadata": metadata}
+ schema.save()
+
+ with patch(_PAUSE_FN) as mock_pause, self.captureOnCommitCallbacks(execute=True):
+ sync_result = sync_old_schemas_with_new_schemas(
+ {"orders": None},
+ source_id=str(source.pk),
+ team_id=self.team.pk,
+ )
+
+ schema.refresh_from_db()
+ assert schema.deleted is False
+ assert sync_result.deleted == []
+ assert schema.should_sync is (should_sync and not expect_disabled)
+ assert mock_pause.called is expect_disabled
+
+ def test_discovery_does_not_revive_a_deleted_migration_schema(self) -> None:
+ source = self._make_source()
+ ref = {"table_id": str(uuid.uuid4()), "url_pattern": "https://b.s3.amazonaws.com/o/*.csv", "format": "CSV"}
+ tombstone = ExternalDataSchema.objects.create(
+ team_id=self.team.pk,
+ source_id=source.pk,
+ name="orders",
+ should_sync=False,
+ deleted=True,
+ sync_type_config={
+ "schema_metadata": {SCHEMA_PINNED_METADATA_KEY: True, SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: ref}
+ },
+ )
+
+ sync_old_schemas_with_new_schemas({"orders": None}, source_id=str(source.pk), team_id=self.team.pk)
+
+ tombstone.refresh_from_db()
+ assert tombstone.deleted is True
+ revived = ExternalDataSchema.objects.get(
+ team_id=self.team.pk, source_id=source.pk, name="orders", deleted=False
+ )
+ assert revived.pk != tombstone.pk
+ assert SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY not in (revived.sync_type_config or {}).get("schema_metadata", {})
+
def test_failed_pause_leaves_the_table_on_and_still_pauses_the_others(self) -> None:
# A row written off while its schedule still runs keeps billing, and the next discovery run
# would skip it as already off, so a failed pause has to leave the row on for that retry.
diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
new file mode 100644
index 000000000000..491ea7f074ac
--- /dev/null
+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
@@ -0,0 +1,666 @@
+from datetime import UTC, datetime
+from io import StringIO
+from typing import Any
+
+from posthog.test.base import APIBaseTest, BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+
+from parameterized import parameterized
+from temporalio.service import RPCError, RPCStatusCode
+
+from posthog.hogql.database.database import Database
+
+from posthog.models.organization import OrganizationMembership
+from posthog.models.team import Team
+
+from products.access_control.backend.models.access_control import AccessControl
+from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
+from products.warehouse_sources.backend.models.external_data_destination import ExternalDataSourceDestination
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import (
+ SCHEMA_PINNED_METADATA_KEY,
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
+ ExternalDataSchema,
+ self_managed_table_ref,
+)
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.self_managed_migration import (
+ MIGRATION_MARKER_KEY,
+ MigrationOptions,
+ TableMigrationState,
+ abandon_tables,
+ evaluate_tables,
+ migrate_self_managed_tables,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store import source_base
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.self_managed import (
+ SelfManagedCredential,
+ parse_self_managed_url,
+ table_definition_for,
+)
+from products.warehouse_sources.backend.types import DataWarehouseTableFormat, ExternalDataJobStatus
+
+MODULE = "products.warehouse_sources.backend.self_managed_migration"
+SOURCE_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_source"
+SCHEMA_VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_schema"
+EXPORTS_URL = "https://exports.s3.eu-west-1.amazonaws.com/orders/*.csv"
+
+
+def _no_region(_bucket: str, _credential: SelfManagedCredential) -> str | None:
+ raise AssertionError("every URL in this test names its region")
+
+
+def _schemas_for(table: DataWarehouseTable) -> list[ExternalDataSchema]:
+ return [
+ schema
+ for schema in ExternalDataSchema.objects.select_related("source").filter(team_id=table.team_id)
+ if (ref := self_managed_table_ref(schema)) is not None and ref.table_id == str(table.id)
+ ]
+
+
+def _schema_for(table: DataWarehouseTable) -> ExternalDataSchema:
+ [schema] = [schema for schema in _schemas_for(table) if not schema.deleted]
+ return schema
+
+
+class _MigrationTest(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ patches = [
+ patch(f"{MODULE}.source_feature_flag_missing", return_value=False),
+ patch(f"{MODULE}.delete_external_data_schedule"),
+ # The location check resolves the bucket host, and these example buckets don't exist.
+ patch.object(source_base, "validate_warehouse_table_url_pattern", return_value=(True, "")),
+ ]
+ mocks = [p.start() for p in patches]
+ for p in patches:
+ self.addCleanup(p.stop)
+ self.flag_missing, self.delete_schedule, _ = mocks
+ self.credential = self._credential("AKIAFIRST")
+
+ def _credential(self, access_key: str) -> DataWarehouseCredential:
+ return DataWarehouseCredential.objects.create(access_key=access_key, access_secret="c2VjcmV0", team=self.team)
+
+ def _table(
+ self,
+ name: str = "orders",
+ url: str = EXPORTS_URL,
+ credential: DataWarehouseCredential | None = None,
+ table_format: str = DataWarehouseTableFormat.CSVWithNames,
+ created_at: datetime | None = None,
+ **fields: Any,
+ ) -> DataWarehouseTable:
+ table = DataWarehouseTable.objects.create(
+ name=name,
+ format=table_format,
+ url_pattern=url,
+ team=self.team,
+ credential=credential or self.credential,
+ columns={"Order ID": {"clickhouse": "Nullable(Int64)", "hogql": "IntegerDatabaseField", "valid": True}},
+ options={"csv_allow_double_quotes": True},
+ **fields,
+ )
+ if created_at is not None:
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(created_at=created_at)
+ table.refresh_from_db()
+ return table
+
+ def _migrate(self, **options: Any) -> list[Any]:
+ resolve_region = options.pop("resolve_region", _no_region)
+ live_run = options.pop("live_run", True)
+ return migrate_self_managed_tables(
+ MigrationOptions(team_id=self.team.pk, live_run=live_run, **options), resolve_region=resolve_region
+ )
+
+ def _abandon(self, table: DataWarehouseTable, **options: Any) -> Any:
+ options.setdefault("live_run", True)
+ [outcome] = abandon_tables([str(table.id)], resolve_region=_no_region, **options)
+ return outcome
+
+ def _status(self, table: DataWarehouseTable) -> Any:
+ [status] = evaluate_tables(team_id=self.team.pk, table_ids=[str(table.id)], resolve_region=_no_region)
+ return status
+
+ def _job(self, table: DataWarehouseTable, status: str, latest_error: str | None = None) -> ExternalDataJob:
+ schema = _schema_for(table)
+ return ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=status, latest_error=latest_error
+ )
+
+ def _adopt(self, table: DataWarehouseTable) -> None:
+ schema = _schema_for(table)
+ table.refresh_from_db()
+ table.adopt_into_source(schema.source)
+ table.url_pattern = f"https://bucket.s3.amazonaws.com/team_{self.team.pk}/{table.name}"
+ table.format = DataWarehouseTableFormat.DeltaS3Wrapper
+ table.save(internally_computed_url_pattern=True)
+ schema.table = table
+ schema.save()
+
+
+class TestSelfManagedMigration(_MigrationTest):
+ def test_tables_group_into_one_source_per_credential_and_bucket(self) -> None:
+ other_credential = self._credential("AKIASECOND")
+ virtual_hosted = self._table("orders")
+ path_style = self._table("refunds", url="https://s3.eu-west-1.amazonaws.com/exports/refunds/*.csv")
+ other_bucket = self._table("archive", url="https://archive.s3.eu-west-1.amazonaws.com/a.parquet")
+ other_key = self._table("mirror", credential=other_credential)
+
+ self._migrate()
+
+ def source_of(table: DataWarehouseTable) -> ExternalDataSource:
+ return _schema_for(table).source
+
+ assert source_of(virtual_hosted).pk == source_of(path_style).pk
+ assert len({source_of(t).pk for t in (virtual_hosted, other_bucket, other_key)}) == 3
+ assert ExternalDataSource.objects.filter(team=self.team).count() == 3
+ # Two sources of one type can't both go without a prefix, or their table names would clash.
+ prefixes = [source_of(t).prefix or "" for t in (virtual_hosted, other_bucket, other_key)]
+ assert prefixes.count("") == 1 and len(set(prefixes)) == 3
+ assert set(prefixes) - {""} <= {"exports", "archive"}
+ assert source_of(other_bucket).job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/archive/"
+ assert source_of(other_key).job_inputs["access_key_id"] == "AKIASECOND"
+ assert (source_of(virtual_hosted).connection_metadata or {})[MIGRATION_MARKER_KEY] == {
+ "group_key": "s3/eu-west-1/exports",
+ "credential_id": str(self.credential.pk),
+ }
+
+ def test_a_live_run_creates_a_turned_off_schema_and_changes_nothing_else(self) -> None:
+ created_at = datetime(2024, 3, 1, tzinfo=UTC)
+ table = self._table(created_at=created_at)
+ before = DataWarehouseTable.raw_objects.values().get(pk=table.pk)
+
+ [outcome] = self._migrate()
+
+ schema = _schema_for(table)
+ expected_contract = table_definition_for(
+ "orders", parse_self_managed_url(EXPORTS_URL), "CSVWithNames", table.columns, True
+ ).schema_metadata()
+ assert (outcome.action, outcome.status.state) == ("linked", TableMigrationState.AWAITING_SYNC)
+ assert (schema.name, schema.should_sync, schema.sync_type, schema.table_id) == (
+ "orders",
+ False,
+ "full_refresh",
+ None,
+ )
+ assert schema.sync_type_config == {
+ "schema_metadata": {
+ **expected_contract,
+ SCHEMA_PINNED_METADATA_KEY: True,
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY: {
+ "table_id": str(table.id),
+ "url_pattern": EXPORTS_URL,
+ "format": "CSVWithNames",
+ },
+ }
+ }
+ assert expected_contract["preserve_column_names"] is True
+ assert schema.source.source_type == "AmazonS3"
+ # A normal new source: its date is not moved to the table's.
+ assert schema.source.created_at > created_at
+ assert schema.source.job_inputs["url_pattern"] == "https://s3.eu-west-1.amazonaws.com/exports/"
+ assert ExternalDataSourceDestination.objects.for_team(self.team.pk).filter(source=schema.source).exists()
+ assert not ExternalDataJob.objects.filter(team=self.team).exists()
+ assert DataWarehouseTable.raw_objects.values().get(pk=table.pk) == before
+
+ def test_a_dry_run_writes_nothing(self) -> None:
+ self._table()
+
+ [outcome] = self._migrate(live_run=False)
+
+ assert (outcome.action, outcome.status.state) == ("would_link", TableMigrationState.PENDING)
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()
+
+ def test_running_again_changes_nothing(self) -> None:
+ table = self._table()
+ self._migrate()
+
+ [again] = self._migrate()
+
+ assert (again.action, again.status.state) == ("none", TableMigrationState.AWAITING_SYNC)
+ assert again.status.schema_id == str(_schema_for(table).id)
+ assert ExternalDataSchema.objects.filter(team=self.team).count() == 1
+ assert ExternalDataSource.objects.filter(team=self.team).count() == 1
+
+ @parameterized.expand(
+ [
+ ("syncing_off", False, None, None, {}, "syncing is off"),
+ ("syncing_on", True, None, None, {}, "syncing is on; waiting for the first sync"),
+ ("first_sync_running", True, ExternalDataJobStatus.RUNNING, None, {}, "the first sync is running"),
+ (
+ "first_sync_failed",
+ True,
+ ExternalDataJobStatus.FAILED,
+ "The bucket doesn't exist",
+ {},
+ "The bucket doesn't exist; the table still reads its own bucket",
+ ),
+ (
+ "table_edited",
+ False,
+ None,
+ None,
+ {"url_pattern": "https://exports.s3.eu-west-1.amazonaws.com/new/*.csv"},
+ "the table changed after the migration, so a sync makes a separate table",
+ ),
+ ]
+ )
+ def test_a_linked_table_reports_why_it_still_reads_its_own_bucket(
+ self,
+ _name: str,
+ should_sync: bool,
+ job_status: str | None,
+ latest_error: str | None,
+ table_change: dict[str, Any],
+ reason: str,
+ ) -> None:
+ table = self._table()
+ self._migrate()
+ ExternalDataSchema.objects.filter(pk=_schema_for(table).pk).update(should_sync=should_sync)
+ if job_status is not None:
+ self._job(table, job_status, latest_error=latest_error)
+ if table_change:
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(**table_change)
+
+ status = self._status(table)
+
+ assert (status.state, status.reason) == (TableMigrationState.AWAITING_SYNC, reason)
+ table.refresh_from_db()
+ assert (table.external_data_source_id, table.credential_id) == (None, self.credential.pk)
+
+ def test_a_schema_that_synced_a_separate_table_is_reported(self) -> None:
+ table = self._table()
+ self._migrate()
+ schema = _schema_for(table)
+ schema.table = DataWarehouseTable.objects.create(
+ name="amazons3_orders", format="Parquet", team=self.team, external_data_source=schema.source
+ )
+ schema.save()
+
+ status = self._status(table)
+
+ assert (status.state, status.reason) == (
+ TableMigrationState.AWAITING_SYNC,
+ "the table changed after the migration, so its schema syncs a separate table",
+ )
+
+ def test_a_migrated_table_is_never_linked_again(self) -> None:
+ table = self._table()
+ self._migrate()
+ self._adopt(table)
+
+ [outcome] = self._migrate()
+
+ assert (outcome.action, outcome.status.state) == ("none", TableMigrationState.MIGRATED)
+ assert ExternalDataSchema.objects.filter(team=self.team).count() == 1
+
+ @parameterized.expand(
+ [
+ ("unknown_host", {"url": "https://minio.example.com/exports/a.csv"}, "isn't a storage host"),
+ ("unsupported_format", {"table_format": "TabSeparated"}, "has no import equivalent"),
+ ("delta_with_glob", {"table_format": "Delta", "url": EXPORTS_URL}, "can't hold a file pattern"),
+ (
+ "azure_key_for_another_account",
+ {"url": "https://examplestore.blob.core.windows.net/exports/a.csv"},
+ "isn't the storage account name",
+ ),
+ ]
+ )
+ def test_a_table_that_cant_be_mapped_is_skipped_with_its_reason(
+ self, _name: str, fields: dict[str, Any], reason: str
+ ) -> None:
+ self._table(**fields)
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert reason in (outcome.status.reason or "")
+ assert outcome.action == "none"
+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()
+
+ def test_teams_without_the_flag_are_skipped(self) -> None:
+ self.flag_missing.return_value = True
+ self._table()
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert "dwh-object-storage-sources" in (outcome.status.reason or "")
+ assert not ExternalDataSchema.objects.filter(team=self.team).exists()
+
+ @parameterized.expand(
+ [
+ ("object_rule_restricts", True, "none", TableMigrationState.SKIPPED),
+ ("object_rule_grants", True, "editor", TableMigrationState.SKIPPED),
+ ("rule_on_another_table", False, "none", TableMigrationState.AWAITING_SYNC),
+ ]
+ )
+ def test_a_table_with_its_own_access_rules_is_skipped(
+ self, _name: str, on_this_table: bool, access_level: str, expected: TableMigrationState
+ ) -> None:
+ table = self._table()
+ other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")
+ AccessControl.objects.create(
+ team=self.team,
+ resource="warehouse_table",
+ resource_id=str(table.id if on_this_table else other.id),
+ access_level=access_level,
+ organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),
+ )
+
+ outcome = next(o for o in self._migrate() if o.status.table_id == str(table.id))
+
+ assert outcome.status.state == expected
+ if expected == TableMigrationState.SKIPPED:
+ assert "own access rules" in (outcome.status.reason or "")
+ assert _schemas_for(table) == []
+
+ @parameterized.expand(
+ [
+ ("member_rule_on_sources", "external_data_source", True),
+ ("default_rule_on_sources", "external_data_source", False),
+ ("member_rule_on_the_warehouse", "warehouse_objects", True),
+ ("default_rule_on_the_warehouse", "warehouse_objects", False),
+ ]
+ )
+ def test_a_project_with_access_rules_for_sources_is_skipped(
+ self, _name: str, resource: str, for_member: bool
+ ) -> None:
+ table = self._table()
+ AccessControl.objects.create(
+ team=self.team,
+ resource=resource,
+ resource_id=None,
+ access_level="none",
+ organization_member=(
+ OrganizationMembership.objects.get(user=self.user, organization=self.organization)
+ if for_member
+ else None
+ ),
+ )
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.table_id == str(table.id)
+ assert outcome.status.state == TableMigrationState.SKIPPED
+ assert "access rules for sources" in (outcome.status.reason or "")
+ assert _schemas_for(table) == []
+
+ def test_status_reports_every_state(self) -> None:
+ pending = self._table("pending_table")
+ self._table("skipped_table", url="https://minio.example.com/exports/a.csv")
+ awaiting = self._table("awaiting_table", url="https://exports.s3.eu-west-1.amazonaws.com/w/*.csv")
+ migrated = self._table("migrated_table", url="https://exports.s3.eu-west-1.amazonaws.com/m/*.csv")
+ abandoned = self._table("abandoned_table", url="https://exports.s3.eu-west-1.amazonaws.com/a/*.csv")
+ self._migrate(table_ids=tuple(str(t.id) for t in (awaiting, migrated, abandoned)))
+ self._adopt(migrated)
+ self._abandon(abandoned, reason="customer asked to keep it")
+
+ statuses = evaluate_tables(team_id=self.team.pk, resolve_region=_no_region)
+
+ assert {status.name: status.state for status in statuses} == {
+ pending.name: TableMigrationState.PENDING,
+ "skipped_table": TableMigrationState.SKIPPED,
+ awaiting.name: TableMigrationState.AWAITING_SYNC,
+ migrated.name: TableMigrationState.MIGRATED,
+ abandoned.name: TableMigrationState.ABANDONED,
+ }
+ assert next(s for s in statuses if s.name == abandoned.name).reason == "customer asked to keep it"
+
+ def test_selection_ignores_synced_deleted_and_other_teams_tables(self) -> None:
+ self._table("deleted_table", deleted=True)
+ self._table("no_credential", credential=None)
+ DataWarehouseTable.raw_objects.filter(name="no_credential").update(credential=None)
+ source = ExternalDataSource.objects.create(team=self.team, source_type="Postgres", job_inputs={})
+ self._table("synced_table", external_data_source=source)
+ other_team = Team.objects.create(organization=self.organization, name="Other team")
+ DataWarehouseTable.objects.create(
+ name="other_team_table", format="CSVWithNames", url_pattern=EXPORTS_URL, team=other_team
+ )
+ kept = self._table("kept")
+
+ assert [status.table_id for status in evaluate_tables(team_id=self.team.pk)] == [str(kept.id)]
+
+ def test_limit_and_table_ids_bound_a_run(self) -> None:
+ first = self._table("first", created_at=datetime(2024, 1, 1, tzinfo=UTC))
+ second = self._table(
+ "second",
+ url="https://exports.s3.eu-west-1.amazonaws.com/b/*.csv",
+ created_at=datetime(2024, 1, 2, tzinfo=UTC),
+ )
+ third = self._table("third", url="https://exports.s3.eu-west-1.amazonaws.com/c/*.csv")
+
+ outcomes = self._migrate(limit=1, table_ids=(str(second.id), str(third.id)))
+
+ assert [(o.status.name, o.action) for o in outcomes] == [("second", "linked"), ("third", "limit_reached")]
+ assert _schemas_for(first) == []
+
+ def test_a_name_already_taken_in_the_bucket_fails_the_link_of_that_table_only(self) -> None:
+ self._table("orders")
+ self._migrate()
+ source = ExternalDataSource.objects.get(team=self.team)
+ ExternalDataSchema.objects.create(name="dupe", team=self.team, source=source)
+ self._table("dupe", url="https://exports.s3.eu-west-1.amazonaws.com/d/*.csv")
+ self._table("fine", url="https://exports.s3.eu-west-1.amazonaws.com/f/*.csv")
+
+ outcomes = self._migrate()
+
+ assert [(o.status.name, o.action) for o in outcomes][1:] == [("dupe", "link_failed"), ("fine", "linked")]
+ assert "already uses the name 'dupe'" in (outcomes[1].error or "")
+
+ @parameterized.expand([("named_region", "eu-west-1", "linked"), ("unknown_region", None, "none")])
+ def test_a_bucket_url_without_a_region_asks_s3_once_per_bucket(
+ self, _name: str, region: str | None, expected_action: str
+ ) -> None:
+ self._table("a", url="https://exports.s3.amazonaws.com/a/*.csv")
+ self._table("b", url="https://s3.amazonaws.com/exports/b/*.csv")
+ resolver = MagicMock(return_value=region)
+
+ outcomes = self._migrate(resolve_region=resolver)
+
+ assert [o.action for o in outcomes] == [expected_action] * 2
+ resolver.assert_called_once()
+ if region is not None:
+ job_inputs = ExternalDataSource.objects.get(team=self.team).job_inputs
+ assert job_inputs["url_pattern"] == f"https://s3.{region}.amazonaws.com/exports/"
+
+ def test_an_adopted_table_is_queried_by_its_bare_name_before_and_after(self) -> None:
+ table = self._table("orders_raw")
+ self._migrate()
+
+ before = Database.create_for(team=self.team)
+ assert before.has_table("orders_raw")
+ assert not before.has_table("amazons3.orders_raw")
+
+ self._adopt(table)
+
+ after = Database.create_for(team=self.team)
+ assert after.has_table("orders_raw")
+ assert after.has_table("amazons3.orders_raw")
+
+
+class TestAbandon(_MigrationTest):
+ def test_abandon_removes_the_schema_and_keeps_the_table(self) -> None:
+ table = self._table()
+ self._migrate()
+ self._job(table, ExternalDataJobStatus.FAILED)
+ schema = _schema_for(table)
+
+ outcome = self._abandon(table, reason="customer asked to keep it")
+
+ assert outcome.action == "abandoned"
+ schema.refresh_from_db()
+ assert schema.deleted is True
+ ref = self_managed_table_ref(schema)
+ assert ref is not None and ref.abandon_reason == "customer asked to keep it"
+ assert ExternalDataSource.objects.get(pk=schema.source_id).deleted is True
+ self.delete_schedule.assert_called_once_with(str(schema.id))
+ table.refresh_from_db()
+ assert (table.deleted, table.credential_id, table.url_pattern) == (False, self.credential.pk, EXPORTS_URL)
+ [again] = self._migrate()
+ assert (again.action, again.status.state) == ("none", TableMigrationState.ABANDONED)
+ assert ExternalDataSchema.objects.filter(team=self.team, deleted=False).count() == 0
+
+ def test_abandon_is_kept_when_the_schedule_delete_fails(self) -> None:
+ table = self._table()
+ self._migrate()
+ self.delete_schedule.side_effect = RPCError("unavailable", RPCStatusCode.UNAVAILABLE, b"")
+
+ assert self._abandon(table).action == "abandoned"
+ assert self._status(table).state == TableMigrationState.ABANDONED
+
+ def test_abandon_keeps_a_source_that_holds_other_tables(self) -> None:
+ table = self._table()
+ other = self._table("other", url="https://exports.s3.eu-west-1.amazonaws.com/o/*.csv")
+ self._migrate()
+
+ self._abandon(table)
+
+ assert _schema_for(other).source.deleted is False
+
+ def test_abandon_of_a_table_that_was_never_linked_records_it_and_removes_the_new_source(self) -> None:
+ table = self._table()
+
+ outcome = self._abandon(table, reason="not wanted")
+
+ assert outcome.action == "abandoned"
+ [schema] = _schemas_for(table)
+ assert (schema.deleted, schema.source.deleted) == (True, True)
+ self.delete_schedule.assert_not_called()
+ assert (self._status(table).state, self._status(table).reason) == (TableMigrationState.ABANDONED, "not wanted")
+ assert self._migrate()[0].action == "none"
+
+ @parameterized.expand(
+ [
+ ("migrated", "the table is already migrated"),
+ ("abandoned", "the table's migration is already abandoned"),
+ ("syncing_on", "the user turned syncing on for the table's schema"),
+ ("running", "the table's first sync is running"),
+ ("separate_table", "the table's schema syncs a separate table"),
+ ]
+ )
+ def test_abandon_refuses_a_table_it_must_not_touch(self, case: str, reason: str) -> None:
+ table = self._table()
+ self._migrate()
+ schema = _schema_for(table)
+ if case == "migrated":
+ self._adopt(table)
+ elif case == "abandoned":
+ self._abandon(table)
+ elif case == "syncing_on":
+ ExternalDataSchema.objects.filter(pk=schema.pk).update(should_sync=True)
+ elif case == "running":
+ self._job(table, ExternalDataJobStatus.RUNNING)
+ else:
+ schema.table = DataWarehouseTable.objects.create(
+ name="amazons3_orders", format="Parquet", team=self.team, external_data_source=schema.source
+ )
+ schema.save()
+ deleted_before = ExternalDataSchema.objects.get(pk=schema.pk).deleted
+
+ outcome = self._abandon(table)
+
+ assert (outcome.action, outcome.reason) == ("refused", reason)
+ assert ExternalDataSchema.objects.get(pk=schema.pk).deleted is deleted_before
+
+ def test_abandon_refuses_a_table_that_cant_be_migrated(self) -> None:
+ table = self._table(url="https://minio.example.com/exports/a.csv")
+
+ outcome = self._abandon(table)
+
+ assert outcome.action == "refused"
+ assert "can't be migrated" in (outcome.reason or "")
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+
+ def test_a_dry_run_abandon_writes_nothing(self) -> None:
+ table = self._table()
+ self._migrate()
+
+ outcomes = abandon_tables(
+ [str(table.id), "00000000-0000-0000-0000-000000000000"], resolve_region=_no_region, live_run=False
+ )
+
+ assert [o.action for o in outcomes] == ["would_abandon", "refused"]
+ assert _schema_for(table).deleted is False
+ self.delete_schedule.assert_not_called()
+
+
+class TestMigrateSelfManagedTablesCommand(BaseTest):
+ @parameterized.expand(
+ [
+ ("zero_limit", {"limit": 0}, "--limit"),
+ ("status_and_abandon", {"status": True, "abandon": "x"}, "only one"),
+ ("empty_abandon", {"abandon": " , "}, "at least one table id"),
+ ]
+ )
+ def test_rejects_invalid_options(self, _name: str, options: dict[str, Any], message: str) -> None:
+ with self.assertRaisesRegex(CommandError, message):
+ call_command("migrate_self_managed_tables", **options)
+
+ def test_status_and_dry_run_report_each_table_and_change_nothing(self) -> None:
+ DataWarehouseTable.objects.create(
+ name="orders",
+ format="CSVWithNames",
+ url_pattern="https://minio.example.com/exports/a.csv",
+ team=self.team,
+ credential=DataWarehouseCredential.objects.create(access_key="k", access_secret="s", team=self.team),
+ )
+
+ for options in ({"status": True}, {}):
+ out = StringIO()
+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=out, **options)
+ assert "name=orders state=skipped" in out.getvalue()
+
+ assert not ExternalDataSource.objects.filter(team=self.team).exists()
+
+
+class TestDeletingAMigrationSchema(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ patches = [
+ patch(f"{MODULE}.source_feature_flag_missing", return_value=False),
+ patch.object(source_base, "validate_warehouse_table_url_pattern", return_value=(True, "")),
+ patch(f"{SOURCE_VIEWS}.base.bulk_delete_external_data_schedules", return_value=[]),
+ patch(f"{SOURCE_VIEWS}.base.delete_external_data_schedule"),
+ patch(f"{SOURCE_VIEWS}.base.delete_discover_schemas_schedule"),
+ patch(f"{SCHEMA_VIEWS}.external_data_workflow_exists", return_value=False),
+ patch("products.data_warehouse.backend.s3.get_s3_client"),
+ ]
+ for p in patches:
+ p.start()
+ self.addCleanup(p.stop)
+ self.credential = DataWarehouseCredential.objects.create(
+ access_key="AKIAFIRST", access_secret="c2VjcmV0", team=self.team
+ )
+
+ @parameterized.expand([("source",), ("schema",), ("schema_data",)])
+ def test_deleting_before_the_first_sync_keeps_the_table_and_ends_its_migration(self, path: str) -> None:
+ table = DataWarehouseTable.objects.create(
+ name="orders",
+ format=DataWarehouseTableFormat.CSVWithNames,
+ url_pattern=EXPORTS_URL,
+ team=self.team,
+ credential=self.credential,
+ )
+ migrate_self_managed_tables(MigrationOptions(team_id=self.team.pk, live_run=True), resolve_region=_no_region)
+ schema = _schema_for(table)
+ urls = {
+ "source": f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}",
+ "schema": f"/api/environments/{self.team.pk}/external_data_schemas/{schema.pk}",
+ "schema_data": f"/api/environments/{self.team.pk}/external_data_schemas/{schema.pk}/delete_data",
+ }
+
+ response = self.client.delete(urls[path])
+
+ assert response.status_code in (200, 204), response.content
+ table.refresh_from_db()
+ assert (table.deleted, table.credential_id, table.url_pattern) == (False, self.credential.pk, EXPORTS_URL)
+ [status] = evaluate_tables(team_id=self.team.pk, resolve_region=_no_region)
+ expected = TableMigrationState.AWAITING_SYNC if path == "schema_data" else TableMigrationState.ABANDONED
+ assert status.state == expected
+ assert Database.create_for(team=self.team).has_table("orders")