:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): migrate self-managed tables with nested columns as json text (#127374)
Tom Owersc62f903a5476
diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.pyindex 2e98e718b70e..9312ae8a8e6b 100644--- a/products/warehouse_sources/backend/self_managed_migration.py+++ b/products/warehouse_sources/backend/self_managed_migration.py@@ -84,6 +84,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objStorageService,UnmappableTableError,needs_preserved_column_names,+ nested_text_columns,parse_self_managed_url,refuse_columns_an_import_changes,table_definition_for,@@ -138,6 +139,7 @@ class TableMigrationStatus:reason: str | None = Nonesource_id: str | None = Noneschema_id: str | None = None+ nested_text_column_types: dict[str, str] | None = None@frozen@@ -497,6 +499,11 @@ def _evaluate(reason=reason,source_id=str(named.source_id) if named is not None else None,schema_id=str(named.id) if named is not None else None,+ nested_text_column_types=(+ nested_text_columns(table.columns)+ if state in (TableMigrationState.PENDING, TableMigrationState.AWAITING_SYNC)+ else None+ ),)if table.external_data_source_id is not None:@@ -799,4 +806,9 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:def describe_status(status: TableMigrationStatus) -> str:reason = f" ({status.reason})" if status.reason else ""- return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}"+ columns = status.nested_text_column_types or {}+ note = ""+ if columns:+ changed = ", ".join(f"{name} ({column_type})" for name, column_type in columns.items())+ note = f" note=(after the first sync these columns are stored as JSON text: {changed})"+ return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}{note}"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.pyindex b8e6e16ff125..4e5dc81c356e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py@@ -64,7 +64,7 @@ _FORMATS: dict[str, FileFormatChoice] = {"Delta": "delta","DeltaS3Wrapper": "delta",}-# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts.+# Column types the import pipeline stores as JSON text._NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})# Columns the import adds to every row. A stored column of the same name would lose its values to them._FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})@@ -168,13 +168,13 @@ def needs_preserved_column_names(column_names: Iterable[str]) -> bool:return any(normalize_column_name(name) != name for name in column_names)-def _refuse_nested_columns(columns: Mapping[str, Any] | None) -> None:- for name, clickhouse_type in stored_clickhouse_types(columns).items():- base = clickhouse_base_type(clickhouse_type)- if base in _NESTED_TYPES:- raise UnmappableTableError(- f"the column '{name}' is a {base}, and the import would store it as text, not as a {base}"- )+def nested_text_columns(columns: Mapping[str, Any] | None) -> dict[str, str]:+ """Return columns whose nested types the import stores as JSON text, with their ClickHouse types."""+ return {+ name: clickhouse_type+ for name, clickhouse_type in stored_clickhouse_types(columns).items()+ if clickhouse_base_type(clickhouse_type) in _NESTED_TYPES+ }def _refuse_file_column_names(columns: Mapping[str, Any] | None) -> None:@@ -195,8 +195,7 @@ def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:def refuse_columns_an_import_changes(columns: Mapping[str, Any] | None, file_format: FileFormatChoice) -> None:- """Refuse a file whose import would change the columns a table read in place exposes."""- _refuse_nested_columns(columns)+ """Refuse column names that the import would change or replace."""_refuse_file_column_names(columns)if file_format == "csv":_refuse_padded_header_names(columns)@@ -211,7 +210,6 @@ def table_definition_for() -> TableDefinition:"""The schema contract that imports exactly the files the table's URL pattern reads."""file_format = source_format_for(table_format)- _refuse_nested_columns(columns)_refuse_file_column_names(columns)preserve = needs_preserved_column_names((columns or {}).keys())if file_format == "delta":diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.pyindex 108ca33d6b37..0778d8029bdb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py@@ -58,6 +58,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objSelfManagedLocation,UnmappableTableError,needs_preserved_column_names,+ nested_text_columns,parse_self_managed_url,refuse_columns_an_import_changes,source_format_for,@@ -502,11 +503,26 @@ class TestTableDefinition:)])- def test_nested_columns_are_refused(self, _name: str, table_format: str, column_type: str, url: str) -> None:+ def test_nested_columns_can_be_linked(self, _name: str, table_format: str, column_type: str, url: str) -> None:columns = {"id": "Int64", "items": {"clickhouse": column_type, "hogql": "StringJSONDatabaseField"}}- with pytest.raises(UnmappableTableError, match="'items'"):- table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)+ definition = table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)++ assert definition.kind == ("delta" if table_format == "Delta" else "files")++ @parameterized.expand(+ [+ ("array", {"items": "Array(String)"}, {"items": "Array(String)"}),+ ("nullable_array", {"items": "Nullable(Array(String))"}, {"items": "Nullable(Array(String))"}),+ ("map", {"items": "Map(String, Int64)"}, {"items": "Map(String, Int64)"}),+ ("tuple", {"items": "Tuple(Int64, String)"}, {"items": "Tuple(Int64, String)"}),+ ("plain_string", {"items": "String"}, {}),+ ("no_columns", None, {}),+ ("invalid_column", {"items": {"clickhouse": "Array(String)", "valid": False}}, {}),+ ]+ )+ def test_nested_text_columns(self, _name: str, columns: dict[str, Any] | None, expected: dict[str, str]) -> None:+ assert nested_text_columns(columns) == expecteddef test_an_invalid_nested_column_does_not_block_the_table(self) -> None:location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.parquet")@@ -549,9 +565,14 @@ class TestTableDefinition:assert [column.name for column in definition.csv.column_types] == ["created"]- def test_json_nested_columns_are_still_refused(self) -> None:- with pytest.raises(UnmappableTableError, match="'tags'"):- refuse_columns_an_import_changes({"tags": "Array(String)", "day": "Date"}, "jsonl")+ def test_json_nested_columns_can_be_linked(self) -> None:+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")+ columns = {"tags": "Array(String)", "day": "Date"}++ refuse_columns_an_import_changes(columns, "jsonl")+ definition = table_definition_for("orders", location, "JSONEachRow", columns, None)++ assert [column.name for column in definition.csv.column_types] == ["day"]def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.pyindex e14aeb70ac6d..6565b45caf8f 100644--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py@@ -741,12 +741,6 @@ class TestUploadedTableMigration(_MigrationTest):[("csv_without_a_header", DataWarehouseTableFormat.CSV, None, "has no import equivalent"),("delta", DataWarehouseTableFormat.Delta, None, "has no import equivalent"),- (- "nested_columns",- DataWarehouseTableFormat.Parquet,- {"tags": {"clickhouse": "Array(Nullable(String))", "hogql": "StringArrayDatabaseField"}},- "'tags' is a Array",- ),("json_unknown_time_zone",DataWarehouseTableFormat.JSON,@@ -780,6 +774,33 @@ class TestUploadedTableMigration(_MigrationTest):assert reason in (outcome.status.reason or "")assert not ExternalDataSource.objects.filter(team=self.team).exists()+ def test_an_uploaded_table_with_nested_columns_is_linked_and_reports_the_type(self) -> None:+ table = self._uploaded("orders", "managed/team_{team}/orders.parquet", DataWarehouseTableFormat.Parquet)+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(+ columns={"tags": {"clickhouse": "Array(Nullable(String))", "hogql": "StringArrayDatabaseField"}}+ )++ dry_run = StringIO()+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=dry_run)++ note = "note=(after the first sync these columns are stored as JSON text: tags (Array(Nullable(String))))"+ dry_run_line = dry_run.getvalue().splitlines()[0]+ assert "state=pending (not linked yet)" in dry_run_line+ assert note in dry_run_line+ assert "action=would_link" in dry_run_line++ [outcome] = self._migrate()++ assert (outcome.action, outcome.status.state) == ("linked", TableMigrationState.AWAITING_SYNC)+ assert _schema_for(table).should_sync is False++ linked_status = StringIO()+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=linked_status)+ linked_line = linked_status.getvalue().splitlines()[0]+ assert "state=awaiting_sync" in linked_line+ assert "action=none" in linked_line+ assert note in linked_line+@parameterized.expand([("date", "Nullable(Date)", "date"),