Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): migrate self-managed tables with nested columns as json text (#127374)

Tom Owersc62f903a5476
diff --git a/products/warehouse_sources/backend/self_managed_migration.py b/products/warehouse_sources/backend/self_managed_migration.py
index 2e98e718b70e..9312ae8a8e6b 100644
--- a/products/warehouse_sources/backend/self_managed_migration.py
+++ b/products/warehouse_sources/backend/self_managed_migration.py
@@ -84,6 +84,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
StorageService,
UnmappableTableError,
needs_preserved_column_names,
+ nested_text_columns,
parse_self_managed_url,
refuse_columns_an_import_changes,
table_definition_for,
@@ -138,6 +139,7 @@ class TableMigrationStatus:
reason: str | None = None
source_id: str | None = None
schema_id: str | None = None
+ nested_text_column_types: dict[str, str] | None = None
@frozen
@@ -497,6 +499,11 @@ def _evaluate(
reason=reason,
source_id=str(named.source_id) if named is not None else None,
schema_id=str(named.id) if named is not None else None,
+ nested_text_column_types=(
+ nested_text_columns(table.columns)
+ if state in (TableMigrationState.PENDING, TableMigrationState.AWAITING_SYNC)
+ else None
+ ),
)
if table.external_data_source_id is not None:
@@ -799,4 +806,9 @@ def _abandon(evaluation: _Evaluation, reason: str) -> None:
def describe_status(status: TableMigrationStatus) -> str:
reason = f" ({status.reason})" if status.reason else ""
- return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}"
+ columns = status.nested_text_column_types or {}
+ note = ""
+ if columns:
+ changed = ", ".join(f"{name} ({column_type})" for name, column_type in columns.items())
+ note = f" note=(after the first sync these columns are stored as JSON text: {changed})"
+ return f"table={status.table_id} team={status.team_id} name={status.name} state={status.state}{reason}{note}"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
index b8e6e16ff125..4e5dc81c356e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
@@ -64,7 +64,7 @@ _FORMATS: dict[str, FileFormatChoice] = {
"Delta": "delta",
"DeltaS3Wrapper": "delta",
}
-# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts.
+# Column types the import pipeline stores as JSON text.
_NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})
# Columns the import adds to every row. A stored column of the same name would lose its values to them.
_FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})
@@ -168,13 +168,13 @@ def needs_preserved_column_names(column_names: Iterable[str]) -> bool:
return any(normalize_column_name(name) != name for name in column_names)
-def _refuse_nested_columns(columns: Mapping[str, Any] | None) -> None:
- for name, clickhouse_type in stored_clickhouse_types(columns).items():
- base = clickhouse_base_type(clickhouse_type)
- if base in _NESTED_TYPES:
- raise UnmappableTableError(
- f"the column '{name}' is a {base}, and the import would store it as text, not as a {base}"
- )
+def nested_text_columns(columns: Mapping[str, Any] | None) -> dict[str, str]:
+ """Return columns whose nested types the import stores as JSON text, with their ClickHouse types."""
+ return {
+ name: clickhouse_type
+ for name, clickhouse_type in stored_clickhouse_types(columns).items()
+ if clickhouse_base_type(clickhouse_type) in _NESTED_TYPES
+ }
def _refuse_file_column_names(columns: Mapping[str, Any] | None) -> None:
@@ -195,8 +195,7 @@ def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:
def refuse_columns_an_import_changes(columns: Mapping[str, Any] | None, file_format: FileFormatChoice) -> None:
- """Refuse a file whose import would change the columns a table read in place exposes."""
- _refuse_nested_columns(columns)
+ """Refuse column names that the import would change or replace."""
_refuse_file_column_names(columns)
if file_format == "csv":
_refuse_padded_header_names(columns)
@@ -211,7 +210,6 @@ def table_definition_for(
) -> TableDefinition:
"""The schema contract that imports exactly the files the table's URL pattern reads."""
file_format = source_format_for(table_format)
- _refuse_nested_columns(columns)
_refuse_file_column_names(columns)
preserve = needs_preserved_column_names((columns or {}).keys())
if file_format == "delta":
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
index 108ca33d6b37..0778d8029bdb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
@@ -58,6 +58,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
SelfManagedLocation,
UnmappableTableError,
needs_preserved_column_names,
+ nested_text_columns,
parse_self_managed_url,
refuse_columns_an_import_changes,
source_format_for,
@@ -502,11 +503,26 @@ class TestTableDefinition:
)
]
)
- def test_nested_columns_are_refused(self, _name: str, table_format: str, column_type: str, url: str) -> None:
+ def test_nested_columns_can_be_linked(self, _name: str, table_format: str, column_type: str, url: str) -> None:
columns = {"id": "Int64", "items": {"clickhouse": column_type, "hogql": "StringJSONDatabaseField"}}
- with pytest.raises(UnmappableTableError, match="'items'"):
- table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)
+ definition = table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)
+
+ assert definition.kind == ("delta" if table_format == "Delta" else "files")
+
+ @parameterized.expand(
+ [
+ ("array", {"items": "Array(String)"}, {"items": "Array(String)"}),
+ ("nullable_array", {"items": "Nullable(Array(String))"}, {"items": "Nullable(Array(String))"}),
+ ("map", {"items": "Map(String, Int64)"}, {"items": "Map(String, Int64)"}),
+ ("tuple", {"items": "Tuple(Int64, String)"}, {"items": "Tuple(Int64, String)"}),
+ ("plain_string", {"items": "String"}, {}),
+ ("no_columns", None, {}),
+ ("invalid_column", {"items": {"clickhouse": "Array(String)", "valid": False}}, {}),
+ ]
+ )
+ def test_nested_text_columns(self, _name: str, columns: dict[str, Any] | None, expected: dict[str, str]) -> None:
+ assert nested_text_columns(columns) == expected
def test_an_invalid_nested_column_does_not_block_the_table(self) -> None:
location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.parquet")
@@ -549,9 +565,14 @@ class TestTableDefinition:
assert [column.name for column in definition.csv.column_types] == ["created"]
- def test_json_nested_columns_are_still_refused(self) -> None:
- with pytest.raises(UnmappableTableError, match="'tags'"):
- refuse_columns_an_import_changes({"tags": "Array(String)", "day": "Date"}, "jsonl")
+ def test_json_nested_columns_can_be_linked(self) -> None:
+ location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")
+ columns = {"tags": "Array(String)", "day": "Date"}
+
+ refuse_columns_an_import_changes(columns, "jsonl")
+ definition = table_definition_for("orders", location, "JSONEachRow", columns, None)
+
+ assert [column.name for column in definition.csv.column_types] == ["day"]
def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:
location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")
diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
index e14aeb70ac6d..6565b45caf8f 100644
--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py
+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
@@ -741,12 +741,6 @@ class TestUploadedTableMigration(_MigrationTest):
[
("csv_without_a_header", DataWarehouseTableFormat.CSV, None, "has no import equivalent"),
("delta", DataWarehouseTableFormat.Delta, None, "has no import equivalent"),
- (
- "nested_columns",
- DataWarehouseTableFormat.Parquet,
- {"tags": {"clickhouse": "Array(Nullable(String))", "hogql": "StringArrayDatabaseField"}},
- "'tags' is a Array",
- ),
(
"json_unknown_time_zone",
DataWarehouseTableFormat.JSON,
@@ -780,6 +774,33 @@ class TestUploadedTableMigration(_MigrationTest):
assert reason in (outcome.status.reason or "")
assert not ExternalDataSource.objects.filter(team=self.team).exists()
+ def test_an_uploaded_table_with_nested_columns_is_linked_and_reports_the_type(self) -> None:
+ table = self._uploaded("orders", "managed/team_{team}/orders.parquet", DataWarehouseTableFormat.Parquet)
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(
+ columns={"tags": {"clickhouse": "Array(Nullable(String))", "hogql": "StringArrayDatabaseField"}}
+ )
+
+ dry_run = StringIO()
+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=dry_run)
+
+ note = "note=(after the first sync these columns are stored as JSON text: tags (Array(Nullable(String))))"
+ dry_run_line = dry_run.getvalue().splitlines()[0]
+ assert "state=pending (not linked yet)" in dry_run_line
+ assert note in dry_run_line
+ assert "action=would_link" in dry_run_line
+
+ [outcome] = self._migrate()
+
+ assert (outcome.action, outcome.status.state) == ("linked", TableMigrationState.AWAITING_SYNC)
+ assert _schema_for(table).should_sync is False
+
+ linked_status = StringIO()
+ call_command("migrate_self_managed_tables", team_id=self.team.pk, stdout=linked_status)
+ linked_line = linked_status.getvalue().splitlines()[0]
+ assert "state=awaiting_sync" in linked_line
+ assert "action=none" in linked_line
+ assert note in linked_line
+
@parameterized.expand(
[
("date", "Nullable(Date)", "date"),