:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): bound long column names in table statistics (#130174)
Tom Owers9782af4cf48b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.pyindex 4ec8cb4eb3f5..9602bf20f643 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py@@ -51,7 +51,10 @@ from products.warehouse_sources.backend.models.external_data_job import Externalfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.models.util import clean_type-from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import (+ NamingConvention,+ fold_column_name,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (retry_on_operational_error,)@@ -71,6 +74,11 @@ MAX_INCREMENTAL_COMMITS = int(os.getenv("WAREHOUSE_STATS_MAX_INCREMENTAL_COMMITS# Files x columns cap for the Add-action scan: Arrow holds about 12 bytes per cell, so the default is ~300 MB.MAX_ADD_ACTION_CELLS = int(os.getenv("WAREHOUSE_STATS_MAX_ADD_ACTION_CELLS", "25000000"))+# Matches WarehouseColumnStatistics.column_name's max_length. Source column names aren't bounded this way+# (a deeply nested JSON schema can produce one far longer), so a name over the limit is trimmed with a+# content hash rather than failing the whole table's stats write.+COLUMN_NAME_DB_MAX_LENGTH = 400+# Product-analytics events — query these to track statistics volume, columns profiled, skips, and errors.EVENT_STARTED = "data warehouse table statistics started"EVENT_COMPLETED = "data warehouse table statistics completed"@@ -839,7 +847,7 @@ def _upsert_statistics(defaults["full_scan_at"] = timezone.now()WarehouseColumnStatistics.objects.for_team(team.id).update_or_create(table=table,- column_name=column_name,+ column_name=NamingConvention.normalize_identifier(column_name, max_length=COLUMN_NAME_DB_MAX_LENGTH),defaults=defaults,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.pyindex 89509a3ac378..dc88f3c6cefd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py@@ -773,6 +773,28 @@ class TestComputeTableStatisticsSync:assert stat.computed_for_delta_version == 12assert stat.column_type == "Int64"+ def test_truncates_a_column_name_too_long_for_the_db(self) -> None:+ # A deeply nested source schema can produce a column name longer than the 400-char limit+ # on WarehouseColumnStatistics.column_name; the whole table's stats write must not fail for it.+ long_name = "deeply_nested_field_" * 25+ team = self._team()+ schema, table, _ = self._schema_table_job(team, columns={long_name: {"clickhouse": "Nullable(Int64)"}})+ add_actions = pa.table(+ {+ "num_records": [10],+ f"null_count.{long_name}": [1],+ f"min.{long_name}": [2],+ f"max.{long_name}": [9],+ }+ )+ with patch(DELTA_HELPER_PATH, return_value=self._mock_delta(add_actions)):+ result = compute_table_statistics_sync(team.id, schema.id)++ assert result["status"] == "done"+ stat = WarehouseColumnStatistics.objects.for_team(team.id).get(table_id=table.id)+ assert len(stat.column_name) == comp.COLUMN_NAME_DB_MAX_LENGTH+ assert stat.min_value == "2"+@pytest.mark.parametrize("saved_during",[