Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): bound long column names in table statistics (#130174)

Tom Owers9782af4cf48b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
index 4ec8cb4eb3f5..9602bf20f643 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
@@ -51,7 +51,10 @@ from products.warehouse_sources.backend.models.external_data_job import External
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.models.util import clean_type
-from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import (
+ NamingConvention,
+ fold_column_name,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (
retry_on_operational_error,
)
@@ -71,6 +74,11 @@ MAX_INCREMENTAL_COMMITS = int(os.getenv("WAREHOUSE_STATS_MAX_INCREMENTAL_COMMITS
# Files x columns cap for the Add-action scan: Arrow holds about 12 bytes per cell, so the default is ~300 MB.
MAX_ADD_ACTION_CELLS = int(os.getenv("WAREHOUSE_STATS_MAX_ADD_ACTION_CELLS", "25000000"))
+# Matches WarehouseColumnStatistics.column_name's max_length. Source column names aren't bounded this way
+# (a deeply nested JSON schema can produce one far longer), so a name over the limit is trimmed with a
+# content hash rather than failing the whole table's stats write.
+COLUMN_NAME_DB_MAX_LENGTH = 400
+
# Product-analytics events — query these to track statistics volume, columns profiled, skips, and errors.
EVENT_STARTED = "data warehouse table statistics started"
EVENT_COMPLETED = "data warehouse table statistics completed"
@@ -839,7 +847,7 @@ def _upsert_statistics(
defaults["full_scan_at"] = timezone.now()
WarehouseColumnStatistics.objects.for_team(team.id).update_or_create(
table=table,
- column_name=column_name,
+ column_name=NamingConvention.normalize_identifier(column_name, max_length=COLUMN_NAME_DB_MAX_LENGTH),
defaults=defaults,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
index 89509a3ac378..dc88f3c6cefd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
@@ -773,6 +773,28 @@ class TestComputeTableStatisticsSync:
assert stat.computed_for_delta_version == 12
assert stat.column_type == "Int64"
+ def test_truncates_a_column_name_too_long_for_the_db(self) -> None:
+ # A deeply nested source schema can produce a column name longer than the 400-char limit
+ # on WarehouseColumnStatistics.column_name; the whole table's stats write must not fail for it.
+ long_name = "deeply_nested_field_" * 25
+ team = self._team()
+ schema, table, _ = self._schema_table_job(team, columns={long_name: {"clickhouse": "Nullable(Int64)"}})
+ add_actions = pa.table(
+ {
+ "num_records": [10],
+ f"null_count.{long_name}": [1],
+ f"min.{long_name}": [2],
+ f"max.{long_name}": [9],
+ }
+ )
+ with patch(DELTA_HELPER_PATH, return_value=self._mock_delta(add_actions)):
+ result = compute_table_statistics_sync(team.id, schema.id)
+
+ assert result["status"] == "done"
+ stat = WarehouseColumnStatistics.objects.for_team(team.id).get(table_id=table.id)
+ assert len(stat.column_name) == comp.COLUMN_NAME_DB_MAX_LENGTH
+ assert stat.min_value == "2"
+
@pytest.mark.parametrize(
"saved_during",
[