:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): stop materializations panicking on decimal columns (#124027)
Jovan Sakovicaf599a6c4de3
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex ea5ca735d127..aac71908ae21 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -12,6 +12,7 @@ import pyarrow as paimport deltalakeimport pyarrow.compute as pcimport pyarrow.parquet as pq+from pyarrow.lib import concat_batchesfrom structlog.contextvars import bind_contextvarsfrom structlog.types import FilteringBoundLoggerfrom temporalio import activity@@ -395,15 +396,6 @@ def get_aws_storage_options() -> dict[str, str]:}-def _combine_batches(batches: list[pa.RecordBatch]) -> pa.RecordBatch:- if len(batches) == 1:- return batches[0]-- table = pa.Table.from_batches(batches)- table = table.combine_chunks()- return table.to_batches(max_chunksize=table.num_rows)[0]--def _transform_date_and_datetimes(batch: pa.RecordBatch, types: list[tuple[str, str]]) -> pa.RecordBatch:"""Transform date/datetimes from ClickHouse UInt representations back to proper types."""new_columns: list[pa.Array] = []@@ -806,14 +798,14 @@ async def hogql_table(await logger.adebug(f"Yielding {len(batches)} batches for total size of {batches_size / 1000 / 1000}MB")- yield (_combine_batches(batches), ch_typings_pairs)+ yield (concat_batches(batches), ch_typings_pairs)yielded_results = Truebatches_size = 0batches = []if len(batches) > 0:await logger.adebug(f"Yielding {len(batches)} batches for total size of {batches_size / 1000 / 1000}MB")- yield (_combine_batches(batches), ch_typings_pairs)+ yield (concat_batches(batches), ch_typings_pairs)yielded_results = Trueif not yielded_results:diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.pyindex 2e58e67a533a..e4bbbc83fed2 100644--- a/posthog/temporal/data_modeling/activities/utils.py+++ b/posthog/temporal/data_modeling/activities/utils.py@@ -107,6 +107,7 @@ EXTERNALLY_ABORTED_MARKERS = (# the trailing colon is load-bearing: it keeps a customer column of the same name out of the match"QueueEmpty: ", # no root node to start from, so the graph was refused rather than the query"Preempted: ",+ "not aligned with the specified scalar type",ABANDONED_ERROR,# the query ran and produced rows; only the publish was refusedQUALITY_BLOCKED_ERROR_PREFIX,diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex 58536897699e..cf0cd5fcd271 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -936,6 +936,7 @@ class TestNodeSuspension:"ch-offline.example.com:9440: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)","Cannot connect to host ch-offline.example.com:8443",'ProxyConnectionError: Failed to connect to proxy URL: "http://proxy.example.com:3128/"',+ "PanicException: Memory pointer from external source (e.g, FFI) is not aligned with the specified scalar type.","Abandoned: the materialization workflow is no longer running","QueueEmpty: Application error","Preempted: a new DAG run started before this job completed",