Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): stop materializations panicking on decimal columns (#124027)

Jovan Sakovicaf599a6c4de3
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index ea5ca735d127..aac71908ae21 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -12,6 +12,7 @@ import pyarrow as pa
import deltalake
import pyarrow.compute as pc
import pyarrow.parquet as pq
+from pyarrow.lib import concat_batches
from structlog.contextvars import bind_contextvars
from structlog.types import FilteringBoundLogger
from temporalio import activity
@@ -395,15 +396,6 @@ def get_aws_storage_options() -> dict[str, str]:
}
-def _combine_batches(batches: list[pa.RecordBatch]) -> pa.RecordBatch:
- if len(batches) == 1:
- return batches[0]
-
- table = pa.Table.from_batches(batches)
- table = table.combine_chunks()
- return table.to_batches(max_chunksize=table.num_rows)[0]
-
-
def _transform_date_and_datetimes(batch: pa.RecordBatch, types: list[tuple[str, str]]) -> pa.RecordBatch:
"""Transform date/datetimes from ClickHouse UInt representations back to proper types."""
new_columns: list[pa.Array] = []
@@ -806,14 +798,14 @@ async def hogql_table(
await logger.adebug(
f"Yielding {len(batches)} batches for total size of {batches_size / 1000 / 1000}MB"
)
- yield (_combine_batches(batches), ch_typings_pairs)
+ yield (concat_batches(batches), ch_typings_pairs)
yielded_results = True
batches_size = 0
batches = []
if len(batches) > 0:
await logger.adebug(f"Yielding {len(batches)} batches for total size of {batches_size / 1000 / 1000}MB")
- yield (_combine_batches(batches), ch_typings_pairs)
+ yield (concat_batches(batches), ch_typings_pairs)
yielded_results = True
if not yielded_results:
diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.py
index 2e58e67a533a..e4bbbc83fed2 100644
--- a/posthog/temporal/data_modeling/activities/utils.py
+++ b/posthog/temporal/data_modeling/activities/utils.py
@@ -107,6 +107,7 @@ EXTERNALLY_ABORTED_MARKERS = (
# the trailing colon is load-bearing: it keeps a customer column of the same name out of the match
"QueueEmpty: ", # no root node to start from, so the graph was refused rather than the query
"Preempted: ",
+ "not aligned with the specified scalar type",
ABANDONED_ERROR,
# the query ran and produced rows; only the publish was refused
QUALITY_BLOCKED_ERROR_PREFIX,
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index 58536897699e..cf0cd5fcd271 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -936,6 +936,7 @@ class TestNodeSuspension:
"ch-offline.example.com:9440: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)",
"Cannot connect to host ch-offline.example.com:8443",
'ProxyConnectionError: Failed to connect to proxy URL: "http://proxy.example.com:3128/"',
+ "PanicException: Memory pointer from external source (e.g, FFI) is not aligned with the specified scalar type.",
"Abandoned: the materialization workflow is no longer running",
"QueueEmpty: Application error",
"Preempted: a new DAG run started before this job completed",