Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): support date and decimal types for Databricks (#126554)

Rosse3dc1f5e441e
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index d5291ab64b17..0c6a221b4d2b 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -984,6 +984,9 @@ def databricks_default_fields() -> list[BatchExportField]:
return batch_export_fields
+DATABRICKS_MAX_DECIMAL_PRECISION = 38
+
+
def _get_databricks_field_type(pa_type: pa.DataType, is_variant: bool) -> str | None:
"""Get the Databricks type for a PyArrow field."""
if pa.types.is_string(pa_type) or isinstance(pa_type, JsonType):
@@ -1013,9 +1016,17 @@ def _get_databricks_field_type(pa_type: pa.DataType, is_variant: bool) -> str |
elif pa.types.is_timestamp(pa_type):
return "TIMESTAMP"
+ elif pa.types.is_date(pa_type):
+ return "DATE"
+
+ elif pa.types.is_decimal(pa_type) and pa_type.precision <= DATABRICKS_MAX_DECIMAL_PRECISION:
+ return f"DECIMAL({pa_type.precision}, {pa_type.scale})"
+
elif pa.types.is_list(pa_type):
assert isinstance(pa_type, pa.ListType)
list_type = _get_databricks_field_type(pa_type.value_type, False)
+ if list_type is None:
+ return None
return f"ARRAY<{list_type}>"
return None
diff --git a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
index ca69703d596d..7c27a20f43dd 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
@@ -9,6 +9,7 @@ import uuid
import typing as t
import datetime as dt
import operator
+import dataclasses
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Callable
@@ -25,6 +26,7 @@ from posthog.models.integration import Integration
from posthog.models.team import Team
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.logger import BATCH_EXPORT_WORKFLOW_TYPES as LOGGER_BATCH_EXPORT_WORKFLOW_TYPES
+from posthog.temporal.tests.utils.events import insert_event_values_in_clickhouse
from products.batch_exports.backend.models.batch_export import BatchExport
from products.batch_exports.backend.service import (
@@ -34,6 +36,7 @@ from products.batch_exports.backend.service import (
BatchExportEventsCustomModel,
BatchExportEventsModel,
BatchExportField,
+ BatchExportHogQLModel,
BatchExportPersonsModel,
BatchExportSchema,
BatchExportSessionsModel,
@@ -43,7 +46,10 @@ from products.batch_exports.backend.temporal.batch_exports import finish_batch_e
from products.batch_exports.backend.temporal.metrics import BATCH_EXPORT_ACTIVITY_TYPES, BATCH_EXPORT_WORKFLOW_TYPES
from products.batch_exports.backend.temporal.pipeline.internal_stage import insert_into_internal_stage_activity
from products.batch_exports.backend.temporal.queue import RecordBatchQueue
-from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel
+from products.batch_exports.backend.temporal.record_batch_model import (
+ HogQLQueryRecordBatchModel,
+ SessionsRecordBatchModel,
+)
from products.batch_exports.backend.tests.temporal.utils.clickhouse_test_producer import ClickHouseTestProducer
from products.batch_exports.backend.tests.temporal.utils.models import (
acreate_batch_export,
@@ -249,6 +255,12 @@ async def _get_records_from_clickhouse(
queue = RecordBatchQueue()
if model_name == "sessions":
producer = ClickHouseTestProducer(model=SessionsRecordBatchModel(team_id))
+ elif isinstance(batch_export_model, BatchExportHogQLModel):
+ producer = ClickHouseTestProducer(
+ model=HogQLQueryRecordBatchModel(
+ team_id=team_id, hogql_query=batch_export_model.query, user_id=batch_export_model.user_id
+ )
+ )
else:
producer = ClickHouseTestProducer()
@@ -403,6 +415,16 @@ class CommonWorkflowTests:
),
BatchExportPersonsModel(),
BatchExportSessionsModel(),
+ # `user_id` must be a member of the test team, so the test replaces this placeholder.
+ # `0.1` has no exact float value, so a DOUBLE column instead of a DECIMAL fails the comparison.
+ BatchExportHogQLModel(
+ query=(
+ "SELECT DISTINCT toString(uuid) AS uuid, toDate(timestamp) AS day, toDecimal(0.1, 2) AS amount "
+ "FROM events WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"
+ ),
+ user_id=0,
+ modifiers={},
+ ),
{
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -492,6 +514,8 @@ class CommonWorkflowTests:
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
ateam,
+ auser,
+ clickhouse_client,
batch_export_for_destination, # This fixture needs to be provided by destination-specific tests
integration,
setup_destination,
@@ -505,6 +529,15 @@ class CommonWorkflowTests:
f"Unnecessary test case as batch export model '{model.name}' is not affected by 'exclude_events'"
)
+ if isinstance(model, BatchExportHogQLModel):
+ model = dataclasses.replace(model, user_id=auser.pk)
+ # The HogQL `events` table reads `sharded_events`, but the fixture writes a recent
+ # interval to `events_recent` only.
+ events_to_export_created, _ = generate_test_data
+ await insert_event_values_in_clickhouse(
+ client=clickhouse_client, events=events_to_export_created, table="sharded_events"
+ )
+
batch_export_model: AnyBatchExportModel | None = None
if isinstance(model, AnyBatchExportModel):
batch_export_model = model
diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
index 20418c99ef50..f8395b9d1cae 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
@@ -7,6 +7,7 @@ import pyarrow as pa
from products.batch_exports.backend.service import (
BatchExportEventsCustomModel,
BatchExportEventsModel,
+ BatchExportHogQLModel,
BatchExportSchema,
)
from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (
@@ -15,6 +16,8 @@ from products.batch_exports.backend.temporal.destinations.databricks_batch_expor
_get_long_running_query_timeout,
databricks_default_fields,
)
+from products.batch_exports.backend.temporal.errors import UnsupportedColumnTypeError
+from products.batch_exports.backend.temporal.utils import JsonType
@pytest.mark.parametrize(
@@ -81,3 +84,50 @@ def test_events_table_fields_follow_a_custom_schema():
}
assert _events_table_field_names(["event", "browser"], schema=schema) == ["event", "browser"]
+
+
+ "field_name, pa_type, expected_databricks_type",
+ [
+ ("col", pa.string(), "STRING"),
+ ("col", JsonType(), "STRING"),
+ ("properties", pa.string(), "VARIANT"),
+ ("col", pa.binary(), "BINARY"),
+ ("col", pa.int32(), "INTEGER"),
+ ("col", pa.uint8(), "INTEGER"),
+ ("col", pa.int64(), "BIGINT"),
+ ("col", pa.uint64(), "BIGINT"),
+ ("col", pa.float32(), "FLOAT"),
+ ("col", pa.float64(), "DOUBLE"),
+ ("col", pa.bool_(), "BOOLEAN"),
+ ("col", pa.timestamp("us", tz="UTC"), "TIMESTAMP"),
+ ("col", pa.date32(), "DATE"),
+ ("col", pa.date64(), "DATE"),
+ ("col", pa.decimal128(38, 2), "DECIMAL(38, 2)"),
+ ("col", pa.decimal256(10, 0), "DECIMAL(10, 0)"),
+ ("col", pa.list_(pa.string()), "ARRAY<STRING>"),
+ ("col", pa.list_(pa.int64()), "ARRAY<BIGINT>"),
+ ],
+)
+def test_hogql_table_fields_map_arrow_types(
+ field_name: str, pa_type: pa.DataType, expected_databricks_type: str
+) -> None:
+ # A HogQL query stages whatever Arrow types its columns produce. A type without a mapping makes
+ # the insert activity raise on every retry, so the run never lands.
+ table_fields, _, _ = _get_databricks_table_settings(
+ model=BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}),
+ record_batch_schema=pa.schema([pa.field(field_name, pa_type)]),
+ use_variant_type=True,
+ )
+
+ assert table_fields == [(field_name, expected_databricks_type)]
+
+
[email protected]("pa_type", [pa.decimal256(76, 0), pa.list_(pa.decimal256(76, 0))])
+def test_hogql_table_fields_reject_unsupported_arrow_types(pa_type: pa.DataType) -> None:
+ with pytest.raises(UnsupportedColumnTypeError, match=r"Column 'col' has type '.*decimal256\(76, 0\)"):
+ _get_databricks_table_settings(
+ model=BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}),
+ record_batch_schema=pa.schema([pa.field("col", pa_type)]),
+ use_variant_type=True,
+ )
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
index f2472fe51ef3..a4166fb45f26 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
@@ -260,7 +260,7 @@ async def _generate_record_batches_from_internal_stage(
continue
-async def _run_activity(
+async def _stage_record_batches(
activity_environment: ActivityEnvironment,
object_storage_client,
team_id,
@@ -269,8 +269,8 @@ async def _run_activity(
exclude_events: list[str] | None = None,
model: AnyBatchExportModel | None = None,
batch_export_id: str | None = None,
-) -> list[dict]:
- """Get rows from the internal stage."""
+) -> list[pa.RecordBatch]:
+ """Run the stage activity and read back the record batches a destination would consume."""
batch_export_id = batch_export_id or str(uuid.uuid4())
insert_inputs = BatchExportInsertIntoInternalStageInputs(
@@ -296,12 +296,36 @@ async def _run_activity(
compression=None,
json_columns=None,
)
- exported_rows = []
- async for record_batch in _generate_record_batches_from_internal_stage(
- batch_export_id, data_interval_start, data_interval_end, stage_folder
- ):
- exported_rows.extend(record_batch.to_pylist())
- return exported_rows
+ return [
+ record_batch
+ async for record_batch in _generate_record_batches_from_internal_stage(
+ batch_export_id, data_interval_start, data_interval_end, stage_folder
+ )
+ ]
+
+
+async def _run_activity(
+ activity_environment: ActivityEnvironment,
+ object_storage_client,
+ team_id,
+ data_interval_start: dt.datetime,
+ data_interval_end: dt.datetime,
+ exclude_events: list[str] | None = None,
+ model: AnyBatchExportModel | None = None,
+ batch_export_id: str | None = None,
+) -> list[dict]:
+ """Get rows from the internal stage."""
+ record_batches = await _stage_record_batches(
+ activity_environment=activity_environment,
+ object_storage_client=object_storage_client,
+ team_id=team_id,
+ data_interval_start=data_interval_start,
+ data_interval_end=data_interval_end,
+ exclude_events=exclude_events,
+ model=model,
+ batch_export_id=batch_export_id,
+ )
+ return [row for record_batch in record_batches for row in record_batch.to_pylist()]
@pytest.mark.parametrize("interval", ["day", "every 5 minutes"], indirect=True)
@@ -1617,6 +1641,37 @@ class TestHogQLModel:
(e["uuid"], e["event"], e["distinct_id"]) for e in events
)
+ async def test_stages_column_types_destinations_map(
+ self,
+ hogql_model_test_data,
+ activity_environment,
+ object_storage_client,
+ ateam,
+ auser,
+ data_interval_start,
+ data_interval_end,
+ ):
+ # Destinations create table columns from the staged Arrow types. If ClickHouse starts to stage
+ # a Date as uint16, for example, a Databricks export creates an integer column, not a DATE.
+ record_batches = await _stage_record_batches(
+ activity_environment=activity_environment,
+ object_storage_client=object_storage_client,
+ team_id=ateam.pk,
+ data_interval_start=data_interval_start,
+ data_interval_end=data_interval_end,
+ model=BatchExportHogQLModel(
+ query="SELECT toDate(timestamp) AS day, toDecimal(1.5, 2) AS amount, uuid AS uuid FROM events",
+ user_id=auser.pk,
+ modifiers={},
+ ),
+ )
+
+ assert {field.name: field.type for field in record_batches[0].schema} == {
+ "day": pa.date32(),
+ "amount": pa.decimal128(18, 2),
+ "uuid": pa.uuid(),
+ }
+
@override_settings(
BATCH_EXPORT_HOGQL_MAX_EXECUTION_TIME=900,
BATCH_EXPORT_HOGQL_MAX_MEMORY_USAGE=30_000_000_000,