:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): support date and decimal types for Databricks (#126554)
Rosse3dc1f5e441e
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex d5291ab64b17..0c6a221b4d2b 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -984,6 +984,9 @@ def databricks_default_fields() -> list[BatchExportField]:return batch_export_fields+DATABRICKS_MAX_DECIMAL_PRECISION = 38++def _get_databricks_field_type(pa_type: pa.DataType, is_variant: bool) -> str | None:"""Get the Databricks type for a PyArrow field."""if pa.types.is_string(pa_type) or isinstance(pa_type, JsonType):@@ -1013,9 +1016,17 @@ def _get_databricks_field_type(pa_type: pa.DataType, is_variant: bool) -> str |elif pa.types.is_timestamp(pa_type):return "TIMESTAMP"+ elif pa.types.is_date(pa_type):+ return "DATE"++ elif pa.types.is_decimal(pa_type) and pa_type.precision <= DATABRICKS_MAX_DECIMAL_PRECISION:+ return f"DECIMAL({pa_type.precision}, {pa_type.scale})"+elif pa.types.is_list(pa_type):assert isinstance(pa_type, pa.ListType)list_type = _get_databricks_field_type(pa_type.value_type, False)+ if list_type is None:+ return Nonereturn f"ARRAY<{list_type}>"return Nonediff --git a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.pyindex ca69703d596d..7c27a20f43dd 100644--- a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py+++ b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py@@ -9,6 +9,7 @@ import uuidimport typing as timport datetime as dtimport operator+import dataclassesfrom abc import ABC, abstractmethodfrom collections.abc import AsyncGenerator, Callable@@ -25,6 +26,7 @@ from posthog.models.integration import Integrationfrom posthog.models.team import Teamfrom posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.logger import BATCH_EXPORT_WORKFLOW_TYPES as LOGGER_BATCH_EXPORT_WORKFLOW_TYPES+from posthog.temporal.tests.utils.events import insert_event_values_in_clickhousefrom products.batch_exports.backend.models.batch_export import BatchExportfrom products.batch_exports.backend.service import (@@ -34,6 +36,7 @@ from products.batch_exports.backend.service import (BatchExportEventsCustomModel,BatchExportEventsModel,BatchExportField,+ BatchExportHogQLModel,BatchExportPersonsModel,BatchExportSchema,BatchExportSessionsModel,@@ -43,7 +46,10 @@ from products.batch_exports.backend.temporal.batch_exports import finish_batch_efrom products.batch_exports.backend.temporal.metrics import BATCH_EXPORT_ACTIVITY_TYPES, BATCH_EXPORT_WORKFLOW_TYPESfrom products.batch_exports.backend.temporal.pipeline.internal_stage import insert_into_internal_stage_activityfrom products.batch_exports.backend.temporal.queue import RecordBatchQueue-from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel+from products.batch_exports.backend.temporal.record_batch_model import (+ HogQLQueryRecordBatchModel,+ SessionsRecordBatchModel,+)from products.batch_exports.backend.tests.temporal.utils.clickhouse_test_producer import ClickHouseTestProducerfrom products.batch_exports.backend.tests.temporal.utils.models import (acreate_batch_export,@@ -249,6 +255,12 @@ async def _get_records_from_clickhouse(queue = RecordBatchQueue()if model_name == "sessions":producer = ClickHouseTestProducer(model=SessionsRecordBatchModel(team_id))+ elif isinstance(batch_export_model, BatchExportHogQLModel):+ producer = ClickHouseTestProducer(+ model=HogQLQueryRecordBatchModel(+ team_id=team_id, hogql_query=batch_export_model.query, user_id=batch_export_model.user_id+ )+ )else:producer = ClickHouseTestProducer()@@ -403,6 +415,16 @@ class CommonWorkflowTests:),BatchExportPersonsModel(),BatchExportSessionsModel(),+ # `user_id` must be a member of the test team, so the test replaces this placeholder.+ # `0.1` has no exact float value, so a DOUBLE column instead of a DECIMAL fails the comparison.+ BatchExportHogQLModel(+ query=(+ "SELECT DISTINCT toString(uuid) AS uuid, toDate(timestamp) AS day, toDecimal(0.1, 2) AS amount "+ "FROM events WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"+ ),+ user_id=0,+ modifiers={},+ ),{"fields": [{"expression": "uuid", "alias": "uuid"},@@ -492,6 +514,8 @@ class CommonWorkflowTests:data_interval_start: dt.datetime,data_interval_end: dt.datetime,ateam,+ auser,+ clickhouse_client,batch_export_for_destination, # This fixture needs to be provided by destination-specific testsintegration,setup_destination,@@ -505,6 +529,15 @@ class CommonWorkflowTests:f"Unnecessary test case as batch export model '{model.name}' is not affected by 'exclude_events'")+ if isinstance(model, BatchExportHogQLModel):+ model = dataclasses.replace(model, user_id=auser.pk)+ # The HogQL `events` table reads `sharded_events`, but the fixture writes a recent+ # interval to `events_recent` only.+ events_to_export_created, _ = generate_test_data+ await insert_event_values_in_clickhouse(+ client=clickhouse_client, events=events_to_export_created, table="sharded_events"+ )+batch_export_model: AnyBatchExportModel | None = Noneif isinstance(model, AnyBatchExportModel):batch_export_model = modeldiff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.pyindex 20418c99ef50..f8395b9d1cae 100644--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py@@ -7,6 +7,7 @@ import pyarrow as pafrom products.batch_exports.backend.service import (BatchExportEventsCustomModel,BatchExportEventsModel,+ BatchExportHogQLModel,BatchExportSchema,)from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (@@ -15,6 +16,8 @@ from products.batch_exports.backend.temporal.destinations.databricks_batch_expor_get_long_running_query_timeout,databricks_default_fields,)+from products.batch_exports.backend.temporal.errors import UnsupportedColumnTypeError+from products.batch_exports.backend.temporal.utils import JsonType@pytest.mark.parametrize(@@ -81,3 +84,50 @@ def test_events_table_fields_follow_a_custom_schema():}assert _events_table_field_names(["event", "browser"], schema=schema) == ["event", "browser"]+++ "field_name, pa_type, expected_databricks_type",+ [+ ("col", pa.string(), "STRING"),+ ("col", JsonType(), "STRING"),+ ("properties", pa.string(), "VARIANT"),+ ("col", pa.binary(), "BINARY"),+ ("col", pa.int32(), "INTEGER"),+ ("col", pa.uint8(), "INTEGER"),+ ("col", pa.int64(), "BIGINT"),+ ("col", pa.uint64(), "BIGINT"),+ ("col", pa.float32(), "FLOAT"),+ ("col", pa.float64(), "DOUBLE"),+ ("col", pa.bool_(), "BOOLEAN"),+ ("col", pa.timestamp("us", tz="UTC"), "TIMESTAMP"),+ ("col", pa.date32(), "DATE"),+ ("col", pa.date64(), "DATE"),+ ("col", pa.decimal128(38, 2), "DECIMAL(38, 2)"),+ ("col", pa.decimal256(10, 0), "DECIMAL(10, 0)"),+ ("col", pa.list_(pa.string()), "ARRAY<STRING>"),+ ("col", pa.list_(pa.int64()), "ARRAY<BIGINT>"),+ ],+)+def test_hogql_table_fields_map_arrow_types(+ field_name: str, pa_type: pa.DataType, expected_databricks_type: str+) -> None:+ # A HogQL query stages whatever Arrow types its columns produce. A type without a mapping makes+ # the insert activity raise on every retry, so the run never lands.+ table_fields, _, _ = _get_databricks_table_settings(+ model=BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}),+ record_batch_schema=pa.schema([pa.field(field_name, pa_type)]),+ use_variant_type=True,+ )++ assert table_fields == [(field_name, expected_databricks_type)]++[email protected]("pa_type", [pa.decimal256(76, 0), pa.list_(pa.decimal256(76, 0))])+def test_hogql_table_fields_reject_unsupported_arrow_types(pa_type: pa.DataType) -> None:+ with pytest.raises(UnsupportedColumnTypeError, match=r"Column 'col' has type '.*decimal256\(76, 0\)"):+ _get_databricks_table_settings(+ model=BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}),+ record_batch_schema=pa.schema([pa.field("col", pa_type)]),+ use_variant_type=True,+ )diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.pyindex f2472fe51ef3..a4166fb45f26 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py@@ -260,7 +260,7 @@ async def _generate_record_batches_from_internal_stage(continue-async def _run_activity(+async def _stage_record_batches(activity_environment: ActivityEnvironment,object_storage_client,team_id,@@ -269,8 +269,8 @@ async def _run_activity(exclude_events: list[str] | None = None,model: AnyBatchExportModel | None = None,batch_export_id: str | None = None,-) -> list[dict]:- """Get rows from the internal stage."""+) -> list[pa.RecordBatch]:+ """Run the stage activity and read back the record batches a destination would consume."""batch_export_id = batch_export_id or str(uuid.uuid4())insert_inputs = BatchExportInsertIntoInternalStageInputs(@@ -296,12 +296,36 @@ async def _run_activity(compression=None,json_columns=None,)- exported_rows = []- async for record_batch in _generate_record_batches_from_internal_stage(- batch_export_id, data_interval_start, data_interval_end, stage_folder- ):- exported_rows.extend(record_batch.to_pylist())- return exported_rows+ return [+ record_batch+ async for record_batch in _generate_record_batches_from_internal_stage(+ batch_export_id, data_interval_start, data_interval_end, stage_folder+ )+ ]+++async def _run_activity(+ activity_environment: ActivityEnvironment,+ object_storage_client,+ team_id,+ data_interval_start: dt.datetime,+ data_interval_end: dt.datetime,+ exclude_events: list[str] | None = None,+ model: AnyBatchExportModel | None = None,+ batch_export_id: str | None = None,+) -> list[dict]:+ """Get rows from the internal stage."""+ record_batches = await _stage_record_batches(+ activity_environment=activity_environment,+ object_storage_client=object_storage_client,+ team_id=team_id,+ data_interval_start=data_interval_start,+ data_interval_end=data_interval_end,+ exclude_events=exclude_events,+ model=model,+ batch_export_id=batch_export_id,+ )+ return [row for record_batch in record_batches for row in record_batch.to_pylist()]@pytest.mark.parametrize("interval", ["day", "every 5 minutes"], indirect=True)@@ -1617,6 +1641,37 @@ class TestHogQLModel:(e["uuid"], e["event"], e["distinct_id"]) for e in events)+ async def test_stages_column_types_destinations_map(+ self,+ hogql_model_test_data,+ activity_environment,+ object_storage_client,+ ateam,+ auser,+ data_interval_start,+ data_interval_end,+ ):+ # Destinations create table columns from the staged Arrow types. If ClickHouse starts to stage+ # a Date as uint16, for example, a Databricks export creates an integer column, not a DATE.+ record_batches = await _stage_record_batches(+ activity_environment=activity_environment,+ object_storage_client=object_storage_client,+ team_id=ateam.pk,+ data_interval_start=data_interval_start,+ data_interval_end=data_interval_end,+ model=BatchExportHogQLModel(+ query="SELECT toDate(timestamp) AS day, toDecimal(1.5, 2) AS amount, uuid AS uuid FROM events",+ user_id=auser.pk,+ modifiers={},+ ),+ )++ assert {field.name: field.type for field in record_batches[0].schema} == {+ "day": pa.date32(),+ "amount": pa.decimal128(18, 2),+ "uuid": pa.uuid(),+ }+@override_settings(BATCH_EXPORT_HOGQL_MAX_EXECUTION_TIME=900,BATCH_EXPORT_HOGQL_MAX_MEMORY_USAGE=30_000_000_000,