:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(staging): Adopt model-specific export types (#116823)
Tomás Farías Santanaebe8f248d759
diff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.pyindex fb93ddfa9266..43932115d0d9 100644--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py@@ -52,6 +52,7 @@ from products.batch_exports.backend.service import (BatchExportField,BatchExportModel,afetch_last_run_records_completed,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import default_fieldsfrom products.batch_exports.backend.temporal.errors import MissingRequiredInputsError@@ -368,6 +369,7 @@ async def insert_into_internal_stage_activity(logger.info("Computed staging partitions", num_partitions=num_partitions)try:+ model = migrate_batch_export_model(inputs.batch_export_model)(_,record_batch_model,@@ -377,7 +379,7 @@ async def insert_into_internal_stage_activity(extra_query_parameters,) = await database_sync_to_async(resolve_batch_exports_model)(inputs.team_id,- inputs.batch_export_model,+ model,inputs.batch_export_id,is_backfill=inputs.backfill_details is not None,)diff --git a/products/batch_exports/backend/temporal/record_batch_model.py b/products/batch_exports/backend/temporal/record_batch_model.pyindex 929869725673..e4a357f4025f 100644--- a/products/batch_exports/backend/temporal/record_batch_model.py+++ b/products/batch_exports/backend/temporal/record_batch_model.py@@ -34,7 +34,14 @@ from products.batch_exports.backend.hogql_source import (serialize_batch_export_query,validate_hogql_batch_export_user,)-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import (+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportHogQLModel,+ BatchExportModelInput,+ BatchExportSessionsModel,+ migrate_batch_export_model,+)from products.batch_exports.backend.temporal.errors import MissingRequiredInputsErrorfrom products.batch_exports.backend.temporal.metrics import log_query_durationfrom products.batch_exports.backend.temporal.sql.common import (@@ -442,7 +449,7 @@ class HogQLQueryRecordBatchModel(RecordBatchModel):def resolve_batch_exports_model(team_id: int,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: BatchExportModelInput | None = None,batch_export_id: str | None = None,is_backfill: bool = False,):@@ -452,35 +459,27 @@ def resolve_batch_exports_model(and new parameters. Eventually, once everything is a `RecordBatchModel`, this couldbe removed."""- model = batch_export_model+ model = migrate_batch_export_model(batch_export_model)+ model_name = model.namerecord_batch_model: RecordBatchModel | None = None- if model is not None:- model_name = model.name- extra_query_parameters = model.schema["values"] if model.schema is not None else None- fields = model.schema["fields"] if model.schema is not None else None- filters = model.filters-- if model_name == "sessions":+ schema = model.schema if isinstance(model, BatchExportEventsCustomModel) else None+ extra_query_parameters = schema["values"] if schema is not None else None+ fields = schema["fields"] if schema is not None else None+ filters = model.filters if isinstance(model, BatchExportEventsModel | BatchExportEventsCustomModel) else None+ match model:+ case BatchExportSessionsModel():record_batch_model = SessionsRecordBatchModel(team_id=team_id, batch_export_id=batch_export_id, is_backfill=is_backfill)- elif model_name == "hogql":- if model.hogql_query is None:- raise UnsupportedHogQLQueryError("Batch export model is 'hogql' but no HogQL query was provided")+ case BatchExportHogQLModel():record_batch_model = HogQLQueryRecordBatchModel(team_id=team_id,- hogql_query=model.hogql_query,+ hogql_query=model.query,batch_export_id=batch_export_id,user_id=model.user_id,- hogql_modifiers=model.hogql_modifiers,+ hogql_modifiers=model.modifiers,)- else:- model_name = "events"- extra_query_parameters = None- fields = None- filters = None- schema = model.schema if model is not None else Noneif schema is not None and (query := schema.get("hogql_query")) and use_new_events_schema(team_id):context = HogQLContext(team_id=team_id,diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.pyindex 2b53a6012d69..dcf1de684fb5 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py@@ -282,7 +282,7 @@ class TestErrorHandling:interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),batch_export_model=BatchExportModel(- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events"+ name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=1),)with patch(diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.pyindex 8661f13db59a..7a9ef257a51e 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py@@ -1313,11 +1313,9 @@ class TestHogQLModel:)with patch("products.batch_exports.backend.temporal.pipeline.internal_stage._execute_query") as execute_query:- result = await activity_environment.run(insert_into_internal_stage_activity, inputs)+ with pytest.raises(ValueError, match="requires user_id"):+ await activity_environment.run(insert_into_internal_stage_activity, inputs)- assert result.error is not None- assert result.error.type == "UnsupportedHogQLQueryError"- assert "needs an active user" in result.error.messageexecute_query.assert_not_called()mock_clickhouse_client.expect_query_count(0)diff --git a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py b/products/batch_exports/backend/tests/temporal/test_record_batch_model.pyindex 9352f0402923..4782cc6374c9 100644--- a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py+++ b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py@@ -28,7 +28,13 @@ from products.batch_exports.backend.hogql_source import (UnsupportedHogQLQueryError,validate_hogql_query_for_batch_export,)-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ BatchExportEventsCustomModel,+ BatchExportHogQLModel,+ BatchExportModel,+ BatchExportSchema,+ migrate_batch_export_model,+)from products.batch_exports.backend.temporal.batch_exports import iter_recordsfrom products.batch_exports.backend.temporal.filters import compose_filters_clausefrom products.batch_exports.backend.temporal.record_batch_model import (@@ -43,9 +49,78 @@ from products.data_modeling.backend.facade.models import DataWarehouseSavedQueryif TYPE_CHECKING:from posthog.models import Organization, Team-pytestmark = [pytest.mark.asyncio, pytest.mark.django_db]+pytestmark = [pytest.mark.asyncio]++[email protected]("migrated", [False, True])+ "legacy",+ [+ None,+ BatchExportModel(name="events", schema=None),+ BatchExportModel(name="events", schema=None, filters=[{"key": "event", "value": "example"}]),+ BatchExportModel(+ name="events",+ schema={"fields": [{"expression": "event", "alias": "name"}], "values": {"value": 1.5}},+ filters=[{"key": "event", "value": "example"}],+ ),+ BatchExportModel(name="persons", schema=None),+ BatchExportModel(name="sessions", schema=None),+ BatchExportModel(+ name="hogql",+ schema=None,+ hogql_query="SELECT event FROM events WHERE timestamp < {data_interval_end}",+ user_id=42,+ hogql_modifiers={"convertToProjectTimezone": False},+ ),+ ],+)+async def test_resolved_model_preserves_query_inputs(legacy: BatchExportModel | None, migrated: bool) -> None:+ model, record_model, name, fields, filters, values = resolve_batch_exports_model(+ team_id=1,+ batch_export_model=migrate_batch_export_model(legacy) if migrated else legacy,+ batch_export_id="export",+ is_backfill=True,+ )++ assert name == (legacy.name if legacy is not None else "events")+ assert model.name == name+ schema = legacy.schema if legacy is not None else None+ assert fields == (schema["fields"] if schema is not None else None)+ assert values == (schema["values"] if schema is not None else None)+ assert (filters or []) == ((legacy.filters or []) if legacy is not None else [])+ if name == "sessions":+ assert isinstance(record_model, SessionsRecordBatchModel)+ assert record_model.is_backfill is True+ assert record_model.team_id == 1+ assert record_model.batch_export_id == "export"+ elif name == "hogql":+ assert isinstance(record_model, HogQLQueryRecordBatchModel)+ assert legacy is not None+ assert record_model.hogql_query == legacy.hogql_query+ assert record_model.hogql_modifiers is not None+ assert record_model.hogql_modifiers.convertToProjectTimezone is False+ assert record_model.user_id == 42+ assert record_model.wait_for_data_interval_end is True+ else:+ assert record_model is None++ "query,user_id,error",+ [(None, 1, "requires a query"), ("SELECT event FROM events", None, "requires user_id")],+)+async def test_resolve_batch_exports_model_rejects_incomplete_hogql_inputs(+ query: str | None, user_id: int | None, error: str+) -> None:+ with pytest.raises(ValueError, match=error):+ resolve_batch_exports_model(+ team_id=1,+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=user_id),+ )+class TestSessionsRecordBatchModel:async def test_get_hogql_query(self, ateam, data_interval_start, data_interval_end):model = SessionsRecordBatchModel(@@ -236,6 +311,7 @@ class TestSessionsRecordBatchModel:assert "PARTITION BY rand() %% 5" in printed_queryclass TestSessionsRecordBatchModelSelection:"""Run the sessions model query against ClickHouse to verify which intervals select a session."""@@ -349,6 +425,7 @@ class TestSessionsRecordBatchModelSelection:assert selected == [session_id]class TestHogQLQueryRecordBatchModel:async def test_as_query_with_parameters(self, ateam, auser, data_interval_start, data_interval_end):model = HogQLQueryRecordBatchModel(@@ -529,13 +606,13 @@ class TestHogQLQueryRecordBatchModel:HogQLQueryRecordBatchModel(team_id=1, hogql_query=hogql_query)async def test_resolve_batch_exports_model_returns_hogql_model(self):- batch_export_model = BatchExportModel(- name="hogql",- schema=None,- hogql_query=(+ batch_export_model = BatchExportHogQLModel(+ query=("SELECT event AS event, timestamp AS timestamp FROM events ""WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"),+ user_id=1,+ modifiers={},)_, record_batch_model, model_name, _, _, _ = resolve_batch_exports_model(@@ -544,7 +621,7 @@ class TestHogQLQueryRecordBatchModel:assert isinstance(record_batch_model, HogQLQueryRecordBatchModel)assert model_name == "hogql"- assert record_batch_model.hogql_query == batch_export_model.hogql_query+ assert record_batch_model.hogql_query == batch_export_model.queryassert record_batch_model.wait_for_data_interval_end is True@pytest.mark.parametrize(@@ -562,12 +639,10 @@ class TestHogQLQueryRecordBatchModel:):ateam.modifiers = team_modifiersawait ateam.asave()- batch_export_model = BatchExportModel(- name="hogql",- schema=None,- hogql_query="SELECT event AS event, timestamp AS timestamp FROM events",+ batch_export_model = BatchExportHogQLModel(+ query="SELECT event AS event, timestamp AS timestamp FROM events",user_id=auser.pk,- hogql_modifiers=hogql_modifiers,+ modifiers=hogql_modifiers or {},)_, record_batch_model, _, _, _, _ = resolve_batch_exports_model(@@ -578,26 +653,17 @@ class TestHogQLQueryRecordBatchModel:assert ("toTimeZone(events.timestamp" in printed_query) is converts_timezone- async def test_resolve_batch_exports_model_raises_without_hogql_query(self):- """Without this, a missing query would fall through to the events template path and export the wrong data."""- with pytest.raises(UnsupportedHogQLQueryError):- resolve_batch_exports_model(team_id=1, batch_export_model=BatchExportModel(name="hogql", schema=None))-- @pytest.mark.parametrize("actor_state", ["missing", "deleted", "inactive", "nonmember", "project-denied"])+ @pytest.mark.parametrize("actor_state", ["deleted", "inactive", "nonmember", "project-denied"])async def test_rechecks_actor_access_at_runtime(self, ateam: "Team", aorganization: "Organization", auser: User, actor_state: str) -> None:query = "SELECT event AS event FROM events"await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)- payload = BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)+ payload = BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={})_, model, _, _, _, _ = resolve_batch_exports_model(ateam.pk, payload)await model.as_query_with_parameters(None, None)- if actor_state == "missing":- _, model, _, _, _, _ = resolve_batch_exports_model(- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query)- )- elif actor_state == "deleted":+ if actor_state == "deleted":await User.objects.filter(pk=auser.pk).adelete()elif actor_state == "inactive":await User.objects.filter(pk=auser.pk).aupdate(is_active=False)@@ -639,7 +705,7 @@ class TestHogQLQueryRecordBatchModel:with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)_, model, _, _, _, _ = resolve_batch_exports_model(- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)+ ateam.pk, BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={}))before, _ = await model.as_query_with_parameters(None, None)assert "NULL AS secret" not in before@@ -674,7 +740,7 @@ class TestHogQLQueryRecordBatchModel:with patch("posthog.hogql.database.database.feature_enabled_or_false", return_value=True):await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)_, model, _, _, _, _ = resolve_batch_exports_model(- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)+ ateam.pk, BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={}))await model.as_query_with_parameters(None, None)membership = await OrganizationMembership.objects.aget(user=auser, organization=aorganization)@@ -690,6 +756,7 @@ class TestHogQLQueryRecordBatchModel:await model.as_query_with_parameters(None, None)@pytest.mark.parametrize("use_new_events_schema", [False, True])async def test_custom_export_recompilation_preserves_column_names(ateam, use_new_events_schema):schema: BatchExportSchema = {@@ -702,7 +769,7 @@ async def test_custom_export_recompilation_preserves_column_names(ateam, use_new}with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):_, _, _, fields, _, values = await database_sync_to_async(resolve_batch_exports_model)(- team_id=ateam.pk, batch_export_model=BatchExportModel(name="events", schema=schema)+ team_id=ateam.pk, batch_export_model=BatchExportEventsCustomModel(schema=schema))assert fields is not None@@ -719,6 +786,7 @@ async def test_custom_export_recompilation_preserves_column_names(ateam, use_new@pytest.mark.parametrize("interval_start_is_none", [False, True])@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)async def test_custom_export_backfill_runs_without_legacy_events_tables(ateam, clickhouse_client, interval_start_is_none):@@ -770,7 +838,7 @@ async def test_custom_export_backfill_runs_without_legacy_events_tables("values": {"unused_old_parameter": "stale"},}_, _, _, fields, _, values = await database_sync_to_async(resolve_batch_exports_model)(- team_id=ateam.pk, batch_export_model=BatchExportModel(name="events", schema=schema)+ team_id=ateam.pk, batch_export_model=BatchExportEventsCustomModel(schema=schema))assert fields is not Noneassert "unused_old_parameter" not in valuesdiff --git a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.pyindex a0252e1fad14..f8f311fffbb4 100644--- a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py+++ b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py@@ -74,7 +74,7 @@ def test_each_run_substitutes_its_own_bounds_without_mutating_the_query() -> Non),("not a valid query", "Failed to parse HogQL query"),("SELECT event FROM events WHERE timestamp >= {unknown}", "Unknown placeholder '{unknown}'"),- (None, "no HogQL query was provided"),+ (None, "requires a query"),],ids=["missing-start", "malformed-query", "unsupported-placeholder", "missing-query"],)@@ -84,7 +84,7 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: stbatch_export_id="00000000-0000-4000-8000-000000000001",data_interval_start=None,data_interval_end=(dt.datetime.now(dt.UTC) - dt.timedelta(hours=1)).isoformat(),- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query),+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),)clickhouse = MockClickHouseClient()@@ -92,10 +92,13 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: st"products.batch_exports.backend.temporal.pipeline.internal_stage.get_client",return_value=clickhouse.mock_client_cm,):- result = await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)-- assert result.error is not None- assert result.error.type == "UnsupportedHogQLQueryError"- assert expected_error in result.error.message- assert result.records_total is None+ if hogql_query is None:+ with pytest.raises(ValueError, match=expected_error):+ await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)+ else:+ result = await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)+ assert result.error is not None+ assert result.error.type == "UnsupportedHogQLQueryError"+ assert expected_error in result.error.message+ assert result.records_total is Noneclickhouse.expect_query_count(0)diff --git a/products/batch_exports/backend/tests/test_nullable_staging.py b/products/batch_exports/backend/tests/test_nullable_staging.pyindex 0f74440e0211..aee5d71c9ba7 100644--- a/products/batch_exports/backend/tests/test_nullable_staging.py+++ b/products/batch_exports/backend/tests/test_nullable_staging.py@@ -76,7 +76,7 @@ async def test_hogql_staging_preserves_optional_bounds_and_cleanup_namespace(data_interval_start=start_str,data_interval_end=end_str,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),)result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)@@ -110,13 +110,16 @@ async def test_unsupported_hogql_returns_nonretryable_staging_error(data_interval_start=None,data_interval_end=None,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query),+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),)- result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)-- assert result.error is not None- assert result.error.type == "UnsupportedHogQLQueryError"+ if hogql_query is None:+ with pytest.raises(ValueError, match="requires a query"):+ await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)+ else:+ result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)+ assert result.error is not None+ assert result.error.type == "UnsupportedHogQLQueryError"staging_clients[0].execute_query_with_summary.assert_not_called()staging_clients[1].list_objects_v2.assert_not_called()@@ -142,7 +145,7 @@ async def test_hogql_staging_rejects_each_future_bound_without_waiting(data_interval_start=start.isoformat() if start is not None else None,data_interval_end=end.isoformat() if end is not None else None,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),)# Patch the helper rather than override settings.TEST, because DatabaseSyncToAsync calls# close_old_connections() whenever settings.TEST is False, and that needs a database.@@ -179,7 +182,7 @@ async def test_missing_staging_inputs_raise(data_interval_start=None,data_interval_end=None,on_demand=on_demand,- batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value"),+ batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value", user_id=1),)with pytest.raises(MissingRequiredInputsError, match="require"):await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)