Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(staging): Adopt model-specific export types (#116823)

Tomás Farías Santanaebe8f248d759
diff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
index fb93ddfa9266..43932115d0d9 100644
--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py
+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
@@ -52,6 +52,7 @@ from products.batch_exports.backend.service import (
BatchExportField,
BatchExportModel,
afetch_last_run_records_completed,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import default_fields
from products.batch_exports.backend.temporal.errors import MissingRequiredInputsError
@@ -368,6 +369,7 @@ async def insert_into_internal_stage_activity(
logger.info("Computed staging partitions", num_partitions=num_partitions)
try:
+ model = migrate_batch_export_model(inputs.batch_export_model)
(
_,
record_batch_model,
@@ -377,7 +379,7 @@ async def insert_into_internal_stage_activity(
extra_query_parameters,
) = await database_sync_to_async(resolve_batch_exports_model)(
inputs.team_id,
- inputs.batch_export_model,
+ model,
inputs.batch_export_id,
is_backfill=inputs.backfill_details is not None,
)
diff --git a/products/batch_exports/backend/temporal/record_batch_model.py b/products/batch_exports/backend/temporal/record_batch_model.py
index 929869725673..e4a357f4025f 100644
--- a/products/batch_exports/backend/temporal/record_batch_model.py
+++ b/products/batch_exports/backend/temporal/record_batch_model.py
@@ -34,7 +34,14 @@ from products.batch_exports.backend.hogql_source import (
serialize_batch_export_query,
validate_hogql_batch_export_user,
)
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import (
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ BatchExportModelInput,
+ BatchExportSessionsModel,
+ migrate_batch_export_model,
+)
from products.batch_exports.backend.temporal.errors import MissingRequiredInputsError
from products.batch_exports.backend.temporal.metrics import log_query_duration
from products.batch_exports.backend.temporal.sql.common import (
@@ -442,7 +449,7 @@ class HogQLQueryRecordBatchModel(RecordBatchModel):
def resolve_batch_exports_model(
team_id: int,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: BatchExportModelInput | None = None,
batch_export_id: str | None = None,
is_backfill: bool = False,
):
@@ -452,35 +459,27 @@ def resolve_batch_exports_model(
and new parameters. Eventually, once everything is a `RecordBatchModel`, this could
be removed.
"""
- model = batch_export_model
+ model = migrate_batch_export_model(batch_export_model)
+ model_name = model.name
record_batch_model: RecordBatchModel | None = None
- if model is not None:
- model_name = model.name
- extra_query_parameters = model.schema["values"] if model.schema is not None else None
- fields = model.schema["fields"] if model.schema is not None else None
- filters = model.filters
-
- if model_name == "sessions":
+ schema = model.schema if isinstance(model, BatchExportEventsCustomModel) else None
+ extra_query_parameters = schema["values"] if schema is not None else None
+ fields = schema["fields"] if schema is not None else None
+ filters = model.filters if isinstance(model, BatchExportEventsModel | BatchExportEventsCustomModel) else None
+ match model:
+ case BatchExportSessionsModel():
record_batch_model = SessionsRecordBatchModel(
team_id=team_id, batch_export_id=batch_export_id, is_backfill=is_backfill
)
- elif model_name == "hogql":
- if model.hogql_query is None:
- raise UnsupportedHogQLQueryError("Batch export model is 'hogql' but no HogQL query was provided")
+ case BatchExportHogQLModel():
record_batch_model = HogQLQueryRecordBatchModel(
team_id=team_id,
- hogql_query=model.hogql_query,
+ hogql_query=model.query,
batch_export_id=batch_export_id,
user_id=model.user_id,
- hogql_modifiers=model.hogql_modifiers,
+ hogql_modifiers=model.modifiers,
)
- else:
- model_name = "events"
- extra_query_parameters = None
- fields = None
- filters = None
- schema = model.schema if model is not None else None
if schema is not None and (query := schema.get("hogql_query")) and use_new_events_schema(team_id):
context = HogQLContext(
team_id=team_id,
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
index 2b53a6012d69..dcf1de684fb5 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
@@ -282,7 +282,7 @@ class TestErrorHandling:
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
batch_export_model=BatchExportModel(
- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events"
+ name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=1
),
)
with patch(
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
index 8661f13db59a..7a9ef257a51e 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
@@ -1313,11 +1313,9 @@ class TestHogQLModel:
)
with patch("products.batch_exports.backend.temporal.pipeline.internal_stage._execute_query") as execute_query:
- result = await activity_environment.run(insert_into_internal_stage_activity, inputs)
+ with pytest.raises(ValueError, match="requires user_id"):
+ await activity_environment.run(insert_into_internal_stage_activity, inputs)
- assert result.error is not None
- assert result.error.type == "UnsupportedHogQLQueryError"
- assert "needs an active user" in result.error.message
execute_query.assert_not_called()
mock_clickhouse_client.expect_query_count(0)
diff --git a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
index 9352f0402923..4782cc6374c9 100644
--- a/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
+++ b/products/batch_exports/backend/tests/temporal/test_record_batch_model.py
@@ -28,7 +28,13 @@ from products.batch_exports.backend.hogql_source import (
UnsupportedHogQLQueryError,
validate_hogql_query_for_batch_export,
)
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ BatchExportEventsCustomModel,
+ BatchExportHogQLModel,
+ BatchExportModel,
+ BatchExportSchema,
+ migrate_batch_export_model,
+)
from products.batch_exports.backend.temporal.batch_exports import iter_records
from products.batch_exports.backend.temporal.filters import compose_filters_clause
from products.batch_exports.backend.temporal.record_batch_model import (
@@ -43,9 +49,78 @@ from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
if TYPE_CHECKING:
from posthog.models import Organization, Team
-pytestmark = [pytest.mark.asyncio, pytest.mark.django_db]
+pytestmark = [pytest.mark.asyncio]
+
+
[email protected]("migrated", [False, True])
+ "legacy",
+ [
+ None,
+ BatchExportModel(name="events", schema=None),
+ BatchExportModel(name="events", schema=None, filters=[{"key": "event", "value": "example"}]),
+ BatchExportModel(
+ name="events",
+ schema={"fields": [{"expression": "event", "alias": "name"}], "values": {"value": 1.5}},
+ filters=[{"key": "event", "value": "example"}],
+ ),
+ BatchExportModel(name="persons", schema=None),
+ BatchExportModel(name="sessions", schema=None),
+ BatchExportModel(
+ name="hogql",
+ schema=None,
+ hogql_query="SELECT event FROM events WHERE timestamp < {data_interval_end}",
+ user_id=42,
+ hogql_modifiers={"convertToProjectTimezone": False},
+ ),
+ ],
+)
+async def test_resolved_model_preserves_query_inputs(legacy: BatchExportModel | None, migrated: bool) -> None:
+ model, record_model, name, fields, filters, values = resolve_batch_exports_model(
+ team_id=1,
+ batch_export_model=migrate_batch_export_model(legacy) if migrated else legacy,
+ batch_export_id="export",
+ is_backfill=True,
+ )
+
+ assert name == (legacy.name if legacy is not None else "events")
+ assert model.name == name
+ schema = legacy.schema if legacy is not None else None
+ assert fields == (schema["fields"] if schema is not None else None)
+ assert values == (schema["values"] if schema is not None else None)
+ assert (filters or []) == ((legacy.filters or []) if legacy is not None else [])
+ if name == "sessions":
+ assert isinstance(record_model, SessionsRecordBatchModel)
+ assert record_model.is_backfill is True
+ assert record_model.team_id == 1
+ assert record_model.batch_export_id == "export"
+ elif name == "hogql":
+ assert isinstance(record_model, HogQLQueryRecordBatchModel)
+ assert legacy is not None
+ assert record_model.hogql_query == legacy.hogql_query
+ assert record_model.hogql_modifiers is not None
+ assert record_model.hogql_modifiers.convertToProjectTimezone is False
+ assert record_model.user_id == 42
+ assert record_model.wait_for_data_interval_end is True
+ else:
+ assert record_model is None
+
+ "query,user_id,error",
+ [(None, 1, "requires a query"), ("SELECT event FROM events", None, "requires user_id")],
+)
+async def test_resolve_batch_exports_model_rejects_incomplete_hogql_inputs(
+ query: str | None, user_id: int | None, error: str
+) -> None:
+ with pytest.raises(ValueError, match=error):
+ resolve_batch_exports_model(
+ team_id=1,
+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=user_id),
+ )
+
class TestSessionsRecordBatchModel:
async def test_get_hogql_query(self, ateam, data_interval_start, data_interval_end):
model = SessionsRecordBatchModel(
@@ -236,6 +311,7 @@ class TestSessionsRecordBatchModel:
assert "PARTITION BY rand() %% 5" in printed_query
class TestSessionsRecordBatchModelSelection:
"""Run the sessions model query against ClickHouse to verify which intervals select a session."""
@@ -349,6 +425,7 @@ class TestSessionsRecordBatchModelSelection:
assert selected == [session_id]
class TestHogQLQueryRecordBatchModel:
async def test_as_query_with_parameters(self, ateam, auser, data_interval_start, data_interval_end):
model = HogQLQueryRecordBatchModel(
@@ -529,13 +606,13 @@ class TestHogQLQueryRecordBatchModel:
HogQLQueryRecordBatchModel(team_id=1, hogql_query=hogql_query)
async def test_resolve_batch_exports_model_returns_hogql_model(self):
- batch_export_model = BatchExportModel(
- name="hogql",
- schema=None,
- hogql_query=(
+ batch_export_model = BatchExportHogQLModel(
+ query=(
"SELECT event AS event, timestamp AS timestamp FROM events "
"WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"
),
+ user_id=1,
+ modifiers={},
)
_, record_batch_model, model_name, _, _, _ = resolve_batch_exports_model(
@@ -544,7 +621,7 @@ class TestHogQLQueryRecordBatchModel:
assert isinstance(record_batch_model, HogQLQueryRecordBatchModel)
assert model_name == "hogql"
- assert record_batch_model.hogql_query == batch_export_model.hogql_query
+ assert record_batch_model.hogql_query == batch_export_model.query
assert record_batch_model.wait_for_data_interval_end is True
@pytest.mark.parametrize(
@@ -562,12 +639,10 @@ class TestHogQLQueryRecordBatchModel:
):
ateam.modifiers = team_modifiers
await ateam.asave()
- batch_export_model = BatchExportModel(
- name="hogql",
- schema=None,
- hogql_query="SELECT event AS event, timestamp AS timestamp FROM events",
+ batch_export_model = BatchExportHogQLModel(
+ query="SELECT event AS event, timestamp AS timestamp FROM events",
user_id=auser.pk,
- hogql_modifiers=hogql_modifiers,
+ modifiers=hogql_modifiers or {},
)
_, record_batch_model, _, _, _, _ = resolve_batch_exports_model(
@@ -578,26 +653,17 @@ class TestHogQLQueryRecordBatchModel:
assert ("toTimeZone(events.timestamp" in printed_query) is converts_timezone
- async def test_resolve_batch_exports_model_raises_without_hogql_query(self):
- """Without this, a missing query would fall through to the events template path and export the wrong data."""
- with pytest.raises(UnsupportedHogQLQueryError):
- resolve_batch_exports_model(team_id=1, batch_export_model=BatchExportModel(name="hogql", schema=None))
-
- @pytest.mark.parametrize("actor_state", ["missing", "deleted", "inactive", "nonmember", "project-denied"])
+ @pytest.mark.parametrize("actor_state", ["deleted", "inactive", "nonmember", "project-denied"])
async def test_rechecks_actor_access_at_runtime(
self, ateam: "Team", aorganization: "Organization", auser: User, actor_state: str
) -> None:
query = "SELECT event AS event FROM events"
await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)
- payload = BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)
+ payload = BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={})
_, model, _, _, _, _ = resolve_batch_exports_model(ateam.pk, payload)
await model.as_query_with_parameters(None, None)
- if actor_state == "missing":
- _, model, _, _, _, _ = resolve_batch_exports_model(
- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query)
- )
- elif actor_state == "deleted":
+ if actor_state == "deleted":
await User.objects.filter(pk=auser.pk).adelete()
elif actor_state == "inactive":
await User.objects.filter(pk=auser.pk).aupdate(is_active=False)
@@ -639,7 +705,7 @@ class TestHogQLQueryRecordBatchModel:
with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):
await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)
_, model, _, _, _, _ = resolve_batch_exports_model(
- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)
+ ateam.pk, BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={})
)
before, _ = await model.as_query_with_parameters(None, None)
assert "NULL AS secret" not in before
@@ -674,7 +740,7 @@ class TestHogQLQueryRecordBatchModel:
with patch("posthog.hogql.database.database.feature_enabled_or_false", return_value=True):
await database_sync_to_async(validate_hogql_query_for_batch_export)(query, ateam, user=auser)
_, model, _, _, _, _ = resolve_batch_exports_model(
- ateam.pk, BatchExportModel(name="hogql", schema=None, hogql_query=query, user_id=auser.pk)
+ ateam.pk, BatchExportHogQLModel(query=query, user_id=auser.pk, modifiers={})
)
await model.as_query_with_parameters(None, None)
membership = await OrganizationMembership.objects.aget(user=auser, organization=aorganization)
@@ -690,6 +756,7 @@ class TestHogQLQueryRecordBatchModel:
await model.as_query_with_parameters(None, None)
@pytest.mark.parametrize("use_new_events_schema", [False, True])
async def test_custom_export_recompilation_preserves_column_names(ateam, use_new_events_schema):
schema: BatchExportSchema = {
@@ -702,7 +769,7 @@ async def test_custom_export_recompilation_preserves_column_names(ateam, use_new
}
with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):
_, _, _, fields, _, values = await database_sync_to_async(resolve_batch_exports_model)(
- team_id=ateam.pk, batch_export_model=BatchExportModel(name="events", schema=schema)
+ team_id=ateam.pk, batch_export_model=BatchExportEventsCustomModel(schema=schema)
)
assert fields is not None
@@ -719,6 +786,7 @@ async def test_custom_export_recompilation_preserves_column_names(ateam, use_new
@pytest.mark.parametrize("interval_start_is_none", [False, True])
@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
async def test_custom_export_backfill_runs_without_legacy_events_tables(
ateam, clickhouse_client, interval_start_is_none
):
@@ -770,7 +838,7 @@ async def test_custom_export_backfill_runs_without_legacy_events_tables(
"values": {"unused_old_parameter": "stale"},
}
_, _, _, fields, _, values = await database_sync_to_async(resolve_batch_exports_model)(
- team_id=ateam.pk, batch_export_model=BatchExportModel(name="events", schema=schema)
+ team_id=ateam.pk, batch_export_model=BatchExportEventsCustomModel(schema=schema)
)
assert fields is not None
assert "unused_old_parameter" not in values
diff --git a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
index a0252e1fad14..f8f311fffbb4 100644
--- a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
+++ b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
@@ -74,7 +74,7 @@ def test_each_run_substitutes_its_own_bounds_without_mutating_the_query() -> Non
),
("not a valid query", "Failed to parse HogQL query"),
("SELECT event FROM events WHERE timestamp >= {unknown}", "Unknown placeholder '{unknown}'"),
- (None, "no HogQL query was provided"),
+ (None, "requires a query"),
],
ids=["missing-start", "malformed-query", "unsupported-placeholder", "missing-query"],
)
@@ -84,7 +84,7 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: st
batch_export_id="00000000-0000-4000-8000-000000000001",
data_interval_start=None,
data_interval_end=(dt.datetime.now(dt.UTC) - dt.timedelta(hours=1)).isoformat(),
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query),
+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),
)
clickhouse = MockClickHouseClient()
@@ -92,10 +92,13 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: st
"products.batch_exports.backend.temporal.pipeline.internal_stage.get_client",
return_value=clickhouse.mock_client_cm,
):
- result = await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)
-
- assert result.error is not None
- assert result.error.type == "UnsupportedHogQLQueryError"
- assert expected_error in result.error.message
- assert result.records_total is None
+ if hogql_query is None:
+ with pytest.raises(ValueError, match=expected_error):
+ await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)
+ else:
+ result = await ActivityEnvironment().run(insert_into_internal_stage_activity, inputs)
+ assert result.error is not None
+ assert result.error.type == "UnsupportedHogQLQueryError"
+ assert expected_error in result.error.message
+ assert result.records_total is None
clickhouse.expect_query_count(0)
diff --git a/products/batch_exports/backend/tests/test_nullable_staging.py b/products/batch_exports/backend/tests/test_nullable_staging.py
index 0f74440e0211..aee5d71c9ba7 100644
--- a/products/batch_exports/backend/tests/test_nullable_staging.py
+++ b/products/batch_exports/backend/tests/test_nullable_staging.py
@@ -76,7 +76,7 @@ async def test_hogql_staging_preserves_optional_bounds_and_cleanup_namespace(
data_interval_start=start_str,
data_interval_end=end_str,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),
+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),
)
result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
@@ -110,13 +110,16 @@ async def test_unsupported_hogql_returns_nonretryable_staging_error(
data_interval_start=None,
data_interval_end=None,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query),
+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),
)
- result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
-
- assert result.error is not None
- assert result.error.type == "UnsupportedHogQLQueryError"
+ if hogql_query is None:
+ with pytest.raises(ValueError, match="requires a query"):
+ await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
+ else:
+ result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
+ assert result.error is not None
+ assert result.error.type == "UnsupportedHogQLQueryError"
staging_clients[0].execute_query_with_summary.assert_not_called()
staging_clients[1].list_objects_v2.assert_not_called()
@@ -142,7 +145,7 @@ async def test_hogql_staging_rejects_each_future_bound_without_waiting(
data_interval_start=start.isoformat() if start is not None else None,
data_interval_end=end.isoformat() if end is not None else None,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),
+ batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),
)
# Patch the helper rather than override settings.TEST, because DatabaseSyncToAsync calls
# close_old_connections() whenever settings.TEST is False, and that needs a database.
@@ -179,7 +182,7 @@ async def test_missing_staging_inputs_raise(
data_interval_start=None,
data_interval_end=None,
on_demand=on_demand,
- batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value"),
+ batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value", user_id=1),
)
with pytest.raises(MissingRequiredInputsError, match="require"):
await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)