:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(service): Emit model-specific export inputs (#120116)
Tomás Farías Santana670f01f3ca2a
diff --git a/products/batch_exports/backend/debug/debugger.py b/products/batch_exports/backend/debug/debugger.pyindex 060ad60e15d6..b1eb207e3f8d 100644--- a/products/batch_exports/backend/debug/debugger.py+++ b/products/batch_exports/backend/debug/debugger.py@@ -25,9 +25,9 @@ from products.batch_exports.backend.service import (DESTINATION_WORKFLOWS,BaseBatchExportInputs,BatchExportField,- BatchExportModel,BigQueryBatchExportInputs,DatabricksBatchExportInputs,+ get_batch_export_model,)from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (BigQueryClient,@@ -293,13 +293,7 @@ class BatchExportsDebugger:team_id=self.batch_export.team_id,batch_export_id=str(self.batch_export.id),interval=str(self.batch_export.interval),- batch_export_model=BatchExportModel(- name=self.batch_export.model or "events",- schema=self.batch_export.schema,- filters=self.batch_export.filters,- hogql_query=self.batch_export.hogql_query,- hogql_modifiers=self.batch_export.hogql_modifiers,- ),+ batch_export_model=get_batch_export_model(self.batch_export),integration_id=self.batch_export.destination.integration_id,**destination_config,)diff --git a/products/batch_exports/backend/logic/file_downloads.py b/products/batch_exports/backend/logic/file_downloads.pyindex f8c54e85d173..c5826c1dc845 100644--- a/products/batch_exports/backend/logic/file_downloads.py+++ b/products/batch_exports/backend/logic/file_downloads.py@@ -39,8 +39,8 @@ from products.batch_exports.backend.models.batch_export import (BatchExportSource,)from products.batch_exports.backend.service import (- BatchExportModel,cancel_running_batch_export_run,+ get_batch_export_model,start_file_download_batch_export,)from products.batch_exports.backend.temporal.record_batch_model import HogQLQueryRecordBatchModel@@ -198,7 +198,6 @@ def start_file_download(team_id: int, user_id: int, request: contracts.FileDownlbatch_export_on_demand = instance.batch_export_on_demandassert batch_export_on_demand is not None- source = batch_export_on_demand.sourcedestination_config = batch_export_on_demand.destination.configtry:start_file_download_batch_export(@@ -207,13 +206,7 @@ def start_file_download(team_id: int, user_id: int, request: contracts.FileDownlbatch_export_run_id=instance.id,data_interval_start=instance.data_interval_start,data_interval_end=instance.data_interval_end,- batch_export_model=BatchExportModel(- name=request.model,- schema=None,- hogql_query=source.hogql_query if source is not None else None,- user_id=batch_export_on_demand.last_modified_by_id if request.model == "hogql" else None,- hogql_modifiers=source.hogql_modifiers if source is not None else None,- ),+ batch_export_model=get_batch_export_model(batch_export_on_demand),compression=destination_config.get("compression", None),format=destination_config.get("format", "Parquet"),max_size_mb=destination_config.get("max_size_mb", 0),diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.pyindex fab473883e2f..8151809a5924 100644--- a/products/batch_exports/backend/service.py+++ b/products/batch_exports/backend/service.py@@ -312,6 +312,38 @@ def migrate_batch_export_model(old: BatchExportModelInput | None, /) -> AnyBatchraise ValueError(f"Invalid model name: '{s}'")+def get_batch_export_model(batch_export: BatchExport | BatchExportOnDemand) -> AnyBatchExportModel:+ match batch_export.model or "events":+ case "events":+ schema = batch_export.schema if isinstance(batch_export, BatchExport) else None+ if schema is not None:+ return BatchExportEventsCustomModel(schema=schema, filters=batch_export.filters or [])+ return BatchExportEventsModel(+ filters=batch_export.filters or [],+ incremental_mode=IncrementalMode(batch_export.incremental_mode),+ )+ case "persons":+ return BatchExportPersonsModel()+ case "sessions":+ return BatchExportSessionsModel()+ case "hogql":+ source = batch_export.source+ if batch_export.last_modified_by_id is None:+ raise ValueError("HogQL model requires user_id")+ if source is None or source.hogql_query is None:+ raise ValueError("HogQL model requires a query")+ return BatchExportHogQLModel(+ query=source.hogql_query,+ user_id=batch_export.last_modified_by_id,+ modifiers=source.hogql_modifiers or {},+ incremental_mode=IncrementalMode(batch_export.incremental_mode),+ primary_key=tuple(source.primary_key or ()),+ version_key=tuple(source.version_key or ()),+ )+ case name:+ raise ValueError(f"Invalid model name: '{name}'")++@dataclassclass BackfillDetails:backfill_id: str | None@@ -1351,16 +1383,7 @@ def sync_batch_export(batch_export: BatchExport, created: bool):batch_export_id=str(batch_export.id),interval=str(batch_export.interval),timezone=str(batch_export.timezone_info),- batch_export_model=BatchExportModel(- name=batch_export.model or "events",- schema=batch_export.schema,- filters=batch_export.filters,- hogql_query=batch_export.hogql_query,- user_id=batch_export.last_modified_by_id- if batch_export.model == BatchExport.Model.HOGQL- else None,- hogql_modifiers=batch_export.hogql_modifiers,- ),+ batch_export_model=get_batch_export_model(batch_export),integration_id=batch_export.destination.integration_id,**destination_config,)diff --git a/products/batch_exports/backend/tests/api/test_create.py b/products/batch_exports/backend/tests/api/test_create.pyindex fb72cb242a4b..52caacce1e07 100644--- a/products/batch_exports/backend/tests/api/test_create.py+++ b/products/batch_exports/backend/tests/api/test_create.py@@ -108,7 +108,7 @@ def test_create_batch_export_with_interval_schedule(assert args["team_id"] == team.pkassert args["batch_export_id"] == data["id"]assert args["interval"] == interval- assert args["batch_export_model"]["user_id"] is None+ assert "user_id" not in args["batch_export_model"]# expected jitter is 15 minutes for hourly exportsassert batch_export.jitter == dt.timedelta(minutes=15)@@ -541,12 +541,12 @@ def test_create_batch_export_with_custom_schema(assert batch_export.schema == expected_schemaassert args["batch_export_model"] == {- "filters": None,+ "filters": [],"name": "events","schema": expected_schema,- "hogql_query": None,- "user_id": None,- "hogql_modifiers": None,+ "primary_key": [],+ "version_key": [],+ "incremental_mode": "APPEND",}@@ -663,12 +663,13 @@ def test_create_batch_export_with_hogql_model(decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)assert args["batch_export_model"] == {- "filters": None,"name": "hogql",- "schema": None,- "hogql_query": hogql_batch_export_data["hogql_query"],+ "query": hogql_batch_export_data["hogql_query"],"user_id": user.pk,- "hogql_modifiers": hogql_modifiers,+ "modifiers": hogql_modifiers,+ "primary_key": [],+ "version_key": [],+ "incremental_mode": "APPEND",}diff --git a/products/batch_exports/backend/tests/api/test_update.py b/products/batch_exports/backend/tests/api/test_update.pyindex 809deb072684..cf58b95afd0e 100644--- a/products/batch_exports/backend/tests/api/test_update.py+++ b/products/batch_exports/backend/tests/api/test_update.py@@ -130,7 +130,7 @@ def test_can_put_config(decoded_payload = async_to_sync(encryption_codec.decode)(new_schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)assert args["bucket_name"] == "my-new-production-s3-bucket"- assert args["batch_export_model"]["user_id"] is None+ assert "user_id" not in args["batch_export_model"]# Credentials are resolved from the integration at run time, never carried in the schedule.assert args["integration_id"] == aws_s3_integration.idassert args.get("aws_secret_access_key") is None@@ -226,7 +226,7 @@ def test_can_patch_config(args = json.loads(decoded_payload[0].data)assert args["bucket_name"] == "my-new-production-s3-bucket"assert new_schedule.schedule.spec.time_zone_name == timezone- assert args["batch_export_model"]["user_id"] is None+ assert "user_id" not in args["batch_export_model"]@pytest.mark.parametrize(@@ -724,7 +724,7 @@ def test_can_patch_hogql_query(assert args["interval"] == "hour"assert args["batch_export_model"] == {"name": "events",- "filters": None,+ "filters": [],"schema": {"fields": [{@@ -743,9 +743,9 @@ def test_can_patch_hogql_query("values": {"hogql_val_0": "test", "hogql_val_1": "Int64"},"hogql_query": "SELECT toString(uuid) AS uuid, 'test' AS test, toInt(plus(1, 1)) AS n FROM events",},- "hogql_query": None,- "user_id": None,- "hogql_modifiers": None,+ "primary_key": [],+ "version_key": [],+ "incremental_mode": "APPEND",}for patch_data, expected_schema in [@@ -758,7 +758,7 @@ def test_can_patch_hogql_query(schedule = describe_schedule(temporal, batch_export["id"])decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)- assert args["batch_export_model"]["schema"] == expected_schema+ assert args["batch_export_model"].get("schema") == expected_schema@pytest.mark.usefixtures("grandfathered_batch_export")@@ -833,7 +833,7 @@ def test_patch_hogql_model_batch_export(decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)assert args["batch_export_model"]["user_id"] == editor.pk- assert args["batch_export_model"]["hogql_query"] == hogql_batch_export_data["hogql_query"]+ assert args["batch_export_model"]["query"] == hogql_batch_export_data["hogql_query"]# A query without interval placeholders is as valid as a bounded one.new_hogql_query = "SELECT uuid AS uuid, created_at AS created_at FROM events"@@ -853,12 +853,13 @@ def test_patch_hogql_model_batch_export(decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)assert args["batch_export_model"] == {- "filters": None,"name": "hogql",- "schema": None,- "hogql_query": new_hogql_query,+ "query": new_hogql_query,"user_id": editor.pk,- "hogql_modifiers": None,+ "modifiers": {},+ "primary_key": [],+ "version_key": [],+ "incremental_mode": "APPEND",}# Modifiers can change without the query, and null removes them.@@ -874,8 +875,8 @@ def test_patch_hogql_model_batch_export(schedule = describe_schedule(temporal, batch_export["id"])decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)- assert args["batch_export_model"]["hogql_query"] == new_hogql_query- assert args["batch_export_model"]["hogql_modifiers"] == hogql_modifiers+ assert args["batch_export_model"]["query"] == new_hogql_query+ assert args["batch_export_model"]["modifiers"] == (hogql_modifiers or {})# A modifier removed from the schema after the export stored it is not returned, so sending back what was read# still saves.@@ -954,7 +955,7 @@ def test_patch_hogql_model_batch_export_checks_editor_table_access(decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)args = json.loads(decoded_payload[0].data)assert args["batch_export_model"]["user_id"] == editor.pk- assert args["batch_export_model"]["hogql_query"] == hogql_query+ assert args["batch_export_model"]["query"] == hogql_query@pytest.mark.usefixtures("hogql_batch_exports_enabled")diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.pyindex 4093c933dff2..d854c7df364f 100644--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py@@ -945,9 +945,9 @@ class TestFileDownloadHogQL:mock_start_file_download_export.assert_called_once()batch_export_model = mock_start_file_download_export.call_args.kwargs["batch_export_model"]assert batch_export_model.name == "hogql"- assert batch_export_model.hogql_query == hogql_query+ assert batch_export_model.query == hogql_queryassert batch_export_model.user_id == user.pk- assert batch_export_model.hogql_modifiers == {"convertToProjectTimezone": False}+ assert batch_export_model.modifiers == {"convertToProjectTimezone": False}assert mock_start_file_download_export.call_args.kwargs["max_size_mb"] == DEFAULT_MAX_SIZE_MBassert mock_start_file_download_export.call_args.kwargs["data_interval_start"] == run.data_interval_startassert mock_start_file_download_export.call_args.kwargs["data_interval_end"] == run.data_interval_enddiff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.pyindex bdafded89fe6..dd19499ecd32 100644--- a/products/batch_exports/backend/tests/test_service.py+++ b/products/batch_exports/backend/tests/test_service.py@@ -10,7 +10,9 @@ from products.batch_exports.backend.models.batch_export import (BatchExport,BatchExportBackfill,BatchExportDestination,+ BatchExportOnDemand,BatchExportRun,+ BatchExportSource,)from products.batch_exports.backend.service import (AnyBatchExportModel,@@ -33,12 +35,85 @@ from products.batch_exports.backend.service import (S3CompatibleBatchExportInputs,aget_or_create_batch_export_backfill,align_timestamp_to_interval,+ get_batch_export_model,migrate_batch_export_model,)from products.batch_exports.backend.temporal.pipeline.internal_stage import BatchExportInsertIntoInternalStageInputsclass TestBatchExportModelMigration:+ @pytest.mark.parametrize(+ "input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]+ )+ @pytest.mark.parametrize(+ "batch_export,expected",+ [+ (BatchExport(model=None), BatchExportEventsModel()),+ (BatchExport(model=""), BatchExportEventsModel()),+ (BatchExport(model="events", filters=None), BatchExportEventsModel()),+ (+ BatchExport(model="events", filters=[{"key": "amount", "value": [1.5, True]}]),+ BatchExportEventsModel(filters=[{"key": "amount", "value": [1.5, True]}]),+ ),+ (+ BatchExport(model="events", schema={"fields": [], "values": {"value": 1.5}}),+ BatchExportEventsCustomModel(schema={"fields": [], "values": {"value": 1.5}}),+ ),+ (BatchExport(model="persons"), BatchExportPersonsModel()),+ (BatchExport(model="sessions"), BatchExportSessionsModel()),+ (+ BatchExport(+ model="hogql",+ last_modified_by_id=42,+ incremental_mode="MERGE",+ source=BatchExportSource(+ hogql_query="SELECT id, updated_at FROM events",+ hogql_modifiers={"convertToProjectTimezone": False},+ primary_key=["id"],+ version_key=["updated_at"],+ ),+ ),+ BatchExportHogQLModel(+ query="SELECT id, updated_at FROM events",+ user_id=42,+ modifiers={"convertToProjectTimezone": False},+ incremental_mode=IncrementalMode.MERGE,+ primary_key=("id",),+ version_key=("updated_at",),+ ),+ ),+ (+ BatchExportOnDemand(+ model="hogql",+ last_modified_by_id=42,+ source=BatchExportSource(hogql_query="SELECT event FROM events"),+ ),+ BatchExportHogQLModel(query="SELECT event FROM events", user_id=42, modifiers={}),+ ),+ ],+ )+ def test_model_from_saved_export(+ self,+ batch_export: BatchExport | BatchExportOnDemand,+ expected: AnyBatchExportModel,+ input_type: type[+ FileDownloadBatchExportInputs | BatchExportInsertInputs | BatchExportInsertIntoInternalStageInputs+ ],+ ) -> None:+ model = get_batch_export_model(batch_export)+ assert model == expected+ inputs = input_type(+ team_id=1,+ batch_export_id="test",+ data_interval_start=None,+ data_interval_end=None,+ batch_export_model=model,+ )+ converter = JSONPlainPayloadConverter()+ payload = converter.to_payload(inputs)+ assert payload is not None+ assert converter.from_payload(payload, input_type).batch_export_model == expected+@pytest.mark.parametrize("model_specific", [False, True])@pytest.mark.parametrize("input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]