Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(service): Emit model-specific export inputs (#120116)

Tomás Farías Santana670f01f3ca2a
diff --git a/products/batch_exports/backend/debug/debugger.py b/products/batch_exports/backend/debug/debugger.py
index 060ad60e15d6..b1eb207e3f8d 100644
--- a/products/batch_exports/backend/debug/debugger.py
+++ b/products/batch_exports/backend/debug/debugger.py
@@ -25,9 +25,9 @@ from products.batch_exports.backend.service import (
DESTINATION_WORKFLOWS,
BaseBatchExportInputs,
BatchExportField,
- BatchExportModel,
BigQueryBatchExportInputs,
DatabricksBatchExportInputs,
+ get_batch_export_model,
)
from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (
BigQueryClient,
@@ -293,13 +293,7 @@ class BatchExportsDebugger:
team_id=self.batch_export.team_id,
batch_export_id=str(self.batch_export.id),
interval=str(self.batch_export.interval),
- batch_export_model=BatchExportModel(
- name=self.batch_export.model or "events",
- schema=self.batch_export.schema,
- filters=self.batch_export.filters,
- hogql_query=self.batch_export.hogql_query,
- hogql_modifiers=self.batch_export.hogql_modifiers,
- ),
+ batch_export_model=get_batch_export_model(self.batch_export),
integration_id=self.batch_export.destination.integration_id,
**destination_config,
)
diff --git a/products/batch_exports/backend/logic/file_downloads.py b/products/batch_exports/backend/logic/file_downloads.py
index f8c54e85d173..c5826c1dc845 100644
--- a/products/batch_exports/backend/logic/file_downloads.py
+++ b/products/batch_exports/backend/logic/file_downloads.py
@@ -39,8 +39,8 @@ from products.batch_exports.backend.models.batch_export import (
BatchExportSource,
)
from products.batch_exports.backend.service import (
- BatchExportModel,
cancel_running_batch_export_run,
+ get_batch_export_model,
start_file_download_batch_export,
)
from products.batch_exports.backend.temporal.record_batch_model import HogQLQueryRecordBatchModel
@@ -198,7 +198,6 @@ def start_file_download(team_id: int, user_id: int, request: contracts.FileDownl
batch_export_on_demand = instance.batch_export_on_demand
assert batch_export_on_demand is not None
- source = batch_export_on_demand.source
destination_config = batch_export_on_demand.destination.config
try:
start_file_download_batch_export(
@@ -207,13 +206,7 @@ def start_file_download(team_id: int, user_id: int, request: contracts.FileDownl
batch_export_run_id=instance.id,
data_interval_start=instance.data_interval_start,
data_interval_end=instance.data_interval_end,
- batch_export_model=BatchExportModel(
- name=request.model,
- schema=None,
- hogql_query=source.hogql_query if source is not None else None,
- user_id=batch_export_on_demand.last_modified_by_id if request.model == "hogql" else None,
- hogql_modifiers=source.hogql_modifiers if source is not None else None,
- ),
+ batch_export_model=get_batch_export_model(batch_export_on_demand),
compression=destination_config.get("compression", None),
format=destination_config.get("format", "Parquet"),
max_size_mb=destination_config.get("max_size_mb", 0),
diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.py
index fab473883e2f..8151809a5924 100644
--- a/products/batch_exports/backend/service.py
+++ b/products/batch_exports/backend/service.py
@@ -312,6 +312,38 @@ def migrate_batch_export_model(old: BatchExportModelInput | None, /) -> AnyBatch
raise ValueError(f"Invalid model name: '{s}'")
+def get_batch_export_model(batch_export: BatchExport | BatchExportOnDemand) -> AnyBatchExportModel:
+ match batch_export.model or "events":
+ case "events":
+ schema = batch_export.schema if isinstance(batch_export, BatchExport) else None
+ if schema is not None:
+ return BatchExportEventsCustomModel(schema=schema, filters=batch_export.filters or [])
+ return BatchExportEventsModel(
+ filters=batch_export.filters or [],
+ incremental_mode=IncrementalMode(batch_export.incremental_mode),
+ )
+ case "persons":
+ return BatchExportPersonsModel()
+ case "sessions":
+ return BatchExportSessionsModel()
+ case "hogql":
+ source = batch_export.source
+ if batch_export.last_modified_by_id is None:
+ raise ValueError("HogQL model requires user_id")
+ if source is None or source.hogql_query is None:
+ raise ValueError("HogQL model requires a query")
+ return BatchExportHogQLModel(
+ query=source.hogql_query,
+ user_id=batch_export.last_modified_by_id,
+ modifiers=source.hogql_modifiers or {},
+ incremental_mode=IncrementalMode(batch_export.incremental_mode),
+ primary_key=tuple(source.primary_key or ()),
+ version_key=tuple(source.version_key or ()),
+ )
+ case name:
+ raise ValueError(f"Invalid model name: '{name}'")
+
+
@dataclass
class BackfillDetails:
backfill_id: str | None
@@ -1351,16 +1383,7 @@ def sync_batch_export(batch_export: BatchExport, created: bool):
batch_export_id=str(batch_export.id),
interval=str(batch_export.interval),
timezone=str(batch_export.timezone_info),
- batch_export_model=BatchExportModel(
- name=batch_export.model or "events",
- schema=batch_export.schema,
- filters=batch_export.filters,
- hogql_query=batch_export.hogql_query,
- user_id=batch_export.last_modified_by_id
- if batch_export.model == BatchExport.Model.HOGQL
- else None,
- hogql_modifiers=batch_export.hogql_modifiers,
- ),
+ batch_export_model=get_batch_export_model(batch_export),
integration_id=batch_export.destination.integration_id,
**destination_config,
)
diff --git a/products/batch_exports/backend/tests/api/test_create.py b/products/batch_exports/backend/tests/api/test_create.py
index fb72cb242a4b..52caacce1e07 100644
--- a/products/batch_exports/backend/tests/api/test_create.py
+++ b/products/batch_exports/backend/tests/api/test_create.py
@@ -108,7 +108,7 @@ def test_create_batch_export_with_interval_schedule(
assert args["team_id"] == team.pk
assert args["batch_export_id"] == data["id"]
assert args["interval"] == interval
- assert args["batch_export_model"]["user_id"] is None
+ assert "user_id" not in args["batch_export_model"]
# expected jitter is 15 minutes for hourly exports
assert batch_export.jitter == dt.timedelta(minutes=15)
@@ -541,12 +541,12 @@ def test_create_batch_export_with_custom_schema(
assert batch_export.schema == expected_schema
assert args["batch_export_model"] == {
- "filters": None,
+ "filters": [],
"name": "events",
"schema": expected_schema,
- "hogql_query": None,
- "user_id": None,
- "hogql_modifiers": None,
+ "primary_key": [],
+ "version_key": [],
+ "incremental_mode": "APPEND",
}
@@ -663,12 +663,13 @@ def test_create_batch_export_with_hogql_model(
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
assert args["batch_export_model"] == {
- "filters": None,
"name": "hogql",
- "schema": None,
- "hogql_query": hogql_batch_export_data["hogql_query"],
+ "query": hogql_batch_export_data["hogql_query"],
"user_id": user.pk,
- "hogql_modifiers": hogql_modifiers,
+ "modifiers": hogql_modifiers,
+ "primary_key": [],
+ "version_key": [],
+ "incremental_mode": "APPEND",
}
diff --git a/products/batch_exports/backend/tests/api/test_update.py b/products/batch_exports/backend/tests/api/test_update.py
index 809deb072684..cf58b95afd0e 100644
--- a/products/batch_exports/backend/tests/api/test_update.py
+++ b/products/batch_exports/backend/tests/api/test_update.py
@@ -130,7 +130,7 @@ def test_can_put_config(
decoded_payload = async_to_sync(encryption_codec.decode)(new_schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
assert args["bucket_name"] == "my-new-production-s3-bucket"
- assert args["batch_export_model"]["user_id"] is None
+ assert "user_id" not in args["batch_export_model"]
# Credentials are resolved from the integration at run time, never carried in the schedule.
assert args["integration_id"] == aws_s3_integration.id
assert args.get("aws_secret_access_key") is None
@@ -226,7 +226,7 @@ def test_can_patch_config(
args = json.loads(decoded_payload[0].data)
assert args["bucket_name"] == "my-new-production-s3-bucket"
assert new_schedule.schedule.spec.time_zone_name == timezone
- assert args["batch_export_model"]["user_id"] is None
+ assert "user_id" not in args["batch_export_model"]
@pytest.mark.parametrize(
@@ -724,7 +724,7 @@ def test_can_patch_hogql_query(
assert args["interval"] == "hour"
assert args["batch_export_model"] == {
"name": "events",
- "filters": None,
+ "filters": [],
"schema": {
"fields": [
{
@@ -743,9 +743,9 @@ def test_can_patch_hogql_query(
"values": {"hogql_val_0": "test", "hogql_val_1": "Int64"},
"hogql_query": "SELECT toString(uuid) AS uuid, 'test' AS test, toInt(plus(1, 1)) AS n FROM events",
},
- "hogql_query": None,
- "user_id": None,
- "hogql_modifiers": None,
+ "primary_key": [],
+ "version_key": [],
+ "incremental_mode": "APPEND",
}
for patch_data, expected_schema in [
@@ -758,7 +758,7 @@ def test_can_patch_hogql_query(
schedule = describe_schedule(temporal, batch_export["id"])
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
- assert args["batch_export_model"]["schema"] == expected_schema
+ assert args["batch_export_model"].get("schema") == expected_schema
@pytest.mark.usefixtures("grandfathered_batch_export")
@@ -833,7 +833,7 @@ def test_patch_hogql_model_batch_export(
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
assert args["batch_export_model"]["user_id"] == editor.pk
- assert args["batch_export_model"]["hogql_query"] == hogql_batch_export_data["hogql_query"]
+ assert args["batch_export_model"]["query"] == hogql_batch_export_data["hogql_query"]
# A query without interval placeholders is as valid as a bounded one.
new_hogql_query = "SELECT uuid AS uuid, created_at AS created_at FROM events"
@@ -853,12 +853,13 @@ def test_patch_hogql_model_batch_export(
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
assert args["batch_export_model"] == {
- "filters": None,
"name": "hogql",
- "schema": None,
- "hogql_query": new_hogql_query,
+ "query": new_hogql_query,
"user_id": editor.pk,
- "hogql_modifiers": None,
+ "modifiers": {},
+ "primary_key": [],
+ "version_key": [],
+ "incremental_mode": "APPEND",
}
# Modifiers can change without the query, and null removes them.
@@ -874,8 +875,8 @@ def test_patch_hogql_model_batch_export(
schedule = describe_schedule(temporal, batch_export["id"])
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
- assert args["batch_export_model"]["hogql_query"] == new_hogql_query
- assert args["batch_export_model"]["hogql_modifiers"] == hogql_modifiers
+ assert args["batch_export_model"]["query"] == new_hogql_query
+ assert args["batch_export_model"]["modifiers"] == (hogql_modifiers or {})
# A modifier removed from the schema after the export stored it is not returned, so sending back what was read
# still saves.
@@ -954,7 +955,7 @@ def test_patch_hogql_model_batch_export_checks_editor_table_access(
decoded_payload = async_to_sync(encryption_codec.decode)(schedule.schedule.action.args)
args = json.loads(decoded_payload[0].data)
assert args["batch_export_model"]["user_id"] == editor.pk
- assert args["batch_export_model"]["hogql_query"] == hogql_query
+ assert args["batch_export_model"]["query"] == hogql_query
@pytest.mark.usefixtures("hogql_batch_exports_enabled")
diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py
index 4093c933dff2..d854c7df364f 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_api.py
@@ -945,9 +945,9 @@ class TestFileDownloadHogQL:
mock_start_file_download_export.assert_called_once()
batch_export_model = mock_start_file_download_export.call_args.kwargs["batch_export_model"]
assert batch_export_model.name == "hogql"
- assert batch_export_model.hogql_query == hogql_query
+ assert batch_export_model.query == hogql_query
assert batch_export_model.user_id == user.pk
- assert batch_export_model.hogql_modifiers == {"convertToProjectTimezone": False}
+ assert batch_export_model.modifiers == {"convertToProjectTimezone": False}
assert mock_start_file_download_export.call_args.kwargs["max_size_mb"] == DEFAULT_MAX_SIZE_MB
assert mock_start_file_download_export.call_args.kwargs["data_interval_start"] == run.data_interval_start
assert mock_start_file_download_export.call_args.kwargs["data_interval_end"] == run.data_interval_end
diff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.py
index bdafded89fe6..dd19499ecd32 100644
--- a/products/batch_exports/backend/tests/test_service.py
+++ b/products/batch_exports/backend/tests/test_service.py
@@ -10,7 +10,9 @@ from products.batch_exports.backend.models.batch_export import (
BatchExport,
BatchExportBackfill,
BatchExportDestination,
+ BatchExportOnDemand,
BatchExportRun,
+ BatchExportSource,
)
from products.batch_exports.backend.service import (
AnyBatchExportModel,
@@ -33,12 +35,85 @@ from products.batch_exports.backend.service import (
S3CompatibleBatchExportInputs,
aget_or_create_batch_export_backfill,
align_timestamp_to_interval,
+ get_batch_export_model,
migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.pipeline.internal_stage import BatchExportInsertIntoInternalStageInputs
class TestBatchExportModelMigration:
+ @pytest.mark.parametrize(
+ "input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]
+ )
+ @pytest.mark.parametrize(
+ "batch_export,expected",
+ [
+ (BatchExport(model=None), BatchExportEventsModel()),
+ (BatchExport(model=""), BatchExportEventsModel()),
+ (BatchExport(model="events", filters=None), BatchExportEventsModel()),
+ (
+ BatchExport(model="events", filters=[{"key": "amount", "value": [1.5, True]}]),
+ BatchExportEventsModel(filters=[{"key": "amount", "value": [1.5, True]}]),
+ ),
+ (
+ BatchExport(model="events", schema={"fields": [], "values": {"value": 1.5}}),
+ BatchExportEventsCustomModel(schema={"fields": [], "values": {"value": 1.5}}),
+ ),
+ (BatchExport(model="persons"), BatchExportPersonsModel()),
+ (BatchExport(model="sessions"), BatchExportSessionsModel()),
+ (
+ BatchExport(
+ model="hogql",
+ last_modified_by_id=42,
+ incremental_mode="MERGE",
+ source=BatchExportSource(
+ hogql_query="SELECT id, updated_at FROM events",
+ hogql_modifiers={"convertToProjectTimezone": False},
+ primary_key=["id"],
+ version_key=["updated_at"],
+ ),
+ ),
+ BatchExportHogQLModel(
+ query="SELECT id, updated_at FROM events",
+ user_id=42,
+ modifiers={"convertToProjectTimezone": False},
+ incremental_mode=IncrementalMode.MERGE,
+ primary_key=("id",),
+ version_key=("updated_at",),
+ ),
+ ),
+ (
+ BatchExportOnDemand(
+ model="hogql",
+ last_modified_by_id=42,
+ source=BatchExportSource(hogql_query="SELECT event FROM events"),
+ ),
+ BatchExportHogQLModel(query="SELECT event FROM events", user_id=42, modifiers={}),
+ ),
+ ],
+ )
+ def test_model_from_saved_export(
+ self,
+ batch_export: BatchExport | BatchExportOnDemand,
+ expected: AnyBatchExportModel,
+ input_type: type[
+ FileDownloadBatchExportInputs | BatchExportInsertInputs | BatchExportInsertIntoInternalStageInputs
+ ],
+ ) -> None:
+ model = get_batch_export_model(batch_export)
+ assert model == expected
+ inputs = input_type(
+ team_id=1,
+ batch_export_id="test",
+ data_interval_start=None,
+ data_interval_end=None,
+ batch_export_model=model,
+ )
+ converter = JSONPlainPayloadConverter()
+ payload = converter.to_payload(inputs)
+ assert payload is not None
+ assert converter.from_payload(payload, input_type).batch_export_model == expected
+
@pytest.mark.parametrize("model_specific", [False, True])
@pytest.mark.parametrize(
"input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]