:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(tests): Migrate batch export model fixtures (#120891)
Tomás Farías Santana51543429f36e
diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.pyindex 918ab5cb85b1..c97a929f306c 100644--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py@@ -6,7 +6,7 @@ from temporalio.testing import ActivityEnvironmentfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsCustomModelfrom products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (AzureBlobInsertInputs,azure_blob_default_fields,@@ -76,10 +76,10 @@ async def test_activity_exports_model_to_azure_blob():"""Test that events, persons, and sessions are exported to Azure Blob Storage."""batch_export_model = None- if isinstance(model, BatchExportModel):+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)batch_export_id = str(uuid.uuid4())diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.pyindex 478c36f0c3cd..27ea7a513cc0 100644--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py@@ -1,6 +1,11 @@import pytest-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.tests.temporal.destinations.azure_blob.utils import (SUPPORTED_FILE_FORMAT_COMPRESSIONS,TEST_AZURE_BLOB_MODELS,@@ -37,10 +42,10 @@ async def test_workflow_exports_model_successfully(data_interval_start,data_interval_end,generate_test_data,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,):"""Test that the workflow exports events, persons, or sessions to Azure Blob Storage."""- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else modelrun = await run_azure_blob_batch_export_workflow(team=ateam,@@ -101,7 +106,7 @@ async def test_workflow_names_parquet_blobs_by_the_extension_setting(integration_id=azure_batch_export.destination.integration.id,file_format=file_format,compression=compression,- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),legacy_parquet_extension=legacy_parquet_extension,)diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.pyindex 063cf7883493..ef28793037be 100644--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py@@ -23,7 +23,12 @@ from azure.storage.blob.aio import BlobServiceClientfrom posthog.models.integration import Integrationfrom products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.tests.temporal.destinations.azure_blob.utils import (SUPPORTED_FILE_FORMAT_COMPRESSIONS,TEST_AZURE_BLOB_MODELS,@@ -132,10 +137,10 @@ async def test_workflow_exports_data_successfully(data_interval_start,data_interval_end,generate_test_data,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,):"""Test workflow exports events, persons, or sessions to real Azure Storage."""- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else modelrun = await run_azure_blob_batch_export_workflow(team=ateam,@@ -170,7 +175,7 @@ async def test_workflow_exports_data_successfully(@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize(("file_format", "compression"), SUPPORTED_FILE_FORMAT_COMPRESSIONS, indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_workflow_handles_formats_and_compression(ateam,azure_batch_export,@@ -183,7 +188,7 @@ async def test_workflow_handles_formats_and_compression(data_interval_start,data_interval_end,generate_test_data,- model: BatchExportModel,+ model: AnyBatchExportModel,):"""Test workflow handles various file formats and compression types with real Azure."""run = await run_azure_blob_batch_export_workflow(diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.pyindex e6e02750bb9a..1758bfad4080 100644--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py@@ -15,7 +15,14 @@ from temporalio.common import RetryPolicyfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (SUPPORTED_COMPRESSIONS,@@ -125,18 +132,20 @@ async def read_manifest(container: ContainerClient, prefix: str) -> dict | None:def extract_model_configuration(- batch_export_model: BatchExportModel | BatchExportSchema | None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,):"""Extract (model_name, fields, filters, extra_query_parameters) from model config."""if batch_export_model is None:return "events", None, None, None- if isinstance(batch_export_model, BatchExportModel):- schema = batch_export_model.schema+ if isinstance(batch_export_model, AnyBatchExportModel):+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else Nonereturn (batch_export_model.name,schema["fields"] if schema else None,- batch_export_model.filters,+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None,schema["values"] if schema else None,)@@ -149,7 +158,7 @@ async def assert_clickhouse_records_in_azure_blob(team_id: int,data_interval_start: dt.datetime,data_interval_end: dt.datetime,- batch_export_model: BatchExportModel | BatchExportSchema | None = None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,compression: str | None = None,file_format: str = "JSONLines",exclude_events: list[str] | None = None,@@ -234,9 +243,8 @@ async def assert_clickhouse_records_in_azure_blob(assert exported_records == expected_records-TEST_AZURE_BLOB_MODELS: list[BatchExportModel | BatchExportSchema | None] = [- BatchExportModel(- name="events",+TEST_AZURE_BLOB_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "uuid", "alias": "uuid"},@@ -248,17 +256,15 @@ TEST_AZURE_BLOB_MODELS: list[BatchExportModel | BatchExportSchema | None] = ["values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},},),- BatchExportModel(name="events", schema=None),- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(),+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},],),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),{"fields": [{"expression": "uuid", "alias": "uuid"},@@ -289,7 +295,7 @@ async def run_azure_blob_batch_export_workflow(integration_id: int,file_format: str = "JSONLines",compression: str | None = None,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,legacy_parquet_extension: bool = True,):"""Run the Azure Blob batch export workflow and return the run result."""diff --git a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.pyindex 8accca2f7d41..ca69703d596d 100644--- a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py+++ b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py@@ -28,11 +28,15 @@ from posthog.temporal.common.logger import BATCH_EXPORT_WORKFLOW_TYPES as LOGGERfrom products.batch_exports.backend.models.batch_export import BatchExportfrom products.batch_exports.backend.service import (+ AnyBatchExportModel,BackfillDetails,BaseBatchExportInputs,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,BatchExportField,- BatchExportModel,+ BatchExportPersonsModel,BatchExportSchema,+ BatchExportSessionsModel,)from products.batch_exports.backend.temporal import ACTIVITIES, WORKFLOWSfrom products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run@@ -102,7 +106,7 @@ class BaseDestinationTest(ABC):data_interval_end: dt.datetime,interval: str,batch_export: BatchExport,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,backfill_details: BackfillDetails | None = None,**override_config,):@@ -211,7 +215,7 @@ async def _get_records_from_clickhouse(data_interval_end: dt.datetime,exclude_events: list[str] | None,include_events: list[str] | None,- batch_export_model: BatchExportModel | BatchExportSchema | None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,backfill_details: BackfillDetails | None,expected_fields: list[str] | None,destination_default_fields: list[BatchExportField],@@ -220,13 +224,16 @@ async def _get_records_from_clickhouse("""Get records from ClickHouse."""json_columns = json_columns or []if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]@@ -297,7 +304,7 @@ async def assert_clickhouse_records_in_destination(data_interval_start: dt.datetime,data_interval_end: dt.datetime,inputs: BaseBatchExportInputs,- batch_export_model: BatchExportModel | BatchExportSchema | None = None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,backfill_details: BackfillDetails | None = None,expected_fields: list[str] | None = None,exclude_events: list[str] | None = None,@@ -329,7 +336,7 @@ async def assert_clickhouse_records_in_destination(# Determine sort key based on modelsort_key = "uuid"- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):if batch_export_model.name == "persons":sort_key = "person_id"elif batch_export_model.name == "sessions":@@ -375,8 +382,7 @@ class CommonWorkflowTests:# Common test models used across destinationsTEST_MODELS = [- BatchExportModel(- name="events",+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "uuid", "alias": "uuid"},@@ -388,17 +394,15 @@ class CommonWorkflowTests:"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},},),- BatchExportModel(name="events", schema=None),- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(),+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},],),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),{"fields": [{"expression": "uuid", "alias": "uuid"},@@ -483,7 +487,7 @@ class CommonWorkflowTests:destination_test: BaseDestinationTest,interval: str,exclude_events: list[str] | None,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start: dt.datetime,data_interval_end: dt.datetime,@@ -496,16 +500,16 @@ class CommonWorkflowTests:We test this for all models using a single interval."""- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:pytest.skip(f"Unnecessary test case as batch export model '{model.name}' is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)inputs = destination_test.create_batch_export_inputs(team_id=ateam.pk,@@ -527,7 +531,7 @@ class CommonWorkflowTests:or run.records_completed == len(persons_to_export_created)or run.records_completed== len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed <= 1)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed <= 1))await assert_clickhouse_records_in_destination(@@ -556,7 +560,7 @@ class CommonWorkflowTests:team_id=ateam.pk,data_interval_end=data_interval_end,interval="hour",- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),batch_export=batch_export_for_destination,)@@ -594,7 +598,7 @@ class CommonWorkflowTests:team_id=ateam.pk,data_interval_end=data_interval_end,interval="hour",- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),batch_export=batch_export_for_destination,)@@ -629,7 +633,7 @@ class CommonWorkflowTests:team_id=ateam.pk,data_interval_end=data_interval_end,interval="hour",- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),batch_export=batch_export_for_destination,)diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.pyindex a523b3ed86bd..3ae2b397e757 100644--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py@@ -13,7 +13,14 @@ from google.cloud import bigqueryfrom posthog.models.integration import Integrationfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (BigQueryInsertInputs,bigquery_default_fields,@@ -64,7 +71,7 @@ async def _run_activity(table_id: str,dataset_id: str,use_json_type: bool = False,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,sort_key: str = "event",@@ -176,7 +183,7 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(exclude_events,bigquery_dataset,use_json_type,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -193,14 +200,14 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(* Are not duplicates of other events that are in the same batch.* Do not have an event name contained in the batch export's `exclude_events`."""- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)sort_key = "event"if batch_export_model is not None:@@ -230,7 +237,7 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(@pytest.mark.parametrize("model",[- BatchExportModel(name="sessions", schema=None),+ BatchExportSessionsModel(),],)async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_into_bigquery_table(@@ -241,7 +248,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_inexclude_events,bigquery_dataset,use_json_type,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,@@ -285,9 +292,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_in@pytest.mark.parametrize("model",[- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},@@ -330,7 +335,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigqueexclude_events,bigquery_dataset,use_json_type,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -349,14 +354,14 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigque* Are not duplicates of other events that are in the same batch.* Do not have an event name contained in the batch export's `exclude_events`."""- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)await _run_activity(activity_environment,@@ -386,7 +391,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigqueexclude_events,bigquery_dataset,use_json_type,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -397,14 +402,14 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigqueFor this test we mock the `check_for_query_permissions_on_table` method to assert thebehavior of the activity function when lacking query permissions in BigQuery."""- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":pytest.skip("Unnecessary test case as person batch export requires query permissions")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)with (unittest.mock.patch(@@ -450,7 +455,7 @@ async def test_insert_into_bigquery_activity_from_stage_merges_persons_data_in_frun of the activity. We expect the new entries to have replaced the old ones in BigQuery afterthe second run."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_id = f"test_insert_activity_mutability_table_persons_{ateam.pk}"await _run_activity(@@ -526,7 +531,7 @@ async def test_insert_into_bigquery_activity_from_stage_merges_sessions_data_in_run of the activity. We expect the new entries to have replaced the old ones in BigQuery afterthe second run with the same time range."""- model = BatchExportModel(name="sessions", schema=None)+ model = BatchExportSessionsModel()table_id = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"result = await _run_activity(@@ -630,7 +635,7 @@ async def test_insert_into_bigquery_activity_from_stage_handles_person_new_columTo replicate this situation we first export the data with the originalschema, then delete a column in the destination and then rerun the export."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_id = f"test_insert_activity_migration_table_{ateam.pk}"await _run_activity(@@ -725,7 +730,7 @@ async def test_insert_into_bigquery_activity_from_stage_handles_datetime_to_int(To replicate this situation we first export the data to create the table, then alter'created_at' to be of type INT64, and then rerun the export."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_id = f"test_insert_activity_migration_table_{ateam.pk}"await _run_activity(@@ -793,7 +798,7 @@ async def test_insert_into_bigquery_activity_creates_tables_with_the_right_particompatibility, this may change in the future), and the consumer table to not bepartitioned."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()table_id = f"test_insert_activity_partitioning_{ateam.pk}"with (diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.pyindex d4b9e91bcb77..ced65636c547 100644--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py@@ -15,8 +15,10 @@ from temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom products.batch_exports.backend.service import (+ AnyBatchExportModel,BackfillDetails,- BatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportPersonsModel,BatchExportSchema,BigQueryBatchExportInputs,)@@ -100,7 +102,7 @@ async def test_bigquery_export_workflow(ateam,table_id,use_json_type,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -110,13 +112,13 @@ async def test_bigquery_export_workflow(The workflow should update the batch export run status to completed and produce the expectedrecords to the configured BigQuery table."""- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None+ batch_export_model: AnyBatchExportModel | None = Nonesort_key = "event"- if isinstance(model, BatchExportModel):+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelif batch_export_model.name == "persons":sort_key = "person_id"@@ -124,7 +126,7 @@ async def test_bigquery_export_workflow(sort_key = "session_id"elif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)workflow_id = str(uuid.uuid4())inputs = BigQueryBatchExportInputs(@@ -198,7 +200,7 @@ async def test_bigquery_export_workflow_without_events(ateam,table_id,use_json_type,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,data_interval_start,data_interval_end,):@@ -206,14 +208,14 @@ async def test_bigquery_export_workflow_without_events(The workflow should update the batch export run status to completed and set 0 as `records_completed`."""- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)workflow_id = str(uuid.uuid4())inputs = BigQueryBatchExportInputs(@@ -264,7 +266,7 @@ async def test_bigquery_export_workflow_without_events()@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize("use_json_type", [True], indirect=True)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])async def test_bigquery_export_workflow_backfill_earliest_persons(ateam,bigquery_client,diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.pyindex e82ae52a2df3..af2dcd9261a5 100644--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py@@ -11,7 +11,14 @@ from google.cloud import bigquery, iam_admin_v1from posthog.models.integration import GoogleCloudServiceAccountIntegration, Integrationfrom posthog.temporal.common.clickhouse import ClickHouseClient-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (bigquery_default_fields,get_our_google_cloud_credentials,@@ -27,9 +34,8 @@ SKIP_IF_MISSING_GOOGLE_APPLICATION_CREDENTIALS = pytest.mark.requires_vendor_creTEST_TIME = dt.datetime.now(dt.UTC).replace(hour=0, minute=0, second=0, microsecond=0)-TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [- BatchExportModel(- name="events",+TEST_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "event", "alias": "event"},@@ -40,8 +46,8 @@ TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = ["values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},},),- BatchExportModel(name="events", schema=None),- BatchExportModel(name="persons", schema=None),+ BatchExportEventsModel(),+ BatchExportPersonsModel(),{"fields": [{"expression": "event", "alias": "event"},@@ -65,7 +71,7 @@ async def assert_clickhouse_records_in_bigquery(min_ingested_timestamp: dt.datetime | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,- batch_export_model: BatchExportModel | BatchExportSchema | None = None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,use_json_type: bool = False,sort_key: str = "event",backfill_details: BackfillDetails | None = None,@@ -127,13 +133,16 @@ async def assert_clickhouse_records_in_bigquery(inserted_records.append(inserted_record)if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.pyindex 079a135b4053..086ca22663c4 100644--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py@@ -22,8 +22,9 @@ from posthog.temporal.common.base import PostHogWorkflowfrom products.batch_exports.backend.service import (BaseBatchExportInputs,+ BatchExportEventsModel,BatchExportField,- BatchExportModel,+ BatchExportPersonsModel,DatabricksBatchExportInputs,)from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (@@ -287,7 +288,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):run of the workflow. We expect the new entries to have replaced the old ones in Databricks afterthe second run."""- batch_export_model = BatchExportModel(name="persons", schema=None)+ batch_export_model = BatchExportPersonsModel()inputs = destination_test.create_batch_export_inputs(team_id=ateam.pk,@@ -442,7 +443,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):with destination_test.cursor(ateam.pk, integration) as cursor:cursor.execute(query)- batch_export_model = BatchExportModel(name=model_name, schema=None)+ batch_export_model = BatchExportPersonsModel() if model_name == "persons" else BatchExportEventsModel()inputs = destination_test.create_batch_export_inputs(team_id=ateam.pk,@@ -516,7 +517,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):with destination_test.cursor(ateam.pk, integration) as cursor:cursor.execute(query)- batch_export_model = BatchExportModel(name="persons", schema=None)+ batch_export_model = BatchExportPersonsModel()inputs = destination_test.create_batch_export_inputs(team_id=ateam.pk,@@ -555,7 +556,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):"Cannot perform Merge as multiple source rows matched and attempted to modify the same target row in the Deltatable in possibly conflicting ways""""- batch_export_model = BatchExportModel(name="persons", schema=None)+ batch_export_model = BatchExportPersonsModel()inputs = destination_test.create_batch_export_inputs(team_id=ateam.pk,diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.pyindex 9fcb23a43855..eff3a7ffac04 100644--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py@@ -7,7 +7,12 @@ from django.test import override_settingsfrom temporalio.testing._activity import ActivityEnvironmentfrom products.batch_exports.backend.models.batch_export import BatchExportFileDownload, BatchExportRun-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportInsertInputs,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.file_download_batch_export import (ExportInputs,GenerateFileDownloadsInputs,@@ -45,7 +50,7 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,ateam,s3_client,@@ -59,11 +64,11 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(S3InsertInputs, and delegates to insert_into_s3_from_stage. We verifythe data ends up in the expected S3 location and matches what ClickHouse produces."""- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)batch_export_id = str(uuid.uuid4())run_id = str(uuid.uuid4())@@ -128,13 +133,13 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(or result.records_completed == len(persons_to_export_created)or result.records_completed== len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= result.records_completed <= 2)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= result.records_completed <= 2))sort_key = "uuid"- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":sort_key = "person_id"- elif isinstance(model, BatchExportModel) and model.name == "sessions":+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":sort_key = "session_id"await assert_clickhouse_records_in_s3(diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.pyindex cee7a746ac85..6ba1ed326016 100644--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py@@ -12,7 +12,7 @@ from temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom products.batch_exports.backend.models.batch_export import BatchExportFileDownload-from products.batch_exports.backend.service import BatchExportModel, FileDownloadBatchExportInputs+from products.batch_exports.backend.service import BatchExportEventsModel, FileDownloadBatchExportInputsfrom products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.file_download_batch_export import (FileDownloadBatchExportWorkflow,@@ -73,7 +73,7 @@ async def file_download_batch_export(@requires_aws_credentials@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("exclude_events", [None], indirect=True)async def test_file_download_workflow_exports_data(clickhouse_client,@@ -87,7 +87,7 @@ async def test_file_download_workflow_exports_data(file_format,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: BatchExportEventsModel,generate_test_data,aws_role_arn,region,@@ -222,9 +222,7 @@ async def test_file_download_workflow_generates_no_downloads_for_a_failed_run(data_interval_end=data_interval_end.isoformat(),data_interval_start=data_interval_start.isoformat(),interval=interval,- batch_export_model=BatchExportModel(- name="events",- schema=None,+ batch_export_model=BatchExportEventsModel(filters=[{"key": "event =", "type": "hogql", "value": None}],),file_format=file_format,diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.pyindex 2483e73455ab..ddd110fd34de 100644--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py@@ -9,7 +9,15 @@ from django.test import override_settingsimport pytest_asynciofrom psycopg import sql-from products.batch_exports.backend.service import BatchExportField, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportField,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.postgres_batch_export import (PostgresInsertInputs,insert_into_postgres_activity_from_stage,@@ -45,7 +53,7 @@ async def _run_activity(data_interval_start,data_interval_end,table_name: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,sort_key: str = "event",@@ -117,13 +125,13 @@ async def _run_activity(return result[email protected]("model", [None, BatchExportModel(name="events", schema=None)])[email protected]("model", [None, BatchExportEventsModel()])async def test_insert_into_postgres_activity_handles_old_staged_events(clickhouse_client,activity_environment,postgres_connection,postgres_config,- model: BatchExportModel | None,+ model: AnyBatchExportModel | None,generate_test_data,data_interval_start,data_interval_end,@@ -151,10 +159,9 @@ async def test_insert_into_postgres_activity_handles_old_staged_events([pytest.param(None, False, id="existing-events-table"),pytest.param(None, True, id="default-without-select"),- pytest.param(BatchExportModel(name="events", schema=None), True, id="events-without-select"),+ pytest.param(BatchExportEventsModel(), True, id="events-without-select"),pytest.param(- BatchExportModel(- name="events",+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "person_id", "alias": "person_id"}], "values": {}},),True,@@ -174,15 +181,15 @@ async def test_insert_into_postgres_activity_handles_existing_events_table(postgres_connection,postgres_config,insert_only_postgres_config,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,ateam,) -> None:table_name = "existing_events"- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model- custom_schema = batch_export_model.schema if batch_export_model else None+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else model+ custom_schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else Noneawait _run_activity(activity_environment=activity_environment,@@ -247,7 +254,7 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(postgres_connection,postgres_config,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -270,17 +277,17 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(to avoid conflicting with PostHog itself."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)sort_key = "event"if batch_export_model is not None:@@ -307,7 +314,7 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(@pytest.mark.parametrize("exclude_events", [None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("test_properties",[@@ -332,7 +339,7 @@ async def test_insert_into_postgres_activity_handles_problematic_json(postgres_connection,postgres_config,exclude_events,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,@@ -377,7 +384,7 @@ async def test_insert_into_postgres_activity_merges_persons_data_in_follow_up_rurun of the activity. We expect the new entries to have replaced the old ones in PostgreSQL afterthe second run."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_name = f"test_insert_activity_mutability_table_persons_{ateam.pk}"# First run@@ -452,7 +459,7 @@ async def test_insert_into_postgres_activity_merges_sessions_data_in_follow_up_rfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse- model = BatchExportModel(name="sessions", schema=None)+ model = BatchExportSessionsModel()table_name = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"# First run@@ -560,12 +567,12 @@ async def persons_table_without_primary_key(postgres_connection, postgres_config)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])async def test_insert_into_postgres_activity_inserts_fails_on_missing_primary_key(activity_environment,postgres_config,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,data_interval_start,data_interval_end,ateam,@@ -579,11 +586,11 @@ async def test_insert_into_postgres_activity_inserts_fails_on_missing_primary_keThis error should only occur if the table is created outside the batch export."""- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)insert_inputs = PostgresInsertInputs(team_id=ateam.pk,@@ -644,7 +651,7 @@ async def test_insert_into_postgres_activity_handles_person_schema_changes(To replicate this situation we first export the data with the originalschema, then delete a column in the destination and then rerun the export."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_name = f"test_insert_activity_migration_table__{ateam.pk}"# First rundiff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.pyindex db2a4013417e..baadea93f8ee 100644--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py@@ -15,7 +15,14 @@ from temporalio.common import RetryPolicyfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.postgres_batch_export import (PostgresBatchExportInputs,@@ -52,7 +59,7 @@ async def test_postgres_export_workflow(exclude_events,ateam,table_name,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -63,17 +70,17 @@ async def test_postgres_export_workflow(records to the local development PostgreSQL database."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)config = postgres_batch_export.destination.config@@ -131,7 +138,7 @@ async def test_postgres_export_workflow(or run.records_completed == len(persons_to_export_created)or run.records_completed== len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed == 1)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed == 1))await assert_clickhouse_records_in_postgres(@@ -150,7 +157,7 @@ async def test_postgres_export_workflow(@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize("exclude_events", [None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("integration", [True], indirect=True)async def test_postgres_export_workflow_with_integration(clickhouse_client,@@ -161,7 +168,7 @@ async def test_postgres_export_workflow_with_integration(exclude_events,ateam,table_name,- model: BatchExportModel | None,+ model: AnyBatchExportModel | None,generate_test_data,data_interval_start,data_interval_end,@@ -245,7 +252,7 @@ async def test_postgres_export_workflow_without_events(exclude_events,ateam,table_name,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,data_interval_start,data_interval_end,):@@ -254,17 +261,17 @@ async def test_postgres_export_workflow_without_events(The workflow should update the batch export run status to completed and set 0 as `records_completed`."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)workflow_id = str(uuid.uuid4())inputs = PostgresBatchExportInputs(@@ -318,7 +325,7 @@ async def test_postgres_export_workflow_without_events(indirect=True,)@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])async def test_postgres_export_workflow_backfill_earliest_persons(ateam,clickhouse_client,@@ -589,7 +596,7 @@ async def test_postgres_export_workflow_with_many_files(can take longer, so we keep the event count low and bump the Workflow timeout."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()workflow_id = str(uuid.uuid4())inputs = PostgresBatchExportInputs(diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.pyindex df4e84715e74..f01d8a6a7bcd 100644--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py@@ -9,7 +9,15 @@ from psycopg import sqlfrom posthog.models import Integrationfrom posthog.temporal.common.clickhouse import ClickHouseClient-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.postgres_batch_export import postgres_default_fieldsfrom products.batch_exports.backend.temporal.queue import RecordBatchQueuefrom products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel@@ -48,18 +56,16 @@ EXPECTED_PERSONS_BATCH_EXPORT_FIELDS = []-TEST_MODELS: list[BatchExportModel | None] = [- BatchExportModel(name="events", schema=None),- BatchExportModel(- name="events",- schema=None,+TEST_MODELS: list[AnyBatchExportModel | None] = [+ BatchExportEventsModel(),+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},],),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),None,]@@ -70,7 +76,7 @@ async def assert_clickhouse_records_in_postgres(schema_name: str,table_name: str,team_id: int,- batch_export_model: BatchExportModel | BatchExportSchema | None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,data_interval_start: dt.datetime,data_interval_end: dt.datetime,exclude_events: list[str] | None = None,@@ -111,13 +117,16 @@ async def assert_clickhouse_records_in_postgres(inserted_records.append(event)if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.pyindex 2717520cc00a..6f08ce794cbf 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py@@ -7,7 +7,15 @@ from psycopg import sqlfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportInsertInputs,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (ConnectionParameters,RedshiftInsertInputs,@@ -45,7 +53,7 @@ async def _run_activity(data_interval_end,table_name: str,properties_data_type: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,sort_key: str = "event",@@ -134,7 +142,7 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(psycopg_connection,redshift_config,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -155,14 +163,14 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(that they appear in the expected Redshift table."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and MISSING_REQUIRED_ENV_VARS):@@ -189,11 +197,11 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(person_properties={"utm_medium": "referral", "$initial_os": "Linux"},)- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)table_name = f"test_insert_activity_table__{ateam.pk}"@@ -240,7 +248,7 @@ async def test_insert_into_redshift_activity_merges_persons_data_in_follow_up_ruif MISSING_REQUIRED_ENV_VARS:pytest.skip("Persons batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()properties_data_type = "varchar"table_name = f"test_insert_activity_mutability_table_{ateam.pk}"@@ -340,7 +348,7 @@ async def test_insert_into_redshift_activity_merges_sessions_data_in_follow_up_rif MISSING_REQUIRED_ENV_VARS:pytest.skip("Sessions batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="sessions", schema=None)+ model = BatchExportSessionsModel()properties_data_type = "varchar"table_name = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"@@ -441,7 +449,7 @@ async def test_insert_into_redshift_activity_handles_person_schema_changes(if MISSING_REQUIRED_ENV_VARS:pytest.skip("Persons batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()properties_data_type = "varchar"table_name = f"test_insert_activity_migration_table__{ateam.pk}"expected_fields = [@@ -523,7 +531,7 @@ async def test_insert_into_redshift_activity_handles_person_schema_changes(@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("properties_data_type", ["super"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("missing_person_id", [False, True])async def test_insert_into_redshift_activity_inserts_data_with_extra_columns(clickhouse_client,@@ -531,7 +539,7 @@ async def test_insert_into_redshift_activity_inserts_data_with_extra_columns(psycopg_connection,redshift_config,exclude_events,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.pyindex c72c50d0c645..72d0175b7401 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py@@ -8,7 +8,15 @@ from psycopg import sqlfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportInsertInputs,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (AWSCredentials,ConnectionParameters,@@ -67,7 +75,7 @@ async def _run_activity(key_prefix: str,credentials: AWSCredentials,properties_data_type: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,sort_key: str = "event",@@ -176,7 +184,7 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table(bucket_name,bucket_region,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -187,7 +195,7 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table():"""Test that the copy_into_redshift_activity function inserts data into a Redshift table."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):@@ -211,11 +219,11 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table(person_properties={"utm_medium": "referral", "$initial_os": "Linux"},)- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)table_name = f"test_copy_activity_table__{ateam.pk}"@@ -270,7 +278,7 @@ async def test_copy_into_redshift_activity_merges_persons_data_in_follow_up_runsif MISSING_REQUIRED_ENV_VARS:pytest.skip("Persons batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()properties_data_type = "super"table_name = f"test_copy_activity_mutability_table_{ateam.pk}"@@ -382,7 +390,7 @@ async def test_copy_into_redshift_activity_merges_sessions_data_in_follow_up_runif MISSING_REQUIRED_ENV_VARS:pytest.skip("Sessions batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="sessions", schema=None)+ model = BatchExportSessionsModel()properties_data_type = "varchar"table_name = f"test_copy_activity_mutability_table_sessions_{ateam.pk}"@@ -495,7 +503,7 @@ async def test_copy_into_redshift_activity_handles_person_schema_changes(if MISSING_REQUIRED_ENV_VARS:pytest.skip("Persons batch export cannot be tested in PostgreSQL")- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()properties_data_type = "varchar"table_name = f"test_copy_activity_migration_table__{ateam.pk}"expected_fields = [@@ -585,7 +593,7 @@ async def test_copy_into_redshift_activity_handles_person_schema_changes(@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("properties_data_type", ["super"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(clickhouse_client,activity_environment,@@ -594,7 +602,7 @@ async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(bucket_name,bucket_region,exclude_events,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,@@ -663,7 +671,7 @@ async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("properties_data_type", ["varchar"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_copy_into_redshift_activity_handles_events_table_missing_columns(clickhouse_client,activity_environment,@@ -672,7 +680,7 @@ async def test_copy_into_redshift_activity_handles_events_table_missing_columns(bucket_name,bucket_region,exclude_events,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,@@ -742,7 +750,7 @@ async def test_copy_into_redshift_activity_handles_events_table_missing_columns(@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("properties_data_type", ["varchar"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_copy_into_redshift_activity_handles_data_over_string_limit(clickhouse_client,activity_environment,@@ -751,7 +759,7 @@ async def test_copy_into_redshift_activity_handles_data_over_string_limit(bucket_name,bucket_region,exclude_events,- model: BatchExportModel,+ model: AnyBatchExportModel,data_interval_start,data_interval_end,properties_data_type,diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.pyindex f2d50e751ed4..3e94d2d66622 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py@@ -16,7 +16,13 @@ from temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom products.batch_exports.backend.models.batch_export import BatchExport-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema, RedshiftCopyInputs+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+ RedshiftCopyInputs,+)from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.redshift_batch_export import (RedshiftBatchExportInputs,@@ -165,7 +171,7 @@ async def test_redshift_export_workflow(ateam,exclude_events,table_name,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -181,14 +187,14 @@ async def test_redshift_export_workflow(records to the provided Redshift instance."""if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and exclude_events is not None):pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")if (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and (model.name == "persons" or model.name == "sessions")and MISSING_REQUIRED_ENV_VARS):@@ -196,11 +202,11 @@ async def test_redshift_export_workflow(copy_inputs = _build_copy_inputs(mode, aws_credentials, bucket_name, bucket_region, key_prefix)- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)inputs = RedshiftBatchExportInputs(team_id=ateam.pk,@@ -223,7 +229,7 @@ async def test_redshift_export_workflow(assert (run.records_completed == len(events_to_export_created)or run.records_completed == len(persons_to_export_created)- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed == 1)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed == 1))sort_key = "event"@@ -324,7 +330,7 @@ async def test_redshift_export_workflow_handles_insert_activity_non_retryable_er@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("mode", ["COPY", "INSERT"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_redshift_export_workflow_handles_undefined_function_error(clickhouse_client,redshift_config,@@ -334,7 +340,7 @@ async def test_redshift_export_workflow_handles_undefined_function_error(ateam,exclude_events,table_name,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,data_interval_start,data_interval_end,diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.pyindex accccd38adf0..e950475f21fc 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py@@ -8,7 +8,15 @@ from psycopg import sqlfrom posthog.temporal.common.clickhouse import ClickHouseClient-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.redshift_batch_export import redshift_default_fieldsfrom products.batch_exports.backend.temporal.queue import RecordBatchQueuefrom products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel@@ -28,10 +36,10 @@ REQUIRED_ENV_VARS = (MISSING_REQUIRED_ENV_VARS = any(env_var not in os.environ for env_var in REQUIRED_ENV_VARS)-TEST_MODELS: list[BatchExportModel | None] = [- BatchExportModel(name="events", schema=None),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+TEST_MODELS: list[AnyBatchExportModel | None] = [+ BatchExportEventsModel(),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),None,]@@ -42,7 +50,7 @@ async def assert_clickhouse_records_in_redshift(schema_name: str,table_name: str,team_id: int,- batch_export_model: BatchExportModel | BatchExportSchema | None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,date_ranges: list[tuple[dt.datetime, dt.datetime]],exclude_events: list[str] | None = None,include_events: list[str] | None = None,@@ -136,13 +144,16 @@ async def assert_clickhouse_records_in_redshift(inserted_records.append(event)if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.pyindex 569834cfad20..5993148e0b59 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py@@ -7,7 +7,12 @@ from temporalio.testing._activity import ActivityEnvironmentfrom posthog.models.integration import Integration, IntegrationErrorfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.constants import (COMPRESSION_EXTENSIONS,FILE_FORMAT_EXTENSIONS,@@ -87,7 +92,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,ateam,s3_compatible_integration,@@ -108,11 +113,11 @@ async def test_insert_into_s3_activity_puts_data_into_s3(prefix = str(uuid.uuid4())- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)batch_export_id = str(uuid.uuid4())@@ -142,16 +147,16 @@ async def test_insert_into_s3_activity_puts_data_into_s3(records_exported == len(events_to_export_created)or records_exported == len(persons_to_export_created)or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2))assert isinstance(bytes_exported, int)assert bytes_exported > 0sort_key = "uuid"- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":sort_key = "person_id"- elif isinstance(model, BatchExportModel) and model.name == "sessions":+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":sort_key = "session_id"await assert_clickhouse_records_in_s3(@@ -173,7 +178,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(@pytest.mark.parametrize("exclude_events", [None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("compression", [None], indirect=True)@pytest.mark.parametrize("file_format", ["JSONLines"], indirect=True)async def test_insert_into_s3_activity_resolves_credentials_from_integration(@@ -186,7 +191,7 @@ async def test_insert_into_s3_activity_resolves_credentials_from_integration(file_format,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,generate_test_data,ateam,s3_compatible_integration,@@ -237,7 +242,7 @@ async def test_insert_into_s3_activity_resolves_credentials_from_integration(@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("compression", [None], indirect=True)@pytest.mark.parametrize("file_format", ["Parquet"], indirect=True)async def test_insert_into_s3_activity_with_exclude_events(@@ -250,7 +255,7 @@ async def test_insert_into_s3_activity_with_exclude_events(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,ateam,s3_compatible_integration,@@ -261,11 +266,11 @@ async def test_insert_into_s3_activity_with_exclude_events(prefix = str(uuid.uuid4())- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)batch_export_id = str(uuid.uuid4())@@ -295,16 +300,16 @@ async def test_insert_into_s3_activity_with_exclude_events(records_exported == len(events_to_export_created)or records_exported == len(persons_to_export_created)or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2))assert isinstance(bytes_exported, int)assert bytes_exported > 0sort_key = "uuid"- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":sort_key = "person_id"- elif isinstance(model, BatchExportModel) and model.name == "sessions":+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":sort_key = "session_id"await assert_clickhouse_records_in_s3(@@ -326,7 +331,7 @@ async def test_insert_into_s3_activity_with_exclude_events(@pytest.mark.parametrize(("file_format", "compression"), SPLIT_FILE_FORMAT_COMPRESSIONS, indirect=["compression"])[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("max_file_size_mb", [None, 1])async def test_insert_into_s3_activity_puts_splitted_files_into_s3(clickhouse_client,@@ -339,7 +344,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_s3(file_format,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,ateam,s3_compatible_integration,):@@ -458,7 +463,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_s3(assert manifest_data["files"] == expected_keys[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("file_format", ["invalid"])async def test_insert_into_s3_activity_fails_on_invalid_file_format(clickhouse_client,@@ -470,7 +475,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_file_format(file_format,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,ateam,s3_compatible_integration,):@@ -500,7 +505,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_file_format(assert result.error_repr == "UnsupportedFileFormatError: 'invalid' is not a supported format for S3 batch exports."[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("compression", ["invalid"])async def test_insert_into_s3_activity_fails_on_invalid_compression(clickhouse_client,@@ -511,7 +516,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_compression(exclude_events,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,ateam,s3_compatible_integration,):diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.pyindex ffd61cb1ead4..61a19e4eb034 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py@@ -9,7 +9,12 @@ from temporalio.testing._activity import ActivityEnvironmentfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.constants import (COMPRESSION_EXTENSIONS,FILE_FORMAT_EXTENSIONS,@@ -61,7 +66,7 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,ateam,gcs_integration,@@ -82,11 +87,11 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(prefix = str(uuid.uuid4())- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)batch_export_id = str(uuid.uuid4())@@ -116,16 +121,16 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(records_exported == len(events_to_export_created)or records_exported == len(persons_to_export_created)or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2))assert isinstance(bytes_exported, int)assert bytes_exported > 0sort_key = "uuid"- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":sort_key = "person_id"- elif isinstance(model, BatchExportModel) and model.name == "sessions":+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":sort_key = "session_id"await assert_clickhouse_records_in_s3(@@ -147,7 +152,7 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(@pytest.mark.parametrize(("file_format", "compression"), SPLIT_FILE_FORMAT_COMPRESSIONS, indirect=["compression"])[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("max_file_size_mb", [None, 6])async def test_insert_into_s3_activity_puts_splitted_files_into_gcs(clickhouse_client,@@ -160,7 +165,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_gcs(file_format,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,ateam,gcs_integration,):diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.pyindex 89293b9e68e4..e996babe21aa 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py@@ -12,7 +12,7 @@ from temporalio.testing._activity import ActivityEnvironmentfrom posthog.models import Organization, Teamfrom posthog.models.integration import Integration-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportEventsModelfrom products.batch_exports.backend.temporal.destinations.s3_batch_export import (PolicyStatement,S3InsertInputs,@@ -92,13 +92,13 @@ def kms_key_value(request) -> str | None:[None, "kms_key_id", "kms_key_arn"],indirect=True,)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_insert_into_s3_activity_puts_data_into_s3_with_role_assumption(clickhouse_client,activity_environment: ActivityEnvironment,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: BatchExportEventsModel,generate_test_data,integration: Integration,s3_client,@@ -159,13 +159,13 @@ async def test_insert_into_s3_activity_puts_data_into_s3_with_role_assumption(@pytest.mark.parametrize("external_id", ["not-an-org-id", None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_insert_into_s3_activity_fails_role_assumption_with_missing_external_id(clickhouse_client,activity_environment: ActivityEnvironment,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: BatchExportEventsModel,generate_test_data,integration: Integration,s3_client,diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.pyindex aba224101d62..58d176886091 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py@@ -17,7 +17,7 @@ from temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom types_aiobotocore_s3.client import S3Client-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportEventsModelfrom products.batch_exports.backend.temporal.batch_exports import finish_batch_export_runfrom products.batch_exports.backend.temporal.destinations.s3_batch_export import (ConcurrentS3Consumer,@@ -246,7 +246,7 @@ async def test_s3_export_workflow_with_request_timeouts(We run the S3 batch export workflow with a mocked session that will raise a `ClientError` dueto a `RequestTimeout` on the first run of the batch export. The second run should work normally."""- batch_export_model = BatchExportModel(name="events", schema=None)+ batch_export_model = BatchExportEventsModel()raised = 0@@ -359,11 +359,6 @@ async def test_s3_export_workflow_with_request_timeouts(assert key.startswith(expected_key_prefix)sort_key = "event"- if batch_export_model is not None:- if batch_export_model.name == "persons":- sort_key = "person_id"- elif batch_export_model.name == "sessions":- sort_key = "session_id"await assert_clickhouse_records_in_s3(s3_compatible_client=object_storage_client,diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.pyindex 769ce9eaca9d..7b333acb5aab 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py@@ -5,7 +5,7 @@ import pytestimport aioboto3import pytest_asyncio-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportSchemafrom products.batch_exports.backend.temporal.destinations.constants import (COMPRESSION_EXTENSIONS,FILE_FORMAT_EXTENSIONS,@@ -85,7 +85,7 @@ async def gcs_batch_export(@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)@pytest.mark.parametrize("compression", [*COMPRESSION_EXTENSIONS.keys(), None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("bucket_name", [os.getenv("GCS_TEST_BUCKET")], indirect=True)@@ -100,7 +100,7 @@ async def test_s3_export_workflow_with_gcs_bucket_with_various_file_formats(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3 Export Workflow end-to-end by using a GCS bucket.diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.pyindex 4e3785a118f4..dade32f83d70 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py@@ -2,7 +2,14 @@ import datetime as dtimport pytest-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsModel,+ BatchExportHogQLModel,+ BatchExportPersonsModel,+ BatchExportSchema,+)from products.batch_exports.backend.tests.temporal.destinations.s3.utils import (SUPPORTED_FILE_FORMAT_COMPRESSIONS,TEST_S3_MODELS,@@ -34,7 +41,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_models(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket instead of S3.@@ -62,7 +69,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_models(@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize(("file_format", "compression"), [*SUPPORTED_FILE_FORMAT_COMPRESSIONS, *UNCOMPRESSED_FILE_FORMATS], indirect=True@@ -80,7 +87,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_compresfile_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket and various compression and file formats."""@@ -101,7 +108,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_compres@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("file_format", ["Parquet"], indirect=True)@pytest.mark.parametrize("compression", ["zstd"], indirect=True)@@ -120,7 +127,7 @@ async def test_s3_export_workflow_names_parquet_files_by_the_extension_setting(legacy_parquet_extension,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):await run_s3_batch_export_workflow(@@ -156,7 +163,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_exclude_events(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket and excluding events."""@@ -183,7 +190,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_exclude_events(indirect=True,)@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])async def test_s3_export_workflow_backfill_earliest_persons_with_local_object_storage(clickhouse_client,object_storage_client,@@ -296,7 +303,7 @@ async def test_s3_export_workflow_with_local_object_storage_and_custom_key_prefis3_key_prefix,data_interval_end,data_interval_start,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test the S3BatchExport Workflow end-to-end by specifying a custom key prefix.@@ -358,7 +365,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_hogql_model(`events_table` is pinned to `sharded_events` because the HogQL `events` table reads it,while the fixture would otherwise default to `events_recent` for a recent interval."""- model = BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=auser.pk)+ model = BatchExportHogQLModel(query=hogql_query, user_id=auser.pk, modifiers={})await run_s3_batch_export_workflow(model=model,diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.pyindex 428246e42543..e8285e56bdd6 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py@@ -8,7 +8,7 @@ import pytest_asynciofrom posthog.models.integration import Integration-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportSchemafrom products.batch_exports.backend.temporal.destinations.constants import (COMPRESSION_EXTENSIONS,FILE_FORMAT_EXTENSIONS,@@ -122,7 +122,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_modefile_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3 Export Workflow end-to-end by using an S3 bucket.@@ -138,7 +138,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_modewill require its prescense in the database when running. This model is indirectly parametrizedby several fixtures. Refer to them for more information."""- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")await run_s3_batch_export_workflow(@@ -159,7 +159,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_mode@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)@pytest.mark.parametrize("encryption", [None, "AES256", "aws:kms"], indirect=True)@pytest.mark.parametrize("compression", [*COMPRESSION_EXTENSIONS.keys(), None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("interval", ["hour"], indirect=True)@pytest.mark.parametrize("exclude_events", [None], indirect=True)@pytest.mark.parametrize("bucket_name", [os.getenv("S3_TEST_BUCKET")], indirect=True)@@ -177,7 +177,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_file_formats(file_format,data_interval_start,data_interval_end,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,):"""Test S3 Export Workflow end-to-end by using an S3 bucket with various file formats, compression, anddiff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.pyindex f98a35cabfc0..f97a92696186 100644--- a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py@@ -22,10 +22,15 @@ from posthog.temporal.common.clickhouse import ClickHouseClientfrom products.batch_exports.backend.hogql_source import parse_hogql_select_for_batch_exportfrom products.batch_exports.backend.service import (+ AnyBatchExportModel,AwsS3BatchExportInputs,BackfillDetails,- BatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportHogQLModel,+ BatchExportPersonsModel,BatchExportSchema,+ BatchExportSessionsModel,S3CompatibleBatchExportInputs,S3FamilyBaseInputs,)@@ -62,9 +67,8 @@ _INPUTS_BY_DESTINATION_TYPE: dict[str, type[S3FamilyBaseInputs]] = {COMPRESSION_OPTIONS = [*COMPRESSION_EXTENSIONS.keys(), None]-TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = [- BatchExportModel(- name="events",+TEST_S3_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "uuid", "alias": "uuid"},@@ -76,17 +80,15 @@ TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = ["values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},},),- BatchExportModel(name="events", schema=None),- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(),+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},],),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),{"fields": [{"expression": "uuid", "alias": "uuid"},@@ -100,10 +102,10 @@ TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = [None,]-TEST_S3_MODELS_AFFECTED_BY_EXCLUDE_EVENTS: list[BatchExportModel | BatchExportSchema | None] = [+TEST_S3_MODELS_AFFECTED_BY_EXCLUDE_EVENTS: list[AnyBatchExportModel | BatchExportSchema | None] = [modelfor model in TEST_S3_MODELS- if not (isinstance(model, BatchExportModel) and model.name in ("persons", "sessions"))+ if not (isinstance(model, AnyBatchExportModel) and model.name in ("persons", "sessions"))]SUPPORTED_FILE_FORMAT_COMPRESSIONS: list[tuple[str, str | None]] = [@@ -234,7 +236,7 @@ async def assert_clickhouse_records_in_s3(data_interval_end: dt.datetime,exclude_events: list[str] | None = None,include_events: list[str] | None = None,- batch_export_model: BatchExportModel | BatchExportSchema | None = None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,compression: str | None = None,file_format: str = "JSONLines",backfill_details: BackfillDetails | None = None,@@ -275,13 +277,16 @@ async def assert_clickhouse_records_in_s3()if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]@@ -295,7 +300,7 @@ async def assert_clickhouse_records_in_s3(if fields is not None:schema_column_names = [field["alias"] for field in fields]- elif isinstance(batch_export_model, BatchExportModel) and batch_export_model.name == "persons":+ elif isinstance(batch_export_model, BatchExportPersonsModel):schema_column_names = ["team_id","distinct_id",@@ -307,10 +312,10 @@ async def assert_clickhouse_records_in_s3("created_at","is_deleted",]- elif isinstance(batch_export_model, BatchExportModel) and batch_export_model.name == "hogql":+ elif isinstance(batch_export_model, BatchExportHogQLModel):# A HogQL model exports whichever columns the query selects, so the query is the schema:# the team_id check and the sort below must key off the query's own columns.- schema_column_names = _hogql_model_column_names(batch_export_model.hogql_query)+ schema_column_names = _hogql_model_column_names(batch_export_model.query)else:schema_column_names = [field["alias"] for field in s3_default_fields()]@@ -322,16 +327,12 @@ async def assert_clickhouse_records_in_s3(queue = RecordBatchQueue()if model_name == "sessions":producer = ClickHouseTestProducer(model=SessionsRecordBatchModel(team_id))- elif (- model_name == "hogql"- and isinstance(batch_export_model, BatchExportModel)- and batch_export_model.hogql_query is not None- ):+ elif isinstance(batch_export_model, BatchExportHogQLModel):# Re-running the query itself yields the expected rows, with the same interval bounds# substituted as the run exported with.producer = ClickHouseTestProducer(model=HogQLQueryRecordBatchModel(- team_id=team_id, hogql_query=batch_export_model.hogql_query, user_id=batch_export_model.user_id+ team_id=team_id, hogql_query=batch_export_model.query, user_id=batch_export_model.user_id))else:@@ -379,7 +380,7 @@ async def assert_clickhouse_records_in_s3(s3_data = sorted(s3_data, key=operator.itemgetter(sort_key))expected_records = sorted(expected_records, key=operator.itemgetter(sort_key))- if isinstance(batch_export_model, BatchExportModel) and batch_export_model.name in ["events", "persons"]:+ if isinstance(batch_export_model, AnyBatchExportModel) and batch_export_model.name in ["events", "persons"]:assert set(s3_data[0].keys()) == set(schema_column_names)if allow_duplicates:@@ -393,7 +394,7 @@ async def assert_clickhouse_records_in_s3(async def run_s3_batch_export_workflow(- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,ateam,batch_export_id,s3_destination_config,@@ -414,11 +415,11 @@ async def run_s3_batch_export_workflow(`destination_type` selects which input dataclass is constructed and passed to the workflow,so the test exercises the same per-destination inputs that production dispatches with."""- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)exclude_events = s3_destination_config.get("exclude_events", None)file_format = s3_destination_config.get("file_format", "JSONLines")@@ -482,7 +483,7 @@ async def run_s3_batch_export_workflow(if expect_no_data:assert run.records_completed == 0 or (- isinstance(model, BatchExportModel)+ isinstance(model, AnyBatchExportModel)and model.name == "sessions"and run.records_completed is not Noneand run.records_completed <= 1@@ -495,9 +496,9 @@ async def run_s3_batch_export_workflow(assert run.bytes_exported > 0sort_key = "uuid"- if isinstance(model, BatchExportModel) and model.name == "persons":+ if isinstance(model, BatchExportPersonsModel):sort_key = "person_id"- elif isinstance(model, BatchExportModel) and model.name == "sessions":+ elif isinstance(model, BatchExportSessionsModel):sort_key = "session_id"await assert_metrics_in_clickhouse(diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.pyindex 6f49ed7f4253..8ce21cd57d9b 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py@@ -16,7 +16,14 @@ from django.test import override_settingsfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (SnowflakeInsertInputs,insert_into_snowflake_activity_from_stage,@@ -54,7 +61,7 @@ async def _run_activity(data_interval_start,data_interval_end,table_name: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events=None,sort_key: str = "event",expected_fields=None,@@ -130,7 +137,7 @@ async def test_insert_into_snowflake_activity_inserts_data_into_snowflake_table(snowflake_cursor,snowflake_activity_inputs,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -150,14 +157,14 @@ async def test_insert_into_snowflake_activity_inserts_data_into_snowflake_table(that they appear in the expected Snowflake table. This function runs against a real Snowflakeinstance, so the environment should be populated with the necessary credentials."""- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)table_name = f"test_insert_activity_table_{ateam.pk}"@@ -203,7 +210,7 @@ async def test_insert_into_snowflake_activity_merges_persons_data_in_follow_up_rrun of the activity. We expect the new entries to have replaced the old ones in Snowflake afterthe second run."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_name = f"test_insert_activity_table_mutable_persons_{ateam.pk}"await _run_activity(@@ -273,7 +280,7 @@ async def test_insert_into_snowflake_activity_merges_sessions_data_in_follow_up_run of the activity. We expect the new entries to have replaced the old ones in Snowflake afterthe second run with the same time range."""- model = BatchExportModel(name="sessions", schema=None)+ model = BatchExportSessionsModel()table_name = f"test_insert_activity_table_mutable_sessions_{ateam.pk}"await _run_activity(@@ -363,7 +370,7 @@ async def test_insert_into_snowflake_activity_removes_internal_stage_files(checking this second run worked and exported data, we also check that no filesare present in the table's internal stage."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()table_name = f"test_insert_activity_table_remove_{ateam.pk}"min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)batch_export_id = str(uuid.uuid4())@@ -534,7 +541,7 @@ async def test_insert_into_snowflake_activity_handles_person_schema_changes(To replicate this situation we first export the data with the originalschema, then delete a column in the destination and then rerun the export."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_name = f"test_insert_activity_migration_table_{ateam.pk}"await _run_activity(@@ -619,7 +626,7 @@ async def test_insert_into_snowflake_activity_from_stage_handles_datetime_to_intTo replicate this situation we first export the data to create the table, then alter'created_at' to be of type INTEGER, and then rerun the export."""- model = BatchExportModel(name="persons", schema=None)+ model = BatchExportPersonsModel()table_name = f"test_insert_activity_persons_table_{ateam.pk}"await _run_activity(@@ -657,9 +664,7 @@ async def test_insert_into_snowflake_activity_from_stage_handles_datetime_to_intassert result.error is None- "model", [BatchExportModel(name="events", schema=None), BatchExportModel(name="persons", schema=None)]-)[email protected]("model", [BatchExportEventsModel(), BatchExportPersonsModel()])async def test_insert_into_snowflake_activity_handles_uppercased_columns(clickhouse_client,activity_environment,@@ -726,7 +731,7 @@ async def test_insert_into_snowflake_activity_handles_uppercased_columns()[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_insert_into_snowflake_activity_handles_extra_columns_in_destination(clickhouse_client,activity_environment,diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.pyindex eb8e24dc6efe..9efc365907b0 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py@@ -20,7 +20,14 @@ from temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (SnowflakeBatchExportInputs,@@ -51,7 +58,7 @@ async def _run_workflow(data_interval_start,data_interval_end,interval: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events=None,backfill_details=None,settings_overrides=None,@@ -142,7 +149,7 @@ async def test_snowflake_export_workflow(snowflake_batch_export,ateam,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -152,14 +159,14 @@ async def test_snowflake_export_workflow(The workflow should update the batch export run status to completed and produce the expectedrecords to the provided Snowflake instance."""- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)@@ -178,7 +185,7 @@ async def test_snowflake_export_workflow(@pytest.mark.parametrize("exclude_events", [None], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_snowflake_export_workflow_with_many_files(clickhouse_client,snowflake_cursor,@@ -186,7 +193,7 @@ async def test_snowflake_export_workflow_with_many_files(snowflake_batch_export,ateam,exclude_events,- model: BatchExportModel | BatchExportSchema | None,+ model: AnyBatchExportModel | BatchExportSchema | None,generate_test_data,data_interval_start,data_interval_end,@@ -198,14 +205,14 @@ async def test_snowflake_export_workflow_with_many_files(means we are uploading one file at a time, which is very innefficient. For this reason, this testcan take longer, so we keep the event count low and bump the Workflow timeout."""- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")- batch_export_model: BatchExportModel | None = None- if isinstance(model, BatchExportModel):+ batch_export_model: AnyBatchExportModel | None = None+ if isinstance(model, AnyBatchExportModel):batch_export_model = modelelif model is not None:- batch_export_model = BatchExportModel(name="events", schema=model)+ batch_export_model = BatchExportEventsCustomModel(schema=model)min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)@@ -232,7 +239,7 @@ async def test_snowflake_export_workflow_with_many_files(indirect=True,)@pytest.mark.parametrize("interval", ["hour"], indirect=True)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])async def test_snowflake_export_workflow_backfill_earliest_persons(ateam,clickhouse_client,diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.pyindex 64238a69aae2..1162ab9fb0fe 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py@@ -18,7 +18,15 @@ from requests.models import PreparedRequestfrom posthog.temporal.common.clickhouse import ClickHouseClient-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSchema,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import snowflake_default_fieldsfrom products.batch_exports.backend.temporal.queue import RecordBatchQueuefrom products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel@@ -60,9 +68,8 @@ EXPECTED_PERSONS_BATCH_EXPORT_FIELDS = ["is_deleted",]-TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [- BatchExportModel(- name="events",+TEST_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "event", "alias": "event"},@@ -73,9 +80,9 @@ TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = ["values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},},),- BatchExportModel(name="events", schema=None),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportEventsModel(),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),{"fields": [{"expression": "event", "alias": "event"},@@ -371,7 +378,7 @@ async def assert_clickhouse_records_in_snowflake(data_interval_end: dt.datetime,exclude_events: list[str] | None = None,include_events: list[str] | None = None,- batch_export_model: BatchExportModel | BatchExportSchema | None = None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,backfill_details: BackfillDetails | None = None,sort_key: str = "event",expected_fields: list[str] | None = None,@@ -428,13 +435,16 @@ async def assert_clickhouse_records_in_snowflake(inserted_records.append(record)if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]diff --git a/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.pyindex db6a5c140127..fb31557ae5f8 100644--- a/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py@@ -22,7 +22,13 @@ from posthog.sync import database_sync_to_asyncfrom posthog.temporal.common.clickhouse import ClickHouseClientfrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel+from products.batch_exports.backend.service import (+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportPersonsModel,+ BatchExportSessionsModel,+)from products.batch_exports.backend.temporal.batch_exports import (finish_batch_export_run,iter_records,@@ -552,29 +558,24 @@ async def test_http_export_workflow("batch_export_model",[None,- BatchExportModel(name="events", schema=None),- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(),+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},],),# this filter should filter out all test events- BatchExportModel(- name="events",- schema=None,+ BatchExportEventsModel(filters=[{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},{"key": "$os", "operator": "exact", "type": "event", "value": ["Linux"]},],),# None of the following are supported so we expect an error to be raised- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),- BatchExportModel(- name="events",+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),+ BatchExportEventsCustomModel(schema={"fields": [{"expression": "uuid", "alias": "uuid"},@@ -645,7 +646,7 @@ async def test_http_export_workflow_with_model(if batch_export_model is not None:if batch_export_model.name != "events":expect_failure = True- if batch_export_model.schema is not None:+ if isinstance(batch_export_model, BatchExportEventsCustomModel):expect_failure = Truemock_server = MockServer()@@ -692,7 +693,7 @@ async def test_http_export_workflow_with_model(assert len(runs) == 1run = runs[0]assert run.status == "FailedRetryable"- if batch_export_model is not None and batch_export_model.schema is not None:+ if isinstance(batch_export_model, BatchExportEventsCustomModel):assert run.latest_error == "NotImplementedError: HTTP export does not support schemas"else:assert run.latest_error == "NotImplementedError: HTTP export only supports the events model"diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.pyindex ae74089987b0..b15b5fd83b5d 100644--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py@@ -4,7 +4,7 @@ import urllib.parseimport pytest-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportInsertInputsfrom products.batch_exports.backend.temporal.destinations import workflows_batch_exportfrom products.batch_exports.backend.temporal.destinations.workflows_batch_export import (WorkflowsInsertInputs,@@ -45,7 +45,7 @@ async def _run_activity(data_interval_start,data_interval_end,sort_key: str,- batch_export_model: BatchExportModel | None = None,+ batch_export_model: AnyBatchExportModel | None = None,exclude_events: list[str] | None = None,include_events: list[str] | None = None,expected_fields=None,@@ -125,7 +125,7 @@ async def test_insert_into_workflows_activity_from_stage_posts_data_to_server(This configures the activity to POST requests to an aiohttp test server, configuredin the server fixture."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()await _run_activity(activity_environment,@@ -158,7 +158,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_on_non_retryable_error,):"""Assert the activity immediately fails on non-retryable error codes."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()if error == 404:expected = "NotFoundErrorGroup"@@ -220,7 +220,7 @@ async def test_insert_into_workflows_activity_from_stage_retries_on_retryable_ererror,):"""Assert the activity retries requests on retryable error codes."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()await _run_activity(activity_environment,@@ -255,7 +255,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_with_empty_url(error,):"""Assert activity fails when an empty URL is passed."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()batch_export_id = str(uuid.uuid4())batch_export_inputs = BatchExportInsertInputs(@@ -306,7 +306,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_when_hog_functionhog_function_id,):"""When every hog function execution errors, the activity should abort after the threshold trips."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()batch_export_id = str(uuid.uuid4())batch_export_inputs = BatchExportInsertInputs(@@ -367,7 +367,7 @@ async def test_insert_into_workflows_activity_from_stage_tolerates_hog_function_hog_function_error_count,):"""A small number of hog function errors (well under the threshold) should not fail the run."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()result = await _run_activity(activity_environment,diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.pyindex 4c5f6fc18788..15de9281239e 100644--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py@@ -10,7 +10,7 @@ from temporalio.common import RetryPolicyfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportEventsModelfrom products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_runfrom products.batch_exports.backend.temporal.destinations.workflows_batch_export import (WorkflowsBatchExportInputs,@@ -81,7 +81,7 @@ async def test_workflows_export_workflow(The workflow should update the batch export run status to completed and post the expectedrecords to a test server."""- model = BatchExportModel(name="events", schema=None)+ model = BatchExportEventsModel()workflow_id = str(uuid.uuid4())inputs = WorkflowsBatchExportInputs(diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.pyindex 5a637719e68b..89bc84bf098f 100644--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py@@ -5,7 +5,13 @@ import operatorfrom posthog.temporal.common.clickhouse import ClickHouseClient-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.workflows_batch_export import workflows_default_fieldsfrom products.batch_exports.backend.temporal.queue import RecordBatchQueuefrom products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel@@ -29,7 +35,7 @@ async def assert_clickhouse_records_were_handled(hog_function_id,team_id: int,sort_key: str,- batch_export_model: BatchExportModel | BatchExportSchema | None,+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,batch_export_id: str,date_ranges: list[tuple[dt.datetime, dt.datetime]],backfill_details: BackfillDetails | None = None,@@ -46,13 +52,16 @@ async def assert_clickhouse_records_were_handled()if batch_export_model is not None:- if isinstance(batch_export_model, BatchExportModel):+ if isinstance(batch_export_model, AnyBatchExportModel):model_name = batch_export_model.name- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None- filters = batch_export_model.filters- extra_query_parameters = (- batch_export_model.schema["values"] if batch_export_model.schema is not None else None+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None+ fields = schema["fields"] if schema is not None else None+ filters = (+ batch_export_model.filters+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)+ else None)+ extra_query_parameters = schema["values"] if schema is not None else Noneelse:model_name = "events"fields = batch_export_model["fields"]diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.pyindex dcf1de684fb5..6e421ee0665a 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py@@ -19,7 +19,12 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.clickhouse import ClickHouseMemoryLimitExceededErrorfrom products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination-from products.batch_exports.backend.service import BaseBatchExportInputs, BatchExportInsertInputs, BatchExportModel+from products.batch_exports.backend.service import (+ BaseBatchExportInputs,+ BatchExportEventsModel,+ BatchExportHogQLModel,+ BatchExportInsertInputs,+)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,finish_batch_export_run,@@ -242,7 +247,7 @@ class TestErrorHandling:batch_export_id=str(batch_export.id),interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),exception_to_raise="DummyNonRetryableError",)run = await self._run_workflow(inputs, expect_workflow_failure=False)@@ -259,7 +264,7 @@ class TestErrorHandling:batch_export_id=str(batch_export.id),interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),exception_to_raise="DummyRetryableError",)run = await self._run_workflow(inputs, expect_workflow_failure=True)@@ -281,8 +286,8 @@ class TestErrorHandling:batch_export_id=str(batch_export.id),interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=1+ batch_export_model=BatchExportHogQLModel(+ query="SELECT event AS event FROM events", user_id=1, modifiers={}),)with patch(@@ -302,7 +307,7 @@ class TestErrorHandling:batch_export_id=str(batch_export.id),interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),)with patch("products.batch_exports.backend.temporal.pipeline.internal_stage._write_batch_export_record_batches_to_internal_stage",@@ -320,7 +325,7 @@ class TestErrorHandling:batch_export_id=str(batch_export_by_interval.id),interval=interval,data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),)run = await self._run_workflow(inputs, expect_workflow_failure=False)assert run.status == "Completed"@@ -333,7 +338,7 @@ class TestErrorHandling:batch_export_id=str(batch_export.id),interval="hour",data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),- batch_export_model=BatchExportModel(name="events", schema=None),+ batch_export_model=BatchExportEventsModel(),records_failed_count=5,)run = await self._run_workflow(inputs, expect_workflow_failure=False)diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.pyindex 7a9ef257a51e..4e914cca6daa 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py@@ -35,7 +35,16 @@ from posthog.temporal.tests.utils.events import ()from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportRun-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, afetch_last_run_records_completed+from products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BackfillDetails,+ BatchExportEventsModel,+ BatchExportHogQLModel,+ BatchExportModel,+ BatchExportPersonsModel,+ BatchExportSessionsModel,+ afetch_last_run_records_completed,+)from products.batch_exports.backend.temporal.pipeline.internal_stage import (BatchExportInsertIntoInternalStageInputs,DataIntervalInFutureError,@@ -77,7 +86,7 @@ def mock_clickhouse_client():@pytest.mark.parametrize("model",[- BatchExportModel(name="events", schema=None),+ BatchExportEventsModel(),],)@pytest.mark.parametrize("is_backfill", [False, True])@@ -90,7 +99,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_eventsdata_interval_start,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,is_backfill: bool,backfill_within_last_6_days: bool,use_native_schema: bool,@@ -156,7 +165,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_events@pytest.mark.parametrize("model",[- BatchExportModel(name="sessions", schema=None),+ BatchExportSessionsModel(),],)@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])@@ -167,7 +176,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_sessiodata_interval_start,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,):"""Test that the insert_into_internal_stage_activity executes the expected ClickHouse query when the model is a sessions model."""@@ -258,7 +267,7 @@ async def _run_activity(data_interval_start: dt.datetime,data_interval_end: dt.datetime,exclude_events: list[str] | None = None,- model: BatchExportModel | None = None,+ model: AnyBatchExportModel | None = None,batch_export_id: str | None = None,) -> list[dict]:"""Get rows from the internal stage."""@@ -301,7 +310,7 @@ async def _run_activity(@pytest.mark.parametrize("model",[- BatchExportModel(name="events", schema=None),+ BatchExportEventsModel(),],)@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])@@ -313,7 +322,7 @@ async def test_insert_into_stage_activity_for_events_model(object_storage_client,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,exclude_events,truncate_clickhouse_tables,):@@ -339,7 +348,7 @@ async def test_insert_into_stage_activity_for_events_model(@pytest.mark.parametrize("model",[- BatchExportModel(name="events", schema=None),+ BatchExportEventsModel(),],)@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])@@ -351,7 +360,7 @@ async def test_insert_into_stage_activity_reports_records_total_for_events_modelobject_storage_client,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,truncate_clickhouse_tables,):"""The activity reports the number of rows written to the stage, from ClickHouse's query summary."""@@ -673,7 +682,7 @@ def assert_exported_rows_match_persons_to_export(exported_rows: list[dict], pers@pytest.mark.parametrize("interval", ["day"], indirect=True)[email protected]("model", [BatchExportModel(name="persons", schema=None)])[email protected]("model", [BatchExportPersonsModel()])@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])@pytest.mark.parametrize("limited_export", [False, True])async def test_insert_into_stage_activity_for_persons_model(@@ -685,7 +694,7 @@ async def test_insert_into_stage_activity_for_persons_model(ateam,test_person_properties,clickhouse_client,- model: BatchExportModel,+ model: AnyBatchExportModel,limited_export: bool,truncate_clickhouse_tables,):@@ -1101,7 +1110,7 @@ async def test_afetch_last_run_records_completed_ignores_runs_older_than(ateam):@pytest.mark.parametrize("interval", ["day"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])async def test_insert_into_stage_activity_writes_dynamic_number_of_files(generate_test_data,@@ -1111,7 +1120,7 @@ async def test_insert_into_stage_activity_writes_dynamic_number_of_files(object_storage_client,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,truncate_clickhouse_tables,):"""A previous run's row count drives how many staging files the activity writes."""@@ -1160,7 +1169,7 @@ async def test_insert_into_stage_activity_writes_dynamic_number_of_files(@pytest.mark.parametrize("interval", ["day"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])async def test_insert_into_stage_activity_uses_static_default_without_previous_run(generate_test_data,@@ -1170,7 +1179,7 @@ async def test_insert_into_stage_activity_uses_static_default_without_previous_robject_storage_client,data_interval_end,ateam,- model: BatchExportModel,+ model: AnyBatchExportModel,truncate_clickhouse_tables,):"""With no previous run to estimate from, the activity writes the static-default number of files."""@@ -1260,9 +1269,9 @@ async def _fetch_staging_query_settings(@pytest.mark.parametrize("model",[- BatchExportModel(name="events", schema=None),- BatchExportModel(name="persons", schema=None),- BatchExportModel(name="sessions", schema=None),+ BatchExportEventsModel(),+ BatchExportPersonsModel(),+ BatchExportSessionsModel(),],ids=["events", "persons", "sessions"],)@@ -1273,7 +1282,7 @@ async def test_insert_into_stage_activity_applies_settings_and_log_comment(clickhouse_client,data_interval_start,data_interval_end,- model: BatchExportModel,+ model: AnyBatchExportModel,):"""Every model's settings and log comment reach the query ClickHouse runs.@@ -1480,7 +1489,7 @@ class TestHogQLModel:team_id=ateam.pk,data_interval_start=data_interval_start,data_interval_end=data_interval_end,- model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=auser.pk),+ model=BatchExportHogQLModel(query=hogql_query, user_id=auser.pk, modifiers={}),)assert all(list(row.keys()) == expected_columns for row in exported_rows)@@ -1521,14 +1530,13 @@ class TestHogQLModel:team_id=ateam.pk,data_interval_start=bounded_start,data_interval_end=data_interval_end,- model=BatchExportModel(- name="hogql",- schema=None,- hogql_query=(+ model=BatchExportHogQLModel(+ query=("SELECT uuid AS uuid, timestamp AS timestamp FROM events ""WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"),user_id=auser.pk,+ modifiers={},),)@@ -1572,11 +1580,10 @@ class TestHogQLModel:team_id=ateam.pk,data_interval_start=data_interval_start,data_interval_end=data_interval_end,- model=BatchExportModel(- name="hogql",- schema=None,- hogql_query="SELECT event_id, event, distinct_id FROM events_view",+ model=BatchExportHogQLModel(+ query="SELECT event_id, event, distinct_id FROM events_view",user_id=auser.pk,+ modifiers={},),)@@ -1592,9 +1599,7 @@ class TestHogQLModel:team_id=ateam.pk,data_interval_start=data_interval_start,data_interval_end=data_interval_end,- model=BatchExportModel(- name="hogql", schema=None, hogql_query="SELECT * FROM events_view", user_id=auser.pk- ),+ model=BatchExportHogQLModel(query="SELECT * FROM events_view", user_id=auser.pk, modifiers={}),)assert sorted((row["event_id"], row["event"], row["distinct_id"]) for row in exported_rows_2) == sorted((e["uuid"], e["event"], e["distinct_id"]) for e in events@@ -1633,9 +1638,7 @@ class TestHogQLModel:team_id=ateam.pk,data_interval_start=data_interval_start,data_interval_end=data_interval_end,- model=BatchExportModel(- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=auser.pk- ),+ model=BatchExportHogQLModel(query="SELECT event AS event FROM events", user_id=auser.pk, modifiers={}),batch_export_id=batch_export_id,)diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py b/products/batch_exports/backend/tests/temporal/pipeline/test_producer.pyindex 81e6800e1d81..6748253daac0 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py@@ -20,7 +20,7 @@ import pyarrow as pafrom posthog.temporal.common import asyncpafrom posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportEventsModelfrom products.batch_exports.backend.temporal.pipeline import producer as producer_modulefrom products.batch_exports.backend.temporal.pipeline.internal_stage import (BatchExportInsertIntoInternalStageInputs,@@ -304,7 +304,7 @@ async def _drain_stage_via_producer(@pytest.mark.django_db@pytest.mark.parametrize("interval", ["day"], indirect=True)[email protected]("model", [BatchExportModel(name="events", schema=None)])[email protected]("model", [BatchExportEventsModel()])async def test_producer_resumes_from_offset_after_mid_file_failure(interval,activity_environment,@@ -313,7 +313,7 @@ async def test_producer_resumes_from_offset_after_mid_file_failure(object_storage_client,ateam,clickhouse_client,- model: BatchExportModel,+ model: BatchExportEventsModel,):"""A mid-file read failure resumes from the last batch boundary instead of re-delivering rows."""diff --git a/products/batch_exports/backend/tests/test_file_download_timeouts.py b/products/batch_exports/backend/tests/test_file_download_timeouts.pyindex 9cf6f3d7465b..f6061935788c 100644--- a/products/batch_exports/backend/tests/test_file_download_timeouts.py+++ b/products/batch_exports/backend/tests/test_file_download_timeouts.py@@ -12,7 +12,11 @@ from temporalio.common import MetricMeterfrom products.batch_exports.backend import servicefrom products.batch_exports.backend.models.batch_export import BatchExportOnDemand-from products.batch_exports.backend.service import BatchExportModel, FileDownloadBatchExportInputs+from products.batch_exports.backend.service import (+ BatchExportEventsModel,+ BatchExportHogQLModel,+ FileDownloadBatchExportInputs,+)from products.batch_exports.backend.temporal.destinations.file_download_batch_export import (FileDownloadBatchExportWorkflow,export_to_file_download_bucket_with_temporary_credentials,@@ -44,7 +48,7 @@ class TestFileDownloadTimeouts(SimpleTestCase):self, _name: str, start: dt.datetime | None, end: dt.datetime | None, main_seconds: int, query_seconds: int) -> None:batch_export = BatchExportOnDemand(id=EXPORT_ID, team_id=1)- model = BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value")+ model = BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})client = AsyncMock(spec=Client)with (@@ -96,7 +100,9 @@ class TestFileDownloadTimeouts(SimpleTestCase):"file-download-test",start,end,- BatchExportModel(name=model_name, schema=None),+ BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})+ if model_name == "hogql"+ else BatchExportEventsModel(),batch_export_run_id=run_id,)connect.assert_not_called()@@ -161,7 +167,7 @@ class TestFileDownloadTimeouts(SimpleTestCase):batch_export_id=str(EXPORT_ID),team_id=1,batch_export_run_id=RUN_ID,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),data_interval_start=start,data_interval_end=end,main_activity_timeout_seconds=28800,diff --git a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.pyindex f8f311fffbb4..92b9be0c70c8 100644--- a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py+++ b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py@@ -13,7 +13,7 @@ from products.batch_exports.backend.hogql_source import (parse_hogql_select_for_batch_export,replace_interval_placeholders,)-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportHogQLModel, BatchExportModelfrom products.batch_exports.backend.temporal.pipeline.internal_stage import (BatchExportInsertIntoInternalStageInputs,insert_into_internal_stage_activity,@@ -84,7 +84,11 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: stbatch_export_id="00000000-0000-4000-8000-000000000001",data_interval_start=None,data_interval_end=(dt.datetime.now(dt.UTC) - dt.timedelta(hours=1)).isoformat(),- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),+ batch_export_model=(+ BatchExportHogQLModel(query=hogql_query, user_id=1, modifiers={})+ if hogql_query is not None+ else BatchExportModel(name="hogql", schema=None, user_id=1)+ ),)clickhouse = MockClickHouseClient()diff --git a/products/batch_exports/backend/tests/test_nullable_staging.py b/products/batch_exports/backend/tests/test_nullable_staging.pyindex aee5d71c9ba7..9ea205b392e5 100644--- a/products/batch_exports/backend/tests/test_nullable_staging.py+++ b/products/batch_exports/backend/tests/test_nullable_staging.py@@ -14,7 +14,7 @@ from posthog.hogql.database.database import Databasefrom posthog.models import Team-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportEventsModel, BatchExportHogQLModel, BatchExportModelfrom products.batch_exports.backend.temporal.errors import MissingRequiredInputsErrorfrom products.batch_exports.backend.temporal.pipeline import internal_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import STAGE_NON_RETRYABLE_ERROR_TYPES@@ -76,7 +76,7 @@ async def test_hogql_staging_preserves_optional_bounds_and_cleanup_namespace(data_interval_start=start_str,data_interval_end=end_str,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),)result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)@@ -110,7 +110,11 @@ async def test_unsupported_hogql_returns_nonretryable_staging_error(data_interval_start=None,data_interval_end=None,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),+ batch_export_model=(+ BatchExportHogQLModel(query=hogql_query, user_id=1, modifiers={})+ if hogql_query is not None+ else BatchExportModel(name="hogql", schema=None, user_id=1)+ ),)if hogql_query is None:@@ -145,7 +149,7 @@ async def test_hogql_staging_rejects_each_future_bound_without_waiting(data_interval_start=start.isoformat() if start is not None else None,data_interval_end=end.isoformat() if end is not None else None,on_demand=True,- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),)# Patch the helper rather than override settings.TEST, because DatabaseSyncToAsync calls# close_old_connections() whenever settings.TEST is False, and that needs a database.@@ -182,7 +186,11 @@ async def test_missing_staging_inputs_raise(data_interval_start=None,data_interval_end=None,on_demand=on_demand,- batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value", user_id=1),+ batch_export_model=(+ BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})+ if model_name == "hogql"+ else BatchExportEventsModel()+ ),)with pytest.raises(MissingRequiredInputsError, match="require"):await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)diff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.pyindex dd19499ecd32..1cf586192197 100644--- a/products/batch_exports/backend/tests/test_service.py+++ b/products/batch_exports/backend/tests/test_service.py@@ -319,9 +319,7 @@ class TestTypeCoercionInBatchExportInputs:table_name="events",use_variant_type=True,use_automatic_schema_evolution=False,- batch_export_model=BatchExportModel(- name="events",- schema=None,+ batch_export_model=BatchExportEventsModel(filters=[{"type": "hogql", "key": "event = 'example_event'", "value": filter_value}],),)@@ -332,8 +330,7 @@ class TestTypeCoercionInBatchExportInputs:assert inputs.use_variant_type is Trueassert inputs.use_automatic_schema_evolution is False- assert inputs.batch_export_model is not None- assert inputs.batch_export_model.filters is not None+ assert isinstance(inputs.batch_export_model, BatchExportEventsModel)assert inputs.batch_export_model.filters[0]["value"] is filter_valuedef test_optional_int_none_is_preserved(self):