Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(tests): Migrate batch export model fixtures (#120891)

Tomás Farías Santana51543429f36e
diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py
index 918ab5cb85b1..c97a929f306c 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_activity.py
@@ -6,7 +6,7 @@ from temporalio.testing import ActivityEnvironment
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsCustomModel
from products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (
AzureBlobInsertInputs,
azure_blob_default_fields,
@@ -76,10 +76,10 @@ async def test_activity_exports_model_to_azure_blob(
):
"""Test that events, persons, and sessions are exported to Azure Blob Storage."""
batch_export_model = None
- if isinstance(model, BatchExportModel):
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
batch_export_id = str(uuid.uuid4())
diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py
index 478c36f0c3cd..27ea7a513cc0 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_execution.py
@@ -1,6 +1,11 @@
import pytest
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.tests.temporal.destinations.azure_blob.utils import (
SUPPORTED_FILE_FORMAT_COMPRESSIONS,
TEST_AZURE_BLOB_MODELS,
@@ -37,10 +42,10 @@ async def test_workflow_exports_model_successfully(
data_interval_start,
data_interval_end,
generate_test_data,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
):
"""Test that the workflow exports events, persons, or sessions to Azure Blob Storage."""
- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model
+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else model
run = await run_azure_blob_batch_export_workflow(
team=ateam,
@@ -101,7 +106,7 @@ async def test_workflow_names_parquet_blobs_by_the_extension_setting(
integration_id=azure_batch_export.destination.integration.id,
file_format=file_format,
compression=compression,
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
legacy_parquet_extension=legacy_parquet_extension,
)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py
index 063cf7883493..ef28793037be 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/test_workflow_with_azure_account.py
@@ -23,7 +23,12 @@ from azure.storage.blob.aio import BlobServiceClient
from posthog.models.integration import Integration
from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.tests.temporal.destinations.azure_blob.utils import (
SUPPORTED_FILE_FORMAT_COMPRESSIONS,
TEST_AZURE_BLOB_MODELS,
@@ -132,10 +137,10 @@ async def test_workflow_exports_data_successfully(
data_interval_start,
data_interval_end,
generate_test_data,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
):
"""Test workflow exports events, persons, or sessions to real Azure Storage."""
- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model
+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else model
run = await run_azure_blob_batch_export_workflow(
team=ateam,
@@ -170,7 +175,7 @@ async def test_workflow_exports_data_successfully(
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize(("file_format", "compression"), SUPPORTED_FILE_FORMAT_COMPRESSIONS, indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_workflow_handles_formats_and_compression(
ateam,
azure_batch_export,
@@ -183,7 +188,7 @@ async def test_workflow_handles_formats_and_compression(
data_interval_start,
data_interval_end,
generate_test_data,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
):
"""Test workflow handles various file formats and compression types with real Azure."""
run = await run_azure_blob_batch_export_workflow(
diff --git a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py
index e6e02750bb9a..1758bfad4080 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/azure_blob/utils.py
@@ -15,7 +15,14 @@ from temporalio.common import RetryPolicy
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.azure_blob_batch_export import (
SUPPORTED_COMPRESSIONS,
@@ -125,18 +132,20 @@ async def read_manifest(container: ContainerClient, prefix: str) -> dict | None:
def extract_model_configuration(
- batch_export_model: BatchExportModel | BatchExportSchema | None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,
):
"""Extract (model_name, fields, filters, extra_query_parameters) from model config."""
if batch_export_model is None:
return "events", None, None, None
- if isinstance(batch_export_model, BatchExportModel):
- schema = batch_export_model.schema
+ if isinstance(batch_export_model, AnyBatchExportModel):
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
return (
batch_export_model.name,
schema["fields"] if schema else None,
- batch_export_model.filters,
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None,
schema["values"] if schema else None,
)
@@ -149,7 +158,7 @@ async def assert_clickhouse_records_in_azure_blob(
team_id: int,
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
- batch_export_model: BatchExportModel | BatchExportSchema | None = None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,
compression: str | None = None,
file_format: str = "JSONLines",
exclude_events: list[str] | None = None,
@@ -234,9 +243,8 @@ async def assert_clickhouse_records_in_azure_blob(
assert exported_records == expected_records
-TEST_AZURE_BLOB_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
- BatchExportModel(
- name="events",
+TEST_AZURE_BLOB_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -248,17 +256,15 @@ TEST_AZURE_BLOB_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},
},
),
- BatchExportModel(name="events", schema=None),
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(),
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
],
),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
{
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -289,7 +295,7 @@ async def run_azure_blob_batch_export_workflow(
integration_id: int,
file_format: str = "JSONLines",
compression: str | None = None,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
legacy_parquet_extension: bool = True,
):
"""Run the Azure Blob batch export workflow and return the run result."""
diff --git a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
index 8accca2f7d41..ca69703d596d 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/base_destination_tests.py
@@ -28,11 +28,15 @@ from posthog.temporal.common.logger import BATCH_EXPORT_WORKFLOW_TYPES as LOGGER
from products.batch_exports.backend.models.batch_export import BatchExport
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
BackfillDetails,
BaseBatchExportInputs,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
BatchExportField,
- BatchExportModel,
+ BatchExportPersonsModel,
BatchExportSchema,
+ BatchExportSessionsModel,
)
from products.batch_exports.backend.temporal import ACTIVITIES, WORKFLOWS
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
@@ -102,7 +106,7 @@ class BaseDestinationTest(ABC):
data_interval_end: dt.datetime,
interval: str,
batch_export: BatchExport,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
backfill_details: BackfillDetails | None = None,
**override_config,
):
@@ -211,7 +215,7 @@ async def _get_records_from_clickhouse(
data_interval_end: dt.datetime,
exclude_events: list[str] | None,
include_events: list[str] | None,
- batch_export_model: BatchExportModel | BatchExportSchema | None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,
backfill_details: BackfillDetails | None,
expected_fields: list[str] | None,
destination_default_fields: list[BatchExportField],
@@ -220,13 +224,16 @@ async def _get_records_from_clickhouse(
"""Get records from ClickHouse."""
json_columns = json_columns or []
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
@@ -297,7 +304,7 @@ async def assert_clickhouse_records_in_destination(
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
inputs: BaseBatchExportInputs,
- batch_export_model: BatchExportModel | BatchExportSchema | None = None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,
backfill_details: BackfillDetails | None = None,
expected_fields: list[str] | None = None,
exclude_events: list[str] | None = None,
@@ -329,7 +336,7 @@ async def assert_clickhouse_records_in_destination(
# Determine sort key based on model
sort_key = "uuid"
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
if batch_export_model.name == "persons":
sort_key = "person_id"
elif batch_export_model.name == "sessions":
@@ -375,8 +382,7 @@ class CommonWorkflowTests:
# Common test models used across destinations
TEST_MODELS = [
- BatchExportModel(
- name="events",
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -388,17 +394,15 @@ class CommonWorkflowTests:
"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},
},
),
- BatchExportModel(name="events", schema=None),
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(),
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
],
),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
{
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -483,7 +487,7 @@ class CommonWorkflowTests:
destination_test: BaseDestinationTest,
interval: str,
exclude_events: list[str] | None,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
@@ -496,16 +500,16 @@ class CommonWorkflowTests:
We test this for all models using a single interval.
"""
- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:
pytest.skip(
f"Unnecessary test case as batch export model '{model.name}' is not affected by 'exclude_events'"
)
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
inputs = destination_test.create_batch_export_inputs(
team_id=ateam.pk,
@@ -527,7 +531,7 @@ class CommonWorkflowTests:
or run.records_completed == len(persons_to_export_created)
or run.records_completed
== len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed <= 1)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed <= 1)
)
await assert_clickhouse_records_in_destination(
@@ -556,7 +560,7 @@ class CommonWorkflowTests:
team_id=ateam.pk,
data_interval_end=data_interval_end,
interval="hour",
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
batch_export=batch_export_for_destination,
)
@@ -594,7 +598,7 @@ class CommonWorkflowTests:
team_id=ateam.pk,
data_interval_end=data_interval_end,
interval="hour",
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
batch_export=batch_export_for_destination,
)
@@ -629,7 +633,7 @@ class CommonWorkflowTests:
team_id=ateam.pk,
data_interval_end=data_interval_end,
interval="hour",
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
batch_export=batch_export_for_destination,
)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py
index a523b3ed86bd..3ae2b397e757 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_activity.py
@@ -13,7 +13,14 @@ from google.cloud import bigquery
from posthog.models.integration import Integration
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (
BigQueryInsertInputs,
bigquery_default_fields,
@@ -64,7 +71,7 @@ async def _run_activity(
table_id: str,
dataset_id: str,
use_json_type: bool = False,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
sort_key: str = "event",
@@ -176,7 +183,7 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(
exclude_events,
bigquery_dataset,
use_json_type,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -193,14 +200,14 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(
* Are not duplicates of other events that are in the same batch.
* Do not have an event name contained in the batch export's `exclude_events`.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
sort_key = "event"
if batch_export_model is not None:
@@ -230,7 +237,7 @@ async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="sessions", schema=None),
+ BatchExportSessionsModel(),
],
)
async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_into_bigquery_table(
@@ -241,7 +248,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_in
exclude_events,
bigquery_dataset,
use_json_type,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -285,9 +292,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_sessions_data_in
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
@@ -330,7 +335,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigque
exclude_events,
bigquery_dataset,
use_json_type,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -349,14 +354,14 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigque
* Are not duplicates of other events that are in the same batch.
* Do not have an event name contained in the batch export's `exclude_events`.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
await _run_activity(
activity_environment,
@@ -386,7 +391,7 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigque
exclude_events,
bigquery_dataset,
use_json_type,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -397,14 +402,14 @@ async def test_insert_into_bigquery_activity_from_stage_inserts_data_into_bigque
For this test we mock the `check_for_query_permissions_on_table` method to assert the
behavior of the activity function when lacking query permissions in BigQuery.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":
pytest.skip("Unnecessary test case as person batch export requires query permissions")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
with (
unittest.mock.patch(
@@ -450,7 +455,7 @@ async def test_insert_into_bigquery_activity_from_stage_merges_persons_data_in_f
run of the activity. We expect the new entries to have replaced the old ones in BigQuery after
the second run.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_id = f"test_insert_activity_mutability_table_persons_{ateam.pk}"
await _run_activity(
@@ -526,7 +531,7 @@ async def test_insert_into_bigquery_activity_from_stage_merges_sessions_data_in_
run of the activity. We expect the new entries to have replaced the old ones in BigQuery after
the second run with the same time range.
"""
- model = BatchExportModel(name="sessions", schema=None)
+ model = BatchExportSessionsModel()
table_id = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"
result = await _run_activity(
@@ -630,7 +635,7 @@ async def test_insert_into_bigquery_activity_from_stage_handles_person_new_colum
To replicate this situation we first export the data with the original
schema, then delete a column in the destination and then rerun the export.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_id = f"test_insert_activity_migration_table_{ateam.pk}"
await _run_activity(
@@ -725,7 +730,7 @@ async def test_insert_into_bigquery_activity_from_stage_handles_datetime_to_int(
To replicate this situation we first export the data to create the table, then alter
'created_at' to be of type INT64, and then rerun the export.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_id = f"test_insert_activity_migration_table_{ateam.pk}"
await _run_activity(
@@ -793,7 +798,7 @@ async def test_insert_into_bigquery_activity_creates_tables_with_the_right_parti
compatibility, this may change in the future), and the consumer table to not be
partitioned.
"""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
table_id = f"test_insert_activity_partitioning_{ateam.pk}"
with (
diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py
index d4b9e91bcb77..ced65636c547 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_workflow.py
@@ -15,8 +15,10 @@ from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
BackfillDetails,
- BatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportPersonsModel,
BatchExportSchema,
BigQueryBatchExportInputs,
)
@@ -100,7 +102,7 @@ async def test_bigquery_export_workflow(
ateam,
table_id,
use_json_type,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -110,13 +112,13 @@ async def test_bigquery_export_workflow(
The workflow should update the batch export run status to completed and produce the expected
records to the configured BigQuery table.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
+ batch_export_model: AnyBatchExportModel | None = None
sort_key = "event"
- if isinstance(model, BatchExportModel):
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
if batch_export_model.name == "persons":
sort_key = "person_id"
@@ -124,7 +126,7 @@ async def test_bigquery_export_workflow(
sort_key = "session_id"
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
workflow_id = str(uuid.uuid4())
inputs = BigQueryBatchExportInputs(
@@ -198,7 +200,7 @@ async def test_bigquery_export_workflow_without_events(
ateam,
table_id,
use_json_type,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
data_interval_start,
data_interval_end,
):
@@ -206,14 +208,14 @@ async def test_bigquery_export_workflow_without_events(
The workflow should update the batch export run status to completed and set 0 as `records_completed`.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
workflow_id = str(uuid.uuid4())
inputs = BigQueryBatchExportInputs(
@@ -264,7 +266,7 @@ async def test_bigquery_export_workflow_without_events(
)
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize("use_json_type", [True], indirect=True)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
async def test_bigquery_export_workflow_backfill_earliest_persons(
ateam,
bigquery_client,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py
index e82ae52a2df3..af2dcd9261a5 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/utils.py
@@ -11,7 +11,14 @@ from google.cloud import bigquery, iam_admin_v1
from posthog.models.integration import GoogleCloudServiceAccountIntegration, Integration
from posthog.temporal.common.clickhouse import ClickHouseClient
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (
bigquery_default_fields,
get_our_google_cloud_credentials,
@@ -27,9 +34,8 @@ SKIP_IF_MISSING_GOOGLE_APPLICATION_CREDENTIALS = pytest.mark.requires_vendor_cre
TEST_TIME = dt.datetime.now(dt.UTC).replace(hour=0, minute=0, second=0, microsecond=0)
-TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
- BatchExportModel(
- name="events",
+TEST_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "event", "alias": "event"},
@@ -40,8 +46,8 @@ TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},
},
),
- BatchExportModel(name="events", schema=None),
- BatchExportModel(name="persons", schema=None),
+ BatchExportEventsModel(),
+ BatchExportPersonsModel(),
{
"fields": [
{"expression": "event", "alias": "event"},
@@ -65,7 +71,7 @@ async def assert_clickhouse_records_in_bigquery(
min_ingested_timestamp: dt.datetime | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
- batch_export_model: BatchExportModel | BatchExportSchema | None = None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,
use_json_type: bool = False,
sort_key: str = "event",
backfill_details: BackfillDetails | None = None,
@@ -127,13 +133,16 @@ async def assert_clickhouse_records_in_bigquery(
inserted_records.append(inserted_record)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py
index 079a135b4053..086ca22663c4 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_workflow.py
@@ -22,8 +22,9 @@ from posthog.temporal.common.base import PostHogWorkflow
from products.batch_exports.backend.service import (
BaseBatchExportInputs,
+ BatchExportEventsModel,
BatchExportField,
- BatchExportModel,
+ BatchExportPersonsModel,
DatabricksBatchExportInputs,
)
from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (
@@ -287,7 +288,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):
run of the workflow. We expect the new entries to have replaced the old ones in Databricks after
the second run.
"""
- batch_export_model = BatchExportModel(name="persons", schema=None)
+ batch_export_model = BatchExportPersonsModel()
inputs = destination_test.create_batch_export_inputs(
team_id=ateam.pk,
@@ -442,7 +443,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):
with destination_test.cursor(ateam.pk, integration) as cursor:
cursor.execute(query)
- batch_export_model = BatchExportModel(name=model_name, schema=None)
+ batch_export_model = BatchExportPersonsModel() if model_name == "persons" else BatchExportEventsModel()
inputs = destination_test.create_batch_export_inputs(
team_id=ateam.pk,
@@ -516,7 +517,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):
with destination_test.cursor(ateam.pk, integration) as cursor:
cursor.execute(query)
- batch_export_model = BatchExportModel(name="persons", schema=None)
+ batch_export_model = BatchExportPersonsModel()
inputs = destination_test.create_batch_export_inputs(
team_id=ateam.pk,
@@ -555,7 +556,7 @@ class TestDatabricksBatchExportWorkflow(CommonWorkflowTests):
"Cannot perform Merge as multiple source rows matched and attempted to modify the same target row in the Delta
table in possibly conflicting ways"
"""
- batch_export_model = BatchExportModel(name="persons", schema=None)
+ batch_export_model = BatchExportPersonsModel()
inputs = destination_test.create_batch_export_inputs(
team_id=ateam.pk,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py
index 9fcb23a43855..eff3a7ffac04 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_activity.py
@@ -7,7 +7,12 @@ from django.test import override_settings
from temporalio.testing._activity import ActivityEnvironment
from products.batch_exports.backend.models.batch_export import BatchExportFileDownload, BatchExportRun
-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportInsertInputs,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.file_download_batch_export import (
ExportInputs,
GenerateFileDownloadsInputs,
@@ -45,7 +50,7 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
ateam,
s3_client,
@@ -59,11 +64,11 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(
S3InsertInputs, and delegates to insert_into_s3_from_stage. We verify
the data ends up in the expected S3 location and matches what ClickHouse produces.
"""
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
batch_export_id = str(uuid.uuid4())
run_id = str(uuid.uuid4())
@@ -128,13 +133,13 @@ async def test_export_to_file_download_bucket_puts_data_into_s3(
or result.records_completed == len(persons_to_export_created)
or result.records_completed
== len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= result.records_completed <= 2)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= result.records_completed <= 2)
)
sort_key = "uuid"
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":
sort_key = "person_id"
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":
sort_key = "session_id"
await assert_clickhouse_records_in_s3(
diff --git a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py
index cee7a746ac85..6ba1ed326016 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/file_download/test_file_download_workflow.py
@@ -12,7 +12,7 @@ from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from products.batch_exports.backend.models.batch_export import BatchExportFileDownload
-from products.batch_exports.backend.service import BatchExportModel, FileDownloadBatchExportInputs
+from products.batch_exports.backend.service import BatchExportEventsModel, FileDownloadBatchExportInputs
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.file_download_batch_export import (
FileDownloadBatchExportWorkflow,
@@ -73,7 +73,7 @@ async def file_download_batch_export(
@requires_aws_credentials
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
async def test_file_download_workflow_exports_data(
clickhouse_client,
@@ -87,7 +87,7 @@ async def test_file_download_workflow_exports_data(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: BatchExportEventsModel,
generate_test_data,
aws_role_arn,
region,
@@ -222,9 +222,7 @@ async def test_file_download_workflow_generates_no_downloads_for_a_failed_run(
data_interval_end=data_interval_end.isoformat(),
data_interval_start=data_interval_start.isoformat(),
interval=interval,
- batch_export_model=BatchExportModel(
- name="events",
- schema=None,
+ batch_export_model=BatchExportEventsModel(
filters=[{"key": "event =", "type": "hogql", "value": None}],
),
file_format=file_format,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py
index 2483e73455ab..ddd110fd34de 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_activity.py
@@ -9,7 +9,15 @@ from django.test import override_settings
import pytest_asyncio
from psycopg import sql
-from products.batch_exports.backend.service import BatchExportField, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportField,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.postgres_batch_export import (
PostgresInsertInputs,
insert_into_postgres_activity_from_stage,
@@ -45,7 +53,7 @@ async def _run_activity(
data_interval_start,
data_interval_end,
table_name: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
sort_key: str = "event",
@@ -117,13 +125,13 @@ async def _run_activity(
return result
[email protected]("model", [None, BatchExportModel(name="events", schema=None)])
[email protected]("model", [None, BatchExportEventsModel()])
async def test_insert_into_postgres_activity_handles_old_staged_events(
clickhouse_client,
activity_environment,
postgres_connection,
postgres_config,
- model: BatchExportModel | None,
+ model: AnyBatchExportModel | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -151,10 +159,9 @@ async def test_insert_into_postgres_activity_handles_old_staged_events(
[
pytest.param(None, False, id="existing-events-table"),
pytest.param(None, True, id="default-without-select"),
- pytest.param(BatchExportModel(name="events", schema=None), True, id="events-without-select"),
+ pytest.param(BatchExportEventsModel(), True, id="events-without-select"),
pytest.param(
- BatchExportModel(
- name="events",
+ BatchExportEventsCustomModel(
schema={"fields": [{"expression": "person_id", "alias": "person_id"}], "values": {}},
),
True,
@@ -174,15 +181,15 @@ async def test_insert_into_postgres_activity_handles_existing_events_table(
postgres_connection,
postgres_config,
insert_only_postgres_config,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
ateam,
) -> None:
table_name = "existing_events"
- batch_export_model = BatchExportModel(name="events", schema=model) if isinstance(model, dict) else model
- custom_schema = batch_export_model.schema if batch_export_model else None
+ batch_export_model = BatchExportEventsCustomModel(schema=model) if isinstance(model, dict) else model
+ custom_schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
await _run_activity(
activity_environment=activity_environment,
@@ -247,7 +254,7 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(
postgres_connection,
postgres_config,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -270,17 +277,17 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(
to avoid conflicting with PostHog itself.
"""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
sort_key = "event"
if batch_export_model is not None:
@@ -307,7 +314,7 @@ async def test_insert_into_postgres_activity_inserts_data_into_postgres_table(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize(
"test_properties",
[
@@ -332,7 +339,7 @@ async def test_insert_into_postgres_activity_handles_problematic_json(
postgres_connection,
postgres_config,
exclude_events,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -377,7 +384,7 @@ async def test_insert_into_postgres_activity_merges_persons_data_in_follow_up_ru
run of the activity. We expect the new entries to have replaced the old ones in PostgreSQL after
the second run.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_name = f"test_insert_activity_mutability_table_persons_{ateam.pk}"
# First run
@@ -452,7 +459,7 @@ async def test_insert_into_postgres_activity_merges_sessions_data_in_follow_up_r
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
- model = BatchExportModel(name="sessions", schema=None)
+ model = BatchExportSessionsModel()
table_name = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"
# First run
@@ -560,12 +567,12 @@ async def persons_table_without_primary_key(postgres_connection, postgres_config
)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
async def test_insert_into_postgres_activity_inserts_fails_on_missing_primary_key(
activity_environment,
postgres_config,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
data_interval_start,
data_interval_end,
ateam,
@@ -579,11 +586,11 @@ async def test_insert_into_postgres_activity_inserts_fails_on_missing_primary_ke
This error should only occur if the table is created outside the batch export.
"""
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
insert_inputs = PostgresInsertInputs(
team_id=ateam.pk,
@@ -644,7 +651,7 @@ async def test_insert_into_postgres_activity_handles_person_schema_changes(
To replicate this situation we first export the data with the original
schema, then delete a column in the destination and then rerun the export.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_name = f"test_insert_activity_migration_table__{ateam.pk}"
# First run
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py
index db2a4013417e..baadea93f8ee 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_workflow.py
@@ -15,7 +15,14 @@ from temporalio.common import RetryPolicy
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.postgres_batch_export import (
PostgresBatchExportInputs,
@@ -52,7 +59,7 @@ async def test_postgres_export_workflow(
exclude_events,
ateam,
table_name,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -63,17 +70,17 @@ async def test_postgres_export_workflow(
records to the local development PostgreSQL database.
"""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
config = postgres_batch_export.destination.config
@@ -131,7 +138,7 @@ async def test_postgres_export_workflow(
or run.records_completed == len(persons_to_export_created)
or run.records_completed
== len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed == 1)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed == 1)
)
await assert_clickhouse_records_in_postgres(
@@ -150,7 +157,7 @@ async def test_postgres_export_workflow(
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("integration", [True], indirect=True)
async def test_postgres_export_workflow_with_integration(
clickhouse_client,
@@ -161,7 +168,7 @@ async def test_postgres_export_workflow_with_integration(
exclude_events,
ateam,
table_name,
- model: BatchExportModel | None,
+ model: AnyBatchExportModel | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -245,7 +252,7 @@ async def test_postgres_export_workflow_without_events(
exclude_events,
ateam,
table_name,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
data_interval_start,
data_interval_end,
):
@@ -254,17 +261,17 @@ async def test_postgres_export_workflow_without_events(
The workflow should update the batch export run status to completed and set 0 as `records_completed`.
"""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
workflow_id = str(uuid.uuid4())
inputs = PostgresBatchExportInputs(
@@ -318,7 +325,7 @@ async def test_postgres_export_workflow_without_events(
indirect=True,
)
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
async def test_postgres_export_workflow_backfill_earliest_persons(
ateam,
clickhouse_client,
@@ -589,7 +596,7 @@ async def test_postgres_export_workflow_with_many_files(
can take longer, so we keep the event count low and bump the Workflow timeout.
"""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
workflow_id = str(uuid.uuid4())
inputs = PostgresBatchExportInputs(
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py
index df4e84715e74..f01d8a6a7bcd 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/utils.py
@@ -9,7 +9,15 @@ from psycopg import sql
from posthog.models import Integration
from posthog.temporal.common.clickhouse import ClickHouseClient
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.postgres_batch_export import postgres_default_fields
from products.batch_exports.backend.temporal.queue import RecordBatchQueue
from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel
@@ -48,18 +56,16 @@ EXPECTED_PERSONS_BATCH_EXPORT_FIELDS = [
]
-TEST_MODELS: list[BatchExportModel | None] = [
- BatchExportModel(name="events", schema=None),
- BatchExportModel(
- name="events",
- schema=None,
+TEST_MODELS: list[AnyBatchExportModel | None] = [
+ BatchExportEventsModel(),
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
],
),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
None,
]
@@ -70,7 +76,7 @@ async def assert_clickhouse_records_in_postgres(
schema_name: str,
table_name: str,
team_id: int,
- batch_export_model: BatchExportModel | BatchExportSchema | None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
exclude_events: list[str] | None = None,
@@ -111,13 +117,16 @@ async def assert_clickhouse_records_in_postgres(
inserted_records.append(event)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py
index 2717520cc00a..6f08ce794cbf 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_activity.py
@@ -7,7 +7,15 @@ from psycopg import sql
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportInsertInputs,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
ConnectionParameters,
RedshiftInsertInputs,
@@ -45,7 +53,7 @@ async def _run_activity(
data_interval_end,
table_name: str,
properties_data_type: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
sort_key: str = "event",
@@ -134,7 +142,7 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(
psycopg_connection,
redshift_config,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -155,14 +163,14 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(
that they appear in the expected Redshift table.
"""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and MISSING_REQUIRED_ENV_VARS
):
@@ -189,11 +197,11 @@ async def test_insert_into_redshift_activity_inserts_data_into_redshift_table(
person_properties={"utm_medium": "referral", "$initial_os": "Linux"},
)
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
table_name = f"test_insert_activity_table__{ateam.pk}"
@@ -240,7 +248,7 @@ async def test_insert_into_redshift_activity_merges_persons_data_in_follow_up_ru
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Persons batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
properties_data_type = "varchar"
table_name = f"test_insert_activity_mutability_table_{ateam.pk}"
@@ -340,7 +348,7 @@ async def test_insert_into_redshift_activity_merges_sessions_data_in_follow_up_r
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Sessions batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="sessions", schema=None)
+ model = BatchExportSessionsModel()
properties_data_type = "varchar"
table_name = f"test_insert_activity_mutability_table_sessions_{ateam.pk}"
@@ -441,7 +449,7 @@ async def test_insert_into_redshift_activity_handles_person_schema_changes(
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Persons batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
properties_data_type = "varchar"
table_name = f"test_insert_activity_migration_table__{ateam.pk}"
expected_fields = [
@@ -523,7 +531,7 @@ async def test_insert_into_redshift_activity_handles_person_schema_changes(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("properties_data_type", ["super"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("missing_person_id", [False, True])
async def test_insert_into_redshift_activity_inserts_data_with_extra_columns(
clickhouse_client,
@@ -531,7 +539,7 @@ async def test_insert_into_redshift_activity_inserts_data_with_extra_columns(
psycopg_connection,
redshift_config,
exclude_events,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py
index c72c50d0c645..72d0175b7401 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_copy_activity.py
@@ -8,7 +8,15 @@ from psycopg import sql
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportInsertInputs,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
AWSCredentials,
ConnectionParameters,
@@ -67,7 +75,7 @@ async def _run_activity(
key_prefix: str,
credentials: AWSCredentials,
properties_data_type: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
sort_key: str = "event",
@@ -176,7 +184,7 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table(
bucket_name,
bucket_region,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -187,7 +195,7 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table(
):
"""Test that the copy_into_redshift_activity function inserts data into a Redshift table."""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
@@ -211,11 +219,11 @@ async def test_copy_into_redshift_activity_inserts_data_into_redshift_table(
person_properties={"utm_medium": "referral", "$initial_os": "Linux"},
)
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
table_name = f"test_copy_activity_table__{ateam.pk}"
@@ -270,7 +278,7 @@ async def test_copy_into_redshift_activity_merges_persons_data_in_follow_up_runs
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Persons batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
properties_data_type = "super"
table_name = f"test_copy_activity_mutability_table_{ateam.pk}"
@@ -382,7 +390,7 @@ async def test_copy_into_redshift_activity_merges_sessions_data_in_follow_up_run
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Sessions batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="sessions", schema=None)
+ model = BatchExportSessionsModel()
properties_data_type = "varchar"
table_name = f"test_copy_activity_mutability_table_sessions_{ateam.pk}"
@@ -495,7 +503,7 @@ async def test_copy_into_redshift_activity_handles_person_schema_changes(
if MISSING_REQUIRED_ENV_VARS:
pytest.skip("Persons batch export cannot be tested in PostgreSQL")
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
properties_data_type = "varchar"
table_name = f"test_copy_activity_migration_table__{ateam.pk}"
expected_fields = [
@@ -585,7 +593,7 @@ async def test_copy_into_redshift_activity_handles_person_schema_changes(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("properties_data_type", ["super"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(
clickhouse_client,
activity_environment,
@@ -594,7 +602,7 @@ async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(
bucket_name,
bucket_region,
exclude_events,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -663,7 +671,7 @@ async def test_copy_into_redshift_activity_inserts_data_with_extra_columns(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("properties_data_type", ["varchar"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_copy_into_redshift_activity_handles_events_table_missing_columns(
clickhouse_client,
activity_environment,
@@ -672,7 +680,7 @@ async def test_copy_into_redshift_activity_handles_events_table_missing_columns(
bucket_name,
bucket_region,
exclude_events,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -742,7 +750,7 @@ async def test_copy_into_redshift_activity_handles_events_table_missing_columns(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("properties_data_type", ["varchar"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_copy_into_redshift_activity_handles_data_over_string_limit(
clickhouse_client,
activity_environment,
@@ -751,7 +759,7 @@ async def test_copy_into_redshift_activity_handles_data_over_string_limit(
bucket_name,
bucket_region,
exclude_events,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
data_interval_start,
data_interval_end,
properties_data_type,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py
index f2d50e751ed4..3e94d2d66622 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_workflow.py
@@ -16,7 +16,13 @@ from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from products.batch_exports.backend.models.batch_export import BatchExport
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema, RedshiftCopyInputs
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+ RedshiftCopyInputs,
+)
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
RedshiftBatchExportInputs,
@@ -165,7 +171,7 @@ async def test_redshift_export_workflow(
ateam,
exclude_events,
table_name,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -181,14 +187,14 @@ async def test_redshift_export_workflow(
records to the provided Redshift instance.
"""
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and exclude_events is not None
):
pytest.skip(f"Unnecessary test case as {model.name} batch export is not affected by 'exclude_events'")
if (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and (model.name == "persons" or model.name == "sessions")
and MISSING_REQUIRED_ENV_VARS
):
@@ -196,11 +202,11 @@ async def test_redshift_export_workflow(
copy_inputs = _build_copy_inputs(mode, aws_credentials, bucket_name, bucket_region, key_prefix)
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
inputs = RedshiftBatchExportInputs(
team_id=ateam.pk,
@@ -223,7 +229,7 @@ async def test_redshift_export_workflow(
assert (
run.records_completed == len(events_to_export_created)
or run.records_completed == len(persons_to_export_created)
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and run.records_completed == 1)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and run.records_completed == 1)
)
sort_key = "event"
@@ -324,7 +330,7 @@ async def test_redshift_export_workflow_handles_insert_activity_non_retryable_er
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("mode", ["COPY", "INSERT"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_redshift_export_workflow_handles_undefined_function_error(
clickhouse_client,
redshift_config,
@@ -334,7 +340,7 @@ async def test_redshift_export_workflow_handles_undefined_function_error(
ateam,
exclude_events,
table_name,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
data_interval_start,
data_interval_end,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py
index accccd38adf0..e950475f21fc 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/utils.py
@@ -8,7 +8,15 @@ from psycopg import sql
from posthog.temporal.common.clickhouse import ClickHouseClient
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import redshift_default_fields
from products.batch_exports.backend.temporal.queue import RecordBatchQueue
from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel
@@ -28,10 +36,10 @@ REQUIRED_ENV_VARS = (
MISSING_REQUIRED_ENV_VARS = any(env_var not in os.environ for env_var in REQUIRED_ENV_VARS)
-TEST_MODELS: list[BatchExportModel | None] = [
- BatchExportModel(name="events", schema=None),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+TEST_MODELS: list[AnyBatchExportModel | None] = [
+ BatchExportEventsModel(),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
None,
]
@@ -42,7 +50,7 @@ async def assert_clickhouse_records_in_redshift(
schema_name: str,
table_name: str,
team_id: int,
- batch_export_model: BatchExportModel | BatchExportSchema | None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,
date_ranges: list[tuple[dt.datetime, dt.datetime]],
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
@@ -136,13 +144,16 @@ async def assert_clickhouse_records_in_redshift(
inserted_records.append(event)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py
index 569834cfad20..5993148e0b59 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity.py
@@ -7,7 +7,12 @@ from temporalio.testing._activity import ActivityEnvironment
from posthog.models.integration import Integration, IntegrationError
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.constants import (
COMPRESSION_EXTENSIONS,
FILE_FORMAT_EXTENSIONS,
@@ -87,7 +92,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
ateam,
s3_compatible_integration,
@@ -108,11 +113,11 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
prefix = str(uuid.uuid4())
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
batch_export_id = str(uuid.uuid4())
@@ -142,16 +147,16 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
records_exported == len(events_to_export_created)
or records_exported == len(persons_to_export_created)
or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
)
assert isinstance(bytes_exported, int)
assert bytes_exported > 0
sort_key = "uuid"
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":
sort_key = "person_id"
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":
sort_key = "session_id"
await assert_clickhouse_records_in_s3(
@@ -173,7 +178,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("compression", [None], indirect=True)
@pytest.mark.parametrize("file_format", ["JSONLines"], indirect=True)
async def test_insert_into_s3_activity_resolves_credentials_from_integration(
@@ -186,7 +191,7 @@ async def test_insert_into_s3_activity_resolves_credentials_from_integration(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
generate_test_data,
ateam,
s3_compatible_integration,
@@ -237,7 +242,7 @@ async def test_insert_into_s3_activity_resolves_credentials_from_integration(
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("compression", [None], indirect=True)
@pytest.mark.parametrize("file_format", ["Parquet"], indirect=True)
async def test_insert_into_s3_activity_with_exclude_events(
@@ -250,7 +255,7 @@ async def test_insert_into_s3_activity_with_exclude_events(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
ateam,
s3_compatible_integration,
@@ -261,11 +266,11 @@ async def test_insert_into_s3_activity_with_exclude_events(
prefix = str(uuid.uuid4())
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
batch_export_id = str(uuid.uuid4())
@@ -295,16 +300,16 @@ async def test_insert_into_s3_activity_with_exclude_events(
records_exported == len(events_to_export_created)
or records_exported == len(persons_to_export_created)
or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
)
assert isinstance(bytes_exported, int)
assert bytes_exported > 0
sort_key = "uuid"
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":
sort_key = "person_id"
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":
sort_key = "session_id"
await assert_clickhouse_records_in_s3(
@@ -326,7 +331,7 @@ async def test_insert_into_s3_activity_with_exclude_events(
@pytest.mark.parametrize(("file_format", "compression"), SPLIT_FILE_FORMAT_COMPRESSIONS, indirect=["compression"])
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("max_file_size_mb", [None, 1])
async def test_insert_into_s3_activity_puts_splitted_files_into_s3(
clickhouse_client,
@@ -339,7 +344,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_s3(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
ateam,
s3_compatible_integration,
):
@@ -458,7 +463,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_s3(
assert manifest_data["files"] == expected_keys
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("file_format", ["invalid"])
async def test_insert_into_s3_activity_fails_on_invalid_file_format(
clickhouse_client,
@@ -470,7 +475,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_file_format(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
ateam,
s3_compatible_integration,
):
@@ -500,7 +505,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_file_format(
assert result.error_repr == "UnsupportedFileFormatError: 'invalid' is not a supported format for S3 batch exports."
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("compression", ["invalid"])
async def test_insert_into_s3_activity_fails_on_invalid_compression(
clickhouse_client,
@@ -511,7 +516,7 @@ async def test_insert_into_s3_activity_fails_on_invalid_compression(
exclude_events,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
ateam,
s3_compatible_integration,
):
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py
index ffd61cb1ead4..61a19e4eb034 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_gcs_bucket.py
@@ -9,7 +9,12 @@ from temporalio.testing._activity import ActivityEnvironment
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.constants import (
COMPRESSION_EXTENSIONS,
FILE_FORMAT_EXTENSIONS,
@@ -61,7 +66,7 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
ateam,
gcs_integration,
@@ -82,11 +87,11 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(
prefix = str(uuid.uuid4())
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
batch_export_id = str(uuid.uuid4())
@@ -116,16 +121,16 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(
records_exported == len(events_to_export_created)
or records_exported == len(persons_to_export_created)
or records_exported == len([event for event in events_to_export_created if event["properties"] is not None])
- or (isinstance(model, BatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
+ or (isinstance(model, AnyBatchExportModel) and model.name == "sessions" and 1 <= records_exported <= 2)
)
assert isinstance(bytes_exported, int)
assert bytes_exported > 0
sort_key = "uuid"
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons":
sort_key = "person_id"
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
+ elif isinstance(model, AnyBatchExportModel) and model.name == "sessions":
sort_key = "session_id"
await assert_clickhouse_records_in_s3(
@@ -147,7 +152,7 @@ async def test_insert_into_s3_activity_puts_data_into_gcs(
@pytest.mark.parametrize(("file_format", "compression"), SPLIT_FILE_FORMAT_COMPRESSIONS, indirect=["compression"])
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("max_file_size_mb", [None, 6])
async def test_insert_into_s3_activity_puts_splitted_files_into_gcs(
clickhouse_client,
@@ -160,7 +165,7 @@ async def test_insert_into_s3_activity_puts_splitted_files_into_gcs(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
ateam,
gcs_integration,
):
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py
index 89293b9e68e4..e996babe21aa 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_activity_with_role_assumption.py
@@ -12,7 +12,7 @@ from temporalio.testing._activity import ActivityEnvironment
from posthog.models import Organization, Team
from posthog.models.integration import Integration
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportEventsModel
from products.batch_exports.backend.temporal.destinations.s3_batch_export import (
PolicyStatement,
S3InsertInputs,
@@ -92,13 +92,13 @@ def kms_key_value(request) -> str | None:
[None, "kms_key_id", "kms_key_arn"],
indirect=True,
)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_insert_into_s3_activity_puts_data_into_s3_with_role_assumption(
clickhouse_client,
activity_environment: ActivityEnvironment,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: BatchExportEventsModel,
generate_test_data,
integration: Integration,
s3_client,
@@ -159,13 +159,13 @@ async def test_insert_into_s3_activity_puts_data_into_s3_with_role_assumption(
@pytest.mark.parametrize("external_id", ["not-an-org-id", None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_insert_into_s3_activity_fails_role_assumption_with_missing_external_id(
clickhouse_client,
activity_environment: ActivityEnvironment,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: BatchExportEventsModel,
generate_test_data,
integration: Integration,
s3_client,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py
index aba224101d62..58d176886091 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_error_handling.py
@@ -17,7 +17,7 @@ from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from types_aiobotocore_s3.client import S3Client
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportEventsModel
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run
from products.batch_exports.backend.temporal.destinations.s3_batch_export import (
ConcurrentS3Consumer,
@@ -246,7 +246,7 @@ async def test_s3_export_workflow_with_request_timeouts(
We run the S3 batch export workflow with a mocked session that will raise a `ClientError` due
to a `RequestTimeout` on the first run of the batch export. The second run should work normally.
"""
- batch_export_model = BatchExportModel(name="events", schema=None)
+ batch_export_model = BatchExportEventsModel()
raised = 0
@@ -359,11 +359,6 @@ async def test_s3_export_workflow_with_request_timeouts(
assert key.startswith(expected_key_prefix)
sort_key = "event"
- if batch_export_model is not None:
- if batch_export_model.name == "persons":
- sort_key = "person_id"
- elif batch_export_model.name == "sessions":
- sort_key = "session_id"
await assert_clickhouse_records_in_s3(
s3_compatible_client=object_storage_client,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py
index 769ce9eaca9d..7b333acb5aab 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_gcs_bucket.py
@@ -5,7 +5,7 @@ import pytest
import aioboto3
import pytest_asyncio
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportSchema
from products.batch_exports.backend.temporal.destinations.constants import (
COMPRESSION_EXTENSIONS,
FILE_FORMAT_EXTENSIONS,
@@ -85,7 +85,7 @@ async def gcs_batch_export(
@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
@pytest.mark.parametrize("compression", [*COMPRESSION_EXTENSIONS.keys(), None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("bucket_name", [os.getenv("GCS_TEST_BUCKET")], indirect=True)
@@ -100,7 +100,7 @@ async def test_s3_export_workflow_with_gcs_bucket_with_various_file_formats(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3 Export Workflow end-to-end by using a GCS bucket.
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py
index 4e3785a118f4..dade32f83d70 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_local_object_storage.py
@@ -2,7 +2,14 @@ import datetime as dt
import pytest
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.tests.temporal.destinations.s3.utils import (
SUPPORTED_FILE_FORMAT_COMPRESSIONS,
TEST_S3_MODELS,
@@ -34,7 +41,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_models(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket instead of S3.
@@ -62,7 +69,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_models(
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize(
("file_format", "compression"), [*SUPPORTED_FILE_FORMAT_COMPRESSIONS, *UNCOMPRESSED_FILE_FORMATS], indirect=True
@@ -80,7 +87,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_compres
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket and various compression and file formats."""
@@ -101,7 +108,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_various_compres
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("file_format", ["Parquet"], indirect=True)
@pytest.mark.parametrize("compression", ["zstd"], indirect=True)
@@ -120,7 +127,7 @@ async def test_s3_export_workflow_names_parquet_files_by_the_extension_setting(
legacy_parquet_extension,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
await run_s3_batch_export_workflow(
@@ -156,7 +163,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_exclude_events(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3BatchExport Workflow end-to-end by using a local object storage bucket and excluding events."""
@@ -183,7 +190,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_exclude_events(
indirect=True,
)
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
async def test_s3_export_workflow_backfill_earliest_persons_with_local_object_storage(
clickhouse_client,
object_storage_client,
@@ -296,7 +303,7 @@ async def test_s3_export_workflow_with_local_object_storage_and_custom_key_prefi
s3_key_prefix,
data_interval_end,
data_interval_start,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test the S3BatchExport Workflow end-to-end by specifying a custom key prefix.
@@ -358,7 +365,7 @@ async def test_s3_export_workflow_with_local_object_storage_with_hogql_model(
`events_table` is pinned to `sharded_events` because the HogQL `events` table reads it,
while the fixture would otherwise default to `events_recent` for a recent interval.
"""
- model = BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=auser.pk)
+ model = BatchExportHogQLModel(query=hogql_query, user_id=auser.pk, modifiers={})
await run_s3_batch_export_workflow(
model=model,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py
index 428246e42543..e8285e56bdd6 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/test_workflow_with_s3_bucket.py
@@ -8,7 +8,7 @@ import pytest_asyncio
from posthog.models.integration import Integration
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportSchema
from products.batch_exports.backend.temporal.destinations.constants import (
COMPRESSION_EXTENSIONS,
FILE_FORMAT_EXTENSIONS,
@@ -122,7 +122,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_mode
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3 Export Workflow end-to-end by using an S3 bucket.
@@ -138,7 +138,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_mode
will require its prescense in the database when running. This model is indirectly parametrized
by several fixtures. Refer to them for more information.
"""
- if isinstance(model, BatchExportModel) and model.name == "persons" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name == "persons" and exclude_events is not None:
pytest.skip("Unnecessary test case as person batch export is not affected by 'exclude_events'")
await run_s3_batch_export_workflow(
@@ -159,7 +159,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_intervals_and_mode
@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
@pytest.mark.parametrize("encryption", [None, "AES256", "aws:kms"], indirect=True)
@pytest.mark.parametrize("compression", [*COMPRESSION_EXTENSIONS.keys(), None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
@pytest.mark.parametrize("bucket_name", [os.getenv("S3_TEST_BUCKET")], indirect=True)
@@ -177,7 +177,7 @@ async def test_s3_export_workflow_with_s3_bucket_with_various_file_formats(
file_format,
data_interval_start,
data_interval_end,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
):
"""Test S3 Export Workflow end-to-end by using an S3 bucket with various file formats, compression, and
diff --git a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
index f98a35cabfc0..f97a92696186 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/s3/utils.py
@@ -22,10 +22,15 @@ from posthog.temporal.common.clickhouse import ClickHouseClient
from products.batch_exports.backend.hogql_source import parse_hogql_select_for_batch_export
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
AwsS3BatchExportInputs,
BackfillDetails,
- BatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ BatchExportPersonsModel,
BatchExportSchema,
+ BatchExportSessionsModel,
S3CompatibleBatchExportInputs,
S3FamilyBaseInputs,
)
@@ -62,9 +67,8 @@ _INPUTS_BY_DESTINATION_TYPE: dict[str, type[S3FamilyBaseInputs]] = {
COMPRESSION_OPTIONS = [*COMPRESSION_EXTENSIONS.keys(), None]
-TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
- BatchExportModel(
- name="events",
+TEST_S3_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -76,17 +80,15 @@ TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},
},
),
- BatchExportModel(name="events", schema=None),
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(),
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
],
),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
{
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -100,10 +102,10 @@ TEST_S3_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
None,
]
-TEST_S3_MODELS_AFFECTED_BY_EXCLUDE_EVENTS: list[BatchExportModel | BatchExportSchema | None] = [
+TEST_S3_MODELS_AFFECTED_BY_EXCLUDE_EVENTS: list[AnyBatchExportModel | BatchExportSchema | None] = [
model
for model in TEST_S3_MODELS
- if not (isinstance(model, BatchExportModel) and model.name in ("persons", "sessions"))
+ if not (isinstance(model, AnyBatchExportModel) and model.name in ("persons", "sessions"))
]
SUPPORTED_FILE_FORMAT_COMPRESSIONS: list[tuple[str, str | None]] = [
@@ -234,7 +236,7 @@ async def assert_clickhouse_records_in_s3(
data_interval_end: dt.datetime,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
- batch_export_model: BatchExportModel | BatchExportSchema | None = None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,
compression: str | None = None,
file_format: str = "JSONLines",
backfill_details: BackfillDetails | None = None,
@@ -275,13 +277,16 @@ async def assert_clickhouse_records_in_s3(
)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
@@ -295,7 +300,7 @@ async def assert_clickhouse_records_in_s3(
if fields is not None:
schema_column_names = [field["alias"] for field in fields]
- elif isinstance(batch_export_model, BatchExportModel) and batch_export_model.name == "persons":
+ elif isinstance(batch_export_model, BatchExportPersonsModel):
schema_column_names = [
"team_id",
"distinct_id",
@@ -307,10 +312,10 @@ async def assert_clickhouse_records_in_s3(
"created_at",
"is_deleted",
]
- elif isinstance(batch_export_model, BatchExportModel) and batch_export_model.name == "hogql":
+ elif isinstance(batch_export_model, BatchExportHogQLModel):
# A HogQL model exports whichever columns the query selects, so the query is the schema:
# the team_id check and the sort below must key off the query's own columns.
- schema_column_names = _hogql_model_column_names(batch_export_model.hogql_query)
+ schema_column_names = _hogql_model_column_names(batch_export_model.query)
else:
schema_column_names = [field["alias"] for field in s3_default_fields()]
@@ -322,16 +327,12 @@ async def assert_clickhouse_records_in_s3(
queue = RecordBatchQueue()
if model_name == "sessions":
producer = ClickHouseTestProducer(model=SessionsRecordBatchModel(team_id))
- elif (
- model_name == "hogql"
- and isinstance(batch_export_model, BatchExportModel)
- and batch_export_model.hogql_query is not None
- ):
+ elif isinstance(batch_export_model, BatchExportHogQLModel):
# Re-running the query itself yields the expected rows, with the same interval bounds
# substituted as the run exported with.
producer = ClickHouseTestProducer(
model=HogQLQueryRecordBatchModel(
- team_id=team_id, hogql_query=batch_export_model.hogql_query, user_id=batch_export_model.user_id
+ team_id=team_id, hogql_query=batch_export_model.query, user_id=batch_export_model.user_id
)
)
else:
@@ -379,7 +380,7 @@ async def assert_clickhouse_records_in_s3(
s3_data = sorted(s3_data, key=operator.itemgetter(sort_key))
expected_records = sorted(expected_records, key=operator.itemgetter(sort_key))
- if isinstance(batch_export_model, BatchExportModel) and batch_export_model.name in ["events", "persons"]:
+ if isinstance(batch_export_model, AnyBatchExportModel) and batch_export_model.name in ["events", "persons"]:
assert set(s3_data[0].keys()) == set(schema_column_names)
if allow_duplicates:
@@ -393,7 +394,7 @@ async def assert_clickhouse_records_in_s3(
async def run_s3_batch_export_workflow(
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
ateam,
batch_export_id,
s3_destination_config,
@@ -414,11 +415,11 @@ async def run_s3_batch_export_workflow(
`destination_type` selects which input dataclass is constructed and passed to the workflow,
so the test exercises the same per-destination inputs that production dispatches with.
"""
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
exclude_events = s3_destination_config.get("exclude_events", None)
file_format = s3_destination_config.get("file_format", "JSONLines")
@@ -482,7 +483,7 @@ async def run_s3_batch_export_workflow(
if expect_no_data:
assert run.records_completed == 0 or (
- isinstance(model, BatchExportModel)
+ isinstance(model, AnyBatchExportModel)
and model.name == "sessions"
and run.records_completed is not None
and run.records_completed <= 1
@@ -495,9 +496,9 @@ async def run_s3_batch_export_workflow(
assert run.bytes_exported > 0
sort_key = "uuid"
- if isinstance(model, BatchExportModel) and model.name == "persons":
+ if isinstance(model, BatchExportPersonsModel):
sort_key = "person_id"
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
+ elif isinstance(model, BatchExportSessionsModel):
sort_key = "session_id"
await assert_metrics_in_clickhouse(
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
index 6f49ed7f4253..8ce21cd57d9b 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
@@ -16,7 +16,14 @@ from django.test import override_settings
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (
SnowflakeInsertInputs,
insert_into_snowflake_activity_from_stage,
@@ -54,7 +61,7 @@ async def _run_activity(
data_interval_start,
data_interval_end,
table_name: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events=None,
sort_key: str = "event",
expected_fields=None,
@@ -130,7 +137,7 @@ async def test_insert_into_snowflake_activity_inserts_data_into_snowflake_table(
snowflake_cursor,
snowflake_activity_inputs,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -150,14 +157,14 @@ async def test_insert_into_snowflake_activity_inserts_data_into_snowflake_table(
that they appear in the expected Snowflake table. This function runs against a real Snowflake
instance, so the environment should be populated with the necessary credentials.
"""
- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:
pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
table_name = f"test_insert_activity_table_{ateam.pk}"
@@ -203,7 +210,7 @@ async def test_insert_into_snowflake_activity_merges_persons_data_in_follow_up_r
run of the activity. We expect the new entries to have replaced the old ones in Snowflake after
the second run.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_name = f"test_insert_activity_table_mutable_persons_{ateam.pk}"
await _run_activity(
@@ -273,7 +280,7 @@ async def test_insert_into_snowflake_activity_merges_sessions_data_in_follow_up_
run of the activity. We expect the new entries to have replaced the old ones in Snowflake after
the second run with the same time range.
"""
- model = BatchExportModel(name="sessions", schema=None)
+ model = BatchExportSessionsModel()
table_name = f"test_insert_activity_table_mutable_sessions_{ateam.pk}"
await _run_activity(
@@ -363,7 +370,7 @@ async def test_insert_into_snowflake_activity_removes_internal_stage_files(
checking this second run worked and exported data, we also check that no files
are present in the table's internal stage.
"""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
table_name = f"test_insert_activity_table_remove_{ateam.pk}"
min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)
batch_export_id = str(uuid.uuid4())
@@ -534,7 +541,7 @@ async def test_insert_into_snowflake_activity_handles_person_schema_changes(
To replicate this situation we first export the data with the original
schema, then delete a column in the destination and then rerun the export.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_name = f"test_insert_activity_migration_table_{ateam.pk}"
await _run_activity(
@@ -619,7 +626,7 @@ async def test_insert_into_snowflake_activity_from_stage_handles_datetime_to_int
To replicate this situation we first export the data to create the table, then alter
'created_at' to be of type INTEGER, and then rerun the export.
"""
- model = BatchExportModel(name="persons", schema=None)
+ model = BatchExportPersonsModel()
table_name = f"test_insert_activity_persons_table_{ateam.pk}"
await _run_activity(
@@ -657,9 +664,7 @@ async def test_insert_into_snowflake_activity_from_stage_handles_datetime_to_int
assert result.error is None
- "model", [BatchExportModel(name="events", schema=None), BatchExportModel(name="persons", schema=None)]
-)
[email protected]("model", [BatchExportEventsModel(), BatchExportPersonsModel()])
async def test_insert_into_snowflake_activity_handles_uppercased_columns(
clickhouse_client,
activity_environment,
@@ -726,7 +731,7 @@ async def test_insert_into_snowflake_activity_handles_uppercased_columns(
)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_insert_into_snowflake_activity_handles_extra_columns_in_destination(
clickhouse_client,
activity_environment,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py
index eb8e24dc6efe..9efc365907b0 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_workflow_e2e.py
@@ -20,7 +20,14 @@ from temporalio.worker import UnsandboxedWorkflowRunner, Worker
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (
SnowflakeBatchExportInputs,
@@ -51,7 +58,7 @@ async def _run_workflow(
data_interval_start,
data_interval_end,
interval: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events=None,
backfill_details=None,
settings_overrides=None,
@@ -142,7 +149,7 @@ async def test_snowflake_export_workflow(
snowflake_batch_export,
ateam,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -152,14 +159,14 @@ async def test_snowflake_export_workflow(
The workflow should update the batch export run status to completed and produce the expected
records to the provided Snowflake instance.
"""
- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:
pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)
@@ -178,7 +185,7 @@ async def test_snowflake_export_workflow(
@pytest.mark.parametrize("exclude_events", [None], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_snowflake_export_workflow_with_many_files(
clickhouse_client,
snowflake_cursor,
@@ -186,7 +193,7 @@ async def test_snowflake_export_workflow_with_many_files(
snowflake_batch_export,
ateam,
exclude_events,
- model: BatchExportModel | BatchExportSchema | None,
+ model: AnyBatchExportModel | BatchExportSchema | None,
generate_test_data,
data_interval_start,
data_interval_end,
@@ -198,14 +205,14 @@ async def test_snowflake_export_workflow_with_many_files(
means we are uploading one file at a time, which is very innefficient. For this reason, this test
can take longer, so we keep the event count low and bump the Workflow timeout.
"""
- if isinstance(model, BatchExportModel) and model.name != "events" and exclude_events is not None:
+ if isinstance(model, AnyBatchExportModel) and model.name != "events" and exclude_events is not None:
pytest.skip("Unnecessary test case as batch export model is not affected by 'exclude_events'")
- batch_export_model: BatchExportModel | None = None
- if isinstance(model, BatchExportModel):
+ batch_export_model: AnyBatchExportModel | None = None
+ if isinstance(model, AnyBatchExportModel):
batch_export_model = model
elif model is not None:
- batch_export_model = BatchExportModel(name="events", schema=model)
+ batch_export_model = BatchExportEventsCustomModel(schema=model)
min_ingested_timestamp = dt.datetime.now(dt.UTC).replace(tzinfo=None)
@@ -232,7 +239,7 @@ async def test_snowflake_export_workflow_with_many_files(
indirect=True,
)
@pytest.mark.parametrize("interval", ["hour"], indirect=True)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
async def test_snowflake_export_workflow_backfill_earliest_persons(
ateam,
clickhouse_client,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
index 64238a69aae2..1162ab9fb0fe 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
@@ -18,7 +18,15 @@ from requests.models import PreparedRequest
from posthog.temporal.common.clickhouse import ClickHouseClient
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSchema,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import snowflake_default_fields
from products.batch_exports.backend.temporal.queue import RecordBatchQueue
from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel
@@ -60,9 +68,8 @@ EXPECTED_PERSONS_BATCH_EXPORT_FIELDS = [
"is_deleted",
]
-TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
- BatchExportModel(
- name="events",
+TEST_MODELS: list[AnyBatchExportModel | BatchExportSchema | None] = [
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "event", "alias": "event"},
@@ -73,9 +80,9 @@ TEST_MODELS: list[BatchExportModel | BatchExportSchema | None] = [
"values": {"hogql_val_0": "$browser", "hogql_val_1": "$os"},
},
),
- BatchExportModel(name="events", schema=None),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportEventsModel(),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
{
"fields": [
{"expression": "event", "alias": "event"},
@@ -371,7 +378,7 @@ async def assert_clickhouse_records_in_snowflake(
data_interval_end: dt.datetime,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
- batch_export_model: BatchExportModel | BatchExportSchema | None = None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None = None,
backfill_details: BackfillDetails | None = None,
sort_key: str = "event",
expected_fields: list[str] | None = None,
@@ -428,13 +435,16 @@ async def assert_clickhouse_records_in_snowflake(
inserted_records.append(record)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
diff --git a/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py
index db6a5c140127..fb31557ae5f8 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/test_http_batch_export_workflow.py
@@ -22,7 +22,13 @@ from posthog.sync import database_sync_to_async
from posthog.temporal.common.clickhouse import ClickHouseClient
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel
+from products.batch_exports.backend.service import (
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportPersonsModel,
+ BatchExportSessionsModel,
+)
from products.batch_exports.backend.temporal.batch_exports import (
finish_batch_export_run,
iter_records,
@@ -552,29 +558,24 @@ async def test_http_export_workflow(
"batch_export_model",
[
None,
- BatchExportModel(name="events", schema=None),
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(),
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Mac OS X"]},
],
),
# this filter should filter out all test events
- BatchExportModel(
- name="events",
- schema=None,
+ BatchExportEventsModel(
filters=[
{"key": "$browser", "operator": "exact", "type": "event", "value": ["Chrome"]},
{"key": "$os", "operator": "exact", "type": "event", "value": ["Linux"]},
],
),
# None of the following are supported so we expect an error to be raised
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
- BatchExportModel(
- name="events",
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
+ BatchExportEventsCustomModel(
schema={
"fields": [
{"expression": "uuid", "alias": "uuid"},
@@ -645,7 +646,7 @@ async def test_http_export_workflow_with_model(
if batch_export_model is not None:
if batch_export_model.name != "events":
expect_failure = True
- if batch_export_model.schema is not None:
+ if isinstance(batch_export_model, BatchExportEventsCustomModel):
expect_failure = True
mock_server = MockServer()
@@ -692,7 +693,7 @@ async def test_http_export_workflow_with_model(
assert len(runs) == 1
run = runs[0]
assert run.status == "FailedRetryable"
- if batch_export_model is not None and batch_export_model.schema is not None:
+ if isinstance(batch_export_model, BatchExportEventsCustomModel):
assert run.latest_error == "NotImplementedError: HTTP export does not support schemas"
else:
assert run.latest_error == "NotImplementedError: HTTP export only supports the events model"
diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py
index ae74089987b0..b15b5fd83b5d 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_activity.py
@@ -4,7 +4,7 @@ import urllib.parse
import pytest
-from products.batch_exports.backend.service import BatchExportInsertInputs, BatchExportModel
+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportEventsModel, BatchExportInsertInputs
from products.batch_exports.backend.temporal.destinations import workflows_batch_export
from products.batch_exports.backend.temporal.destinations.workflows_batch_export import (
WorkflowsInsertInputs,
@@ -45,7 +45,7 @@ async def _run_activity(
data_interval_start,
data_interval_end,
sort_key: str,
- batch_export_model: BatchExportModel | None = None,
+ batch_export_model: AnyBatchExportModel | None = None,
exclude_events: list[str] | None = None,
include_events: list[str] | None = None,
expected_fields=None,
@@ -125,7 +125,7 @@ async def test_insert_into_workflows_activity_from_stage_posts_data_to_server(
This configures the activity to POST requests to an aiohttp test server, configured
in the server fixture.
"""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
await _run_activity(
activity_environment,
@@ -158,7 +158,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_on_non_retryable_
error,
):
"""Assert the activity immediately fails on non-retryable error codes."""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
if error == 404:
expected = "NotFoundErrorGroup"
@@ -220,7 +220,7 @@ async def test_insert_into_workflows_activity_from_stage_retries_on_retryable_er
error,
):
"""Assert the activity retries requests on retryable error codes."""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
await _run_activity(
activity_environment,
@@ -255,7 +255,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_with_empty_url(
error,
):
"""Assert activity fails when an empty URL is passed."""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
batch_export_id = str(uuid.uuid4())
batch_export_inputs = BatchExportInsertInputs(
@@ -306,7 +306,7 @@ async def test_insert_into_workflows_activity_from_stage_fails_when_hog_function
hog_function_id,
):
"""When every hog function execution errors, the activity should abort after the threshold trips."""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
batch_export_id = str(uuid.uuid4())
batch_export_inputs = BatchExportInsertInputs(
@@ -367,7 +367,7 @@ async def test_insert_into_workflows_activity_from_stage_tolerates_hog_function_
hog_function_error_count,
):
"""A small number of hog function errors (well under the threshold) should not fail the run."""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
result = await _run_activity(
activity_environment,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py
index 4c5f6fc18788..15de9281239e 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/test_workflow.py
@@ -10,7 +10,7 @@ from temporalio.common import RetryPolicy
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportEventsModel
from products.batch_exports.backend.temporal.batch_exports import finish_batch_export_run, start_batch_export_run
from products.batch_exports.backend.temporal.destinations.workflows_batch_export import (
WorkflowsBatchExportInputs,
@@ -81,7 +81,7 @@ async def test_workflows_export_workflow(
The workflow should update the batch export run status to completed and post the expected
records to a test server.
"""
- model = BatchExportModel(name="events", schema=None)
+ model = BatchExportEventsModel()
workflow_id = str(uuid.uuid4())
inputs = WorkflowsBatchExportInputs(
diff --git a/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py b/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py
index 5a637719e68b..89bc84bf098f 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/workflows/utils.py
@@ -5,7 +5,13 @@ import operator
from posthog.temporal.common.clickhouse import ClickHouseClient
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.workflows_batch_export import workflows_default_fields
from products.batch_exports.backend.temporal.queue import RecordBatchQueue
from products.batch_exports.backend.temporal.record_batch_model import SessionsRecordBatchModel
@@ -29,7 +35,7 @@ async def assert_clickhouse_records_were_handled(
hog_function_id,
team_id: int,
sort_key: str,
- batch_export_model: BatchExportModel | BatchExportSchema | None,
+ batch_export_model: AnyBatchExportModel | BatchExportSchema | None,
batch_export_id: str,
date_ranges: list[tuple[dt.datetime, dt.datetime]],
backfill_details: BackfillDetails | None = None,
@@ -46,13 +52,16 @@ async def assert_clickhouse_records_were_handled(
)
if batch_export_model is not None:
- if isinstance(batch_export_model, BatchExportModel):
+ if isinstance(batch_export_model, AnyBatchExportModel):
model_name = batch_export_model.name
- fields = batch_export_model.schema["fields"] if batch_export_model.schema is not None else None
- filters = batch_export_model.filters
- extra_query_parameters = (
- batch_export_model.schema["values"] if batch_export_model.schema is not None else None
+ schema = batch_export_model.schema if isinstance(batch_export_model, BatchExportEventsCustomModel) else None
+ fields = schema["fields"] if schema is not None else None
+ filters = (
+ batch_export_model.filters
+ if isinstance(batch_export_model, BatchExportEventsModel | BatchExportEventsCustomModel)
+ else None
)
+ extra_query_parameters = schema["values"] if schema is not None else None
else:
model_name = "events"
fields = batch_export_model["fields"]
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
index dcf1de684fb5..6e421ee0665a 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_entrypoint.py
@@ -19,7 +19,12 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.clickhouse import ClickHouseMemoryLimitExceededError
from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination
-from products.batch_exports.backend.service import BaseBatchExportInputs, BatchExportInsertInputs, BatchExportModel
+from products.batch_exports.backend.service import (
+ BaseBatchExportInputs,
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ BatchExportInsertInputs,
+)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
finish_batch_export_run,
@@ -242,7 +247,7 @@ class TestErrorHandling:
batch_export_id=str(batch_export.id),
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
exception_to_raise="DummyNonRetryableError",
)
run = await self._run_workflow(inputs, expect_workflow_failure=False)
@@ -259,7 +264,7 @@ class TestErrorHandling:
batch_export_id=str(batch_export.id),
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
exception_to_raise="DummyRetryableError",
)
run = await self._run_workflow(inputs, expect_workflow_failure=True)
@@ -281,8 +286,8 @@ class TestErrorHandling:
batch_export_id=str(batch_export.id),
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(
- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=1
+ batch_export_model=BatchExportHogQLModel(
+ query="SELECT event AS event FROM events", user_id=1, modifiers={}
),
)
with patch(
@@ -302,7 +307,7 @@ class TestErrorHandling:
batch_export_id=str(batch_export.id),
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
)
with patch(
"products.batch_exports.backend.temporal.pipeline.internal_stage._write_batch_export_record_batches_to_internal_stage",
@@ -320,7 +325,7 @@ class TestErrorHandling:
batch_export_id=str(batch_export_by_interval.id),
interval=interval,
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
)
run = await self._run_workflow(inputs, expect_workflow_failure=False)
assert run.status == "Completed"
@@ -333,7 +338,7 @@ class TestErrorHandling:
batch_export_id=str(batch_export.id),
interval="hour",
data_interval_end=dt.datetime(2025, 7, 21, 13, 0, 0, tzinfo=dt.UTC).isoformat(),
- batch_export_model=BatchExportModel(name="events", schema=None),
+ batch_export_model=BatchExportEventsModel(),
records_failed_count=5,
)
run = await self._run_workflow(inputs, expect_workflow_failure=False)
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
index 7a9ef257a51e..4e914cca6daa 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_internal_stage.py
@@ -35,7 +35,16 @@ from posthog.temporal.tests.utils.events import (
)
from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportDestination, BatchExportRun
-from products.batch_exports.backend.service import BackfillDetails, BatchExportModel, afetch_last_run_records_completed
+from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BackfillDetails,
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ BatchExportModel,
+ BatchExportPersonsModel,
+ BatchExportSessionsModel,
+ afetch_last_run_records_completed,
+)
from products.batch_exports.backend.temporal.pipeline.internal_stage import (
BatchExportInsertIntoInternalStageInputs,
DataIntervalInFutureError,
@@ -77,7 +86,7 @@ def mock_clickhouse_client():
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="events", schema=None),
+ BatchExportEventsModel(),
],
)
@pytest.mark.parametrize("is_backfill", [False, True])
@@ -90,7 +99,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_events
data_interval_start,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
is_backfill: bool,
backfill_within_last_6_days: bool,
use_native_schema: bool,
@@ -156,7 +165,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_events
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="sessions", schema=None),
+ BatchExportSessionsModel(),
],
)
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
@@ -167,7 +176,7 @@ async def test_insert_into_stage_activity_executes_the_expected_query_for_sessio
data_interval_start,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
):
"""Test that the insert_into_internal_stage_activity executes the expected ClickHouse query when the model is a sessions model."""
@@ -258,7 +267,7 @@ async def _run_activity(
data_interval_start: dt.datetime,
data_interval_end: dt.datetime,
exclude_events: list[str] | None = None,
- model: BatchExportModel | None = None,
+ model: AnyBatchExportModel | None = None,
batch_export_id: str | None = None,
) -> list[dict]:
"""Get rows from the internal stage."""
@@ -301,7 +310,7 @@ async def _run_activity(
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="events", schema=None),
+ BatchExportEventsModel(),
],
)
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
@@ -313,7 +322,7 @@ async def test_insert_into_stage_activity_for_events_model(
object_storage_client,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
exclude_events,
truncate_clickhouse_tables,
):
@@ -339,7 +348,7 @@ async def test_insert_into_stage_activity_for_events_model(
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="events", schema=None),
+ BatchExportEventsModel(),
],
)
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
@@ -351,7 +360,7 @@ async def test_insert_into_stage_activity_reports_records_total_for_events_model
object_storage_client,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
truncate_clickhouse_tables,
):
"""The activity reports the number of rows written to the stage, from ClickHouse's query summary."""
@@ -673,7 +682,7 @@ def assert_exported_rows_match_persons_to_export(exported_rows: list[dict], pers
@pytest.mark.parametrize("interval", ["day"], indirect=True)
[email protected]("model", [BatchExportModel(name="persons", schema=None)])
[email protected]("model", [BatchExportPersonsModel()])
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
@pytest.mark.parametrize("limited_export", [False, True])
async def test_insert_into_stage_activity_for_persons_model(
@@ -685,7 +694,7 @@ async def test_insert_into_stage_activity_for_persons_model(
ateam,
test_person_properties,
clickhouse_client,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
limited_export: bool,
truncate_clickhouse_tables,
):
@@ -1101,7 +1110,7 @@ async def test_afetch_last_run_records_completed_ignores_runs_older_than(ateam):
@pytest.mark.parametrize("interval", ["day"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
async def test_insert_into_stage_activity_writes_dynamic_number_of_files(
generate_test_data,
@@ -1111,7 +1120,7 @@ async def test_insert_into_stage_activity_writes_dynamic_number_of_files(
object_storage_client,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
truncate_clickhouse_tables,
):
"""A previous run's row count drives how many staging files the activity writes."""
@@ -1160,7 +1169,7 @@ async def test_insert_into_stage_activity_writes_dynamic_number_of_files(
@pytest.mark.parametrize("interval", ["day"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
@pytest.mark.parametrize("data_interval_end", [TEST_DATA_INTERVAL_END])
async def test_insert_into_stage_activity_uses_static_default_without_previous_run(
generate_test_data,
@@ -1170,7 +1179,7 @@ async def test_insert_into_stage_activity_uses_static_default_without_previous_r
object_storage_client,
data_interval_end,
ateam,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
truncate_clickhouse_tables,
):
"""With no previous run to estimate from, the activity writes the static-default number of files."""
@@ -1260,9 +1269,9 @@ async def _fetch_staging_query_settings(
@pytest.mark.parametrize(
"model",
[
- BatchExportModel(name="events", schema=None),
- BatchExportModel(name="persons", schema=None),
- BatchExportModel(name="sessions", schema=None),
+ BatchExportEventsModel(),
+ BatchExportPersonsModel(),
+ BatchExportSessionsModel(),
],
ids=["events", "persons", "sessions"],
)
@@ -1273,7 +1282,7 @@ async def test_insert_into_stage_activity_applies_settings_and_log_comment(
clickhouse_client,
data_interval_start,
data_interval_end,
- model: BatchExportModel,
+ model: AnyBatchExportModel,
):
"""Every model's settings and log comment reach the query ClickHouse runs.
@@ -1480,7 +1489,7 @@ class TestHogQLModel:
team_id=ateam.pk,
data_interval_start=data_interval_start,
data_interval_end=data_interval_end,
- model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=auser.pk),
+ model=BatchExportHogQLModel(query=hogql_query, user_id=auser.pk, modifiers={}),
)
assert all(list(row.keys()) == expected_columns for row in exported_rows)
@@ -1521,14 +1530,13 @@ class TestHogQLModel:
team_id=ateam.pk,
data_interval_start=bounded_start,
data_interval_end=data_interval_end,
- model=BatchExportModel(
- name="hogql",
- schema=None,
- hogql_query=(
+ model=BatchExportHogQLModel(
+ query=(
"SELECT uuid AS uuid, timestamp AS timestamp FROM events "
"WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}"
),
user_id=auser.pk,
+ modifiers={},
),
)
@@ -1572,11 +1580,10 @@ class TestHogQLModel:
team_id=ateam.pk,
data_interval_start=data_interval_start,
data_interval_end=data_interval_end,
- model=BatchExportModel(
- name="hogql",
- schema=None,
- hogql_query="SELECT event_id, event, distinct_id FROM events_view",
+ model=BatchExportHogQLModel(
+ query="SELECT event_id, event, distinct_id FROM events_view",
user_id=auser.pk,
+ modifiers={},
),
)
@@ -1592,9 +1599,7 @@ class TestHogQLModel:
team_id=ateam.pk,
data_interval_start=data_interval_start,
data_interval_end=data_interval_end,
- model=BatchExportModel(
- name="hogql", schema=None, hogql_query="SELECT * FROM events_view", user_id=auser.pk
- ),
+ model=BatchExportHogQLModel(query="SELECT * FROM events_view", user_id=auser.pk, modifiers={}),
)
assert sorted((row["event_id"], row["event"], row["distinct_id"]) for row in exported_rows_2) == sorted(
(e["uuid"], e["event"], e["distinct_id"]) for e in events
@@ -1633,9 +1638,7 @@ class TestHogQLModel:
team_id=ateam.pk,
data_interval_start=data_interval_start,
data_interval_end=data_interval_end,
- model=BatchExportModel(
- name="hogql", schema=None, hogql_query="SELECT event AS event FROM events", user_id=auser.pk
- ),
+ model=BatchExportHogQLModel(query="SELECT event AS event FROM events", user_id=auser.pk, modifiers={}),
batch_export_id=batch_export_id,
)
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py b/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py
index 81e6800e1d81..6748253daac0 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_producer.py
@@ -20,7 +20,7 @@ import pyarrow as pa
from posthog.temporal.common import asyncpa
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportEventsModel
from products.batch_exports.backend.temporal.pipeline import producer as producer_module
from products.batch_exports.backend.temporal.pipeline.internal_stage import (
BatchExportInsertIntoInternalStageInputs,
@@ -304,7 +304,7 @@ async def _drain_stage_via_producer(
@pytest.mark.django_db
@pytest.mark.parametrize("interval", ["day"], indirect=True)
[email protected]("model", [BatchExportModel(name="events", schema=None)])
[email protected]("model", [BatchExportEventsModel()])
async def test_producer_resumes_from_offset_after_mid_file_failure(
interval,
activity_environment,
@@ -313,7 +313,7 @@ async def test_producer_resumes_from_offset_after_mid_file_failure(
object_storage_client,
ateam,
clickhouse_client,
- model: BatchExportModel,
+ model: BatchExportEventsModel,
):
"""A mid-file read failure resumes from the last batch boundary instead of re-delivering rows."""
diff --git a/products/batch_exports/backend/tests/test_file_download_timeouts.py b/products/batch_exports/backend/tests/test_file_download_timeouts.py
index 9cf6f3d7465b..f6061935788c 100644
--- a/products/batch_exports/backend/tests/test_file_download_timeouts.py
+++ b/products/batch_exports/backend/tests/test_file_download_timeouts.py
@@ -12,7 +12,11 @@ from temporalio.common import MetricMeter
from products.batch_exports.backend import service
from products.batch_exports.backend.models.batch_export import BatchExportOnDemand
-from products.batch_exports.backend.service import BatchExportModel, FileDownloadBatchExportInputs
+from products.batch_exports.backend.service import (
+ BatchExportEventsModel,
+ BatchExportHogQLModel,
+ FileDownloadBatchExportInputs,
+)
from products.batch_exports.backend.temporal.destinations.file_download_batch_export import (
FileDownloadBatchExportWorkflow,
export_to_file_download_bucket_with_temporary_credentials,
@@ -44,7 +48,7 @@ class TestFileDownloadTimeouts(SimpleTestCase):
self, _name: str, start: dt.datetime | None, end: dt.datetime | None, main_seconds: int, query_seconds: int
) -> None:
batch_export = BatchExportOnDemand(id=EXPORT_ID, team_id=1)
- model = BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value")
+ model = BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})
client = AsyncMock(spec=Client)
with (
@@ -96,7 +100,9 @@ class TestFileDownloadTimeouts(SimpleTestCase):
"file-download-test",
start,
end,
- BatchExportModel(name=model_name, schema=None),
+ BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})
+ if model_name == "hogql"
+ else BatchExportEventsModel(),
batch_export_run_id=run_id,
)
connect.assert_not_called()
@@ -161,7 +167,7 @@ class TestFileDownloadTimeouts(SimpleTestCase):
batch_export_id=str(EXPORT_ID),
team_id=1,
batch_export_run_id=RUN_ID,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value"),
+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),
data_interval_start=start,
data_interval_end=end,
main_activity_timeout_seconds=28800,
diff --git a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
index f8f311fffbb4..92b9be0c70c8 100644
--- a/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
+++ b/products/batch_exports/backend/tests/test_hogql_interval_placeholders.py
@@ -13,7 +13,7 @@ from products.batch_exports.backend.hogql_source import (
parse_hogql_select_for_batch_export,
replace_interval_placeholders,
)
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportHogQLModel, BatchExportModel
from products.batch_exports.backend.temporal.pipeline.internal_stage import (
BatchExportInsertIntoInternalStageInputs,
insert_into_internal_stage_activity,
@@ -84,7 +84,11 @@ async def test_invalid_hogql_returns_non_retryable_staging_error(hogql_query: st
batch_export_id="00000000-0000-4000-8000-000000000001",
data_interval_start=None,
data_interval_end=(dt.datetime.now(dt.UTC) - dt.timedelta(hours=1)).isoformat(),
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),
+ batch_export_model=(
+ BatchExportHogQLModel(query=hogql_query, user_id=1, modifiers={})
+ if hogql_query is not None
+ else BatchExportModel(name="hogql", schema=None, user_id=1)
+ ),
)
clickhouse = MockClickHouseClient()
diff --git a/products/batch_exports/backend/tests/test_nullable_staging.py b/products/batch_exports/backend/tests/test_nullable_staging.py
index aee5d71c9ba7..9ea205b392e5 100644
--- a/products/batch_exports/backend/tests/test_nullable_staging.py
+++ b/products/batch_exports/backend/tests/test_nullable_staging.py
@@ -14,7 +14,7 @@ from posthog.hogql.database.database import Database
from posthog.models import Team
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportEventsModel, BatchExportHogQLModel, BatchExportModel
from products.batch_exports.backend.temporal.errors import MissingRequiredInputsError
from products.batch_exports.backend.temporal.pipeline import internal_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import STAGE_NON_RETRYABLE_ERROR_TYPES
@@ -76,7 +76,7 @@ async def test_hogql_staging_preserves_optional_bounds_and_cleanup_namespace(
data_interval_start=start_str,
data_interval_end=end_str,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),
+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),
)
result = await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
@@ -110,7 +110,11 @@ async def test_unsupported_hogql_returns_nonretryable_staging_error(
data_interval_start=None,
data_interval_end=None,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query=hogql_query, user_id=1),
+ batch_export_model=(
+ BatchExportHogQLModel(query=hogql_query, user_id=1, modifiers={})
+ if hogql_query is not None
+ else BatchExportModel(name="hogql", schema=None, user_id=1)
+ ),
)
if hogql_query is None:
@@ -145,7 +149,7 @@ async def test_hogql_staging_rejects_each_future_bound_without_waiting(
data_interval_start=start.isoformat() if start is not None else None,
data_interval_end=end.isoformat() if end is not None else None,
on_demand=True,
- batch_export_model=BatchExportModel(name="hogql", schema=None, hogql_query="SELECT 1 AS value", user_id=1),
+ batch_export_model=BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={}),
)
# Patch the helper rather than override settings.TEST, because DatabaseSyncToAsync calls
# close_old_connections() whenever settings.TEST is False, and that needs a database.
@@ -182,7 +186,11 @@ async def test_missing_staging_inputs_raise(
data_interval_start=None,
data_interval_end=None,
on_demand=on_demand,
- batch_export_model=BatchExportModel(name=model_name, schema=None, hogql_query="SELECT 1 AS value", user_id=1),
+ batch_export_model=(
+ BatchExportHogQLModel(query="SELECT 1 AS value", user_id=1, modifiers={})
+ if model_name == "hogql"
+ else BatchExportEventsModel()
+ ),
)
with pytest.raises(MissingRequiredInputsError, match="require"):
await ActivityEnvironment().run(internal_stage.insert_into_internal_stage_activity, inputs)
diff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.py
index dd19499ecd32..1cf586192197 100644
--- a/products/batch_exports/backend/tests/test_service.py
+++ b/products/batch_exports/backend/tests/test_service.py
@@ -319,9 +319,7 @@ class TestTypeCoercionInBatchExportInputs:
table_name="events",
use_variant_type=True,
use_automatic_schema_evolution=False,
- batch_export_model=BatchExportModel(
- name="events",
- schema=None,
+ batch_export_model=BatchExportEventsModel(
filters=[{"type": "hogql", "key": "event = 'example_event'", "value": filter_value}],
),
)
@@ -332,8 +330,7 @@ class TestTypeCoercionInBatchExportInputs:
assert inputs.use_variant_type is True
assert inputs.use_automatic_schema_evolution is False
- assert inputs.batch_export_model is not None
- assert inputs.batch_export_model.filters is not None
+ assert isinstance(inputs.batch_export_model, BatchExportEventsModel)
assert inputs.batch_export_model.filters[0]["value"] is filter_value
def test_optional_int_none_is_preserved(self):