Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(consumers): Accept model-specific export inputs (#120240)

Tomás Farías Santanab195d950ac35
diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.py
index b38a2a1b5124..fab473883e2f 100644
--- a/products/batch_exports/backend/service.py
+++ b/products/batch_exports/backend/service.py
@@ -359,7 +359,7 @@ class BaseBatchExportInputs:
is_backfill: bool = False
is_earliest_backfill: bool = False
backfill_details: BackfillDetails | None = None
- batch_export_model: BatchExportModel | None = None
+ batch_export_model: BatchExportModelInput | None = None
integration_id: int | None = None
def __post_init__(self) -> None:
@@ -1121,7 +1121,7 @@ def start_file_download_batch_export(
workflow_id: str,
data_interval_start: dt.datetime | None,
data_interval_end: dt.datetime | None,
- batch_export_model: BatchExportModel,
+ batch_export_model: BatchExportModelInput,
batch_export_run_id: UUID | None = None,
compression: str | None = None,
format: str = "Parquet",
@@ -1576,7 +1576,7 @@ class BatchExportInsertInputs:
include_events: list[str] | None = None
run_id: str | None = None
backfill_details: BackfillDetails | None = None
- batch_export_model: BatchExportModel | None = None
+ batch_export_model: BatchExportModelInput | None = None
# TODO: Remove after updating existing batch exports to use backfill_details
is_backfill: bool = False
# TODO - pass these in to all inherited classes
diff --git a/products/batch_exports/backend/temporal/pipeline/entrypoint.py b/products/batch_exports/backend/temporal/pipeline/entrypoint.py
index 9256c0dee391..fa2463504ddf 100644
--- a/products/batch_exports/backend/temporal/pipeline/entrypoint.py
+++ b/products/batch_exports/backend/temporal/pipeline/entrypoint.py
@@ -13,7 +13,7 @@ from posthog.temporal.common.logger import get_write_only_logger
from products.batch_exports.backend.facade.enums import BatchExportRunStatus
from products.batch_exports.backend.models.batch_export import BatchExportRun
-from products.batch_exports.backend.service import BackfillDetails, BatchExportField, BatchExportModel
+from products.batch_exports.backend.service import BackfillDetails, BatchExportField, BatchExportModelInput
from products.batch_exports.backend.temporal.batch_exports import FinishBatchExportRunInputs, finish_batch_export_run
from products.batch_exports.backend.temporal.metrics import get_export_finished_metric, get_export_started_metric
from products.batch_exports.backend.temporal.pipeline.internal_stage import (
@@ -44,7 +44,7 @@ class _BatchExportInputsProtocol(typing.Protocol):
include_events: list[str] | None = None
run_id: str | None = None
backfill_details: BackfillDetails | None = None
- batch_export_model: BatchExportModel | None = None
+ batch_export_model: BatchExportModelInput | None = None
is_backfill: bool = False
batch_export_id: str | None = None
destination_default_fields: list[BatchExportField] | None = None
diff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
index 43932115d0d9..651abb2cf84b 100644
--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py
+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py
@@ -50,7 +50,7 @@ from products.batch_exports.backend.models.batch_export import BatchExport
from products.batch_exports.backend.service import (
BackfillDetails,
BatchExportField,
- BatchExportModel,
+ BatchExportModelInput,
afetch_last_run_records_completed,
migrate_batch_export_model,
)
@@ -293,7 +293,7 @@ class BatchExportInsertIntoInternalStageInputs:
include_events: list[str] | None = None
run_id: str | None = None
backfill_details: BackfillDetails | None = None
- batch_export_model: BatchExportModel | None = None
+ batch_export_model: BatchExportModelInput | None = None
is_workflows: bool = False
on_demand: bool = False
destination_default_fields: list[BatchExportField] | None = None
diff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.py
index a90e73113c87..bdafded89fe6 100644
--- a/products/batch_exports/backend/tests/test_service.py
+++ b/products/batch_exports/backend/tests/test_service.py
@@ -19,11 +19,13 @@ from products.batch_exports.backend.service import (
BatchExportEventsCustomModel,
BatchExportEventsModel,
BatchExportHogQLModel,
+ BatchExportInsertInputs,
BatchExportModel,
BatchExportPersonsModel,
BatchExportSessionsModel,
BigQueryBatchExportInputs,
DatabricksBatchExportInputs,
+ FileDownloadBatchExportInputs,
PostgresBatchExportInputs,
RedshiftBatchExportInputs,
RedshiftCopyInputs,
@@ -33,9 +35,14 @@ from products.batch_exports.backend.service import (
align_timestamp_to_interval,
migrate_batch_export_model,
)
+from products.batch_exports.backend.temporal.pipeline.internal_stage import BatchExportInsertIntoInternalStageInputs
class TestBatchExportModelMigration:
+ @pytest.mark.parametrize("model_specific", [False, True])
+ @pytest.mark.parametrize(
+ "input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]
+ )
@pytest.mark.parametrize(
"legacy,expected",
[
@@ -72,14 +79,26 @@ class TestBatchExportModelMigration:
],
)
def test_migration_preserves_legacy_payload(
- self, legacy: BatchExportModel | None, expected: AnyBatchExportModel
+ self,
+ legacy: BatchExportModel | None,
+ expected: AnyBatchExportModel,
+ input_type: type[
+ FileDownloadBatchExportInputs | BatchExportInsertInputs | BatchExportInsertIntoInternalStageInputs
+ ],
+ model_specific: bool,
) -> None:
converter = JSONPlainPayloadConverter()
- if legacy is not None:
- payload = converter.to_payload(legacy)
- assert payload is not None
- legacy = converter.from_payload(payload, BatchExportModel)
- migrated = migrate_batch_export_model(legacy)
+ inputs = input_type(
+ team_id=1,
+ batch_export_id="test",
+ data_interval_start=None,
+ data_interval_end=None,
+ batch_export_model=expected if model_specific else legacy,
+ )
+ payload = converter.to_payload(inputs)
+ assert payload is not None
+ decoded = converter.from_payload(payload, input_type)
+ migrated = migrate_batch_export_model(decoded.batch_export_model)
assert migrated == expected
assert migrate_batch_export_model(migrated) is migrated