:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(consumers): Accept model-specific export inputs (#120240)
Tomás Farías Santanab195d950ac35
diff --git a/products/batch_exports/backend/service.py b/products/batch_exports/backend/service.pyindex b38a2a1b5124..fab473883e2f 100644--- a/products/batch_exports/backend/service.py+++ b/products/batch_exports/backend/service.py@@ -359,7 +359,7 @@ class BaseBatchExportInputs:is_backfill: bool = Falseis_earliest_backfill: bool = Falsebackfill_details: BackfillDetails | None = None- batch_export_model: BatchExportModel | None = None+ batch_export_model: BatchExportModelInput | None = Noneintegration_id: int | None = Nonedef __post_init__(self) -> None:@@ -1121,7 +1121,7 @@ def start_file_download_batch_export(workflow_id: str,data_interval_start: dt.datetime | None,data_interval_end: dt.datetime | None,- batch_export_model: BatchExportModel,+ batch_export_model: BatchExportModelInput,batch_export_run_id: UUID | None = None,compression: str | None = None,format: str = "Parquet",@@ -1576,7 +1576,7 @@ class BatchExportInsertInputs:include_events: list[str] | None = Nonerun_id: str | None = Nonebackfill_details: BackfillDetails | None = None- batch_export_model: BatchExportModel | None = None+ batch_export_model: BatchExportModelInput | None = None# TODO: Remove after updating existing batch exports to use backfill_detailsis_backfill: bool = False# TODO - pass these in to all inherited classesdiff --git a/products/batch_exports/backend/temporal/pipeline/entrypoint.py b/products/batch_exports/backend/temporal/pipeline/entrypoint.pyindex 9256c0dee391..fa2463504ddf 100644--- a/products/batch_exports/backend/temporal/pipeline/entrypoint.py+++ b/products/batch_exports/backend/temporal/pipeline/entrypoint.py@@ -13,7 +13,7 @@ from posthog.temporal.common.logger import get_write_only_loggerfrom products.batch_exports.backend.facade.enums import BatchExportRunStatusfrom products.batch_exports.backend.models.batch_export import BatchExportRun-from products.batch_exports.backend.service import BackfillDetails, BatchExportField, BatchExportModel+from products.batch_exports.backend.service import BackfillDetails, BatchExportField, BatchExportModelInputfrom products.batch_exports.backend.temporal.batch_exports import FinishBatchExportRunInputs, finish_batch_export_runfrom products.batch_exports.backend.temporal.metrics import get_export_finished_metric, get_export_started_metricfrom products.batch_exports.backend.temporal.pipeline.internal_stage import (@@ -44,7 +44,7 @@ class _BatchExportInputsProtocol(typing.Protocol):include_events: list[str] | None = Nonerun_id: str | None = Nonebackfill_details: BackfillDetails | None = None- batch_export_model: BatchExportModel | None = None+ batch_export_model: BatchExportModelInput | None = Noneis_backfill: bool = Falsebatch_export_id: str | None = Nonedestination_default_fields: list[BatchExportField] | None = Nonediff --git a/products/batch_exports/backend/temporal/pipeline/internal_stage.py b/products/batch_exports/backend/temporal/pipeline/internal_stage.pyindex 43932115d0d9..651abb2cf84b 100644--- a/products/batch_exports/backend/temporal/pipeline/internal_stage.py+++ b/products/batch_exports/backend/temporal/pipeline/internal_stage.py@@ -50,7 +50,7 @@ from products.batch_exports.backend.models.batch_export import BatchExportfrom products.batch_exports.backend.service import (BackfillDetails,BatchExportField,- BatchExportModel,+ BatchExportModelInput,afetch_last_run_records_completed,migrate_batch_export_model,)@@ -293,7 +293,7 @@ class BatchExportInsertIntoInternalStageInputs:include_events: list[str] | None = Nonerun_id: str | None = Nonebackfill_details: BackfillDetails | None = None- batch_export_model: BatchExportModel | None = None+ batch_export_model: BatchExportModelInput | None = Noneis_workflows: bool = Falseon_demand: bool = Falsedestination_default_fields: list[BatchExportField] | None = Nonediff --git a/products/batch_exports/backend/tests/test_service.py b/products/batch_exports/backend/tests/test_service.pyindex a90e73113c87..bdafded89fe6 100644--- a/products/batch_exports/backend/tests/test_service.py+++ b/products/batch_exports/backend/tests/test_service.py@@ -19,11 +19,13 @@ from products.batch_exports.backend.service import (BatchExportEventsCustomModel,BatchExportEventsModel,BatchExportHogQLModel,+ BatchExportInsertInputs,BatchExportModel,BatchExportPersonsModel,BatchExportSessionsModel,BigQueryBatchExportInputs,DatabricksBatchExportInputs,+ FileDownloadBatchExportInputs,PostgresBatchExportInputs,RedshiftBatchExportInputs,RedshiftCopyInputs,@@ -33,9 +35,14 @@ from products.batch_exports.backend.service import (align_timestamp_to_interval,migrate_batch_export_model,)+from products.batch_exports.backend.temporal.pipeline.internal_stage import BatchExportInsertIntoInternalStageInputsclass TestBatchExportModelMigration:+ @pytest.mark.parametrize("model_specific", [False, True])+ @pytest.mark.parametrize(+ "input_type", [FileDownloadBatchExportInputs, BatchExportInsertInputs, BatchExportInsertIntoInternalStageInputs]+ )@pytest.mark.parametrize("legacy,expected",[@@ -72,14 +79,26 @@ class TestBatchExportModelMigration:],)def test_migration_preserves_legacy_payload(- self, legacy: BatchExportModel | None, expected: AnyBatchExportModel+ self,+ legacy: BatchExportModel | None,+ expected: AnyBatchExportModel,+ input_type: type[+ FileDownloadBatchExportInputs | BatchExportInsertInputs | BatchExportInsertIntoInternalStageInputs+ ],+ model_specific: bool,) -> None:converter = JSONPlainPayloadConverter()- if legacy is not None:- payload = converter.to_payload(legacy)- assert payload is not None- legacy = converter.from_payload(payload, BatchExportModel)- migrated = migrate_batch_export_model(legacy)+ inputs = input_type(+ team_id=1,+ batch_export_id="test",+ data_interval_start=None,+ data_interval_end=None,+ batch_export_model=expected if model_specific else legacy,+ )+ payload = converter.to_payload(inputs)+ assert payload is not None+ decoded = converter.from_payload(payload, input_type)+ migrated = migrate_batch_export_model(decoded.batch_export_model)assert migrated == expectedassert migrate_batch_export_model(migrated) is migrated