:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(destinations): Adopt model-specific export types (#117000)
Tomás Farías Santana05051783bce6
diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.pyindex df2cdd6949e0..22db2333d3d8 100644--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py@@ -22,7 +22,8 @@ from products.batch_exports.backend.service import (AzureBlobBatchExportInputs,BatchExportField,BatchExportInsertInputs,- BatchExportModel,+ BatchExportModelInput,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -142,20 +143,21 @@ class AzureBlobConsumer(Consumer):prefix: str,data_interval_start: str | None,data_interval_end: str,- batch_export_model: BatchExportModel | None,+ batch_export_model: BatchExportModelInput | None,file_format: str,compression: str | None = None,max_file_size_mb: int | None = None,max_concurrency: int = 5,legacy_parquet_extension: bool = True,):- super().__init__(model=batch_export_model.name if batch_export_model else "events")+ model = migrate_batch_export_model(batch_export_model)+ super().__init__(model=model.name)self.container_client = container_clientself.prefix = prefixself.data_interval_start = data_interval_startself.data_interval_end = data_interval_end- self.batch_export_model = batch_export_model+ self.batch_export_model = modelself.file_format = file_formatself.compression = compressionself.max_file_size_mb = max_file_size_mbdiff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.pyindex 7e4c671f30f6..29ce56be5b20 100644--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py@@ -46,11 +46,12 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.service import (BatchExportField,BatchExportInsertInputs,- BatchExportModel,BigQueryBatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -595,6 +596,21 @@ def _is_contention_exception(exc: Exception) -> bool:return "concurrent update" in str(exc)+def _quote_bigquery_identifier(identifier: str) -> str:+ escaped = identifier.translate(+ {+ ord("\\"): r"\\",+ ord("`"): r"\`",+ ord("\n"): r"\n",+ ord("\r"): r"\r",+ ord("\t"): r"\t",+ ord("\b"): r"\b",+ ord("\f"): r"\f",+ }+ )+ return f"`{escaped}`"++class BigQueryClient:"""Async client to interact with BigQuery.@@ -791,7 +807,7 @@ class BigQueryClient:"""Attempt to SELECT from table to check for query permissions."""if isinstance(table, BigQueryTable) and "timestamp" in table:query = f"""- SELECT 1 FROM `{table.fully_qualified_name}` TABLESAMPLE SYSTEM (0.0001 PERCENT) WHERE timestamp IS NOT NULL+ SELECT 1 FROM {_quote_bigquery_identifier(table.fully_qualified_name)} TABLESAMPLE SYSTEM (0.0001 PERCENT) WHERE timestamp IS NOT NULL"""if table.time_partitioning is not None and table.time_partitioning.field == "timestamp":@@ -802,7 +818,7 @@ class BigQueryClient:else:query = f"""- SELECT 1 FROM `{table.fully_qualified_name}` TABLESAMPLE SYSTEM (0.0001 PERCENT) LIMIT 1+ SELECT 1 FROM {_quote_bigquery_identifier(table.fully_qualified_name)} TABLESAMPLE SYSTEM (0.0001 PERCENT) LIMIT 1"""try:@@ -875,7 +891,7 @@ class BigQueryClient:stage: list[BigQueryTable],):"""Insert data from one or more `stage` tables into `final`."""- into_table_fields = ",".join(f"`{field.name}`" for field in final.fields)+ into_table_fields = ",".join(_quote_bigquery_identifier(field.name) for field in final.fields)fields_to_cast = {field.name@@ -894,7 +910,7 @@ class BigQueryClient:REGEXP_REPLACE(REGEXP_REPLACE(REGEXP_REPLACE(- `{field.name}`,+ {_quote_bigquery_identifier(field.name)},r'(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}}\\\\u[dD][c-fC-F][0-9A-Fa-f]{{2}})|(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}})','\\\\1'),@@ -908,16 +924,17 @@ class BigQueryClient:)"""if field.name in fields_to_cast- else f"`{field.name}`"+ else _quote_bigquery_identifier(field.name)for field in final.fields)query = f"""- INSERT INTO `{final.fully_qualified_name}`+ INSERT INTO {_quote_bigquery_identifier(final.fully_qualified_name)}({into_table_fields})"""query += "\nUNION ALL\n".join(- f"SELECT {stage_table_fields} FROM `{table.fully_qualified_name}`" for table in stage+ f"SELECT {stage_table_fields} FROM {_quote_bigquery_identifier(table.fully_qualified_name)}"+ for table in stage)self.logger.info(@@ -954,15 +971,18 @@ class BigQueryClient:for n, field_name in enumerate(final.primary_key):if n > 0:merge_condition += " AND "- merge_condition += f"final.`{field_name}` = stage.`{field_name}`"+ quoted_field = _quote_bigquery_identifier(field_name)+ merge_condition += f"final.{quoted_field} = stage.{quoted_field}"- update_condition = "AND ("+ update_condition = ""for index, field_name in enumerate(final.version_key):if index > 0:update_condition += " OR "- update_condition += f"final.`{field_name}` < stage.`{field_name}`"- update_condition += ")"+ quoted_field = _quote_bigquery_identifier(field_name)+ update_condition += f"final.{quoted_field} < stage.{quoted_field}"+ if update_condition:+ update_condition = f"AND ({update_condition})"update_clause = ""values = ""@@ -985,7 +1005,7 @@ class BigQueryClient:REGEXP_REPLACE(REGEXP_REPLACE(REGEXP_REPLACE(- stage.`{field.name}`,+ stage.{_quote_bigquery_identifier(field.name)},r'(\\\\u[dD][89A-Ba-b][0-9A-Fa-f]{{2}}\\\\u[dD][c-fC-F][0-9A-Fa-f]{{2}})|(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}})','\\\\1'),@@ -999,24 +1019,31 @@ class BigQueryClient:)"""if field.name in fields_to_cast- else f"stage.`{field.name}`"+ else f"stage.{_quote_bigquery_identifier(field.name)}")- update_clause += f"final.`{field.name}` = {stage_field}"- field_names += f"`{field.name}`"+ update_clause += f"final.{_quote_bigquery_identifier(field.name)} = {stage_field}"+ field_names += _quote_bigquery_identifier(field.name)values += stage_fieldif not update_clause:raise ValueError("Empty update clause")- inner_union_query = "\nUNION ALL\n".join(f"SELECT * FROM `{table.fully_qualified_name}`" for table in stage)+ inner_union_query = "\nUNION ALL\n".join(+ f"SELECT * FROM {_quote_bigquery_identifier(table.fully_qualified_name)}" for table in stage+ )+ order_by = (+ "ORDER BY " + ",".join(f"{_quote_bigquery_identifier(field_name)} DESC" for field_name in final.version_key)+ if final.version_key+ else ""+ )union_query = f"""SELECT*,ROW_NUMBER() OVER (- PARTITION BY {",".join(field_name for field_name in final.primary_key)}- ORDER BY {",".join(f"{field_name} DESC" for field_name in final.version_key)}+ PARTITION BY {",".join(_quote_bigquery_identifier(field_name) for field_name in final.primary_key)}+ {order_by}) row_numFROM ({inner_union_query}@@ -1024,7 +1051,7 @@ class BigQueryClient:"""merge_query = f"""- MERGE `{final.fully_qualified_name}` final+ MERGE {_quote_bigquery_identifier(final.fully_qualified_name)} finalUSING (SELECT * FROM(@@ -1427,31 +1454,6 @@ async def merge_all_consumer_tables()-class MergeSettings(typing.NamedTuple):- primary_key: collections.abc.Sequence[str]- version_key: collections.abc.Sequence[str]---def _get_merge_settings(- model: BatchExportModel | None,-) -> MergeSettings | None:- """Return merge settings for models that require merging."""- if isinstance(model, BatchExportModel):- if model.name == "persons":- primary_key: collections.abc.Sequence[str] = ("team_id", "distinct_id")- version_key: collections.abc.Sequence[str] = ("person_version", "person_distinct_id_version")- elif model.name == "sessions":- primary_key = ("team_id", "session_id")- version_key = ("end_timestamp",)- # TODO: Support merges in 'events'?- else:- return None- else:- return None-- return MergeSettings(primary_key, version_key)--@dataclasses.dataclass(frozen=False, kw_only=True)class BigQueryInsertInputs(BatchExportInsertInputs):"""Inputs for BigQuery."""@@ -1528,7 +1530,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs))async with Heartbeater():- model = inputs.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -1569,14 +1571,14 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)else:json_fields = set()- merge_settings = _get_merge_settings(model)+ requires_merge = model.incremental_mode == IncrementalMode.MERGEtarget_table = BigQueryTable.from_arrow_schema(record_batch_schema,table_id=inputs.table_id,project_id=project_id,dataset_id=inputs.dataset_id,- primary_key=merge_settings.primary_key if merge_settings else (),- version_key=merge_settings.version_key if merge_settings else (),+ primary_key=model.primary_key if requires_merge else (),+ version_key=model.version_key if requires_merge else (),)data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")@@ -1697,7 +1699,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)run_consumer(client=bq_client,consumer_table=consumer_table,- model=model.name if isinstance(model, BatchExportModel) else "events",+ model=model.name,file_format=file_format,queue=queue,transformer=transformer,diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex fb4f6ae54587..3f77faefad6c 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -29,11 +29,14 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsModel,BatchExportField,BatchExportInsertInputs,- BatchExportModel,DatabricksBatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -223,6 +226,10 @@ class DatabricksInsertInputs(BatchExportInsertInputs):use_automatic_schema_evolution: bool = True+def _quote_databricks_identifier(identifier: str) -> str:+ return "`" + identifier.replace("`", "``") + "`"++class DatabricksClient:# How often to poll for query status. This is a trade-off between responsiveness and number of# queries we make to Databricks. 1 second has been chosen rather arbitrarily.@@ -637,7 +644,7 @@ class DatabricksClient:async def acreate_table(self, table_name: str, fields: list[DatabricksField]):"""Asynchronously create the Databricks delta table if it doesn't exist."""timeout = FIVE_MINUTES- field_ddl = ", ".join(f"`{field[0]}` {field[1]}" for field in fields)+ field_ddl = ", ".join(f"{_quote_databricks_identifier(field[0])} {field[1]}" for field in fields)query = f"""CREATE TABLE IF NOT EXISTS `{table_name}` ({field_ddl}@@ -710,14 +717,15 @@ class DatabricksClient:"""select_fields = []for field in fields:+ quoted_field = _quote_databricks_identifier(field[0])if field[1] == "VARIANT":- select_fields.append(f"PARSE_JSON(`{field[0]}`) as `{field[0]}`")+ select_fields.append(f"PARSE_JSON({quoted_field}) as {quoted_field}")elif field[1] == "BIGINT":- select_fields.append(f"CAST(`{field[0]}` as BIGINT) as `{field[0]}`")+ select_fields.append(f"CAST({quoted_field} as BIGINT) as {quoted_field}")elif field[1] == "INTEGER":- select_fields.append(f"CAST(`{field[0]}` as INTEGER) as `{field[0]}`")+ select_fields.append(f"CAST({quoted_field} as INTEGER) as {quoted_field}")else:- select_fields.append(f"`{field[0]}`")+ select_fields.append(quoted_field)select_fields_str = ", ".join(select_fields)merge_schema = f"true" if with_schema_evolution else "false"@@ -823,7 +831,6 @@ class DatabricksClient:"""assert merge_key, "Merge key must be defined"- assert update_key, "Update key must be defined"if with_schema_evolution is True:self.logger.info(@@ -880,14 +887,20 @@ class DatabricksClient:merge_key: collections.abc.Iterable[str],update_key: collections.abc.Iterable[str],) -> str:- merge_condition = " AND ".join([f"target.`{field}` = source.`{field}`" for field in merge_key])- update_condition = " OR ".join([f"target.`{field}` < source.`{field}`" for field in update_key])+ merge_condition = " AND ".join(+ f"target.{quoted} = source.{quoted}" for quoted in map(_quote_databricks_identifier, merge_key)+ )+ update_condition = " OR ".join(+ f"target.{quoted} < source.{quoted}" for quoted in map(_quote_databricks_identifier, update_key)+ )+ if update_condition:+ update_condition = f" AND ({update_condition})"return f"""MERGE WITH SCHEMA EVOLUTION INTO `{target_table}` AS targetUSING `{source_table}` AS sourceON {merge_condition}- WHEN MATCHED AND ({update_condition}) THEN+ WHEN MATCHED{update_condition} THENUPDATE SET *WHEN NOT MATCHED THENINSERT *@@ -902,27 +915,41 @@ class DatabricksClient:source_table_fields: collections.abc.Iterable[DatabricksField],target_table_field_names: list[str],) -> str:- merge_condition = " AND ".join([f"target.`{field}` = source.`{field}`" for field in merge_key])- update_condition = " OR ".join([f"target.`{field}` < source.`{field}`" for field in update_key])+ merge_condition = " AND ".join(+ f"target.{quoted} = source.{quoted}" for quoted in map(_quote_databricks_identifier, merge_key)+ )+ update_condition = " OR ".join(+ f"target.{quoted} < source.{quoted}" for quoted in map(_quote_databricks_identifier, update_key)+ )+ if update_condition:+ update_condition = f" AND ({update_condition})"update_clause = ", ".join([- f"target.`{field[0]}` = source.`{field[0]}`"+ f"target.{_quote_databricks_identifier(field[0])} = source.{_quote_databricks_identifier(field[0])}"for field in source_table_fieldsif field[0] in target_table_field_names])field_names = ", ".join(- [f"`{field[0]}`" for field in source_table_fields if field[0] in target_table_field_names]+ [+ _quote_databricks_identifier(field[0])+ for field in source_table_fields+ if field[0] in target_table_field_names+ ])values = ", ".join(- [f"source.`{field[0]}`" for field in source_table_fields if field[0] in target_table_field_names]+ [+ f"source.{_quote_databricks_identifier(field[0])}"+ for field in source_table_fields+ if field[0] in target_table_field_names+ ])return f"""MERGE INTO `{target_table}` AS targetUSING `{source_table}` AS sourceON {merge_condition}- WHEN MATCHED AND ({update_condition}) THEN+ WHEN MATCHED{update_condition} THENUPDATE SET{update_clause}WHEN NOT MATCHED THEN@@ -1035,7 +1062,7 @@ class TableSettings(t.NamedTuple):def _get_databricks_table_settings(- model: BatchExportModel | None,+ model: AnyBatchExportModel,record_batch_schema: pa.Schema,use_variant_type: bool,) -> TableSettings:@@ -1058,7 +1085,7 @@ def _get_databricks_table_settings(json_type = "STRING"known_variant_columns = []- if model is None or (isinstance(model, BatchExportModel) and model.name == "events" and model.schema is None):+ if isinstance(model, BatchExportEventsModel):table_fields = _events_table_fields(json_type)# Staging and copying are two activities, so a deploy of a schema change between them could@@ -1083,24 +1110,6 @@ def _get_databricks_table_settings(return TableSettings(table_fields, record_batch_schema, known_variant_columns)-def _get_databricks_merge_config(- model: BatchExportModel | None,-) -> tuple[bool, list[str], list[str]]:- requires_merge = False- merge_key = []- update_key = []- if isinstance(model, BatchExportModel):- if model.name == "persons":- requires_merge = True- merge_key = ["team_id", "distinct_id"]- update_key = ["person_version", "person_distinct_id_version"]- elif model.name == "sessions":- requires_merge = True- merge_key = ["team_id", "session_id"]- update_key = ["end_timestamp"]- return requires_merge, merge_key, update_key--async def _get_databricks_integration(inputs: DatabricksInsertInputs) -> DatabricksIntegration:"""Get the Databricks integration.@@ -1300,7 +1309,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInp)async with Heartbeater():- model = inputs.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_DATABRICKS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -1330,7 +1339,9 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInpuse_variant_type=inputs.use_variant_type,)- requires_merge, merge_key, update_key = _get_databricks_merge_config(model=model)+ requires_merge = model.incremental_mode == IncrementalMode.MERGE+ merge_key = list(model.primary_key) if requires_merge else []+ update_key = list(model.version_key) if requires_merge else []data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")# include attempt in the stage table & volume names to avoid collisions if multiple attempts are running at the@@ -1375,7 +1386,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInpconsumer = DatabricksConsumer(client=databricks_client,volume_path=volume_path,- model=model.name if isinstance(model, BatchExportModel) else "events",+ model=model.name,)transformer: ChunkTransformerProtocol = ParquetStreamTransformer(diff --git a/products/batch_exports/backend/temporal/destinations/http_batch_export.py b/products/batch_exports/backend/temporal/destinations/http_batch_export.pyindex 91cd13efdddc..ac0251c22f9d 100644--- a/products/batch_exports/backend/temporal/destinations/http_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/http_batch_export.py@@ -18,7 +18,14 @@ from posthog.temporal.common.clickhouse import get_clientfrom posthog.temporal.common.logger import get_loggerfrom products.batch_exports.backend.models.batch_export import BatchExportRun-from products.batch_exports.backend.service import BatchExportField, BatchExportInsertInputs, HttpBatchExportInputs+from products.batch_exports.backend.service import (+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportField,+ BatchExportInsertInputs,+ HttpBatchExportInputs,+ migrate_batch_export_model,+)from products.batch_exports.backend.temporal.batch_exports import (FinishBatchExportRunInputs,StartBatchExportRunInputs,@@ -203,11 +210,11 @@ async def insert_into_http_activity(inputs: HttpInsertInputs) -> BatchExportResuif not await client.is_alive():raise ConnectionError("Cannot establish connection to ClickHouse")- if inputs.batch_export_model is not None:- if inputs.batch_export_model.name != "events":- raise NotImplementedError("HTTP export only supports the events model")- if inputs.batch_export_model.schema is not None:- raise NotImplementedError("HTTP export does not support schemas")+ model = migrate_batch_export_model(inputs.batch_export_model)+ if isinstance(model, BatchExportEventsCustomModel):+ raise NotImplementedError("HTTP export does not support schemas")+ if not isinstance(model, BatchExportEventsModel):+ raise NotImplementedError("HTTP export only supports the events model")use_native_schema = await database_sync_to_async(use_new_events_schema)(inputs.team_id)@@ -228,8 +235,8 @@ async def insert_into_http_activity(inputs: HttpInsertInputs) -> BatchExportResufields = http_default_fields(native_source)columns = [field["alias"] for field in fields]- filters = inputs.batch_export_model.filters if inputs.batch_export_model is not None else None- if filters is not None and len(filters) > 0:+ filters = model.filters+ if filters:filters_str, extra_query_parameters = await database_sync_to_async(compose_filters_clause)(filters, team_id=inputs.team_id, values=None)diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.pyindex 6848c05f602e..70e513be3fcc 100644--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py@@ -33,11 +33,14 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.service import (+ AnyBatchExportModel,+ BatchExportEventsModel,BatchExportField,BatchExportInsertInputs,- BatchExportModel,PostgresBatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -399,7 +402,7 @@ class PostgreSQLClient:table_name: str,fields: Fields,exists_ok: bool = True,- primary_key: Fields | None = None,+ primary_key: collections.abc.Sequence[str] | None = None,log_statements: bool = False,) -> None:"""Create a table in PostgreSQL.@@ -424,7 +427,7 @@ class PostgreSQLClient:if primary_key is not None:primary_key_clause = sql.SQL(", PRIMARY KEY ({fields})").format(- fields=sql.SQL(",").join(sql.Identifier(field[0]) for field in primary_key)+ fields=sql.SQL(",").join(sql.Identifier(field) for field in primary_key))async with self.connection.transaction():@@ -499,7 +502,7 @@ class PostgreSQLClient:schema: str,table_name: str,fields: Fields,- primary_key: Fields | None = None,+ primary_key: collections.abc.Sequence[str] | None = None,exists_ok: bool = True,not_found_ok: bool = True,delete: bool = True,@@ -537,18 +540,19 @@ class PostgreSQLClient:def _build_merge_query(final_table: sql.Identifier,stage_table: sql.Identifier,- merge_key: Fields,- update_key: Fields,+ merge_key: collections.abc.Sequence[str],+ update_key: collections.abc.Sequence[str],update_when_matched: Fields,) -> sql.Composed:- merge_key_names = _join_field_names(merge_key)+ merge_key_names = sql.SQL(",").join(sql.Identifier(field) for field in merge_key)update_condition = sql.SQL(" OR ").join(sql.SQL("EXCLUDED.{stage_field} > final.{final_field}").format(- final_field=sql.Identifier(field[0]),- stage_field=sql.Identifier(field[0]),+ final_field=sql.Identifier(field),+ stage_field=sql.Identifier(field),)for field in update_key)+ update_filter = sql.SQL("WHERE ({})").format(update_condition) if update_key else sql.SQL("")update_clause = sql.SQL(",").join(sql.SQL("{final_field} = EXCLUDED.{stage_field}").format(final_field=sql.Identifier(field[0]),@@ -564,14 +568,14 @@ class PostgreSQLClient:ORDER BY {order_by}ON CONFLICT ({conflict_fields}) DO UPDATE SET{update_clause}- WHERE ({update_condition})+ {update_filter}""").format(final_table=final_table,order_by=merge_key_names,conflict_fields=merge_key_names,stage_table=stage_table,- update_condition=update_condition,+ update_filter=update_filter,update_clause=update_clause,field_names=_join_field_names(update_when_matched),)@@ -606,8 +610,8 @@ class PostgreSQLClient:final_table_name: str,stage_table_name: str,schema: str,- merge_key: Fields,- update_key: Fields,+ merge_key: collections.abc.Sequence[str],+ update_key: collections.abc.Sequence[str],update_when_matched: Fields,timeout: float | int | None = None,lock_timeout: float | int = MERGE_LOCK_TIMEOUT_SECONDS,@@ -642,7 +646,7 @@ class PostgreSQLClient:async with asyncio.timeout(timeout):await merge_with_retries(merge_query, schema, lock_timeout)except psycopg.errors.InvalidColumnReference:- raise MissingPrimaryKeyError(final_table, _join_field_names(merge_key))+ raise MissingPrimaryKeyError(final_table, sql.SQL(",").join(sql.Identifier(field) for field in merge_key))except TimeoutError as e:self.external_logger.exception("Final merge into '%s.%s' is taking too long to complete and will be rolled-back. Perhaps the database is under too much load?",@@ -861,11 +865,11 @@ class PostgreSQLConsumer(Consumer):def _get_table_fields(- model: BatchExportModel | None,+ model: AnyBatchExportModel,record_batch_schema: pa.Schema,) -> Fields:"""Extract table field definitions from model and schema."""- if model is None or model.name == "events":+ if model.name == "events":table_fields: Fields = [("uuid", "VARCHAR(200)"),("event", "VARCHAR(200)"),@@ -890,48 +894,6 @@ def _get_table_fields()-class MergeSettings(typing.NamedTuple):- requires_merge: bool- merge_key: Fields- update_key: Fields- primary_key: Fields | None---def _get_merge_settings(- model: BatchExportModel | None,-) -> MergeSettings:- requires_merge = False- merge_key: Fields = []- update_key: Fields = []- primary_key: Fields | None = None-- if isinstance(model, BatchExportModel):- if model.name == "persons":- requires_merge = True- merge_key = [- ("team_id", "INT"),- ("distinct_id", "TEXT"),- ]- update_key = [- ("person_version", "INT"),- ("person_distinct_id_version", "INT"),- ]- primary_key = (("team_id", "INTEGER"), ("distinct_id", "VARCHAR(200)"))-- elif model.name == "sessions":- requires_merge = True- merge_key = [- ("team_id", "INT"),- ("session_id", "TEXT"),- ]- update_key = [- ("end_timestamp", "TIMESTAMP"),- ]- primary_key = (("team_id", "INTEGER"), ("session_id", "TEXT"))-- return MergeSettings(requires_merge, merge_key, update_key, primary_key)--class PostgreSQLIntegrationNotFoundError(Exception):"""Error raised when the PostgreSQL integration is not found."""@@ -976,7 +938,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs))async with Heartbeater():- model = inputs.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_POSTGRES_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -1004,7 +966,8 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs))table_fields = _get_table_fields(model, record_batch_schema)- merge_settings = _get_merge_settings(model)+ requires_merge = model.incremental_mode == IncrementalMode.MERGE+ primary_key = model.primary_key if requires_merge else Nonedata_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")@@ -1014,7 +977,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)# TODO: That should be enough, but we should add a proper check and alert on larger inputs.stage_table_name = (f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"- if merge_settings.requires_merge+ if requires_mergeelse inputs.table_name)[:63]@@ -1037,9 +1000,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)f"No matching columns found in the destination table '{inputs.schema}.{inputs.table_name}'")except psycopg.errors.InsufficientPrivilege:- if model is None or (- isinstance(model, BatchExportModel) and model.name == "events" and model.schema is None- ):+ if isinstance(model, BatchExportEventsModel):# Without introspection, do not require this column on older destination tables.table_fields = [field for field in table_fields if field[0] != "person_id"]external_logger.warning(@@ -1066,24 +1027,24 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)table_fields,create=not table_exists,delete=False,- primary_key=merge_settings.primary_key,+ primary_key=primary_key,log_statements=True,) as pg_table,pg_client.managed_table(inputs.schema,stage_table_name,table_fields,- create=merge_settings.requires_merge,- delete=merge_settings.requires_merge,- primary_key=merge_settings.primary_key,+ create=requires_merge,+ delete=requires_merge,+ primary_key=primary_key,) as pg_stage_table,):consumer = PostgreSQLConsumer(client=pg_client,schema=inputs.schema,- table_name=pg_stage_table if merge_settings.requires_merge else pg_table,+ table_name=pg_stage_table if requires_merge else pg_table,schema_columns=schema_columns,- model=model.name if isinstance(model, BatchExportModel) else "events",+ model=model.name,)transformer = CSVStreamTransformer(@@ -1119,7 +1080,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)raisefinally:- if merge_settings.requires_merge:+ if requires_merge:merge_query_timeout = get_query_timeout(dt.datetime.fromisoformat(inputs.data_interval_start)if inputs.data_interval_start@@ -1132,8 +1093,8 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)stage_table_name=pg_stage_table,schema=inputs.schema,update_when_matched=table_fields,- merge_key=merge_settings.merge_key,- update_key=merge_settings.update_key,+ merge_key=model.primary_key,+ update_key=model.version_key,timeout=merge_query_timeout,)diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.pyindex 79ee68d8ccec..e4366943d84f 100644--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py@@ -38,14 +38,16 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.service import (+ AnyBatchExportModel,AWSCredentials,BatchExportField,BatchExportInsertInputs,- BatchExportModel,IAMRole,IntegrationID,RedshiftBatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -303,8 +305,8 @@ class RedshiftClient(PostgreSQLClient):final_table_name: str,stage_table_name: str,schema: str,- merge_key: Fields,- update_key: Fields,+ merge_key: collections.abc.Sequence[str],+ update_key: collections.abc.Sequence[str],final_table_fields: Fields,stage_fields_cast_to_super: collections.abc.Container[str] | None = None,remove_duplicates: bool = True,@@ -326,8 +328,8 @@ class RedshiftClient(PostgreSQLClient):and_separator = sql.SQL("AND")merge_condition = and_separator.join(sql.SQL("{final_field} = {stage_field}").format(- final_field=sql.Identifier(schema, final_table_name, field[0]),- stage_field=sql.Identifier("stage", field[0]),+ final_field=sql.Identifier(schema, final_table_name, field),+ stage_field=sql.Identifier("stage", field),)for field in merge_key)@@ -338,8 +340,8 @@ class RedshiftClient(PostgreSQLClient):# versions back.delete_condition = and_separator.join(sql.SQL("{final_field} = {stage_field}").format(- final_field=sql.Identifier("final", field[0]),- stage_field=sql.Identifier(schema, stage_table_name, field[0]),+ final_field=sql.Identifier("final", field),+ stage_field=sql.Identifier(schema, stage_table_name, field),)for field in merge_key)@@ -347,8 +349,8 @@ class RedshiftClient(PostgreSQLClient):or_separator = sql.SQL(" OR ")delete_extra_conditions = or_separator.join(sql.SQL("{stage_field} < {final_field}").format(- final_field=sql.Identifier("final", field[0]),- stage_field=sql.Identifier(schema, stage_table_name, field[0]),+ final_field=sql.Identifier("final", field),+ stage_field=sql.Identifier(schema, stage_table_name, field),)for field in update_key)@@ -428,7 +430,7 @@ class RedshiftClient(PostgreSQLClient):self.logger.info("Table locked")try:- if not skip_delete:+ if update_key and not skip_delete:await cursor.execute(delete_query)except psycopg.errors.UndefinedFunction:self.logger.exception(@@ -810,7 +812,7 @@ class TableSchemas(typing.NamedTuple):def _get_table_schemas(- model: BatchExportModel | None,+ model: AnyBatchExportModel,record_batch_schema: pa.Schema,properties_data_type: str,) -> TableSchemas:@@ -824,7 +826,7 @@ def _get_table_schemas(properties_type = "VARCHAR(65535)"use_super = False- if model is None or model.name == "events":+ if model.name == "events":table_schema: Fields = [("uuid", "VARCHAR(200)"),("event", "VARCHAR(200)"),@@ -862,9 +864,9 @@ def _get_table_schemas(class RequiredMergeSettings(typing.NamedTuple):requires_merge: typing.Literal[True]- merge_key: Fields- update_key: Fields- primary_key: Fields+ merge_key: collections.abc.Sequence[str]+ update_key: collections.abc.Sequence[str]+ primary_key: collections.abc.Sequence[str]class NotRequiredMergeSettings(typing.NamedTuple):@@ -875,49 +877,20 @@ MergeSettings = RequiredMergeSettings | NotRequiredMergeSettingsdef _get_merge_settings(- model: BatchExportModel | None,+ model: AnyBatchExportModel,use_super: bool = False,) -> MergeSettings:"""Return merge settings for models that require merging."""- merge_key: Fields- update_key: Fields- primary_key: Fields-- if isinstance(model, BatchExportModel) and model.name == "persons":- merge_key = [- ("team_id", "INT"),- ("distinct_id", "TEXT"),- ]- update_key = [- ("person_version", "INT"),- ("person_distinct_id_version", "INT"),- ]- primary_key = (("team_id", "INTEGER"), ("distinct_id", "VARCHAR(200)"))-- return RequiredMergeSettings(- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key- )-- elif isinstance(model, BatchExportModel) and model.name == "sessions":- merge_key = [- ("team_id", "INT"),- ("session_id", "TEXT"),- ]- update_key = [- ("end_timestamp", "TIMESTAMP"),- ]- primary_key = (("team_id", "INTEGER"), ("session_id", "TEXT"))-+ if model.incremental_mode == IncrementalMode.MERGE:return RequiredMergeSettings(- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key+ requires_merge=True,+ merge_key=model.primary_key,+ update_key=model.version_key,+ primary_key=model.primary_key,)- elif (model is None or model.name == "events") and use_super is True:- merge_key = [("uuid", "TEXT")]- update_key = [("timestamp", "TIMESTAMP")]- primary_key = [("uuid", "TEXT")]-+ elif model.name == "events" and use_super is True:return RequiredMergeSettings(- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key+ requires_merge=True, merge_key=("uuid",), update_key=("timestamp",), primary_key=("uuid",))else:return NotRequiredMergeSettings(requires_merge=False)@@ -1302,7 +1275,7 @@ async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs))async with Heartbeater():- model = inputs.batch_export.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_REDSHIFT_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -1818,7 +1791,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInp)async with Heartbeater():- model = inputs.batch_export.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -2035,7 +2008,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInpupdate_key=merge_settings.update_key,stage_fields_cast_to_super=table_schemas.super_columns if table_schemas.use_super else None,remove_duplicates=remove_duplicates,- skip_delete=isinstance(model, BatchExportModel)+ skip_delete=inputs.batch_export.batch_export_model is not Noneand model.name == "events"and str(inputs.batch_export.team_id) in settings.BATCH_EXPORT_REDSHIFT_SKIP_DELETE_TEAM_IDS,)diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.pyindex bf95de0d8f56..1c816ebe2f2c 100644--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py@@ -43,8 +43,9 @@ from products.batch_exports.backend.service import (AWSCredentials,BatchExportField,BatchExportInsertInputs,- BatchExportModel,+ BatchExportModelInput,S3BatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -852,7 +853,7 @@ class ConcurrentS3Consumer(Consumer):prefix: str,data_interval_start: str | None,data_interval_end: str | None,- batch_export_model: BatchExportModel | None,+ batch_export_model: BatchExportModelInput | None,file_format: str,checksum_algorithm: str | None = None,kms_key_id: str | None = None,@@ -865,7 +866,8 @@ class ConcurrentS3Consumer(Consumer):max_concurrent_uploads: int = 5,file_name_prefix: str | None = None,):- super().__init__(model=batch_export_model.name if batch_export_model else "events")+ model = migrate_batch_export_model(batch_export_model)+ super().__init__(model=model.name)self.bucket = bucketself.region_name = region_name@@ -874,7 +876,7 @@ class ConcurrentS3Consumer(Consumer):self.data_interval_start = data_interval_startself.data_interval_end = data_interval_endself.file_name_prefix = file_name_prefix- self.batch_export_model = batch_export_model+ self.batch_export_model = modelself.checksum_algorithm = checksum_algorithmdiff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex c126a956c863..9aa5c1cb8cc1 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -30,11 +30,12 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger+from products.batch_exports.backend.facade.enums import IncrementalModefrom products.batch_exports.backend.service import (BatchExportField,BatchExportInsertInputs,- BatchExportModel,SnowflakeBatchExportInputs,+ migrate_batch_export_model,)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,@@ -548,6 +549,10 @@ def _is_connect_error_retryable(err: Exception) -> bool:return isinstance(err, OperationalError) and err.errno in (ER_FAILED_TO_CONNECT_TO_DB, ER_RETRYABLE_CODE)+def _quote_snowflake_identifier(identifier: str) -> str:+ return '"' + identifier.replace('"', '""') + '"'++class SnowflakeClient:"""Snowflake connection client used in batch exports."""@@ -953,7 +958,9 @@ class SnowflakeClient:Whether to include an 'IF NOT EXISTS' clause in the query such that itwon't fail if the table already exists."""- field_ddl = ", ".join(f'"{field.name}" {field.snowflake_type_name}' for field in table)+ field_ddl = ", ".join(+ f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_name}" for field in table+ )if_not_exists = ""if exists_ok:@@ -1086,13 +1093,13 @@ class SnowflakeClient:perform this operation."""select_fields = ", ".join(- f"PARSE_JSON($1:\"{field.name}\", 'd')"+ f"PARSE_JSON($1:{_quote_snowflake_identifier(field.name)}, 'd')"if field.data_type == JsonType() and field.name.lower() != "elements"- else f'$1:"{field.name}"'+ else f"$1:{_quote_snowflake_identifier(field.name)}"for field in table)query = f"""- COPY INTO "{table.name}" ({", ".join(f'"{field.name}"' for field in table)})+ COPY INTO "{table.name}" ({", ".join(_quote_snowflake_identifier(field.name) for field in table)})FROM (SELECT {select_fields} FROM '@%"{table.name}"/{table.stage_prefix}'@@ -1201,16 +1208,19 @@ class SnowflakeClient:field = final[primary_key_field_name]if n > 0:merge_condition += " AND "- merge_condition += f'final."{field.name}" = stage."{field.name}"'+ quoted_field = _quote_snowflake_identifier(field.name)+ merge_condition += f"final.{quoted_field} = stage.{quoted_field}"- update_condition = "AND ("+ update_condition = ""for index, version_key_field_name in enumerate(final.version_key):field = final[version_key_field_name]if index > 0:update_condition += " OR "- update_condition += f'final."{field.name}" < stage."{field.name}"'- update_condition += ")"+ quoted_field = _quote_snowflake_identifier(field.name)+ update_condition += f"final.{quoted_field} < stage.{quoted_field}"+ if update_condition:+ update_condition = f"AND ({update_condition})"update_clause = ""values = ""@@ -1221,9 +1231,10 @@ class SnowflakeClient:values += ", "field_names += ", "- update_clause += f'final."{field.name}" = stage."{field.name}"'- field_names += f'"{field.name}"'- values += f'stage."{field.name}"'+ quoted_field = _quote_snowflake_identifier(field.name)+ update_clause += f"final.{quoted_field} = stage.{quoted_field}"+ field_names += quoted_field+ values += f"stage.{quoted_field}"merge_query = f"""MERGE INTO "{final.name}" AS final@@ -1277,33 +1288,6 @@ def snowflake_default_fields() -> list[BatchExportField]:return batch_export_fields-class MergeSettings(typing.NamedTuple):- primary_key: collections.abc.Sequence[str]- version_key: collections.abc.Sequence[str]---def _get_merge_settings(- model: BatchExportModel | None,-) -> MergeSettings | None:- if isinstance(model, BatchExportModel):- if model.name == "persons":- primary_key: collections.abc.Sequence[str] = ("team_id", "distinct_id")- version_key: collections.abc.Sequence[str] = (- "person_version",- "person_distinct_id_version",- )-- elif model.name == "sessions":- primary_key = ("team_id", "session_id")- version_key = ("end_timestamp",)- else:- return None- else:- return None-- return MergeSettings(primary_key, version_key)--class SnowflakeConsumer(Consumer):"""A consumer that uploads data to Snowflake from the internal stage.@@ -1415,7 +1399,7 @@ async def insert_into_snowflake_activity_from_stage(integration = await _get_snowflake_integration(inputs.integration_id, inputs.team_id)- model = inputs.batch_export_model+ model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_SNOWFLAKE_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -1458,14 +1442,14 @@ async def insert_into_snowflake_activity_from_stage(attempt = activity.info().attemptstage_table_name = f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"- merge_settings = _get_merge_settings(model)+ requires_merge = model.incremental_mode == IncrementalMode.MERGEtarget_table = SnowflakeTable.from_arrow_schema(record_batch_schema,name=inputs.table_name,table_schema=inputs.schema,database=inputs.database,- primary_key=merge_settings.primary_key if merge_settings else (),- version_key=merge_settings.version_key if merge_settings else (),+ primary_key=model.primary_key if requires_merge else (),+ version_key=model.version_key if requires_merge else (),stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",)if "elements" in target_table:@@ -1510,7 +1494,7 @@ async def insert_into_snowflake_activity_from_stage(consumer = SnowflakeConsumer(snowflake_client=snow_client,snowflake_table=snow_consumer_table,- model=model.name if isinstance(model, BatchExportModel) else "events",+ model=model.name,)transformer = PipelineTransformer(diff --git a/products/batch_exports/backend/temporal/destinations/utils.py b/products/batch_exports/backend/temporal/destinations/utils.pyindex ad231ccce4d9..dda61306662f 100644--- a/products/batch_exports/backend/temporal/destinations/utils.py+++ b/products/batch_exports/backend/temporal/destinations/utils.py@@ -3,7 +3,7 @@ import posixpathfrom posthog.temporal.common.logger import get_logger-from products.batch_exports.backend.service import BatchExportModel+from products.batch_exports.backend.service import BatchExportModelInputfrom products.batch_exports.backend.temporal.destinations.constants import (COMPRESSION_EXTENSIONS,FILE_FORMAT_EXTENSIONS,@@ -14,7 +14,7 @@ EXTERNAL_LOGGER = get_logger("EXTERNAL")def get_allowed_template_variables(- data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None+ data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModelInput | None) -> dict[str, str]:"""Derive from inputs a dictionary of supported template variables for key prefixes."""export_datetime = dt.datetime.fromisoformat(data_interval_end)@@ -33,7 +33,10 @@ def get_allowed_template_variables(def get_key_prefix(- prefix: str, data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None+ prefix: str,+ data_interval_start: str | None,+ data_interval_end: str,+ batch_export_model: BatchExportModelInput | None,) -> str:"""Format a key prefix with template variables."""template_variables = get_allowed_template_variables(data_interval_start, data_interval_end, batch_export_model)@@ -48,7 +51,10 @@ def get_key_prefix(def get_absolute_key_prefix(- prefix: str, data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None+ prefix: str,+ data_interval_start: str | None,+ data_interval_end: str,+ batch_export_model: BatchExportModelInput | None,) -> str:"""Like `get_key_prefix`, but ensure key is absolute and ends on a slash."""key_prefix = get_key_prefix(prefix, data_interval_start, data_interval_end, batch_export_model)@@ -66,7 +72,7 @@ def get_manifest_key(prefix: str,data_interval_start: str | None,data_interval_end: str | None,- batch_export_model: BatchExportModel | None,+ batch_export_model: BatchExportModelInput | None,file_name_prefix: str | None = None,) -> str:"""Generate manifest file key."""@@ -99,7 +105,7 @@ def get_object_key(prefix: str,data_interval_start: str | None,data_interval_end: str | None,- batch_export_model: BatchExportModel | None,+ batch_export_model: BatchExportModelInput | None,file_format: str,compression: str | None = None,legacy_parquet_extension: bool = False,diff --git a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.pyindex 3b1ccfd2a9ef..64504ee3e15d 100644--- a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py@@ -18,7 +18,12 @@ from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger-from products.batch_exports.backend.service import BatchExportField, BatchExportInsertInputs, WorkflowsBatchExportInputs+from products.batch_exports.backend.service import (+ BatchExportField,+ BatchExportInsertInputs,+ WorkflowsBatchExportInputs,+ migrate_batch_export_model,+)from products.batch_exports.backend.temporal.batch_exports import (StartBatchExportRunInputs,get_data_interval,@@ -410,6 +415,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInput)async with Heartbeater():+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_WORKFLOWS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()assert inputs.batch_export.batch_export_id is not None@@ -458,9 +464,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInputteam_id=inputs.batch_export.team_id,session=session,request_task_group=tg,- model=inputs.batch_export.batch_export_model.name- if inputs.batch_export.batch_export_model- else "events",+ model=model.name,max_concurrent_requests=settings.BATCH_EXPORT_WORKFLOWS_MAX_CONCURRENT_REQUESTS,max_pending_requests=settings.BATCH_EXPORT_WORKFLOWS_MAX_PENDING_REQUESTS,)diff --git a/products/batch_exports/backend/temporal/pipeline/table.py b/products/batch_exports/backend/temporal/pipeline/table.pyindex fd50369450bc..eaf9b4c1498d 100644--- a/products/batch_exports/backend/temporal/pipeline/table.py+++ b/products/batch_exports/backend/temporal/pipeline/table.py@@ -425,4 +425,4 @@ class Table(TableBase, typing.Generic[FieldType]):raise KeyError(key)def is_mutable(self) -> bool:- return len(self.primary_key) > 0 and len(self.version_key) > 0+ return len(self.primary_key) > 0diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.pyindex c6ff294e7ecb..1423518e66d1 100644--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py@@ -7,7 +7,7 @@ import weakrefimport collections.abcimport pytest-from unittest.mock import MagicMock, patch+from unittest.mock import AsyncMock, MagicMock, patchimport pyarrow as paimport requests@@ -219,9 +219,90 @@ async def test_create_table(fields, time_partitioning, bigquery_client, bigqueryawait client.delete_table(table)[email protected]("versioned", [True, False])+ "key,quoted_key,version,quoted_version",+ [+ ("id", "`id`", "version", "`version`"),+ ("group", "`group`", "updated at", "`updated at`"),+ ("key`name", r"`key\`name`", "version\\name", r"`version\\name`"),+ ("key\\`name", r"`key\\\`name`", "version\nname", r"`version\nname`"),+ ],+)+async def test_merge_query_with_optional_version_key(+ versioned: bool, key: str, quoted_key: str, version: str, quoted_version: str+) -> None:+ fields = (+ BigQueryField(key, BigQueryType("INT64", False), False),+ BigQueryField(version, BigQueryType("INT64", False), False),+ )+ final = BigQueryTable(+ "final", fields, ("project", "dataset"), primary_key=(key,), version_key=(version,) if versioned else ()+ )+ stage = BigQueryTable("stage", fields, ("project", "dataset"))+ client = BigQueryClient(MagicMock())++ with patch.object(client, "execute_query", new_callable=AsyncMock) as execute:+ await client.merge_tables(final, stage)++ query = " ".join(execute.call_args.args[0].split())+ assert query.startswith("MERGE `project.dataset.final`")+ assert f"ON final.{quoted_key} = stage.{quoted_key}" in query+ assert (+ f"UPDATE SET final.{quoted_key} = stage.{quoted_key}, final.{quoted_version} = stage.{quoted_version}" in query+ )+ assert f"WHEN NOT MATCHED BY TARGET THEN INSERT ({quoted_key}, {quoted_version})" in query+ assert f"VALUES (stage.{quoted_key}, stage.{quoted_version})" in query+ if versioned:+ assert f"WHEN MATCHED AND (final.{quoted_version} < stage.{quoted_version}) THEN UPDATE" in query+ assert f"PARTITION BY {quoted_key} ORDER BY {quoted_version} DESC" in query+ else:+ assert "WHEN MATCHED THEN UPDATE" in query+ assert f"ROW_NUMBER() OVER ( PARTITION BY {quoted_key} )" in query++[email protected]("merge", [True, False])+async def test_write_query_quotes_fields_and_tables_with_json_casts(merge: bool) -> None:+ final = BigQueryTable(+ "final`table",+ (+ BigQueryField("group", BigQueryType("INT64", False), False),+ BigQueryField("json\\`data", BigQueryType("JSON", False), True),+ ),+ ("project", "dataset"),+ primary_key=("group",) if merge else (),+ )+ stage = BigQueryTable(+ "stage\\`table",+ (+ BigQueryField("group", BigQueryType("INT64", False), False),+ BigQueryField("json\\`data", BigQueryType("STRING", False), True),+ ),+ ("project", "dataset"),+ )+ client = BigQueryClient(MagicMock())+ with patch.object(client, "execute_query", new_callable=AsyncMock) as execute:+ await client.merge_tables(final, stage)++ query = " ".join(execute.call_args.args[0].split())+ assert r"`project.dataset.final\`table`" in query+ assert r"FROM `project.dataset.stage\\\`table`" in query+ assert r"`json\\\`data`" in query+ prefix = "stage." if merge else ""+ assert f"REGEXP_REPLACE( {prefix}" + r"`json\\\`data`," in query+ if merge:+ assert r"INSERT (`group`, `json\\\`data`)" in query+ else:+ assert r"(`group`,`json\\\`data`)" in query+ assert "SELECT `group`, SAFE.PARSE_JSON(" in query++@SKIP_IF_MISSING_GOOGLE_APPLICATION_CREDENTIALS@pytest.mark.asyncio-async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery_dataset):[email protected]("version_key", [("version",), ()])+async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery_dataset, version_key):fields = (BigQueryField("id", BigQueryType("INT64", False), False),BigQueryField("version", BigQueryType("INT64", False), False),@@ -230,11 +311,11 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquerysuffix = "".join(random.choices(string.ascii_letters, k=10))parents = (bigquery_client.project, bigquery_dataset.dataset_id)final_table = BigQueryTable(- f"test_merge_final_{suffix}", fields, parents, primary_key=("id",), version_key=("version",)+ f"test_merge_final_{suffix}", fields, parents, primary_key=("id",), version_key=version_key)stage_tables = [BigQueryTable(- f"test_merge_stage_{index}_{suffix}", fields, parents, primary_key=("id",), version_key=("version",)+ f"test_merge_stage_{index}_{suffix}", fields, parents, primary_key=("id",), version_key=version_key)for index in range(2)]@@ -244,9 +325,7 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigqueryawait client.create_table(table)try:- # Key 1 exists in final and in both stage tables: the highest version across all- # stage tables must win. Key 4 is newer in final than in stage: it must not be- # overwritten.+ # Key 1 tests cross-stage deduplication; key 4 tests whether versions gate replacement.await client.execute_query(f"INSERT INTO `{final_table.fully_qualified_name}` (id, version, value)"" VALUES (1, 1, 'final-v1'), (4, 5, 'final-v5')"@@ -262,14 +341,17 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigqueryawait client.merge_tables(final=final_table, stage=stage_tables)- result = await client.execute_query(f"SELECT id, version, value FROM `{final_table.fully_qualified_name}`")+ result = list(+ await client.execute_query(f"SELECT id, version, value FROM `{final_table.fully_qualified_name}`")+ )rows = {row["id"]: (row["version"], row["value"]) for row in result}+ assert len(rows) == len(result) == 4+ assert rows.pop(1) in ({(3, "stage1-v3")} if version_key else {(2, "stage0-v2"), (3, "stage1-v3")})assert rows == {- 1: (3, "stage1-v3"),2: (1, "stage0-only"),3: (1, "stage1-only"),- 4: (5, "final-v5"),+ 4: (5, "final-v5") if version_key else (1, "stage0-v1"),}finally:for table in (final_table, *stage_tables):diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.pyindex b7fd6a856982..a749f281c415 100644--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py@@ -270,31 +270,76 @@ class TestConnect:class TestQueryBuilders:- async def test_merge_query_with_schema_evolution(self, client: DatabricksClient):+ @pytest.mark.parametrize("schema_evolution", [True, False])+ @pytest.mark.parametrize("versioned", [True, False])+ async def test_merge_escapes_column_names(+ self, client: DatabricksClient, schema_evolution: bool, versioned: bool+ ) -> None:+ with (+ patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute,+ patch.object(+ client, "aget_table_columns", new_callable=AsyncMock, return_value=["key`name", "version`name"]+ ),+ ):+ await client.amerge_tables(+ target_table="test_target",+ source_table="test_source",+ merge_key=["key`name"],+ update_key=["version`name"] if versioned else [],+ source_table_fields=[("key`name", "BIGINT"), ("version`name", "BIGINT")],+ with_schema_evolution=schema_evolution,+ )++ query = " ".join(execute.call_args.args[0].split())+ assert "ON target.`key``name` = source.`key``name`" in query+ if versioned:+ assert "WHEN MATCHED AND (target.`version``name` < source.`version``name`) THEN" in query+ else:+ assert "WHEN MATCHED THEN" in query+ if schema_evolution:+ assert "UPDATE SET * WHEN NOT MATCHED THEN INSERT *" in query+ else:+ assert (+ "UPDATE SET target.`key``name` = source.`key``name`, target.`version``name` = source.`version``name`"+ in query+ )+ assert "INSERT (`key``name`, `version``name`) VALUES (source.`key``name`, source.`version``name`)" in query++ @pytest.mark.parametrize("versioned", [True, False])+ async def test_merge_query_with_schema_evolution(self, client: DatabricksClient, versioned: bool) -> None:merge_key = ["team_id", "distinct_id"]- update_key = ["person_version", "person_distinct_id_version"]- merge_query = client._get_merge_query_with_schema_evolution(- target_table="test_target",- source_table="test_source",- merge_key=merge_key,- update_key=update_key,+ update_key = ["person_version", "person_distinct_id_version"] if versioned else []+ with patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute:+ await client.amerge_tables(+ target_table="test_target",+ source_table="test_source",+ merge_key=merge_key,+ update_key=update_key,+ source_table_fields=[],+ )+ merge_query = execute.call_args.args[0]+ expected_condition = (+ " AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`)"+ if versioned+ else "")assert (merge_query- == """+ == f"""MERGE WITH SCHEMA EVOLUTION INTO `test_target` AS targetUSING `test_source` AS sourceON target.`team_id` = source.`team_id` AND target.`distinct_id` = source.`distinct_id`- WHEN MATCHED AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`) THEN+ WHEN MATCHED{expected_condition} THENUPDATE SET *WHEN NOT MATCHED THENINSERT *""")- async def test_merge_query_without_schema_evolution(self, client: DatabricksClient):+ @pytest.mark.parametrize("versioned", [True, False])+ async def test_merge_query_without_schema_evolution(self, client: DatabricksClient, versioned: bool) -> None:merge_key = ["team_id", "distinct_id"]- update_key = ["person_version", "person_distinct_id_version"]+ update_key = ["person_version", "person_distinct_id_version"] if versioned else []merge_query = client._get_merge_query_without_schema_evolution(target_table="test_target",source_table="test_source",@@ -315,13 +360,18 @@ class TestQueryBuilders:"properties",],)+ expected_condition = (+ " AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`)"+ if versioned+ else ""+ )assert (merge_query- == """+ == f"""MERGE INTO `test_target` AS targetUSING `test_source` AS sourceON target.`team_id` = source.`team_id` AND target.`distinct_id` = source.`distinct_id`- WHEN MATCHED AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`) THEN+ WHEN MATCHED{expected_condition} THENUPDATE SETtarget.`team_id` = source.`team_id`, target.`distinct_id` = source.`distinct_id`, target.`person_version` = source.`person_version`, target.`person_distinct_id_version` = source.`person_distinct_id_version`, target.`properties` = source.`properties`WHEN NOT MATCHED THEN@@ -376,12 +426,17 @@ class TestQueryBuilders:""")- async def test_copy_into_table_from_volume_query(self, client: DatabricksClient):+ @pytest.mark.parametrize(+ "json_field,quoted_json_field", [("properties", "`properties`"), ("json`data", "`json``data`")]+ )+ async def test_copy_into_table_from_volume_query(+ self, client: DatabricksClient, json_field: str, quoted_json_field: str+ ) -> None:"""Test that we construct the correct SQL for COPY INTO from a volume, including VARIANT/BIGINT casts."""fields = [("uuid", "STRING"),("event", "STRING"),- ("properties", "VARIANT"),+ (json_field, "VARIANT"),("distinct_id", "STRING"),("team_id", "BIGINT"),("timestamp", "TIMESTAMP"),@@ -394,10 +449,10 @@ class TestQueryBuilders:)assert (query- == """+ == f"""COPY INTO `test_table`FROM (- SELECT `uuid`, `event`, PARSE_JSON(`properties`) as `properties`, `distinct_id`, CAST(`team_id` as BIGINT) as `team_id`, `timestamp`, `databricks_ingested_timestamp` FROM '/Volumes/my_volume/path/file.parquet'+ SELECT `uuid`, `event`, PARSE_JSON({quoted_json_field}) as {quoted_json_field}, `distinct_id`, CAST(`team_id` as BIGINT) as `team_id`, `timestamp`, `databricks_ingested_timestamp` FROM '/Volumes/my_volume/path/file.parquet')FILEFORMAT = PARQUETCOPY_OPTIONS ('force' = 'true', 'mergeSchema' = 'true')diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.pyindex e446e4f66ee2..20418c99ef50 100644--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py@@ -4,7 +4,11 @@ import pytestimport pyarrow as pa-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema+from products.batch_exports.backend.service import (+ BatchExportEventsCustomModel,+ BatchExportEventsModel,+ BatchExportSchema,+)from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (_events_table_fields,_get_databricks_table_settings,@@ -37,7 +41,7 @@ def _staged_events_schema(column_names: list[str]) -> pa.Schema:def _events_table_field_names(staged_column_names: list[str], schema: BatchExportSchema | None) -> list[str]:table_fields, _, _ = _get_databricks_table_settings(- model=BatchExportModel(name="events", schema=schema),+ model=BatchExportEventsCustomModel(schema=schema) if schema is not None else BatchExportEventsModel(),record_batch_schema=_staged_events_schema(staged_column_names),use_variant_type=True,)diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.pynew file mode 100644index 000000000000..f354ceb07220--- /dev/null+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.py@@ -0,0 +1,32 @@+import pytest+from unittest.mock import AsyncMock, MagicMock++from products.batch_exports.backend.temporal.destinations.postgres_batch_export import PostgreSQLClient++[email protected]("version_key", [("version",), ()])+async def test_merge_with_optional_version_key(version_key: tuple[str, ...]) -> None:+ client = PostgreSQLClient(+ user="test", password="test", host="localhost", port=5432, database="test", ssl_mode="prefer"+ )+ cursor = AsyncMock()+ connection = MagicMock()+ connection.cursor.return_value.__aenter__.return_value = cursor+ client._connection = connection++ await client.amerge_mutable_tables(+ final_table_name="final",+ stage_table_name="stage",+ schema="test",+ merge_key=("id",),+ update_key=version_key,+ update_when_matched=[("id", "BIGINT"), ("version", "BIGINT")],+ )++ query = " ".join(cursor.execute.call_args.args[0].as_string().split())+ assert 'ON CONFLICT ("id") DO UPDATE SET "id" = EXCLUDED."id","version" = EXCLUDED."version"' in query+ if version_key:+ assert 'WHERE (EXCLUDED."version" > final."version")' in query+ else:+ assert "WHERE" not in querydiff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.pyindex 4f20f323c92c..bb535db17bfe 100644--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py@@ -206,6 +206,59 @@ async def test_client_from_integration(postgres_config, setup_postgres_test_db,assert results == [(1,)][email protected]("version_key", [("version",), ()])+async def test_amerge_mutable_tables_replaces_rows_with_optional_version_key(+ postgres_config: dict[str, typing.Any],+ postgres_connection: psycopg.AsyncConnection,+ version_key: tuple[str, ...],+) -> None:+ schema = postgres_config["schema"]+ final_table = sql.Identifier(schema, "merge_final")+ stage_table = sql.Identifier(schema, "merge_stage")+ async with postgres_connection.cursor() as cursor:+ for table in (final_table, stage_table):+ await cursor.execute(+ sql.SQL("CREATE TABLE {} (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, value TEXT)").format(table)+ )+ await cursor.execute(+ sql.SQL("INSERT INTO {} VALUES (1, 5, 'original'), (2, 1, 'original'), (3, 1, 'untouched')").format(+ final_table+ )+ )+ await cursor.execute(+ sql.SQL("INSERT INTO {} VALUES (1, 1, 'replacement'), (2, 1, 'replacement'), (4, 1, 'new')").format(+ stage_table+ )+ )++ client = PostgreSQLClient(+ user=postgres_config["user"],+ password=postgres_config["password"],+ database=postgres_config["database"],+ host=postgres_config["host"],+ port=postgres_config["port"],+ ssl_mode="prefer",+ )+ async with client.connect():+ await client.amerge_mutable_tables(+ final_table_name="merge_final",+ stage_table_name="merge_stage",+ schema=schema,+ merge_key=("id",),+ update_key=version_key,+ update_when_matched=[("id", "INTEGER"), ("version", "INTEGER"), ("value", "TEXT")],+ )++ async with postgres_connection.cursor() as cursor:+ await cursor.execute(sql.SQL("SELECT id, version, value FROM {} ORDER BY id").format(final_table))+ assert await cursor.fetchall() == [+ (1, 5, "original") if version_key else (1, 1, "replacement"),+ (2, 1, "original") if version_key else (2, 1, "replacement"),+ (3, 1, "untouched"),+ (4, 1, "new"),+ ]++async def test_amerge_mutable_tables_raises_non_retryable_error_when_a_row_lock_blocks_it(postgres_config: dict[str, typing.Any], postgres_connection: psycopg.AsyncConnection) -> None:@@ -240,8 +293,8 @@ async def test_amerge_mutable_tables_raises_non_retryable_error_when_a_row_lock_final_table_name="merge_lock_final",stage_table_name="merge_lock_stage",schema=schema,- merge_key=[("id", "INTEGER")],- update_key=[("version", "INTEGER")],+ merge_key=["id"],+ update_key=["version"],update_when_matched=[("id", "INTEGER"), ("version", "INTEGER")],timeout=60,lock_timeout=1,diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.pynew file mode 100644index 000000000000..cd29caafd8f7--- /dev/null+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.py@@ -0,0 +1,44 @@+import pytest+from unittest.mock import AsyncMock, MagicMock++from products.batch_exports.backend.temporal.destinations.redshift_batch_export import RedshiftClient++[email protected]("version_key", [("version",), ()])[email protected]("remove_duplicates", [True, False])+async def test_merge_with_optional_version_key(version_key: tuple[str, ...], remove_duplicates: bool) -> None:+ client = RedshiftClient(+ user="test", password="test", host="localhost", port=5439, database="test", ssl_mode="prefer"+ )+ cursor = AsyncMock()+ cursor.fetchone.return_value = ("snapshot isolation",)+ connection = MagicMock()+ connection.cursor.return_value.__aenter__.return_value = cursor+ client._connection = connection++ await client.amerge_tables(+ final_table_name="final",+ stage_table_name="stage",+ schema="test",+ merge_key=("id",),+ update_key=version_key,+ final_table_fields=[("id", "BIGINT"), ("version", "BIGINT")],+ remove_duplicates=remove_duplicates,+ )++ queries = [" ".join(call.args[0].as_string().split()) for call in cursor.execute.call_args_list]+ deletes = [query for query in queries if query.startswith("DELETE")]+ if version_key:+ assert len(deletes) == 1+ assert 'AND ("test"."stage"."version" < "final"."version")' in deletes[0]+ else:+ assert deletes == []+ merge = queries[-1]+ assert merge.startswith('MERGE INTO "test"."final"')+ assert 'ON "test"."final"."id" = "stage"."id"' in merge+ if remove_duplicates:+ assert merge.endswith("REMOVE DUPLICATES")+ else:+ assert "WHEN MATCHED THEN UPDATE SET" in merge+ assert "WHEN NOT MATCHED THEN INSERT" in mergediff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.pyindex 4db30cd2507f..24fb129c4ea8 100644--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py@@ -15,7 +15,7 @@ import botocore.exceptionsfrom posthog.models.integration import AWSRedshiftRoleBasedIntegration, Integration, IntegrationError-from products.batch_exports.backend.service import AWSCredentials, BatchExportModel+from products.batch_exports.backend.service import AWSCredentials, BatchExportEventsModelfrom products.batch_exports.backend.temporal.destinations.redshift_batch_export import (ClientErrorGroup,InsufficientS3PermissionsError,@@ -43,7 +43,7 @@ def test_events_table_schemas_match_staged_person_id(has_person_id: bool, propercolumns.append("person_id")schema = pa.schema([(name, pa.string()) for name in columns])- table_schemas = _get_table_schemas(BatchExportModel(name="events", schema=None), schema, properties_data_type)+ table_schemas = _get_table_schemas(BatchExportEventsModel(), schema, properties_data_type)for fields in (table_schemas.table_schema, table_schemas.stage_table_schema):assert {name for name, _ in fields} == set(columns)diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.pynew file mode 100644index 000000000000..b8db18504549--- /dev/null+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py@@ -0,0 +1,75 @@+import pytest+from unittest.mock import AsyncMock, patch++import pyarrow as pa++from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (+ SnowflakeClient,+ SnowflakeField,+ SnowflakeTable,+ SnowflakeType,+)+from products.batch_exports.backend.temporal.utils import JsonType++[email protected]("versioned", [True, False])+ "key,quoted_key,version,quoted_version",+ [+ ("id", '"id"', "version", '"version"'),+ ('key"name', '"key""name"', 'version"name', '"version""name"'),+ ],+)+async def test_merge_with_optional_version_key(+ versioned: bool, key: str, quoted_key: str, version: str, quoted_version: str+) -> None:+ fields = tuple(SnowflakeField(name, SnowflakeType("INT", False), pa.int64(), False) for name in (key, version))+ final = SnowflakeTable("final", fields, primary_key=(key,), version_key=(version,) if versioned else ())+ stage = SnowflakeTable("stage", fields)+ client = SnowflakeClient(+ user="test", password="test", account="test", warehouse="test", database="test", schema="test"+ )++ assert final.is_mutable()+ with patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute:+ await client.merge_into_final_from_stage(final, stage, timeout=60)++ query = " ".join(execute.call_args.args[0].split())+ assert f"ON final.{quoted_key} = stage.{quoted_key}" in query+ assert (+ f"UPDATE SET final.{quoted_key} = stage.{quoted_key}, final.{quoted_version} = stage.{quoted_version}" in query+ )+ assert f"WHEN NOT MATCHED THEN INSERT ({quoted_key}, {quoted_version})" in query+ assert f"VALUES (stage.{quoted_key}, stage.{quoted_version})" in query+ if versioned:+ assert f"WHEN MATCHED AND (final.{quoted_version} < stage.{quoted_version}) THEN UPDATE" in query+ else:+ assert "WHEN MATCHED THEN UPDATE" in query++[email protected]("create", [True, False])+async def test_staging_quotes_column_names(create: bool) -> None:+ table = SnowflakeTable(+ "stage",+ (+ SnowflakeField('key"name', SnowflakeType("INT", False), pa.int64(), False),+ SnowflakeField('json"data', SnowflakeType("VARIANT", False), JsonType(), True),+ ),+ )+ client = SnowflakeClient(+ user="test", password="test", account="test", warehouse="test", database="test", schema="test"+ )+ with patch.object(client, "execute_async_query", new_callable=AsyncMock, return_value=([], None)) as execute:+ if create:+ await client.create_table(table)+ else:+ await client.copy_loaded_files_to_snowflake_table(table, timeout=60)++ query = " ".join(execute.call_args.args[0].split())+ if create:+ assert 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT )' in query+ else:+ assert 'COPY INTO "stage" ("key""name", "json""data")' in query+ assert 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\')' in querydiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.pyindex 2d3b7c485e73..83097bfe3f54 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py@@ -361,7 +361,7 @@ class PostgresDestinationWriter:# Declared at creation rather than added later, so the merge target carries its own# constraint. `_ensure_unique_index` still covers a table the customer already had,# which this never touches.- key = [(name, type_name) for name, type_name in fields if name in set(primary_keys or ())]+ key = [name for name, _ in fields if name in set(primary_keys or ())]await client.acreate_table(self._schema,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.pyindex eec3c9bee03d..e3a14c78c5ce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py@@ -214,13 +214,12 @@ class RedshiftDestinationWriter(PostgresDestinationWriter):# The stage was created from the destination, so both share this order. Only the name# in each field is read, which is why the types are left empty.final_table_fields: Fields = [(name, "") for name in await client.aget_table_columns(self._schema, target)]- merge_key: Fields = [(key, "") for key in primary_keys]await cast(RedshiftClient, client).amerge_tables(final_table_name=target,stage_table_name=stage,schema=self._schema,- merge_key=merge_key,+ merge_key=primary_keys,# Only read to build the delete that `skip_delete` suppresses.update_key=[],final_table_fields=final_table_fields,