Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(destinations): Adopt model-specific export types (#117000)

Tomás Farías Santana05051783bce6
diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
index df2cdd6949e0..22db2333d3d8 100644
--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py
@@ -22,7 +22,8 @@ from products.batch_exports.backend.service import (
AzureBlobBatchExportInputs,
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
+ BatchExportModelInput,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -142,20 +143,21 @@ class AzureBlobConsumer(Consumer):
prefix: str,
data_interval_start: str | None,
data_interval_end: str,
- batch_export_model: BatchExportModel | None,
+ batch_export_model: BatchExportModelInput | None,
file_format: str,
compression: str | None = None,
max_file_size_mb: int | None = None,
max_concurrency: int = 5,
legacy_parquet_extension: bool = True,
):
- super().__init__(model=batch_export_model.name if batch_export_model else "events")
+ model = migrate_batch_export_model(batch_export_model)
+ super().__init__(model=model.name)
self.container_client = container_client
self.prefix = prefix
self.data_interval_start = data_interval_start
self.data_interval_end = data_interval_end
- self.batch_export_model = batch_export_model
+ self.batch_export_model = model
self.file_format = file_format
self.compression = compression
self.max_file_size_mb = max_file_size_mb
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
index 7e4c671f30f6..29ce56be5b20 100644
--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
@@ -46,11 +46,12 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.service import (
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
BigQueryBatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -595,6 +596,21 @@ def _is_contention_exception(exc: Exception) -> bool:
return "concurrent update" in str(exc)
+def _quote_bigquery_identifier(identifier: str) -> str:
+ escaped = identifier.translate(
+ {
+ ord("\\"): r"\\",
+ ord("`"): r"\`",
+ ord("\n"): r"\n",
+ ord("\r"): r"\r",
+ ord("\t"): r"\t",
+ ord("\b"): r"\b",
+ ord("\f"): r"\f",
+ }
+ )
+ return f"`{escaped}`"
+
+
class BigQueryClient:
"""Async client to interact with BigQuery.
@@ -791,7 +807,7 @@ class BigQueryClient:
"""Attempt to SELECT from table to check for query permissions."""
if isinstance(table, BigQueryTable) and "timestamp" in table:
query = f"""
- SELECT 1 FROM `{table.fully_qualified_name}` TABLESAMPLE SYSTEM (0.0001 PERCENT) WHERE timestamp IS NOT NULL
+ SELECT 1 FROM {_quote_bigquery_identifier(table.fully_qualified_name)} TABLESAMPLE SYSTEM (0.0001 PERCENT) WHERE timestamp IS NOT NULL
"""
if table.time_partitioning is not None and table.time_partitioning.field == "timestamp":
@@ -802,7 +818,7 @@ class BigQueryClient:
else:
query = f"""
- SELECT 1 FROM `{table.fully_qualified_name}` TABLESAMPLE SYSTEM (0.0001 PERCENT) LIMIT 1
+ SELECT 1 FROM {_quote_bigquery_identifier(table.fully_qualified_name)} TABLESAMPLE SYSTEM (0.0001 PERCENT) LIMIT 1
"""
try:
@@ -875,7 +891,7 @@ class BigQueryClient:
stage: list[BigQueryTable],
):
"""Insert data from one or more `stage` tables into `final`."""
- into_table_fields = ",".join(f"`{field.name}`" for field in final.fields)
+ into_table_fields = ",".join(_quote_bigquery_identifier(field.name) for field in final.fields)
fields_to_cast = {
field.name
@@ -894,7 +910,7 @@ class BigQueryClient:
REGEXP_REPLACE(
REGEXP_REPLACE(
REGEXP_REPLACE(
- `{field.name}`,
+ {_quote_bigquery_identifier(field.name)},
r'(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}}\\\\u[dD][c-fC-F][0-9A-Fa-f]{{2}})|(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}})',
'\\\\1'
),
@@ -908,16 +924,17 @@ class BigQueryClient:
)
"""
if field.name in fields_to_cast
- else f"`{field.name}`"
+ else _quote_bigquery_identifier(field.name)
for field in final.fields
)
query = f"""
- INSERT INTO `{final.fully_qualified_name}`
+ INSERT INTO {_quote_bigquery_identifier(final.fully_qualified_name)}
({into_table_fields})
"""
query += "\nUNION ALL\n".join(
- f"SELECT {stage_table_fields} FROM `{table.fully_qualified_name}`" for table in stage
+ f"SELECT {stage_table_fields} FROM {_quote_bigquery_identifier(table.fully_qualified_name)}"
+ for table in stage
)
self.logger.info(
@@ -954,15 +971,18 @@ class BigQueryClient:
for n, field_name in enumerate(final.primary_key):
if n > 0:
merge_condition += " AND "
- merge_condition += f"final.`{field_name}` = stage.`{field_name}`"
+ quoted_field = _quote_bigquery_identifier(field_name)
+ merge_condition += f"final.{quoted_field} = stage.{quoted_field}"
- update_condition = "AND ("
+ update_condition = ""
for index, field_name in enumerate(final.version_key):
if index > 0:
update_condition += " OR "
- update_condition += f"final.`{field_name}` < stage.`{field_name}`"
- update_condition += ")"
+ quoted_field = _quote_bigquery_identifier(field_name)
+ update_condition += f"final.{quoted_field} < stage.{quoted_field}"
+ if update_condition:
+ update_condition = f"AND ({update_condition})"
update_clause = ""
values = ""
@@ -985,7 +1005,7 @@ class BigQueryClient:
REGEXP_REPLACE(
REGEXP_REPLACE(
REGEXP_REPLACE(
- stage.`{field.name}`,
+ stage.{_quote_bigquery_identifier(field.name)},
r'(\\\\u[dD][89A-Ba-b][0-9A-Fa-f]{{2}}\\\\u[dD][c-fC-F][0-9A-Fa-f]{{2}})|(\\\\u[dD][89A-Fa-f][0-9A-Fa-f]{{2}})',
'\\\\1'
),
@@ -999,24 +1019,31 @@ class BigQueryClient:
)
"""
if field.name in fields_to_cast
- else f"stage.`{field.name}`"
+ else f"stage.{_quote_bigquery_identifier(field.name)}"
)
- update_clause += f"final.`{field.name}` = {stage_field}"
- field_names += f"`{field.name}`"
+ update_clause += f"final.{_quote_bigquery_identifier(field.name)} = {stage_field}"
+ field_names += _quote_bigquery_identifier(field.name)
values += stage_field
if not update_clause:
raise ValueError("Empty update clause")
- inner_union_query = "\nUNION ALL\n".join(f"SELECT * FROM `{table.fully_qualified_name}`" for table in stage)
+ inner_union_query = "\nUNION ALL\n".join(
+ f"SELECT * FROM {_quote_bigquery_identifier(table.fully_qualified_name)}" for table in stage
+ )
+ order_by = (
+ "ORDER BY " + ",".join(f"{_quote_bigquery_identifier(field_name)} DESC" for field_name in final.version_key)
+ if final.version_key
+ else ""
+ )
union_query = f"""
SELECT
*,
ROW_NUMBER() OVER (
- PARTITION BY {",".join(field_name for field_name in final.primary_key)}
- ORDER BY {",".join(f"{field_name} DESC" for field_name in final.version_key)}
+ PARTITION BY {",".join(_quote_bigquery_identifier(field_name) for field_name in final.primary_key)}
+ {order_by}
) row_num
FROM (
{inner_union_query}
@@ -1024,7 +1051,7 @@ class BigQueryClient:
"""
merge_query = f"""
- MERGE `{final.fully_qualified_name}` final
+ MERGE {_quote_bigquery_identifier(final.fully_qualified_name)} final
USING (
SELECT * FROM
(
@@ -1427,31 +1454,6 @@ async def merge_all_consumer_tables(
)
-class MergeSettings(typing.NamedTuple):
- primary_key: collections.abc.Sequence[str]
- version_key: collections.abc.Sequence[str]
-
-
-def _get_merge_settings(
- model: BatchExportModel | None,
-) -> MergeSettings | None:
- """Return merge settings for models that require merging."""
- if isinstance(model, BatchExportModel):
- if model.name == "persons":
- primary_key: collections.abc.Sequence[str] = ("team_id", "distinct_id")
- version_key: collections.abc.Sequence[str] = ("person_version", "person_distinct_id_version")
- elif model.name == "sessions":
- primary_key = ("team_id", "session_id")
- version_key = ("end_timestamp",)
- # TODO: Support merges in 'events'?
- else:
- return None
- else:
- return None
-
- return MergeSettings(primary_key, version_key)
-
-
@dataclasses.dataclass(frozen=False, kw_only=True)
class BigQueryInsertInputs(BatchExportInsertInputs):
"""Inputs for BigQuery."""
@@ -1528,7 +1530,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)
)
async with Heartbeater():
- model = inputs.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -1569,14 +1571,14 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)
else:
json_fields = set()
- merge_settings = _get_merge_settings(model)
+ requires_merge = model.incremental_mode == IncrementalMode.MERGE
target_table = BigQueryTable.from_arrow_schema(
record_batch_schema,
table_id=inputs.table_id,
project_id=project_id,
dataset_id=inputs.dataset_id,
- primary_key=merge_settings.primary_key if merge_settings else (),
- version_key=merge_settings.version_key if merge_settings else (),
+ primary_key=model.primary_key if requires_merge else (),
+ version_key=model.version_key if requires_merge else (),
)
data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")
@@ -1697,7 +1699,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)
run_consumer(
client=bq_client,
consumer_table=consumer_table,
- model=model.name if isinstance(model, BatchExportModel) else "events",
+ model=model.name,
file_format=file_format,
queue=queue,
transformer=transformer,
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index fb4f6ae54587..3f77faefad6c 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -29,11 +29,14 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsModel,
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
DatabricksBatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -223,6 +226,10 @@ class DatabricksInsertInputs(BatchExportInsertInputs):
use_automatic_schema_evolution: bool = True
+def _quote_databricks_identifier(identifier: str) -> str:
+ return "`" + identifier.replace("`", "``") + "`"
+
+
class DatabricksClient:
# How often to poll for query status. This is a trade-off between responsiveness and number of
# queries we make to Databricks. 1 second has been chosen rather arbitrarily.
@@ -637,7 +644,7 @@ class DatabricksClient:
async def acreate_table(self, table_name: str, fields: list[DatabricksField]):
"""Asynchronously create the Databricks delta table if it doesn't exist."""
timeout = FIVE_MINUTES
- field_ddl = ", ".join(f"`{field[0]}` {field[1]}" for field in fields)
+ field_ddl = ", ".join(f"{_quote_databricks_identifier(field[0])} {field[1]}" for field in fields)
query = f"""
CREATE TABLE IF NOT EXISTS `{table_name}` (
{field_ddl}
@@ -710,14 +717,15 @@ class DatabricksClient:
"""
select_fields = []
for field in fields:
+ quoted_field = _quote_databricks_identifier(field[0])
if field[1] == "VARIANT":
- select_fields.append(f"PARSE_JSON(`{field[0]}`) as `{field[0]}`")
+ select_fields.append(f"PARSE_JSON({quoted_field}) as {quoted_field}")
elif field[1] == "BIGINT":
- select_fields.append(f"CAST(`{field[0]}` as BIGINT) as `{field[0]}`")
+ select_fields.append(f"CAST({quoted_field} as BIGINT) as {quoted_field}")
elif field[1] == "INTEGER":
- select_fields.append(f"CAST(`{field[0]}` as INTEGER) as `{field[0]}`")
+ select_fields.append(f"CAST({quoted_field} as INTEGER) as {quoted_field}")
else:
- select_fields.append(f"`{field[0]}`")
+ select_fields.append(quoted_field)
select_fields_str = ", ".join(select_fields)
merge_schema = f"true" if with_schema_evolution else "false"
@@ -823,7 +831,6 @@ class DatabricksClient:
"""
assert merge_key, "Merge key must be defined"
- assert update_key, "Update key must be defined"
if with_schema_evolution is True:
self.logger.info(
@@ -880,14 +887,20 @@ class DatabricksClient:
merge_key: collections.abc.Iterable[str],
update_key: collections.abc.Iterable[str],
) -> str:
- merge_condition = " AND ".join([f"target.`{field}` = source.`{field}`" for field in merge_key])
- update_condition = " OR ".join([f"target.`{field}` < source.`{field}`" for field in update_key])
+ merge_condition = " AND ".join(
+ f"target.{quoted} = source.{quoted}" for quoted in map(_quote_databricks_identifier, merge_key)
+ )
+ update_condition = " OR ".join(
+ f"target.{quoted} < source.{quoted}" for quoted in map(_quote_databricks_identifier, update_key)
+ )
+ if update_condition:
+ update_condition = f" AND ({update_condition})"
return f"""
MERGE WITH SCHEMA EVOLUTION INTO `{target_table}` AS target
USING `{source_table}` AS source
ON {merge_condition}
- WHEN MATCHED AND ({update_condition}) THEN
+ WHEN MATCHED{update_condition} THEN
UPDATE SET *
WHEN NOT MATCHED THEN
INSERT *
@@ -902,27 +915,41 @@ class DatabricksClient:
source_table_fields: collections.abc.Iterable[DatabricksField],
target_table_field_names: list[str],
) -> str:
- merge_condition = " AND ".join([f"target.`{field}` = source.`{field}`" for field in merge_key])
- update_condition = " OR ".join([f"target.`{field}` < source.`{field}`" for field in update_key])
+ merge_condition = " AND ".join(
+ f"target.{quoted} = source.{quoted}" for quoted in map(_quote_databricks_identifier, merge_key)
+ )
+ update_condition = " OR ".join(
+ f"target.{quoted} < source.{quoted}" for quoted in map(_quote_databricks_identifier, update_key)
+ )
+ if update_condition:
+ update_condition = f" AND ({update_condition})"
update_clause = ", ".join(
[
- f"target.`{field[0]}` = source.`{field[0]}`"
+ f"target.{_quote_databricks_identifier(field[0])} = source.{_quote_databricks_identifier(field[0])}"
for field in source_table_fields
if field[0] in target_table_field_names
]
)
field_names = ", ".join(
- [f"`{field[0]}`" for field in source_table_fields if field[0] in target_table_field_names]
+ [
+ _quote_databricks_identifier(field[0])
+ for field in source_table_fields
+ if field[0] in target_table_field_names
+ ]
)
values = ", ".join(
- [f"source.`{field[0]}`" for field in source_table_fields if field[0] in target_table_field_names]
+ [
+ f"source.{_quote_databricks_identifier(field[0])}"
+ for field in source_table_fields
+ if field[0] in target_table_field_names
+ ]
)
return f"""
MERGE INTO `{target_table}` AS target
USING `{source_table}` AS source
ON {merge_condition}
- WHEN MATCHED AND ({update_condition}) THEN
+ WHEN MATCHED{update_condition} THEN
UPDATE SET
{update_clause}
WHEN NOT MATCHED THEN
@@ -1035,7 +1062,7 @@ class TableSettings(t.NamedTuple):
def _get_databricks_table_settings(
- model: BatchExportModel | None,
+ model: AnyBatchExportModel,
record_batch_schema: pa.Schema,
use_variant_type: bool,
) -> TableSettings:
@@ -1058,7 +1085,7 @@ def _get_databricks_table_settings(
json_type = "STRING"
known_variant_columns = []
- if model is None or (isinstance(model, BatchExportModel) and model.name == "events" and model.schema is None):
+ if isinstance(model, BatchExportEventsModel):
table_fields = _events_table_fields(json_type)
# Staging and copying are two activities, so a deploy of a schema change between them could
@@ -1083,24 +1110,6 @@ def _get_databricks_table_settings(
return TableSettings(table_fields, record_batch_schema, known_variant_columns)
-def _get_databricks_merge_config(
- model: BatchExportModel | None,
-) -> tuple[bool, list[str], list[str]]:
- requires_merge = False
- merge_key = []
- update_key = []
- if isinstance(model, BatchExportModel):
- if model.name == "persons":
- requires_merge = True
- merge_key = ["team_id", "distinct_id"]
- update_key = ["person_version", "person_distinct_id_version"]
- elif model.name == "sessions":
- requires_merge = True
- merge_key = ["team_id", "session_id"]
- update_key = ["end_timestamp"]
- return requires_merge, merge_key, update_key
-
-
async def _get_databricks_integration(inputs: DatabricksInsertInputs) -> DatabricksIntegration:
"""Get the Databricks integration.
@@ -1300,7 +1309,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInp
)
async with Heartbeater():
- model = inputs.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_DATABRICKS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -1330,7 +1339,9 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInp
use_variant_type=inputs.use_variant_type,
)
- requires_merge, merge_key, update_key = _get_databricks_merge_config(model=model)
+ requires_merge = model.incremental_mode == IncrementalMode.MERGE
+ merge_key = list(model.primary_key) if requires_merge else []
+ update_key = list(model.version_key) if requires_merge else []
data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")
# include attempt in the stage table & volume names to avoid collisions if multiple attempts are running at the
@@ -1375,7 +1386,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInp
consumer = DatabricksConsumer(
client=databricks_client,
volume_path=volume_path,
- model=model.name if isinstance(model, BatchExportModel) else "events",
+ model=model.name,
)
transformer: ChunkTransformerProtocol = ParquetStreamTransformer(
diff --git a/products/batch_exports/backend/temporal/destinations/http_batch_export.py b/products/batch_exports/backend/temporal/destinations/http_batch_export.py
index 91cd13efdddc..ac0251c22f9d 100644
--- a/products/batch_exports/backend/temporal/destinations/http_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/http_batch_export.py
@@ -18,7 +18,14 @@ from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import get_logger
from products.batch_exports.backend.models.batch_export import BatchExportRun
-from products.batch_exports.backend.service import BatchExportField, BatchExportInsertInputs, HttpBatchExportInputs
+from products.batch_exports.backend.service import (
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportField,
+ BatchExportInsertInputs,
+ HttpBatchExportInputs,
+ migrate_batch_export_model,
+)
from products.batch_exports.backend.temporal.batch_exports import (
FinishBatchExportRunInputs,
StartBatchExportRunInputs,
@@ -203,11 +210,11 @@ async def insert_into_http_activity(inputs: HttpInsertInputs) -> BatchExportResu
if not await client.is_alive():
raise ConnectionError("Cannot establish connection to ClickHouse")
- if inputs.batch_export_model is not None:
- if inputs.batch_export_model.name != "events":
- raise NotImplementedError("HTTP export only supports the events model")
- if inputs.batch_export_model.schema is not None:
- raise NotImplementedError("HTTP export does not support schemas")
+ model = migrate_batch_export_model(inputs.batch_export_model)
+ if isinstance(model, BatchExportEventsCustomModel):
+ raise NotImplementedError("HTTP export does not support schemas")
+ if not isinstance(model, BatchExportEventsModel):
+ raise NotImplementedError("HTTP export only supports the events model")
use_native_schema = await database_sync_to_async(use_new_events_schema)(inputs.team_id)
@@ -228,8 +235,8 @@ async def insert_into_http_activity(inputs: HttpInsertInputs) -> BatchExportResu
fields = http_default_fields(native_source)
columns = [field["alias"] for field in fields]
- filters = inputs.batch_export_model.filters if inputs.batch_export_model is not None else None
- if filters is not None and len(filters) > 0:
+ filters = model.filters
+ if filters:
filters_str, extra_query_parameters = await database_sync_to_async(compose_filters_clause)(
filters, team_id=inputs.team_id, values=None
)
diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
index 6848c05f602e..70e513be3fcc 100644
--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
@@ -33,11 +33,14 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
+ BatchExportEventsModel,
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
PostgresBatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -399,7 +402,7 @@ class PostgreSQLClient:
table_name: str,
fields: Fields,
exists_ok: bool = True,
- primary_key: Fields | None = None,
+ primary_key: collections.abc.Sequence[str] | None = None,
log_statements: bool = False,
) -> None:
"""Create a table in PostgreSQL.
@@ -424,7 +427,7 @@ class PostgreSQLClient:
if primary_key is not None:
primary_key_clause = sql.SQL(", PRIMARY KEY ({fields})").format(
- fields=sql.SQL(",").join(sql.Identifier(field[0]) for field in primary_key)
+ fields=sql.SQL(",").join(sql.Identifier(field) for field in primary_key)
)
async with self.connection.transaction():
@@ -499,7 +502,7 @@ class PostgreSQLClient:
schema: str,
table_name: str,
fields: Fields,
- primary_key: Fields | None = None,
+ primary_key: collections.abc.Sequence[str] | None = None,
exists_ok: bool = True,
not_found_ok: bool = True,
delete: bool = True,
@@ -537,18 +540,19 @@ class PostgreSQLClient:
def _build_merge_query(
final_table: sql.Identifier,
stage_table: sql.Identifier,
- merge_key: Fields,
- update_key: Fields,
+ merge_key: collections.abc.Sequence[str],
+ update_key: collections.abc.Sequence[str],
update_when_matched: Fields,
) -> sql.Composed:
- merge_key_names = _join_field_names(merge_key)
+ merge_key_names = sql.SQL(",").join(sql.Identifier(field) for field in merge_key)
update_condition = sql.SQL(" OR ").join(
sql.SQL("EXCLUDED.{stage_field} > final.{final_field}").format(
- final_field=sql.Identifier(field[0]),
- stage_field=sql.Identifier(field[0]),
+ final_field=sql.Identifier(field),
+ stage_field=sql.Identifier(field),
)
for field in update_key
)
+ update_filter = sql.SQL("WHERE ({})").format(update_condition) if update_key else sql.SQL("")
update_clause = sql.SQL(",").join(
sql.SQL("{final_field} = EXCLUDED.{stage_field}").format(
final_field=sql.Identifier(field[0]),
@@ -564,14 +568,14 @@ class PostgreSQLClient:
ORDER BY {order_by}
ON CONFLICT ({conflict_fields}) DO UPDATE SET
{update_clause}
- WHERE ({update_condition})
+ {update_filter}
"""
).format(
final_table=final_table,
order_by=merge_key_names,
conflict_fields=merge_key_names,
stage_table=stage_table,
- update_condition=update_condition,
+ update_filter=update_filter,
update_clause=update_clause,
field_names=_join_field_names(update_when_matched),
)
@@ -606,8 +610,8 @@ class PostgreSQLClient:
final_table_name: str,
stage_table_name: str,
schema: str,
- merge_key: Fields,
- update_key: Fields,
+ merge_key: collections.abc.Sequence[str],
+ update_key: collections.abc.Sequence[str],
update_when_matched: Fields,
timeout: float | int | None = None,
lock_timeout: float | int = MERGE_LOCK_TIMEOUT_SECONDS,
@@ -642,7 +646,7 @@ class PostgreSQLClient:
async with asyncio.timeout(timeout):
await merge_with_retries(merge_query, schema, lock_timeout)
except psycopg.errors.InvalidColumnReference:
- raise MissingPrimaryKeyError(final_table, _join_field_names(merge_key))
+ raise MissingPrimaryKeyError(final_table, sql.SQL(",").join(sql.Identifier(field) for field in merge_key))
except TimeoutError as e:
self.external_logger.exception(
"Final merge into '%s.%s' is taking too long to complete and will be rolled-back. Perhaps the database is under too much load?",
@@ -861,11 +865,11 @@ class PostgreSQLConsumer(Consumer):
def _get_table_fields(
- model: BatchExportModel | None,
+ model: AnyBatchExportModel,
record_batch_schema: pa.Schema,
) -> Fields:
"""Extract table field definitions from model and schema."""
- if model is None or model.name == "events":
+ if model.name == "events":
table_fields: Fields = [
("uuid", "VARCHAR(200)"),
("event", "VARCHAR(200)"),
@@ -890,48 +894,6 @@ def _get_table_fields(
)
-class MergeSettings(typing.NamedTuple):
- requires_merge: bool
- merge_key: Fields
- update_key: Fields
- primary_key: Fields | None
-
-
-def _get_merge_settings(
- model: BatchExportModel | None,
-) -> MergeSettings:
- requires_merge = False
- merge_key: Fields = []
- update_key: Fields = []
- primary_key: Fields | None = None
-
- if isinstance(model, BatchExportModel):
- if model.name == "persons":
- requires_merge = True
- merge_key = [
- ("team_id", "INT"),
- ("distinct_id", "TEXT"),
- ]
- update_key = [
- ("person_version", "INT"),
- ("person_distinct_id_version", "INT"),
- ]
- primary_key = (("team_id", "INTEGER"), ("distinct_id", "VARCHAR(200)"))
-
- elif model.name == "sessions":
- requires_merge = True
- merge_key = [
- ("team_id", "INT"),
- ("session_id", "TEXT"),
- ]
- update_key = [
- ("end_timestamp", "TIMESTAMP"),
- ]
- primary_key = (("team_id", "INTEGER"), ("session_id", "TEXT"))
-
- return MergeSettings(requires_merge, merge_key, update_key, primary_key)
-
-
class PostgreSQLIntegrationNotFoundError(Exception):
"""Error raised when the PostgreSQL integration is not found."""
@@ -976,7 +938,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
)
async with Heartbeater():
- model = inputs.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_POSTGRES_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -1004,7 +966,8 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
)
table_fields = _get_table_fields(model, record_batch_schema)
- merge_settings = _get_merge_settings(model)
+ requires_merge = model.incremental_mode == IncrementalMode.MERGE
+ primary_key = model.primary_key if requires_merge else None
data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")
@@ -1014,7 +977,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
# TODO: That should be enough, but we should add a proper check and alert on larger inputs.
stage_table_name = (
f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"
- if merge_settings.requires_merge
+ if requires_merge
else inputs.table_name
)[:63]
@@ -1037,9 +1000,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
f"No matching columns found in the destination table '{inputs.schema}.{inputs.table_name}'"
)
except psycopg.errors.InsufficientPrivilege:
- if model is None or (
- isinstance(model, BatchExportModel) and model.name == "events" and model.schema is None
- ):
+ if isinstance(model, BatchExportEventsModel):
# Without introspection, do not require this column on older destination tables.
table_fields = [field for field in table_fields if field[0] != "person_id"]
external_logger.warning(
@@ -1066,24 +1027,24 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
table_fields,
create=not table_exists,
delete=False,
- primary_key=merge_settings.primary_key,
+ primary_key=primary_key,
log_statements=True,
) as pg_table,
pg_client.managed_table(
inputs.schema,
stage_table_name,
table_fields,
- create=merge_settings.requires_merge,
- delete=merge_settings.requires_merge,
- primary_key=merge_settings.primary_key,
+ create=requires_merge,
+ delete=requires_merge,
+ primary_key=primary_key,
) as pg_stage_table,
):
consumer = PostgreSQLConsumer(
client=pg_client,
schema=inputs.schema,
- table_name=pg_stage_table if merge_settings.requires_merge else pg_table,
+ table_name=pg_stage_table if requires_merge else pg_table,
schema_columns=schema_columns,
- model=model.name if isinstance(model, BatchExportModel) else "events",
+ model=model.name,
)
transformer = CSVStreamTransformer(
@@ -1119,7 +1080,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
raise
finally:
- if merge_settings.requires_merge:
+ if requires_merge:
merge_query_timeout = get_query_timeout(
dt.datetime.fromisoformat(inputs.data_interval_start)
if inputs.data_interval_start
@@ -1132,8 +1093,8 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)
stage_table_name=pg_stage_table,
schema=inputs.schema,
update_when_matched=table_fields,
- merge_key=merge_settings.merge_key,
- update_key=merge_settings.update_key,
+ merge_key=model.primary_key,
+ update_key=model.version_key,
timeout=merge_query_timeout,
)
diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
index 79ee68d8ccec..e4366943d84f 100644
--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
@@ -38,14 +38,16 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.service import (
+ AnyBatchExportModel,
AWSCredentials,
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
IAMRole,
IntegrationID,
RedshiftBatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -303,8 +305,8 @@ class RedshiftClient(PostgreSQLClient):
final_table_name: str,
stage_table_name: str,
schema: str,
- merge_key: Fields,
- update_key: Fields,
+ merge_key: collections.abc.Sequence[str],
+ update_key: collections.abc.Sequence[str],
final_table_fields: Fields,
stage_fields_cast_to_super: collections.abc.Container[str] | None = None,
remove_duplicates: bool = True,
@@ -326,8 +328,8 @@ class RedshiftClient(PostgreSQLClient):
and_separator = sql.SQL("AND")
merge_condition = and_separator.join(
sql.SQL("{final_field} = {stage_field}").format(
- final_field=sql.Identifier(schema, final_table_name, field[0]),
- stage_field=sql.Identifier("stage", field[0]),
+ final_field=sql.Identifier(schema, final_table_name, field),
+ stage_field=sql.Identifier("stage", field),
)
for field in merge_key
)
@@ -338,8 +340,8 @@ class RedshiftClient(PostgreSQLClient):
# versions back.
delete_condition = and_separator.join(
sql.SQL("{final_field} = {stage_field}").format(
- final_field=sql.Identifier("final", field[0]),
- stage_field=sql.Identifier(schema, stage_table_name, field[0]),
+ final_field=sql.Identifier("final", field),
+ stage_field=sql.Identifier(schema, stage_table_name, field),
)
for field in merge_key
)
@@ -347,8 +349,8 @@ class RedshiftClient(PostgreSQLClient):
or_separator = sql.SQL(" OR ")
delete_extra_conditions = or_separator.join(
sql.SQL("{stage_field} < {final_field}").format(
- final_field=sql.Identifier("final", field[0]),
- stage_field=sql.Identifier(schema, stage_table_name, field[0]),
+ final_field=sql.Identifier("final", field),
+ stage_field=sql.Identifier(schema, stage_table_name, field),
)
for field in update_key
)
@@ -428,7 +430,7 @@ class RedshiftClient(PostgreSQLClient):
self.logger.info("Table locked")
try:
- if not skip_delete:
+ if update_key and not skip_delete:
await cursor.execute(delete_query)
except psycopg.errors.UndefinedFunction:
self.logger.exception(
@@ -810,7 +812,7 @@ class TableSchemas(typing.NamedTuple):
def _get_table_schemas(
- model: BatchExportModel | None,
+ model: AnyBatchExportModel,
record_batch_schema: pa.Schema,
properties_data_type: str,
) -> TableSchemas:
@@ -824,7 +826,7 @@ def _get_table_schemas(
properties_type = "VARCHAR(65535)"
use_super = False
- if model is None or model.name == "events":
+ if model.name == "events":
table_schema: Fields = [
("uuid", "VARCHAR(200)"),
("event", "VARCHAR(200)"),
@@ -862,9 +864,9 @@ def _get_table_schemas(
class RequiredMergeSettings(typing.NamedTuple):
requires_merge: typing.Literal[True]
- merge_key: Fields
- update_key: Fields
- primary_key: Fields
+ merge_key: collections.abc.Sequence[str]
+ update_key: collections.abc.Sequence[str]
+ primary_key: collections.abc.Sequence[str]
class NotRequiredMergeSettings(typing.NamedTuple):
@@ -875,49 +877,20 @@ MergeSettings = RequiredMergeSettings | NotRequiredMergeSettings
def _get_merge_settings(
- model: BatchExportModel | None,
+ model: AnyBatchExportModel,
use_super: bool = False,
) -> MergeSettings:
"""Return merge settings for models that require merging."""
- merge_key: Fields
- update_key: Fields
- primary_key: Fields
-
- if isinstance(model, BatchExportModel) and model.name == "persons":
- merge_key = [
- ("team_id", "INT"),
- ("distinct_id", "TEXT"),
- ]
- update_key = [
- ("person_version", "INT"),
- ("person_distinct_id_version", "INT"),
- ]
- primary_key = (("team_id", "INTEGER"), ("distinct_id", "VARCHAR(200)"))
-
- return RequiredMergeSettings(
- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key
- )
-
- elif isinstance(model, BatchExportModel) and model.name == "sessions":
- merge_key = [
- ("team_id", "INT"),
- ("session_id", "TEXT"),
- ]
- update_key = [
- ("end_timestamp", "TIMESTAMP"),
- ]
- primary_key = (("team_id", "INTEGER"), ("session_id", "TEXT"))
-
+ if model.incremental_mode == IncrementalMode.MERGE:
return RequiredMergeSettings(
- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key
+ requires_merge=True,
+ merge_key=model.primary_key,
+ update_key=model.version_key,
+ primary_key=model.primary_key,
)
- elif (model is None or model.name == "events") and use_super is True:
- merge_key = [("uuid", "TEXT")]
- update_key = [("timestamp", "TIMESTAMP")]
- primary_key = [("uuid", "TEXT")]
-
+ elif model.name == "events" and use_super is True:
return RequiredMergeSettings(
- requires_merge=True, merge_key=merge_key, update_key=update_key, primary_key=primary_key
+ requires_merge=True, merge_key=("uuid",), update_key=("timestamp",), primary_key=("uuid",)
)
else:
return NotRequiredMergeSettings(requires_merge=False)
@@ -1302,7 +1275,7 @@ async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs)
)
async with Heartbeater():
- model = inputs.batch_export.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_REDSHIFT_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -1818,7 +1791,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInp
)
async with Heartbeater():
- model = inputs.batch_export.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -2035,7 +2008,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInp
update_key=merge_settings.update_key,
stage_fields_cast_to_super=table_schemas.super_columns if table_schemas.use_super else None,
remove_duplicates=remove_duplicates,
- skip_delete=isinstance(model, BatchExportModel)
+ skip_delete=inputs.batch_export.batch_export_model is not None
and model.name == "events"
and str(inputs.batch_export.team_id) in settings.BATCH_EXPORT_REDSHIFT_SKIP_DELETE_TEAM_IDS,
)
diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
index bf95de0d8f56..1c816ebe2f2c 100644
--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py
@@ -43,8 +43,9 @@ from products.batch_exports.backend.service import (
AWSCredentials,
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
+ BatchExportModelInput,
S3BatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -852,7 +853,7 @@ class ConcurrentS3Consumer(Consumer):
prefix: str,
data_interval_start: str | None,
data_interval_end: str | None,
- batch_export_model: BatchExportModel | None,
+ batch_export_model: BatchExportModelInput | None,
file_format: str,
checksum_algorithm: str | None = None,
kms_key_id: str | None = None,
@@ -865,7 +866,8 @@ class ConcurrentS3Consumer(Consumer):
max_concurrent_uploads: int = 5,
file_name_prefix: str | None = None,
):
- super().__init__(model=batch_export_model.name if batch_export_model else "events")
+ model = migrate_batch_export_model(batch_export_model)
+ super().__init__(model=model.name)
self.bucket = bucket
self.region_name = region_name
@@ -874,7 +876,7 @@ class ConcurrentS3Consumer(Consumer):
self.data_interval_start = data_interval_start
self.data_interval_end = data_interval_end
self.file_name_prefix = file_name_prefix
- self.batch_export_model = batch_export_model
+ self.batch_export_model = model
self.checksum_algorithm = checksum_algorithm
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index c126a956c863..9aa5c1cb8cc1 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -30,11 +30,12 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
+from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.service import (
BatchExportField,
BatchExportInsertInputs,
- BatchExportModel,
SnowflakeBatchExportInputs,
+ migrate_batch_export_model,
)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
@@ -548,6 +549,10 @@ def _is_connect_error_retryable(err: Exception) -> bool:
return isinstance(err, OperationalError) and err.errno in (ER_FAILED_TO_CONNECT_TO_DB, ER_RETRYABLE_CODE)
+def _quote_snowflake_identifier(identifier: str) -> str:
+ return '"' + identifier.replace('"', '""') + '"'
+
+
class SnowflakeClient:
"""Snowflake connection client used in batch exports."""
@@ -953,7 +958,9 @@ class SnowflakeClient:
Whether to include an 'IF NOT EXISTS' clause in the query such that it
won't fail if the table already exists.
"""
- field_ddl = ", ".join(f'"{field.name}" {field.snowflake_type_name}' for field in table)
+ field_ddl = ", ".join(
+ f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_name}" for field in table
+ )
if_not_exists = ""
if exists_ok:
@@ -1086,13 +1093,13 @@ class SnowflakeClient:
perform this operation.
"""
select_fields = ", ".join(
- f"PARSE_JSON($1:\"{field.name}\", 'd')"
+ f"PARSE_JSON($1:{_quote_snowflake_identifier(field.name)}, 'd')"
if field.data_type == JsonType() and field.name.lower() != "elements"
- else f'$1:"{field.name}"'
+ else f"$1:{_quote_snowflake_identifier(field.name)}"
for field in table
)
query = f"""
- COPY INTO "{table.name}" ({", ".join(f'"{field.name}"' for field in table)})
+ COPY INTO "{table.name}" ({", ".join(_quote_snowflake_identifier(field.name) for field in table)})
FROM (
SELECT {select_fields} FROM '@%"{table.name}"/{table.stage_prefix}'
@@ -1201,16 +1208,19 @@ class SnowflakeClient:
field = final[primary_key_field_name]
if n > 0:
merge_condition += " AND "
- merge_condition += f'final."{field.name}" = stage."{field.name}"'
+ quoted_field = _quote_snowflake_identifier(field.name)
+ merge_condition += f"final.{quoted_field} = stage.{quoted_field}"
- update_condition = "AND ("
+ update_condition = ""
for index, version_key_field_name in enumerate(final.version_key):
field = final[version_key_field_name]
if index > 0:
update_condition += " OR "
- update_condition += f'final."{field.name}" < stage."{field.name}"'
- update_condition += ")"
+ quoted_field = _quote_snowflake_identifier(field.name)
+ update_condition += f"final.{quoted_field} < stage.{quoted_field}"
+ if update_condition:
+ update_condition = f"AND ({update_condition})"
update_clause = ""
values = ""
@@ -1221,9 +1231,10 @@ class SnowflakeClient:
values += ", "
field_names += ", "
- update_clause += f'final."{field.name}" = stage."{field.name}"'
- field_names += f'"{field.name}"'
- values += f'stage."{field.name}"'
+ quoted_field = _quote_snowflake_identifier(field.name)
+ update_clause += f"final.{quoted_field} = stage.{quoted_field}"
+ field_names += quoted_field
+ values += f"stage.{quoted_field}"
merge_query = f"""
MERGE INTO "{final.name}" AS final
@@ -1277,33 +1288,6 @@ def snowflake_default_fields() -> list[BatchExportField]:
return batch_export_fields
-class MergeSettings(typing.NamedTuple):
- primary_key: collections.abc.Sequence[str]
- version_key: collections.abc.Sequence[str]
-
-
-def _get_merge_settings(
- model: BatchExportModel | None,
-) -> MergeSettings | None:
- if isinstance(model, BatchExportModel):
- if model.name == "persons":
- primary_key: collections.abc.Sequence[str] = ("team_id", "distinct_id")
- version_key: collections.abc.Sequence[str] = (
- "person_version",
- "person_distinct_id_version",
- )
-
- elif model.name == "sessions":
- primary_key = ("team_id", "session_id")
- version_key = ("end_timestamp",)
- else:
- return None
- else:
- return None
-
- return MergeSettings(primary_key, version_key)
-
-
class SnowflakeConsumer(Consumer):
"""A consumer that uploads data to Snowflake from the internal stage.
@@ -1415,7 +1399,7 @@ async def insert_into_snowflake_activity_from_stage(
integration = await _get_snowflake_integration(inputs.integration_id, inputs.team_id)
- model = inputs.batch_export_model
+ model = migrate_batch_export_model(inputs.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_SNOWFLAKE_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
@@ -1458,14 +1442,14 @@ async def insert_into_snowflake_activity_from_stage(
attempt = activity.info().attempt
stage_table_name = f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"
- merge_settings = _get_merge_settings(model)
+ requires_merge = model.incremental_mode == IncrementalMode.MERGE
target_table = SnowflakeTable.from_arrow_schema(
record_batch_schema,
name=inputs.table_name,
table_schema=inputs.schema,
database=inputs.database,
- primary_key=merge_settings.primary_key if merge_settings else (),
- version_key=merge_settings.version_key if merge_settings else (),
+ primary_key=model.primary_key if requires_merge else (),
+ version_key=model.version_key if requires_merge else (),
stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",
)
if "elements" in target_table:
@@ -1510,7 +1494,7 @@ async def insert_into_snowflake_activity_from_stage(
consumer = SnowflakeConsumer(
snowflake_client=snow_client,
snowflake_table=snow_consumer_table,
- model=model.name if isinstance(model, BatchExportModel) else "events",
+ model=model.name,
)
transformer = PipelineTransformer(
diff --git a/products/batch_exports/backend/temporal/destinations/utils.py b/products/batch_exports/backend/temporal/destinations/utils.py
index ad231ccce4d9..dda61306662f 100644
--- a/products/batch_exports/backend/temporal/destinations/utils.py
+++ b/products/batch_exports/backend/temporal/destinations/utils.py
@@ -3,7 +3,7 @@ import posixpath
from posthog.temporal.common.logger import get_logger
-from products.batch_exports.backend.service import BatchExportModel
+from products.batch_exports.backend.service import BatchExportModelInput
from products.batch_exports.backend.temporal.destinations.constants import (
COMPRESSION_EXTENSIONS,
FILE_FORMAT_EXTENSIONS,
@@ -14,7 +14,7 @@ EXTERNAL_LOGGER = get_logger("EXTERNAL")
def get_allowed_template_variables(
- data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None
+ data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModelInput | None
) -> dict[str, str]:
"""Derive from inputs a dictionary of supported template variables for key prefixes."""
export_datetime = dt.datetime.fromisoformat(data_interval_end)
@@ -33,7 +33,10 @@ def get_allowed_template_variables(
def get_key_prefix(
- prefix: str, data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None
+ prefix: str,
+ data_interval_start: str | None,
+ data_interval_end: str,
+ batch_export_model: BatchExportModelInput | None,
) -> str:
"""Format a key prefix with template variables."""
template_variables = get_allowed_template_variables(data_interval_start, data_interval_end, batch_export_model)
@@ -48,7 +51,10 @@ def get_key_prefix(
def get_absolute_key_prefix(
- prefix: str, data_interval_start: str | None, data_interval_end: str, batch_export_model: BatchExportModel | None
+ prefix: str,
+ data_interval_start: str | None,
+ data_interval_end: str,
+ batch_export_model: BatchExportModelInput | None,
) -> str:
"""Like `get_key_prefix`, but ensure key is absolute and ends on a slash."""
key_prefix = get_key_prefix(prefix, data_interval_start, data_interval_end, batch_export_model)
@@ -66,7 +72,7 @@ def get_manifest_key(
prefix: str,
data_interval_start: str | None,
data_interval_end: str | None,
- batch_export_model: BatchExportModel | None,
+ batch_export_model: BatchExportModelInput | None,
file_name_prefix: str | None = None,
) -> str:
"""Generate manifest file key."""
@@ -99,7 +105,7 @@ def get_object_key(
prefix: str,
data_interval_start: str | None,
data_interval_end: str | None,
- batch_export_model: BatchExportModel | None,
+ batch_export_model: BatchExportModelInput | None,
file_format: str,
compression: str | None = None,
legacy_parquet_extension: bool = False,
diff --git a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py
index 3b1ccfd2a9ef..64504ee3e15d 100644
--- a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py
@@ -18,7 +18,12 @@ from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
-from products.batch_exports.backend.service import BatchExportField, BatchExportInsertInputs, WorkflowsBatchExportInputs
+from products.batch_exports.backend.service import (
+ BatchExportField,
+ BatchExportInsertInputs,
+ WorkflowsBatchExportInputs,
+ migrate_batch_export_model,
+)
from products.batch_exports.backend.temporal.batch_exports import (
StartBatchExportRunInputs,
get_data_interval,
@@ -410,6 +415,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInput
)
async with Heartbeater():
+ model = migrate_batch_export_model(inputs.batch_export.batch_export_model)
queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_WORKFLOWS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)
producer = Producer()
assert inputs.batch_export.batch_export_id is not None
@@ -458,9 +464,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInput
team_id=inputs.batch_export.team_id,
session=session,
request_task_group=tg,
- model=inputs.batch_export.batch_export_model.name
- if inputs.batch_export.batch_export_model
- else "events",
+ model=model.name,
max_concurrent_requests=settings.BATCH_EXPORT_WORKFLOWS_MAX_CONCURRENT_REQUESTS,
max_pending_requests=settings.BATCH_EXPORT_WORKFLOWS_MAX_PENDING_REQUESTS,
)
diff --git a/products/batch_exports/backend/temporal/pipeline/table.py b/products/batch_exports/backend/temporal/pipeline/table.py
index fd50369450bc..eaf9b4c1498d 100644
--- a/products/batch_exports/backend/temporal/pipeline/table.py
+++ b/products/batch_exports/backend/temporal/pipeline/table.py
@@ -425,4 +425,4 @@ class Table(TableBase, typing.Generic[FieldType]):
raise KeyError(key)
def is_mutable(self) -> bool:
- return len(self.primary_key) > 0 and len(self.version_key) > 0
+ return len(self.primary_key) > 0
diff --git a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py
index c6ff294e7ecb..1423518e66d1 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/bigquery/test_client.py
@@ -7,7 +7,7 @@ import weakref
import collections.abc
import pytest
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pyarrow as pa
import requests
@@ -219,9 +219,90 @@ async def test_create_table(fields, time_partitioning, bigquery_client, bigquery
await client.delete_table(table)
[email protected]("versioned", [True, False])
+ "key,quoted_key,version,quoted_version",
+ [
+ ("id", "`id`", "version", "`version`"),
+ ("group", "`group`", "updated at", "`updated at`"),
+ ("key`name", r"`key\`name`", "version\\name", r"`version\\name`"),
+ ("key\\`name", r"`key\\\`name`", "version\nname", r"`version\nname`"),
+ ],
+)
+async def test_merge_query_with_optional_version_key(
+ versioned: bool, key: str, quoted_key: str, version: str, quoted_version: str
+) -> None:
+ fields = (
+ BigQueryField(key, BigQueryType("INT64", False), False),
+ BigQueryField(version, BigQueryType("INT64", False), False),
+ )
+ final = BigQueryTable(
+ "final", fields, ("project", "dataset"), primary_key=(key,), version_key=(version,) if versioned else ()
+ )
+ stage = BigQueryTable("stage", fields, ("project", "dataset"))
+ client = BigQueryClient(MagicMock())
+
+ with patch.object(client, "execute_query", new_callable=AsyncMock) as execute:
+ await client.merge_tables(final, stage)
+
+ query = " ".join(execute.call_args.args[0].split())
+ assert query.startswith("MERGE `project.dataset.final`")
+ assert f"ON final.{quoted_key} = stage.{quoted_key}" in query
+ assert (
+ f"UPDATE SET final.{quoted_key} = stage.{quoted_key}, final.{quoted_version} = stage.{quoted_version}" in query
+ )
+ assert f"WHEN NOT MATCHED BY TARGET THEN INSERT ({quoted_key}, {quoted_version})" in query
+ assert f"VALUES (stage.{quoted_key}, stage.{quoted_version})" in query
+ if versioned:
+ assert f"WHEN MATCHED AND (final.{quoted_version} < stage.{quoted_version}) THEN UPDATE" in query
+ assert f"PARTITION BY {quoted_key} ORDER BY {quoted_version} DESC" in query
+ else:
+ assert "WHEN MATCHED THEN UPDATE" in query
+ assert f"ROW_NUMBER() OVER ( PARTITION BY {quoted_key} )" in query
+
+
[email protected]("merge", [True, False])
+async def test_write_query_quotes_fields_and_tables_with_json_casts(merge: bool) -> None:
+ final = BigQueryTable(
+ "final`table",
+ (
+ BigQueryField("group", BigQueryType("INT64", False), False),
+ BigQueryField("json\\`data", BigQueryType("JSON", False), True),
+ ),
+ ("project", "dataset"),
+ primary_key=("group",) if merge else (),
+ )
+ stage = BigQueryTable(
+ "stage\\`table",
+ (
+ BigQueryField("group", BigQueryType("INT64", False), False),
+ BigQueryField("json\\`data", BigQueryType("STRING", False), True),
+ ),
+ ("project", "dataset"),
+ )
+ client = BigQueryClient(MagicMock())
+ with patch.object(client, "execute_query", new_callable=AsyncMock) as execute:
+ await client.merge_tables(final, stage)
+
+ query = " ".join(execute.call_args.args[0].split())
+ assert r"`project.dataset.final\`table`" in query
+ assert r"FROM `project.dataset.stage\\\`table`" in query
+ assert r"`json\\\`data`" in query
+ prefix = "stage." if merge else ""
+ assert f"REGEXP_REPLACE( {prefix}" + r"`json\\\`data`," in query
+ if merge:
+ assert r"INSERT (`group`, `json\\\`data`)" in query
+ else:
+ assert r"(`group`,`json\\\`data`)" in query
+ assert "SELECT `group`, SAFE.PARSE_JSON(" in query
+
+
@SKIP_IF_MISSING_GOOGLE_APPLICATION_CREDENTIALS
@pytest.mark.asyncio
-async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery_dataset):
[email protected]("version_key", [("version",), ()])
+async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery_dataset, version_key):
fields = (
BigQueryField("id", BigQueryType("INT64", False), False),
BigQueryField("version", BigQueryType("INT64", False), False),
@@ -230,11 +311,11 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery
suffix = "".join(random.choices(string.ascii_letters, k=10))
parents = (bigquery_client.project, bigquery_dataset.dataset_id)
final_table = BigQueryTable(
- f"test_merge_final_{suffix}", fields, parents, primary_key=("id",), version_key=("version",)
+ f"test_merge_final_{suffix}", fields, parents, primary_key=("id",), version_key=version_key
)
stage_tables = [
BigQueryTable(
- f"test_merge_stage_{index}_{suffix}", fields, parents, primary_key=("id",), version_key=("version",)
+ f"test_merge_stage_{index}_{suffix}", fields, parents, primary_key=("id",), version_key=version_key
)
for index in range(2)
]
@@ -244,9 +325,7 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery
await client.create_table(table)
try:
- # Key 1 exists in final and in both stage tables: the highest version across all
- # stage tables must win. Key 4 is newer in final than in stage: it must not be
- # overwritten.
+ # Key 1 tests cross-stage deduplication; key 4 tests whether versions gate replacement.
await client.execute_query(
f"INSERT INTO `{final_table.fully_qualified_name}` (id, version, value)"
" VALUES (1, 1, 'final-v1'), (4, 5, 'final-v5')"
@@ -262,14 +341,17 @@ async def test_merge_tables_with_multiple_stage_tables(bigquery_client, bigquery
await client.merge_tables(final=final_table, stage=stage_tables)
- result = await client.execute_query(f"SELECT id, version, value FROM `{final_table.fully_qualified_name}`")
+ result = list(
+ await client.execute_query(f"SELECT id, version, value FROM `{final_table.fully_qualified_name}`")
+ )
rows = {row["id"]: (row["version"], row["value"]) for row in result}
+ assert len(rows) == len(result) == 4
+ assert rows.pop(1) in ({(3, "stage1-v3")} if version_key else {(2, "stage0-v2"), (3, "stage1-v3")})
assert rows == {
- 1: (3, "stage1-v3"),
2: (1, "stage0-only"),
3: (1, "stage1-only"),
- 4: (5, "final-v5"),
+ 4: (5, "final-v5") if version_key else (1, "stage0-v1"),
}
finally:
for table in (final_table, *stage_tables):
diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
index b7fd6a856982..a749f281c415 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
@@ -270,31 +270,76 @@ class TestConnect:
class TestQueryBuilders:
- async def test_merge_query_with_schema_evolution(self, client: DatabricksClient):
+ @pytest.mark.parametrize("schema_evolution", [True, False])
+ @pytest.mark.parametrize("versioned", [True, False])
+ async def test_merge_escapes_column_names(
+ self, client: DatabricksClient, schema_evolution: bool, versioned: bool
+ ) -> None:
+ with (
+ patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute,
+ patch.object(
+ client, "aget_table_columns", new_callable=AsyncMock, return_value=["key`name", "version`name"]
+ ),
+ ):
+ await client.amerge_tables(
+ target_table="test_target",
+ source_table="test_source",
+ merge_key=["key`name"],
+ update_key=["version`name"] if versioned else [],
+ source_table_fields=[("key`name", "BIGINT"), ("version`name", "BIGINT")],
+ with_schema_evolution=schema_evolution,
+ )
+
+ query = " ".join(execute.call_args.args[0].split())
+ assert "ON target.`key``name` = source.`key``name`" in query
+ if versioned:
+ assert "WHEN MATCHED AND (target.`version``name` < source.`version``name`) THEN" in query
+ else:
+ assert "WHEN MATCHED THEN" in query
+ if schema_evolution:
+ assert "UPDATE SET * WHEN NOT MATCHED THEN INSERT *" in query
+ else:
+ assert (
+ "UPDATE SET target.`key``name` = source.`key``name`, target.`version``name` = source.`version``name`"
+ in query
+ )
+ assert "INSERT (`key``name`, `version``name`) VALUES (source.`key``name`, source.`version``name`)" in query
+
+ @pytest.mark.parametrize("versioned", [True, False])
+ async def test_merge_query_with_schema_evolution(self, client: DatabricksClient, versioned: bool) -> None:
merge_key = ["team_id", "distinct_id"]
- update_key = ["person_version", "person_distinct_id_version"]
- merge_query = client._get_merge_query_with_schema_evolution(
- target_table="test_target",
- source_table="test_source",
- merge_key=merge_key,
- update_key=update_key,
+ update_key = ["person_version", "person_distinct_id_version"] if versioned else []
+ with patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute:
+ await client.amerge_tables(
+ target_table="test_target",
+ source_table="test_source",
+ merge_key=merge_key,
+ update_key=update_key,
+ source_table_fields=[],
+ )
+ merge_query = execute.call_args.args[0]
+ expected_condition = (
+ " AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`)"
+ if versioned
+ else ""
)
assert (
merge_query
- == """
+ == f"""
MERGE WITH SCHEMA EVOLUTION INTO `test_target` AS target
USING `test_source` AS source
ON target.`team_id` = source.`team_id` AND target.`distinct_id` = source.`distinct_id`
- WHEN MATCHED AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`) THEN
+ WHEN MATCHED{expected_condition} THEN
UPDATE SET *
WHEN NOT MATCHED THEN
INSERT *
"""
)
- async def test_merge_query_without_schema_evolution(self, client: DatabricksClient):
+ @pytest.mark.parametrize("versioned", [True, False])
+ async def test_merge_query_without_schema_evolution(self, client: DatabricksClient, versioned: bool) -> None:
merge_key = ["team_id", "distinct_id"]
- update_key = ["person_version", "person_distinct_id_version"]
+ update_key = ["person_version", "person_distinct_id_version"] if versioned else []
merge_query = client._get_merge_query_without_schema_evolution(
target_table="test_target",
source_table="test_source",
@@ -315,13 +360,18 @@ class TestQueryBuilders:
"properties",
],
)
+ expected_condition = (
+ " AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`)"
+ if versioned
+ else ""
+ )
assert (
merge_query
- == """
+ == f"""
MERGE INTO `test_target` AS target
USING `test_source` AS source
ON target.`team_id` = source.`team_id` AND target.`distinct_id` = source.`distinct_id`
- WHEN MATCHED AND (target.`person_version` < source.`person_version` OR target.`person_distinct_id_version` < source.`person_distinct_id_version`) THEN
+ WHEN MATCHED{expected_condition} THEN
UPDATE SET
target.`team_id` = source.`team_id`, target.`distinct_id` = source.`distinct_id`, target.`person_version` = source.`person_version`, target.`person_distinct_id_version` = source.`person_distinct_id_version`, target.`properties` = source.`properties`
WHEN NOT MATCHED THEN
@@ -376,12 +426,17 @@ class TestQueryBuilders:
"""
)
- async def test_copy_into_table_from_volume_query(self, client: DatabricksClient):
+ @pytest.mark.parametrize(
+ "json_field,quoted_json_field", [("properties", "`properties`"), ("json`data", "`json``data`")]
+ )
+ async def test_copy_into_table_from_volume_query(
+ self, client: DatabricksClient, json_field: str, quoted_json_field: str
+ ) -> None:
"""Test that we construct the correct SQL for COPY INTO from a volume, including VARIANT/BIGINT casts."""
fields = [
("uuid", "STRING"),
("event", "STRING"),
- ("properties", "VARIANT"),
+ (json_field, "VARIANT"),
("distinct_id", "STRING"),
("team_id", "BIGINT"),
("timestamp", "TIMESTAMP"),
@@ -394,10 +449,10 @@ class TestQueryBuilders:
)
assert (
query
- == """
+ == f"""
COPY INTO `test_table`
FROM (
- SELECT `uuid`, `event`, PARSE_JSON(`properties`) as `properties`, `distinct_id`, CAST(`team_id` as BIGINT) as `team_id`, `timestamp`, `databricks_ingested_timestamp` FROM '/Volumes/my_volume/path/file.parquet'
+ SELECT `uuid`, `event`, PARSE_JSON({quoted_json_field}) as {quoted_json_field}, `distinct_id`, CAST(`team_id` as BIGINT) as `team_id`, `timestamp`, `databricks_ingested_timestamp` FROM '/Volumes/my_volume/path/file.parquet'
)
FILEFORMAT = PARQUET
COPY_OPTIONS ('force' = 'true', 'mergeSchema' = 'true')
diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
index e446e4f66ee2..20418c99ef50 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_utils.py
@@ -4,7 +4,11 @@ import pytest
import pyarrow as pa
-from products.batch_exports.backend.service import BatchExportModel, BatchExportSchema
+from products.batch_exports.backend.service import (
+ BatchExportEventsCustomModel,
+ BatchExportEventsModel,
+ BatchExportSchema,
+)
from products.batch_exports.backend.temporal.destinations.databricks_batch_export import (
_events_table_fields,
_get_databricks_table_settings,
@@ -37,7 +41,7 @@ def _staged_events_schema(column_names: list[str]) -> pa.Schema:
def _events_table_field_names(staged_column_names: list[str], schema: BatchExportSchema | None) -> list[str]:
table_fields, _, _ = _get_databricks_table_settings(
- model=BatchExportModel(name="events", schema=schema),
+ model=BatchExportEventsCustomModel(schema=schema) if schema is not None else BatchExportEventsModel(),
record_batch_schema=_staged_events_schema(staged_column_names),
use_variant_type=True,
)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.py
new file mode 100644
index 000000000000..f354ceb07220
--- /dev/null
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_merge.py
@@ -0,0 +1,32 @@
+import pytest
+from unittest.mock import AsyncMock, MagicMock
+
+from products.batch_exports.backend.temporal.destinations.postgres_batch_export import PostgreSQLClient
+
+
[email protected]("version_key", [("version",), ()])
+async def test_merge_with_optional_version_key(version_key: tuple[str, ...]) -> None:
+ client = PostgreSQLClient(
+ user="test", password="test", host="localhost", port=5432, database="test", ssl_mode="prefer"
+ )
+ cursor = AsyncMock()
+ connection = MagicMock()
+ connection.cursor.return_value.__aenter__.return_value = cursor
+ client._connection = connection
+
+ await client.amerge_mutable_tables(
+ final_table_name="final",
+ stage_table_name="stage",
+ schema="test",
+ merge_key=("id",),
+ update_key=version_key,
+ update_when_matched=[("id", "BIGINT"), ("version", "BIGINT")],
+ )
+
+ query = " ".join(cursor.execute.call_args.args[0].as_string().split())
+ assert 'ON CONFLICT ("id") DO UPDATE SET "id" = EXCLUDED."id","version" = EXCLUDED."version"' in query
+ if version_key:
+ assert 'WHERE (EXCLUDED."version" > final."version")' in query
+ else:
+ assert "WHERE" not in query
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
index 4f20f323c92c..bb535db17bfe 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
@@ -206,6 +206,59 @@ async def test_client_from_integration(postgres_config, setup_postgres_test_db,
assert results == [(1,)]
[email protected]("version_key", [("version",), ()])
+async def test_amerge_mutable_tables_replaces_rows_with_optional_version_key(
+ postgres_config: dict[str, typing.Any],
+ postgres_connection: psycopg.AsyncConnection,
+ version_key: tuple[str, ...],
+) -> None:
+ schema = postgres_config["schema"]
+ final_table = sql.Identifier(schema, "merge_final")
+ stage_table = sql.Identifier(schema, "merge_stage")
+ async with postgres_connection.cursor() as cursor:
+ for table in (final_table, stage_table):
+ await cursor.execute(
+ sql.SQL("CREATE TABLE {} (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, value TEXT)").format(table)
+ )
+ await cursor.execute(
+ sql.SQL("INSERT INTO {} VALUES (1, 5, 'original'), (2, 1, 'original'), (3, 1, 'untouched')").format(
+ final_table
+ )
+ )
+ await cursor.execute(
+ sql.SQL("INSERT INTO {} VALUES (1, 1, 'replacement'), (2, 1, 'replacement'), (4, 1, 'new')").format(
+ stage_table
+ )
+ )
+
+ client = PostgreSQLClient(
+ user=postgres_config["user"],
+ password=postgres_config["password"],
+ database=postgres_config["database"],
+ host=postgres_config["host"],
+ port=postgres_config["port"],
+ ssl_mode="prefer",
+ )
+ async with client.connect():
+ await client.amerge_mutable_tables(
+ final_table_name="merge_final",
+ stage_table_name="merge_stage",
+ schema=schema,
+ merge_key=("id",),
+ update_key=version_key,
+ update_when_matched=[("id", "INTEGER"), ("version", "INTEGER"), ("value", "TEXT")],
+ )
+
+ async with postgres_connection.cursor() as cursor:
+ await cursor.execute(sql.SQL("SELECT id, version, value FROM {} ORDER BY id").format(final_table))
+ assert await cursor.fetchall() == [
+ (1, 5, "original") if version_key else (1, 1, "replacement"),
+ (2, 1, "original") if version_key else (2, 1, "replacement"),
+ (3, 1, "untouched"),
+ (4, 1, "new"),
+ ]
+
+
async def test_amerge_mutable_tables_raises_non_retryable_error_when_a_row_lock_blocks_it(
postgres_config: dict[str, typing.Any], postgres_connection: psycopg.AsyncConnection
) -> None:
@@ -240,8 +293,8 @@ async def test_amerge_mutable_tables_raises_non_retryable_error_when_a_row_lock_
final_table_name="merge_lock_final",
stage_table_name="merge_lock_stage",
schema=schema,
- merge_key=[("id", "INTEGER")],
- update_key=[("version", "INTEGER")],
+ merge_key=["id"],
+ update_key=["version"],
update_when_matched=[("id", "INTEGER"), ("version", "INTEGER")],
timeout=60,
lock_timeout=1,
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.py
new file mode 100644
index 000000000000..cd29caafd8f7
--- /dev/null
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_merge.py
@@ -0,0 +1,44 @@
+import pytest
+from unittest.mock import AsyncMock, MagicMock
+
+from products.batch_exports.backend.temporal.destinations.redshift_batch_export import RedshiftClient
+
+
[email protected]("version_key", [("version",), ()])
[email protected]("remove_duplicates", [True, False])
+async def test_merge_with_optional_version_key(version_key: tuple[str, ...], remove_duplicates: bool) -> None:
+ client = RedshiftClient(
+ user="test", password="test", host="localhost", port=5439, database="test", ssl_mode="prefer"
+ )
+ cursor = AsyncMock()
+ cursor.fetchone.return_value = ("snapshot isolation",)
+ connection = MagicMock()
+ connection.cursor.return_value.__aenter__.return_value = cursor
+ client._connection = connection
+
+ await client.amerge_tables(
+ final_table_name="final",
+ stage_table_name="stage",
+ schema="test",
+ merge_key=("id",),
+ update_key=version_key,
+ final_table_fields=[("id", "BIGINT"), ("version", "BIGINT")],
+ remove_duplicates=remove_duplicates,
+ )
+
+ queries = [" ".join(call.args[0].as_string().split()) for call in cursor.execute.call_args_list]
+ deletes = [query for query in queries if query.startswith("DELETE")]
+ if version_key:
+ assert len(deletes) == 1
+ assert 'AND ("test"."stage"."version" < "final"."version")' in deletes[0]
+ else:
+ assert deletes == []
+ merge = queries[-1]
+ assert merge.startswith('MERGE INTO "test"."final"')
+ assert 'ON "test"."final"."id" = "stage"."id"' in merge
+ if remove_duplicates:
+ assert merge.endswith("REMOVE DUPLICATES")
+ else:
+ assert "WHEN MATCHED THEN UPDATE SET" in merge
+ assert "WHEN NOT MATCHED THEN INSERT" in merge
diff --git a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
index 4db30cd2507f..24fb129c4ea8 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/redshift/test_redshift_utils.py
@@ -15,7 +15,7 @@ import botocore.exceptions
from posthog.models.integration import AWSRedshiftRoleBasedIntegration, Integration, IntegrationError
-from products.batch_exports.backend.service import AWSCredentials, BatchExportModel
+from products.batch_exports.backend.service import AWSCredentials, BatchExportEventsModel
from products.batch_exports.backend.temporal.destinations.redshift_batch_export import (
ClientErrorGroup,
InsufficientS3PermissionsError,
@@ -43,7 +43,7 @@ def test_events_table_schemas_match_staged_person_id(has_person_id: bool, proper
columns.append("person_id")
schema = pa.schema([(name, pa.string()) for name in columns])
- table_schemas = _get_table_schemas(BatchExportModel(name="events", schema=None), schema, properties_data_type)
+ table_schemas = _get_table_schemas(BatchExportEventsModel(), schema, properties_data_type)
for fields in (table_schemas.table_schema, table_schemas.stage_table_schema):
assert {name for name, _ in fields} == set(columns)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py
new file mode 100644
index 000000000000..b8db18504549
--- /dev/null
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py
@@ -0,0 +1,75 @@
+import pytest
+from unittest.mock import AsyncMock, patch
+
+import pyarrow as pa
+
+from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (
+ SnowflakeClient,
+ SnowflakeField,
+ SnowflakeTable,
+ SnowflakeType,
+)
+from products.batch_exports.backend.temporal.utils import JsonType
+
+
[email protected]("versioned", [True, False])
+ "key,quoted_key,version,quoted_version",
+ [
+ ("id", '"id"', "version", '"version"'),
+ ('key"name', '"key""name"', 'version"name', '"version""name"'),
+ ],
+)
+async def test_merge_with_optional_version_key(
+ versioned: bool, key: str, quoted_key: str, version: str, quoted_version: str
+) -> None:
+ fields = tuple(SnowflakeField(name, SnowflakeType("INT", False), pa.int64(), False) for name in (key, version))
+ final = SnowflakeTable("final", fields, primary_key=(key,), version_key=(version,) if versioned else ())
+ stage = SnowflakeTable("stage", fields)
+ client = SnowflakeClient(
+ user="test", password="test", account="test", warehouse="test", database="test", schema="test"
+ )
+
+ assert final.is_mutable()
+ with patch.object(client, "execute_async_query", new_callable=AsyncMock) as execute:
+ await client.merge_into_final_from_stage(final, stage, timeout=60)
+
+ query = " ".join(execute.call_args.args[0].split())
+ assert f"ON final.{quoted_key} = stage.{quoted_key}" in query
+ assert (
+ f"UPDATE SET final.{quoted_key} = stage.{quoted_key}, final.{quoted_version} = stage.{quoted_version}" in query
+ )
+ assert f"WHEN NOT MATCHED THEN INSERT ({quoted_key}, {quoted_version})" in query
+ assert f"VALUES (stage.{quoted_key}, stage.{quoted_version})" in query
+ if versioned:
+ assert f"WHEN MATCHED AND (final.{quoted_version} < stage.{quoted_version}) THEN UPDATE" in query
+ else:
+ assert "WHEN MATCHED THEN UPDATE" in query
+
+
[email protected]("create", [True, False])
+async def test_staging_quotes_column_names(create: bool) -> None:
+ table = SnowflakeTable(
+ "stage",
+ (
+ SnowflakeField('key"name', SnowflakeType("INT", False), pa.int64(), False),
+ SnowflakeField('json"data', SnowflakeType("VARIANT", False), JsonType(), True),
+ ),
+ )
+ client = SnowflakeClient(
+ user="test", password="test", account="test", warehouse="test", database="test", schema="test"
+ )
+ with patch.object(client, "execute_async_query", new_callable=AsyncMock, return_value=([], None)) as execute:
+ if create:
+ await client.create_table(table)
+ else:
+ await client.copy_loaded_files_to_snowflake_table(table, timeout=60)
+
+ query = " ".join(execute.call_args.args[0].split())
+ if create:
+ assert 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT )' in query
+ else:
+ assert 'COPY INTO "stage" ("key""name", "json""data")' in query
+ assert 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\')' in query
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
index 2d3b7c485e73..83097bfe3f54 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
@@ -361,7 +361,7 @@ class PostgresDestinationWriter:
# Declared at creation rather than added later, so the merge target carries its own
# constraint. `_ensure_unique_index` still covers a table the customer already had,
# which this never touches.
- key = [(name, type_name) for name, type_name in fields if name in set(primary_keys or ())]
+ key = [name for name, _ in fields if name in set(primary_keys or ())]
await client.acreate_table(
self._schema,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
index eec3c9bee03d..e3a14c78c5ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/redshift.py
@@ -214,13 +214,12 @@ class RedshiftDestinationWriter(PostgresDestinationWriter):
# The stage was created from the destination, so both share this order. Only the name
# in each field is read, which is why the types are left empty.
final_table_fields: Fields = [(name, "") for name in await client.aget_table_columns(self._schema, target)]
- merge_key: Fields = [(key, "") for key in primary_keys]
await cast(RedshiftClient, client).amerge_tables(
final_table_name=target,
stage_table_name=stage,
schema=self._schema,
- merge_key=merge_key,
+ merge_key=primary_keys,
# Only read to build the delete that `skip_delete` suppresses.
update_key=[],
final_table_fields=final_table_fields,