:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): partition new history tables by open and closed month (#123554)
Daniel RC696b7d41cecf
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex bb690d085a01..ee03ebbb3ac2 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -188,10 +188,12 @@ type DataWarehouseTableIntrospectedColumns = dict[str, DataWarehouseTableIntrosp# and never user-facing.HIDDEN_COLUMNS: frozenset[str] = frozenset({"_dlt_id", "_dlt_load_id", "_ph_debug", PARTITION_KEY})-# Capture bookkeeping that a history table carries next to `valid_from`, `valid_to` and `_ph_deleted`,+# Capture and layout bookkeeping that a history table carries next to its validity columns and `_ph_deleted`,# which already tell a reader when a version held and whether it is a delete. The names mirror the# constants in `cdc/batcher.py`, which this module does not import because that pulls in pyarrow.-HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at"})+HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset(+ {"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at", "_ph_history_partition"}+)# Set in `options` when the sync registers a table as a schema's history table. Column names cannot# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.HISTORY_TABLE_OPTION = "history_table"diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.pyindex ff0d01cc2fd6..37ca5f5db4e3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py@@ -29,7 +29,7 @@ CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}@frozenclass HistoryNaming:- """The table name suffix and the validity column names of one history table.+ """The table name suffix, the validity column names and the partition column of one history table.A history table keeps the naming it was created with, so code reads it from the table and neverfrom a constant.@@ -38,15 +38,31 @@ class HistoryNaming:suffix: strvalid_from: strvalid_to: str+ # The SCD2 writer files open rows under `CURRENT_HISTORY_PARTITION` and closed rows under the month their+ # version ended. A close then rewrites only files of the current partition, which grows with the live+ # table, not with history.+ partition_column: str | None = None@propertydef validity_columns(self) -> frozenset[str]:return frozenset({self.valid_from, self.valid_to})+ @property+ def stamped_columns(self) -> frozenset[str]:+ """Every column a history table carries next to the source's own columns."""+ return self.validity_columns | ({self.partition_column} if self.partition_column else set())+LEGACY_HISTORY_NAMING = HistoryNaming(suffix="_cdc", valid_from="valid_from", valid_to="valid_to")# The `_ph_` names leave a source's own `valid_from` and `valid_to` free to sync.-HISTORY_NAMING = HistoryNaming(suffix="_history", valid_from="_ph_valid_from", valid_to="_ph_valid_to")+HISTORY_PARTITION_COLUMN = "_ph_history_partition"+HISTORY_NAMING = HistoryNaming(+ suffix="_history",+ valid_from="_ph_valid_from",+ valid_to="_ph_valid_to",+ partition_column=HISTORY_PARTITION_COLUMN,+)+CURRENT_HISTORY_PARTITION = "current"def history_naming_of(column_names: Iterable[str]) -> HistoryNaming:@@ -94,7 +110,7 @@ def history_columns(naming: HistoryNaming) -> frozenset[str]:`_ph_valid_*` stays reserved on a legacy history table too, because `history_naming_of` reads thenaming from it."""- return HISTORY_NAMING.validity_columns | naming.validity_columns+ return HISTORY_NAMING.stamped_columns | naming.stamped_columnsdef reserved_column_names(history: HistoryNaming | None) -> frozenset[str]:@@ -124,10 +140,10 @@ def reserved_column_collisions(def metadata_columns(history: HistoryNaming | None) -> frozenset[str]:"""Columns always filled from the change event itself, never from the last known row.- `history` names the validity columns stamped on a history batch. A source column that only shares a- validity name with another naming is data.+ `history` names the columns a history table carries next to the source's. A source column that only+ shares a validity name with another naming is data."""- return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.validity_columns+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.stamped_columns# Micro-batch thresholds for CDC WAL processing. Flushing periodicallydiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.pyindex f7d96399faa2..5f6db18fdc98 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py@@ -181,11 +181,13 @@ async def _seed_cdc_companion_from_snapshot(from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_OP_COLUMN,CDC_TIMESTAMP_COLUMN,+ CURRENT_HISTORY_PARTITION,DELETED_AT_COLUMN,DELETED_COLUMN,companion_resource_name as build_companion_resource_name,)from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_naming+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.scd2 import Scd2DeltaWriterfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableReffrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import DeltaWriterfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema@@ -203,7 +205,7 @@ async def _seed_cdc_companion_from_snapshot(CDC_TIMESTAMP_COLUMN,DELETED_COLUMN,DELETED_AT_COLUMN,- *history.validity_columns,+ *history.stamped_columns,}read_columns = [c for c in dataset.schema.names if c not in cdc_meta_cols]@@ -259,12 +261,22 @@ async def _seed_cdc_companion_from_snapshot(# Plain append — the companion table is freshly reset so there are no existing# rows to close, making SCD2 merge unnecessary.- await DeltaWriter(companion_ref).write(- data=batch_table,- write_type="append",- should_overwrite_table=False,- primary_keys=None,- )+ if history.partition_column is None:+ await DeltaWriter(companion_ref).write(+ data=batch_table,+ write_type="append",+ should_overwrite_table=False,+ primary_keys=None,+ )+ else:+ # The SCD2 writer creates the table partitioned and files each row. Without keys it closes nothing.+ await Scd2DeltaWriter(+ companion_ref,+ valid_from_column=history.valid_from,+ valid_to_column=history.valid_to,+ partition_column=history.partition_column,+ current_partition=CURRENT_HISTORY_PARTITION,+ ).write(data=batch_table, primary_keys=[])hogql_schema.add_pyarrow_table(batch_table)total_rows += n@@ -291,6 +303,7 @@ async def _run_delta_maintenance(is_cdc_companion: bool,logger: FilteringBoundLogger,) -> None:+ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_PARTITION_COLUMNfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import ( # noqa: PLC0415 — keeps the heavy deltalake dep off this module's top-level import pathDeltaMaintenance,)@@ -307,6 +320,8 @@ async def _run_delta_maintenance(is_cdc_companion=is_cdc_companion,compact_small_files=not schema.is_cdc,)+ if is_cdc_companion:+ await DeltaMaintenance(delta_table_ref).compact_history_partitions(HISTORY_PARTITION_COLUMN)def _stored_sync_type_config(schema_id: Any, team_id: int) -> Any:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.pyindex c3e97c31f7d3..ecf65af4013d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py@@ -8,19 +8,28 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom django.db import OperationalErrorimport pyarrow as pa+import deltalakefrom parameterized import parameterizedfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ HISTORY_NAMING,+ HISTORY_PARTITION_COLUMN,+)from products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errorsfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.load import (IncrementalFieldMissingFromDataError,PostLoadResult,+ _seed_cdc_companion_from_snapshot,get_incremental_field_value,notify_revenue_analytics_that_sync_has_completed,run_post_load_operations,update_job_row_count,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import (+ make_local_table_ref,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.post_load_phases import (SUMMARY_EVENT,record_post_load_phases,@@ -35,6 +44,8 @@ _LOAD_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelin_DB_RETRY_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry"_PIPELINE_SYNC_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync"_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller"+_HISTORY_TABLE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table"+_DELTA_TABLE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"def _make_schema(@@ -537,3 +548,33 @@ class TestNotifyRevenueAnalyticsThatSyncHasCompleted:assert schema.team.revenue_analytics_config.notified_first_sync is Trueschema.team.revenue_analytics_config.save.assert_called_once()logger.aexception.assert_not_called()+++class TestSeedHistoryTableFromSnapshot:+ @pytest.mark.asyncio+ async def test_a_new_history_table_is_seeded_partitioned_with_every_row_current(self, tmp_path):+ # Seeded through the plain writer instead, every new history table would start unpartitioned.+ snapshot_path = str(tmp_path / "snapshot")+ deltalake.write_deltalake(snapshot_path, pa.table({"id": [1, 2], "name": ["a", "b"]}))+ history_path = str(tmp_path / "history")+ history_ref = make_local_table_ref(history_path)+ history_ref.reset_table = AsyncMock() # type: ignore[method-assign]+ schema = MagicMock()+ schema.name = "orders"++ with (+ patch(f"{_HISTORY_TABLE_MODULE}.history_naming", return_value=HISTORY_NAMING),+ patch(f"{_DELTA_TABLE_MODULE}.DeltaTableRef", return_value=history_ref),+ patch(f"{_LOAD_MODULE}.run_post_load_operations", AsyncMock()),+ ):+ await _seed_cdc_companion_from_snapshot(+ schema=schema,+ job=MagicMock(),+ source=MagicMock(),+ snapshot_delta_table_ref=make_local_table_ref(snapshot_path),+ logger=MagicMock(),+ )++ table = deltalake.DeltaTable(history_path)+ assert table.metadata().partition_columns == [HISTORY_PARTITION_COLUMN]+ assert table.to_pyarrow_table().column(HISTORY_PARTITION_COLUMN).to_pylist() == ["current", "current"]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.pyindex be182b8d6390..9659845767d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py@@ -480,7 +480,12 @@ class DeltaMaintenance:@recorded_phase("compact")async def _compact(- self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None, *, partitions: list[str] | None = None+ self,+ table: deltalake.DeltaTable,+ plan: CompactionPlan | None = None,+ *,+ partitions: list[str] | None = None,+ partition_column: str = PARTITION_KEY,) -> bool:plan = plan or await self._plan_compaction(table)target_size = plan.target_size@@ -510,7 +515,7 @@ class DeltaMaintenance:def _compact_op(size: int = target_size, tasks: int | None = max_concurrent_tasks) -> dict[str, Any]:if partitions:return table.optimize.compact(- partition_filters=[(PARTITION_KEY, "in", partitions)],+ partition_filters=[(partition_column, "in", partitions)],target_size=size,max_concurrent_tasks=tasks,)@@ -974,6 +979,46 @@ class DeltaMaintenance:except Exception as e:await self._log_maintenance_failure(e, step="compact")+ async def compact_history_partitions(self, partition_column: str) -> bool:+ """Compact each partition of a partitioned history table that holds enough small files.++ Every append and every close merge adds a file to the current partition, and each close moves the+ rows it closes into the month that is still filling. A CDC table runs without the small-file+ triggers of `compact_if_fragmented`, because a CDC load lands on every tick, and its count triggers+ average over every month partition. Never raises, like `run_scheduled`.+ """+ try:+ table = await self._table.get_delta_table()+ if table is None or partition_column not in (table.metadata().partition_columns or []):+ return False+ prefix = f"{partition_column}="+ sizes: defaultdict[str, list[int]] = defaultdict(list)+ for path, size in table._table.get_add_file_sizes().items():+ directory = path.rpartition("/")[0]+ if directory.startswith(prefix):+ sizes[directory[len(prefix) :]].append(size or 0)++ def fragmented(target_size: int) -> list[str]:+ return sorted(+ value+ for value, file_sizes in sizes.items()+ if removable_file_count(file_sizes, target_size)+ >= DEFAULT_COMPACT_REMOVABLE_FILES_PER_PARTITION_THRESHOLD+ )++ if not fragmented(table_target_file_size(table)):+ return False+ # The memory plan can shrink the bin below the table's target, and a smaller bin can merge+ # fewer files. Without the recheck every sync would plan a compaction that removes nothing.+ plan = await self._plan_compaction(table)+ selected = [] if plan.target_size is None else fragmented(plan.target_size)+ if not selected:+ return False+ return await self._compact(table, plan, partitions=selected, partition_column=partition_column)+ except Exception as e:+ await self._log_maintenance_failure(e, step="compact_history_partitions")+ return False+async def _log_maintenance_failure(self, e: Exception, *, step: str) -> None:if isinstance(e, ObjectStorePermissionDeniedError):await self._logger.awarning(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.pyindex 26b7123ffbcb..4f5227269c3f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Anyimport pyarrow as paimport deltalake+import pyarrow.compute as pcfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (first_per_pk_table,@@ -23,6 +24,8 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delif TYPE_CHECKING:from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef+_CLOSING_PARTITION = "_ph_closing_partition"+class Scd2DeltaWriter:"""SCD Type 2 writer over one schema's Delta table.@@ -33,11 +36,24 @@ class Scd2DeltaWriter:via `build_scd2_table`."""- def __init__(self, table: "DeltaTableRef", *, valid_from_column: str, valid_to_column: str) -> None:+ def __init__(+ self,+ table: "DeltaTableRef",+ *,+ valid_from_column: str,+ valid_to_column: str,+ partition_column: str | None = None,+ current_partition: str = "current",+ ) -> None:+ """`partition_column` partitions a new table: open rows in `current_partition`, closed rows in the+ `YYYY-MM` month their version ended. The writer sets the column itself, so a batch that lacks it+ cannot put open rows where no close looks. A table created without it stays unpartitioned."""self._table = tableself._logger = table.loggerself._valid_from_column = valid_from_columnself._valid_to_column = valid_to_column+ self._partition_column = partition_column+ self._current_partition = current_partitionasync def write(self,@@ -66,6 +82,14 @@ class Scd2DeltaWriter:data = relax_batch_nullability(data)delta_table = await self._table.get_delta_table()+ # A table created before partitioning stays unpartitioned, and its rows get no partition column.+ partition_column = self._partition_column if delta_table is None or self._partitioned(delta_table) else None+ if partition_column is not None:+ if partition_column in data.column_names:+ data = data.drop_columns([partition_column])+ data = data.append_column(+ pa.field(partition_column, pa.string()), self._partitions(data.column(self._valid_to_column))+ )if delta_table:delta_table = await evolve_delta_schema(delta_table, data.schema)@@ -90,6 +114,17 @@ class Scd2DeltaWriter:predicate_parts = [f"source.{col} = target.{col}" for col in normalized_pks]predicate_parts.append(f"target.{self._valid_to_column} IS NULL")+ updates = {self._valid_to_column: f"source.{self._valid_from_column}"}+ if partition_column is not None:+ # Open rows only sit in the current partition, so the merge reads and rewrites no other.+ predicate_parts.append(f"target.{partition_column} = '{self._current_partition}'")+ # The merge reads only the keys and `valid_from`, and a source column cannot then+ # collide with the closing month.+ first_per_pk = first_per_pk.select([*normalized_pks, self._valid_from_column])+ first_per_pk = first_per_pk.append_column(+ _CLOSING_PARTITION, self._partitions(first_per_pk.column(self._valid_from_column))+ )+ updates[partition_column] = f"source.{_CLOSING_PARTITION}"predicate = " AND ".join(predicate_parts)# NOTE: do NOT tag this intermediate merge with `commit_properties`. SCD2 is a@@ -107,7 +142,7 @@ class Scd2DeltaWriter:streamed_exec=False,**delta_merge_spill_kwargs(),)- .when_matched_update(updates={self._valid_to_column: f"source.{self._valid_from_column}"})+ .when_matched_update(updates=updates).execute())@@ -127,6 +162,7 @@ class Scd2DeltaWriter:deltalake.DeltaTable.create,table_uri=delta_uri,schema=data.schema,+ partition_by=[partition_column] if partition_column is not None else None,storage_options=storage_options,configuration=DELTA_TABLE_PROPERTIES,)@@ -146,3 +182,10 @@ class Scd2DeltaWriter:await ensure_table_properties(delta_table, self._logger)return delta_table++ def _partitions(self, version_ends: pa.Array | pa.ChunkedArray) -> pa.Array | pa.ChunkedArray:+ """The current partition for a version that has not ended, else the `YYYY-MM` month it ended in."""+ return pc.if_else(pc.is_null(version_ends), self._current_partition, pc.strftime(version_ends, format="%Y-%m"))++ def _partitioned(self, delta_table: deltalake.DeltaTable) -> bool:+ return self._partition_column in (delta_table.metadata().partition_columns or [])diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.pyindex 9131bfebf3f3..f9e3c7af8e9e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py@@ -1485,3 +1485,50 @@ class TestVacuumOnARealTable:assert os.path.exists(sibling)assert recently_tombstoned <= _parquet_files(root)assert deltalake.DeltaTable(root).to_pyarrow_table().num_rows == 1+++class TestCompactHistoryPartitions:+ @parameterized.expand(+ [+ ("below_the_threshold", 4, 4, _DEFAULT_PLAN, 4, 4),+ # The month that is still filling takes a file from every close, so it fragments too.+ ("current_and_month_above_the_threshold", 12, 12, _DEFAULT_PLAN, 1, 1),+ ("only_the_month_above_the_threshold", 4, 12, _DEFAULT_PLAN, 4, 1),+ # Every file is larger than the planned bin, so a compaction would remove none of them.+ (+ "planned_bin_smaller_than_the_files",+ 12,+ 12,+ CompactionPlan(target_size=1, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1.0),+ 12,+ 12,+ ),+ ]+ )+ @pytest.mark.asyncio+ async def test_compacts_each_fragmented_partition(+ self,+ _name: str,+ current_files: int,+ month_files: int,+ plan: CompactionPlan,+ current_after: int,+ month_after: int,+ ):+ with tempfile.TemporaryDirectory() as path:+ for partition, count in (("current", current_files), ("2026-01", month_files)):+ for i in range(count):+ deltalake.write_deltalake(+ path,+ pa.table({"id": [i], "_ph_history_partition": [partition]}),+ mode="append",+ partition_by=["_ph_history_partition"],+ )+ maintenance = _make_maintenance(deltalake.DeltaTable(path))+ with patch.object(maintenance, "_plan_compaction", AsyncMock(return_value=plan)):+ ran = await maintenance.compact_history_partitions("_ph_history_partition")++ files = deltalake.DeltaTable(path).file_uris()+ assert ran is (current_after < current_files or month_after < month_files)+ assert sum("_ph_history_partition=current/" in f for f in files) == current_after+ assert sum("_ph_history_partition=2026-01/" in f for f in files) == month_afterdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.pyindex 1e05e2b5262b..bb7a089da3fe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py@@ -23,7 +23,69 @@ def _make_writer(delta_uri: str) -> Scd2DeltaWriter:return Scd2DeltaWriter(make_local_table_ref(delta_uri), valid_from_column="valid_from", valid_to_column="valid_to")+def _history_writer(delta_uri: str, partition_column: str | None) -> Scd2DeltaWriter:+ return Scd2DeltaWriter(+ make_local_table_ref(delta_uri),+ valid_from_column="_ph_valid_from",+ valid_to_column="_ph_valid_to",+ partition_column=partition_column,+ )+++def _history_batch(rows: list[tuple[int, datetime, datetime | None]]) -> pa.Table:+ return pa.table(+ {+ "id": pa.array([row[0] for row in rows]),+ "_ph_valid_from": pa.array([row[1] for row in rows], type=_TS_TYPE),+ "_ph_valid_to": pa.array([row[2] for row in rows], type=_TS_TYPE),+ # A source column under the merge's own closing-month name must not collide with it.+ "_ph_closing_partition": pa.array(["source value"] * len(rows)),+ }+ )++class TestScd2Write:+ @pytest.mark.asyncio+ async def test_a_partitioned_table_files_each_row_by_when_its_version_ended(self, tmp_path: Path) -> None:+ # The close merge reads only the current partition, so a closed row left there would stay open to+ # every later merge. The batches carry no partition column: the writer files every row itself.+ delta_path = str(tmp_path / "scd2_table")+ writer = _history_writer(delta_path, "_ph_history_partition")++ jan, mar, apr = (datetime(2026, month, day, tzinfo=UTC) for month, day in ((1, 1), (3, 15), (4, 2)))+ await writer.write(data=_history_batch([(1, jan, None), (2, jan, None)]), primary_keys=["id"])+ # Key 1 closes against the table, and key 3 closes inside its own batch.+ await writer.write(data=_history_batch([(1, mar, None), (3, mar, apr), (3, apr, None)]), primary_keys=["id"])++ table = deltalake.DeltaTable(delta_path)+ rows = table.to_pyarrow_table().sort_by([("id", "ascending"), ("_ph_valid_from", "ascending")]).to_pylist()+ assert table.metadata().partition_columns == ["_ph_history_partition"]+ assert [(r["id"], r["_ph_valid_to"], r["_ph_history_partition"]) for r in rows] == [+ (1, mar, "2026-03"),+ (1, None, "current"),+ (2, None, "current"),+ (3, apr, "2026-04"),+ (3, None, "current"),+ ]++ @pytest.mark.asyncio+ async def test_a_table_created_before_partitioning_keeps_closing_its_open_rows(self, tmp_path: Path) -> None:+ # Stamping the partition column here would leave the old open rows null in it, where the close merge's+ # `= 'current'` match never finds them.+ delta_path = str(tmp_path / "scd2_table")+ jan, mar = datetime(2026, 1, 1, tzinfo=UTC), datetime(2026, 3, 15, tzinfo=UTC)+ await _history_writer(delta_path, None).write(data=_history_batch([(1, jan, None)]), primary_keys=["id"])++ await _history_writer(delta_path, "_ph_history_partition").write(+ data=_history_batch([(1, mar, None)]), primary_keys=["id"]+ )++ table = deltalake.DeltaTable(delta_path)+ rows = table.to_pyarrow_table().sort_by([("_ph_valid_from", "ascending")]).to_pylist()+ assert table.metadata().partition_columns == []+ assert [(r["_ph_valid_from"], r["_ph_valid_to"]) for r in rows] == [(jan, mar), (mar, None)]+ assert "_ph_history_partition" not in rows[0]+@pytest.mark.asyncioasync def test_first_write_creates_the_table_with_a_short_checkpoint_interval(self, tmp_path: Path) -> None:# The SCD2 writer has its own create path; a table it creates on delta-rs's default intervaldiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex a29ec944b343..3306239ad933 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.models.external_data_schema import Exterfrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_OP_COLUMN,+ CURRENT_HISTORY_PARTITION,TOAST_OMITTED_COLUMN,enrich_delete_rows,enrich_toast_omitted_rows,@@ -169,6 +170,7 @@ def _read_existing_rows_by_first_pk(existing_delta_table: deltalake.DeltaTable,first_pk: str,first_components: list[Any],+ partition: tuple[str, str] | None = None,) -> pa.Table:"""Read the existing rows whose `first_pk` value is in `first_components`.@@ -177,12 +179,15 @@ def _read_existing_rows_by_first_pk(unimplemented, so pushing an `in` predicate down onto a string primary key raises`ArrowNotImplementedError`. When that happens, read the table and filter in PyArrowafter casting the key to `string`, which has working kernels.++ `partition` limits both reads to one partition, as `(column, value)`."""+ partitions = None if partition is None else [(partition[0], "=", partition[1])]try:- return existing_delta_table.to_pyarrow_table(filters=[(first_pk, "in", first_components)])+ return existing_delta_table.to_pyarrow_table(filters=[(first_pk, "in", first_components), *(partitions or [])])except pa.lib.ArrowNotImplementedError:logger.warning("cdc_delete_enrichment_pushdown_fallback", primary_key=first_pk)- existing = existing_delta_table.to_pyarrow_table()+ existing = existing_delta_table.to_pyarrow_table(partitions=partitions)key = existing.column(first_pk).cast(pa.string())wanted = pa.array(first_components, pa.string())return existing.filter(pc.is_in(key, value_set=wanted))@@ -241,7 +246,18 @@ def _enrich_cdc_rows(# For composite PKs that IN is a superset — narrow in PyArrow below.first_pk = present_pks[0]first_components = list({t[0] for t in enrich_key_set})- existing_rows = _read_existing_rows_by_first_pk(existing_delta_table, first_pk, first_components)+ history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else None+ # The open rows of a partitioned history table all sit in its current partition.+ partition_column = history.partition_column if history is not None else None+ current_partition = (+ (partition_column, CURRENT_HISTORY_PARTITION)+ if partition_column is not None+ and partition_column in (existing_delta_table.metadata().partition_columns or [])+ else None+ )+ existing_rows = _read_existing_rows_by_first_pk(+ existing_delta_table, first_pk, first_components, current_partition+ )# For composite PKs the IN filter is a superset — narrow to exact matches.if len(present_pks) > 1 and existing_rows.num_rows > 0:@@ -259,7 +275,6 @@ def _enrich_cdc_rows(# For SCD2 tables, keep only "current" rows (valid_to IS NULL) so we# enrich with the most recent state rather than a historical one.- history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else Noneif (history is not Noneand existing_rows.num_rows > 0@@ -1664,7 +1679,11 @@ def _process_message_reported(# The lane stamped the batch with its table's validity columns, so the batch names them.history = history_naming_of(pa_table.column_names)scd2_writer = Scd2DeltaWriter(- delta_table_ref, valid_from_column=history.valid_from, valid_to_column=history.valid_to+ delta_table_ref,+ valid_from_column=history.valid_from,+ valid_to_column=history.valid_to,+ partition_column=history.partition_column,+ current_partition=CURRENT_HISTORY_PARTITION,)delta_table = async_to_sync(scd2_writer.write)(data=pa_table,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 87e047858c38..cbe95f9d26f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_OP_COLUMN,CDC_SEQ_COLUMN,HISTORY_NAMING,+ HISTORY_PARTITION_COLUMN,LEGACY_HISTORY_NAMING,TOAST_OMITTED_COLUMN,HistoryNaming,@@ -1739,22 +1740,35 @@ class TestEnrichCdcRows:assert result.column("name").to_pylist() == ["renamed", "two"]assert TOAST_OMITTED_COLUMN not in result.column_names- @parameterized.expand([("legacy_table", LEGACY_HISTORY_NAMING), ("new_table", HISTORY_NAMING)])- def test_scd2_fill_uses_current_row_not_history(self, _name: str, history: HistoryNaming) -> None:+ @parameterized.expand(+ [+ ("legacy_table", LEGACY_HISTORY_NAMING, False),+ ("new_table", HISTORY_NAMING, False),+ # The read keeps to the current partition, which must still hold the open row.+ ("partitioned_new_table", HISTORY_NAMING, True),+ ]+ )+ def test_scd2_fill_uses_current_row_not_history(+ self, _name: str, history: HistoryNaming, partitioned: bool+ ) -> None:with tempfile.TemporaryDirectory() as path:ts = pa.timestamp("us", tz="UTC")+ existing = pa.table(+ {+ "id": pa.array([1, 1], pa.int64()),+ "name": pa.array(["v1", "v2"], pa.string()),+ "big": pa.array(["old-toast", "current-toast"], pa.string()),+ history.valid_from: pa.array([0, 1_000_000], ts),+ history.valid_to: pa.array([1_000_000, None], ts),+ }+ )+ if partitioned:+ existing = existing.append_column(HISTORY_PARTITION_COLUMN, pa.array(["1970-01", "current"]))write_deltalake(path,- pa.table(- {- "id": pa.array([1, 1], pa.int64()),- "name": pa.array(["v1", "v2"], pa.string()),- "big": pa.array(["old-toast", "current-toast"], pa.string()),- history.valid_from: pa.array([0, 1_000_000], ts),- history.valid_to: pa.array([1_000_000, None], ts),- }- ),+ existing,mode="overwrite",+ partition_by=[HISTORY_PARTITION_COLUMN] if partitioned else None,)# The lane stamps the batch with its table's validity columns before the loader sees it.diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.pyindex ccd357b0bd17..5b91764d3d28 100644--- a/products/warehouse_sources/backend/tests/test_table.py+++ b/products/warehouse_sources/backend/tests/test_table.py@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_SEQ_COLUMN,CDC_TIMESTAMP_COLUMN,DELETED_AT_COLUMN,+ HISTORY_NAMING,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,@@ -756,7 +757,13 @@ class TestSchemaInferenceMode(BaseTest):class TestHistoryHiddenColumns(SimpleTestCase):def test_names_match_the_columns_capture_writes(self) -> None:- assert HISTORY_HIDDEN_COLUMNS == {CDC_OP_COLUMN, CDC_SEQ_COLUMN, CDC_TIMESTAMP_COLUMN, DELETED_AT_COLUMN}+ assert HISTORY_HIDDEN_COLUMNS == {+ CDC_OP_COLUMN,+ CDC_SEQ_COLUMN,+ CDC_TIMESTAMP_COLUMN,+ DELETED_AT_COLUMN,+ HISTORY_NAMING.partition_column,+ }class TestGetHogqlFieldForColumn(SimpleTestCase):