Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): partition new history tables by open and closed month (#123554)

Daniel RC696b7d41cecf
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index bb690d085a01..ee03ebbb3ac2 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -188,10 +188,12 @@ type DataWarehouseTableIntrospectedColumns = dict[str, DataWarehouseTableIntrosp
# and never user-facing.
HIDDEN_COLUMNS: frozenset[str] = frozenset({"_dlt_id", "_dlt_load_id", "_ph_debug", PARTITION_KEY})
-# Capture bookkeeping that a history table carries next to `valid_from`, `valid_to` and `_ph_deleted`,
+# Capture and layout bookkeeping that a history table carries next to its validity columns and `_ph_deleted`,
# which already tell a reader when a version held and whether it is a delete. The names mirror the
# constants in `cdc/batcher.py`, which this module does not import because that pulls in pyarrow.
-HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at"})
+HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset(
+ {"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at", "_ph_history_partition"}
+)
# Set in `options` when the sync registers a table as a schema's history table. Column names cannot
# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.
HISTORY_TABLE_OPTION = "history_table"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
index ff0d01cc2fd6..37ca5f5db4e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
@@ -29,7 +29,7 @@ CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}
@frozen
class HistoryNaming:
- """The table name suffix and the validity column names of one history table.
+ """The table name suffix, the validity column names and the partition column of one history table.
A history table keeps the naming it was created with, so code reads it from the table and never
from a constant.
@@ -38,15 +38,31 @@ class HistoryNaming:
suffix: str
valid_from: str
valid_to: str
+ # The SCD2 writer files open rows under `CURRENT_HISTORY_PARTITION` and closed rows under the month their
+ # version ended. A close then rewrites only files of the current partition, which grows with the live
+ # table, not with history.
+ partition_column: str | None = None
@property
def validity_columns(self) -> frozenset[str]:
return frozenset({self.valid_from, self.valid_to})
+ @property
+ def stamped_columns(self) -> frozenset[str]:
+ """Every column a history table carries next to the source's own columns."""
+ return self.validity_columns | ({self.partition_column} if self.partition_column else set())
+
LEGACY_HISTORY_NAMING = HistoryNaming(suffix="_cdc", valid_from="valid_from", valid_to="valid_to")
# The `_ph_` names leave a source's own `valid_from` and `valid_to` free to sync.
-HISTORY_NAMING = HistoryNaming(suffix="_history", valid_from="_ph_valid_from", valid_to="_ph_valid_to")
+HISTORY_PARTITION_COLUMN = "_ph_history_partition"
+HISTORY_NAMING = HistoryNaming(
+ suffix="_history",
+ valid_from="_ph_valid_from",
+ valid_to="_ph_valid_to",
+ partition_column=HISTORY_PARTITION_COLUMN,
+)
+CURRENT_HISTORY_PARTITION = "current"
def history_naming_of(column_names: Iterable[str]) -> HistoryNaming:
@@ -94,7 +110,7 @@ def history_columns(naming: HistoryNaming) -> frozenset[str]:
`_ph_valid_*` stays reserved on a legacy history table too, because `history_naming_of` reads the
naming from it.
"""
- return HISTORY_NAMING.validity_columns | naming.validity_columns
+ return HISTORY_NAMING.stamped_columns | naming.stamped_columns
def reserved_column_names(history: HistoryNaming | None) -> frozenset[str]:
@@ -124,10 +140,10 @@ def reserved_column_collisions(
def metadata_columns(history: HistoryNaming | None) -> frozenset[str]:
"""Columns always filled from the change event itself, never from the last known row.
- `history` names the validity columns stamped on a history batch. A source column that only shares a
- validity name with another naming is data.
+ `history` names the columns a history table carries next to the source's. A source column that only
+ shares a validity name with another naming is data.
"""
- return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.validity_columns
+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.stamped_columns
# Micro-batch thresholds for CDC WAL processing. Flushing periodically
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
index f7d96399faa2..5f6db18fdc98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
@@ -181,11 +181,13 @@ async def _seed_cdc_companion_from_snapshot(
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_OP_COLUMN,
CDC_TIMESTAMP_COLUMN,
+ CURRENT_HISTORY_PARTITION,
DELETED_AT_COLUMN,
DELETED_COLUMN,
companion_resource_name as build_companion_resource_name,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_naming
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.scd2 import Scd2DeltaWriter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import DeltaWriter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema
@@ -203,7 +205,7 @@ async def _seed_cdc_companion_from_snapshot(
CDC_TIMESTAMP_COLUMN,
DELETED_COLUMN,
DELETED_AT_COLUMN,
- *history.validity_columns,
+ *history.stamped_columns,
}
read_columns = [c for c in dataset.schema.names if c not in cdc_meta_cols]
@@ -259,12 +261,22 @@ async def _seed_cdc_companion_from_snapshot(
# Plain append — the companion table is freshly reset so there are no existing
# rows to close, making SCD2 merge unnecessary.
- await DeltaWriter(companion_ref).write(
- data=batch_table,
- write_type="append",
- should_overwrite_table=False,
- primary_keys=None,
- )
+ if history.partition_column is None:
+ await DeltaWriter(companion_ref).write(
+ data=batch_table,
+ write_type="append",
+ should_overwrite_table=False,
+ primary_keys=None,
+ )
+ else:
+ # The SCD2 writer creates the table partitioned and files each row. Without keys it closes nothing.
+ await Scd2DeltaWriter(
+ companion_ref,
+ valid_from_column=history.valid_from,
+ valid_to_column=history.valid_to,
+ partition_column=history.partition_column,
+ current_partition=CURRENT_HISTORY_PARTITION,
+ ).write(data=batch_table, primary_keys=[])
hogql_schema.add_pyarrow_table(batch_table)
total_rows += n
@@ -291,6 +303,7 @@ async def _run_delta_maintenance(
is_cdc_companion: bool,
logger: FilteringBoundLogger,
) -> None:
+ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_PARTITION_COLUMN
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import ( # noqa: PLC0415 — keeps the heavy deltalake dep off this module's top-level import path
DeltaMaintenance,
)
@@ -307,6 +320,8 @@ async def _run_delta_maintenance(
is_cdc_companion=is_cdc_companion,
compact_small_files=not schema.is_cdc,
)
+ if is_cdc_companion:
+ await DeltaMaintenance(delta_table_ref).compact_history_partitions(HISTORY_PARTITION_COLUMN)
def _stored_sync_type_config(schema_id: Any, team_id: int) -> Any:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py
index c3e97c31f7d3..ecf65af4013d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_load.py
@@ -8,19 +8,28 @@ from unittest.mock import AsyncMock, MagicMock, patch
from django.db import OperationalError
import pyarrow as pa
+import deltalake
from parameterized import parameterized
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ HISTORY_NAMING,
+ HISTORY_PARTITION_COLUMN,
+)
from products.warehouse_sources.backend.temporal.data_imports.external_data_job import Any_Source_Errors
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.load import (
IncrementalFieldMissingFromDataError,
PostLoadResult,
+ _seed_cdc_companion_from_snapshot,
get_incremental_field_value,
notify_revenue_analytics_that_sync_has_completed,
run_post_load_operations,
update_job_row_count,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import (
+ make_local_table_ref,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.post_load_phases import (
SUMMARY_EVENT,
record_post_load_phases,
@@ -35,6 +44,8 @@ _LOAD_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelin
_DB_RETRY_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry"
_PIPELINE_SYNC_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_sync"
_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller"
+_HISTORY_TABLE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table"
+_DELTA_TABLE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"
def _make_schema(
@@ -537,3 +548,33 @@ class TestNotifyRevenueAnalyticsThatSyncHasCompleted:
assert schema.team.revenue_analytics_config.notified_first_sync is True
schema.team.revenue_analytics_config.save.assert_called_once()
logger.aexception.assert_not_called()
+
+
+class TestSeedHistoryTableFromSnapshot:
+ @pytest.mark.asyncio
+ async def test_a_new_history_table_is_seeded_partitioned_with_every_row_current(self, tmp_path):
+ # Seeded through the plain writer instead, every new history table would start unpartitioned.
+ snapshot_path = str(tmp_path / "snapshot")
+ deltalake.write_deltalake(snapshot_path, pa.table({"id": [1, 2], "name": ["a", "b"]}))
+ history_path = str(tmp_path / "history")
+ history_ref = make_local_table_ref(history_path)
+ history_ref.reset_table = AsyncMock() # type: ignore[method-assign]
+ schema = MagicMock()
+ schema.name = "orders"
+
+ with (
+ patch(f"{_HISTORY_TABLE_MODULE}.history_naming", return_value=HISTORY_NAMING),
+ patch(f"{_DELTA_TABLE_MODULE}.DeltaTableRef", return_value=history_ref),
+ patch(f"{_LOAD_MODULE}.run_post_load_operations", AsyncMock()),
+ ):
+ await _seed_cdc_companion_from_snapshot(
+ schema=schema,
+ job=MagicMock(),
+ source=MagicMock(),
+ snapshot_delta_table_ref=make_local_table_ref(snapshot_path),
+ logger=MagicMock(),
+ )
+
+ table = deltalake.DeltaTable(history_path)
+ assert table.metadata().partition_columns == [HISTORY_PARTITION_COLUMN]
+ assert table.to_pyarrow_table().column(HISTORY_PARTITION_COLUMN).to_pylist() == ["current", "current"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
index be182b8d6390..9659845767d1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
@@ -480,7 +480,12 @@ class DeltaMaintenance:
@recorded_phase("compact")
async def _compact(
- self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None, *, partitions: list[str] | None = None
+ self,
+ table: deltalake.DeltaTable,
+ plan: CompactionPlan | None = None,
+ *,
+ partitions: list[str] | None = None,
+ partition_column: str = PARTITION_KEY,
) -> bool:
plan = plan or await self._plan_compaction(table)
target_size = plan.target_size
@@ -510,7 +515,7 @@ class DeltaMaintenance:
def _compact_op(size: int = target_size, tasks: int | None = max_concurrent_tasks) -> dict[str, Any]:
if partitions:
return table.optimize.compact(
- partition_filters=[(PARTITION_KEY, "in", partitions)],
+ partition_filters=[(partition_column, "in", partitions)],
target_size=size,
max_concurrent_tasks=tasks,
)
@@ -974,6 +979,46 @@ class DeltaMaintenance:
except Exception as e:
await self._log_maintenance_failure(e, step="compact")
+ async def compact_history_partitions(self, partition_column: str) -> bool:
+ """Compact each partition of a partitioned history table that holds enough small files.
+
+ Every append and every close merge adds a file to the current partition, and each close moves the
+ rows it closes into the month that is still filling. A CDC table runs without the small-file
+ triggers of `compact_if_fragmented`, because a CDC load lands on every tick, and its count triggers
+ average over every month partition. Never raises, like `run_scheduled`.
+ """
+ try:
+ table = await self._table.get_delta_table()
+ if table is None or partition_column not in (table.metadata().partition_columns or []):
+ return False
+ prefix = f"{partition_column}="
+ sizes: defaultdict[str, list[int]] = defaultdict(list)
+ for path, size in table._table.get_add_file_sizes().items():
+ directory = path.rpartition("/")[0]
+ if directory.startswith(prefix):
+ sizes[directory[len(prefix) :]].append(size or 0)
+
+ def fragmented(target_size: int) -> list[str]:
+ return sorted(
+ value
+ for value, file_sizes in sizes.items()
+ if removable_file_count(file_sizes, target_size)
+ >= DEFAULT_COMPACT_REMOVABLE_FILES_PER_PARTITION_THRESHOLD
+ )
+
+ if not fragmented(table_target_file_size(table)):
+ return False
+ # The memory plan can shrink the bin below the table's target, and a smaller bin can merge
+ # fewer files. Without the recheck every sync would plan a compaction that removes nothing.
+ plan = await self._plan_compaction(table)
+ selected = [] if plan.target_size is None else fragmented(plan.target_size)
+ if not selected:
+ return False
+ return await self._compact(table, plan, partitions=selected, partition_column=partition_column)
+ except Exception as e:
+ await self._log_maintenance_failure(e, step="compact_history_partitions")
+ return False
+
async def _log_maintenance_failure(self, e: Exception, *, step: str) -> None:
if isinstance(e, ObjectStorePermissionDeniedError):
await self._logger.awarning(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py
index 26b7123ffbcb..4f5227269c3f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/scd2.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any
import pyarrow as pa
import deltalake
+import pyarrow.compute as pc
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
first_per_pk_table,
@@ -23,6 +24,8 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
if TYPE_CHECKING:
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
+_CLOSING_PARTITION = "_ph_closing_partition"
+
class Scd2DeltaWriter:
"""SCD Type 2 writer over one schema's Delta table.
@@ -33,11 +36,24 @@ class Scd2DeltaWriter:
via `build_scd2_table`.
"""
- def __init__(self, table: "DeltaTableRef", *, valid_from_column: str, valid_to_column: str) -> None:
+ def __init__(
+ self,
+ table: "DeltaTableRef",
+ *,
+ valid_from_column: str,
+ valid_to_column: str,
+ partition_column: str | None = None,
+ current_partition: str = "current",
+ ) -> None:
+ """`partition_column` partitions a new table: open rows in `current_partition`, closed rows in the
+ `YYYY-MM` month their version ended. The writer sets the column itself, so a batch that lacks it
+ cannot put open rows where no close looks. A table created without it stays unpartitioned."""
self._table = table
self._logger = table.logger
self._valid_from_column = valid_from_column
self._valid_to_column = valid_to_column
+ self._partition_column = partition_column
+ self._current_partition = current_partition
async def write(
self,
@@ -66,6 +82,14 @@ class Scd2DeltaWriter:
data = relax_batch_nullability(data)
delta_table = await self._table.get_delta_table()
+ # A table created before partitioning stays unpartitioned, and its rows get no partition column.
+ partition_column = self._partition_column if delta_table is None or self._partitioned(delta_table) else None
+ if partition_column is not None:
+ if partition_column in data.column_names:
+ data = data.drop_columns([partition_column])
+ data = data.append_column(
+ pa.field(partition_column, pa.string()), self._partitions(data.column(self._valid_to_column))
+ )
if delta_table:
delta_table = await evolve_delta_schema(delta_table, data.schema)
@@ -90,6 +114,17 @@ class Scd2DeltaWriter:
predicate_parts = [f"source.{col} = target.{col}" for col in normalized_pks]
predicate_parts.append(f"target.{self._valid_to_column} IS NULL")
+ updates = {self._valid_to_column: f"source.{self._valid_from_column}"}
+ if partition_column is not None:
+ # Open rows only sit in the current partition, so the merge reads and rewrites no other.
+ predicate_parts.append(f"target.{partition_column} = '{self._current_partition}'")
+ # The merge reads only the keys and `valid_from`, and a source column cannot then
+ # collide with the closing month.
+ first_per_pk = first_per_pk.select([*normalized_pks, self._valid_from_column])
+ first_per_pk = first_per_pk.append_column(
+ _CLOSING_PARTITION, self._partitions(first_per_pk.column(self._valid_from_column))
+ )
+ updates[partition_column] = f"source.{_CLOSING_PARTITION}"
predicate = " AND ".join(predicate_parts)
# NOTE: do NOT tag this intermediate merge with `commit_properties`. SCD2 is a
@@ -107,7 +142,7 @@ class Scd2DeltaWriter:
streamed_exec=False,
**delta_merge_spill_kwargs(),
)
- .when_matched_update(updates={self._valid_to_column: f"source.{self._valid_from_column}"})
+ .when_matched_update(updates=updates)
.execute()
)
@@ -127,6 +162,7 @@ class Scd2DeltaWriter:
deltalake.DeltaTable.create,
table_uri=delta_uri,
schema=data.schema,
+ partition_by=[partition_column] if partition_column is not None else None,
storage_options=storage_options,
configuration=DELTA_TABLE_PROPERTIES,
)
@@ -146,3 +182,10 @@ class Scd2DeltaWriter:
await ensure_table_properties(delta_table, self._logger)
return delta_table
+
+ def _partitions(self, version_ends: pa.Array | pa.ChunkedArray) -> pa.Array | pa.ChunkedArray:
+ """The current partition for a version that has not ended, else the `YYYY-MM` month it ended in."""
+ return pc.if_else(pc.is_null(version_ends), self._current_partition, pc.strftime(version_ends, format="%Y-%m"))
+
+ def _partitioned(self, delta_table: deltalake.DeltaTable) -> bool:
+ return self._partition_column in (delta_table.metadata().partition_columns or [])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
index 9131bfebf3f3..f9e3c7af8e9e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
@@ -1485,3 +1485,50 @@ class TestVacuumOnARealTable:
assert os.path.exists(sibling)
assert recently_tombstoned <= _parquet_files(root)
assert deltalake.DeltaTable(root).to_pyarrow_table().num_rows == 1
+
+
+class TestCompactHistoryPartitions:
+ @parameterized.expand(
+ [
+ ("below_the_threshold", 4, 4, _DEFAULT_PLAN, 4, 4),
+ # The month that is still filling takes a file from every close, so it fragments too.
+ ("current_and_month_above_the_threshold", 12, 12, _DEFAULT_PLAN, 1, 1),
+ ("only_the_month_above_the_threshold", 4, 12, _DEFAULT_PLAN, 4, 1),
+ # Every file is larger than the planned bin, so a compaction would remove none of them.
+ (
+ "planned_bin_smaller_than_the_files",
+ 12,
+ 12,
+ CompactionPlan(target_size=1, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1.0),
+ 12,
+ 12,
+ ),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_compacts_each_fragmented_partition(
+ self,
+ _name: str,
+ current_files: int,
+ month_files: int,
+ plan: CompactionPlan,
+ current_after: int,
+ month_after: int,
+ ):
+ with tempfile.TemporaryDirectory() as path:
+ for partition, count in (("current", current_files), ("2026-01", month_files)):
+ for i in range(count):
+ deltalake.write_deltalake(
+ path,
+ pa.table({"id": [i], "_ph_history_partition": [partition]}),
+ mode="append",
+ partition_by=["_ph_history_partition"],
+ )
+ maintenance = _make_maintenance(deltalake.DeltaTable(path))
+ with patch.object(maintenance, "_plan_compaction", AsyncMock(return_value=plan)):
+ ran = await maintenance.compact_history_partitions("_ph_history_partition")
+
+ files = deltalake.DeltaTable(path).file_uris()
+ assert ran is (current_after < current_files or month_after < month_files)
+ assert sum("_ph_history_partition=current/" in f for f in files) == current_after
+ assert sum("_ph_history_partition=2026-01/" in f for f in files) == month_after
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py
index 1e05e2b5262b..bb7a089da3fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_scd2.py
@@ -23,7 +23,69 @@ def _make_writer(delta_uri: str) -> Scd2DeltaWriter:
return Scd2DeltaWriter(make_local_table_ref(delta_uri), valid_from_column="valid_from", valid_to_column="valid_to")
+def _history_writer(delta_uri: str, partition_column: str | None) -> Scd2DeltaWriter:
+ return Scd2DeltaWriter(
+ make_local_table_ref(delta_uri),
+ valid_from_column="_ph_valid_from",
+ valid_to_column="_ph_valid_to",
+ partition_column=partition_column,
+ )
+
+
+def _history_batch(rows: list[tuple[int, datetime, datetime | None]]) -> pa.Table:
+ return pa.table(
+ {
+ "id": pa.array([row[0] for row in rows]),
+ "_ph_valid_from": pa.array([row[1] for row in rows], type=_TS_TYPE),
+ "_ph_valid_to": pa.array([row[2] for row in rows], type=_TS_TYPE),
+ # A source column under the merge's own closing-month name must not collide with it.
+ "_ph_closing_partition": pa.array(["source value"] * len(rows)),
+ }
+ )
+
+
class TestScd2Write:
+ @pytest.mark.asyncio
+ async def test_a_partitioned_table_files_each_row_by_when_its_version_ended(self, tmp_path: Path) -> None:
+ # The close merge reads only the current partition, so a closed row left there would stay open to
+ # every later merge. The batches carry no partition column: the writer files every row itself.
+ delta_path = str(tmp_path / "scd2_table")
+ writer = _history_writer(delta_path, "_ph_history_partition")
+
+ jan, mar, apr = (datetime(2026, month, day, tzinfo=UTC) for month, day in ((1, 1), (3, 15), (4, 2)))
+ await writer.write(data=_history_batch([(1, jan, None), (2, jan, None)]), primary_keys=["id"])
+ # Key 1 closes against the table, and key 3 closes inside its own batch.
+ await writer.write(data=_history_batch([(1, mar, None), (3, mar, apr), (3, apr, None)]), primary_keys=["id"])
+
+ table = deltalake.DeltaTable(delta_path)
+ rows = table.to_pyarrow_table().sort_by([("id", "ascending"), ("_ph_valid_from", "ascending")]).to_pylist()
+ assert table.metadata().partition_columns == ["_ph_history_partition"]
+ assert [(r["id"], r["_ph_valid_to"], r["_ph_history_partition"]) for r in rows] == [
+ (1, mar, "2026-03"),
+ (1, None, "current"),
+ (2, None, "current"),
+ (3, apr, "2026-04"),
+ (3, None, "current"),
+ ]
+
+ @pytest.mark.asyncio
+ async def test_a_table_created_before_partitioning_keeps_closing_its_open_rows(self, tmp_path: Path) -> None:
+ # Stamping the partition column here would leave the old open rows null in it, where the close merge's
+ # `= 'current'` match never finds them.
+ delta_path = str(tmp_path / "scd2_table")
+ jan, mar = datetime(2026, 1, 1, tzinfo=UTC), datetime(2026, 3, 15, tzinfo=UTC)
+ await _history_writer(delta_path, None).write(data=_history_batch([(1, jan, None)]), primary_keys=["id"])
+
+ await _history_writer(delta_path, "_ph_history_partition").write(
+ data=_history_batch([(1, mar, None)]), primary_keys=["id"]
+ )
+
+ table = deltalake.DeltaTable(delta_path)
+ rows = table.to_pyarrow_table().sort_by([("_ph_valid_from", "ascending")]).to_pylist()
+ assert table.metadata().partition_columns == []
+ assert [(r["_ph_valid_from"], r["_ph_valid_to"]) for r in rows] == [(jan, mar), (mar, None)]
+ assert "_ph_history_partition" not in rows[0]
+
@pytest.mark.asyncio
async def test_first_write_creates_the_table_with_a_short_checkpoint_interval(self, tmp_path: Path) -> None:
# The SCD2 writer has its own create path; a table it creates on delta-rs's default interval
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index a29ec944b343..3306239ad933 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.models.external_data_schema import Exter
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_OP_COLUMN,
+ CURRENT_HISTORY_PARTITION,
TOAST_OMITTED_COLUMN,
enrich_delete_rows,
enrich_toast_omitted_rows,
@@ -169,6 +170,7 @@ def _read_existing_rows_by_first_pk(
existing_delta_table: deltalake.DeltaTable,
first_pk: str,
first_components: list[Any],
+ partition: tuple[str, str] | None = None,
) -> pa.Table:
"""Read the existing rows whose `first_pk` value is in `first_components`.
@@ -177,12 +179,15 @@ def _read_existing_rows_by_first_pk(
unimplemented, so pushing an `in` predicate down onto a string primary key raises
`ArrowNotImplementedError`. When that happens, read the table and filter in PyArrow
after casting the key to `string`, which has working kernels.
+
+ `partition` limits both reads to one partition, as `(column, value)`.
"""
+ partitions = None if partition is None else [(partition[0], "=", partition[1])]
try:
- return existing_delta_table.to_pyarrow_table(filters=[(first_pk, "in", first_components)])
+ return existing_delta_table.to_pyarrow_table(filters=[(first_pk, "in", first_components), *(partitions or [])])
except pa.lib.ArrowNotImplementedError:
logger.warning("cdc_delete_enrichment_pushdown_fallback", primary_key=first_pk)
- existing = existing_delta_table.to_pyarrow_table()
+ existing = existing_delta_table.to_pyarrow_table(partitions=partitions)
key = existing.column(first_pk).cast(pa.string())
wanted = pa.array(first_components, pa.string())
return existing.filter(pc.is_in(key, value_set=wanted))
@@ -241,7 +246,18 @@ def _enrich_cdc_rows(
# For composite PKs that IN is a superset — narrow in PyArrow below.
first_pk = present_pks[0]
first_components = list({t[0] for t in enrich_key_set})
- existing_rows = _read_existing_rows_by_first_pk(existing_delta_table, first_pk, first_components)
+ history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else None
+ # The open rows of a partitioned history table all sit in its current partition.
+ partition_column = history.partition_column if history is not None else None
+ current_partition = (
+ (partition_column, CURRENT_HISTORY_PARTITION)
+ if partition_column is not None
+ and partition_column in (existing_delta_table.metadata().partition_columns or [])
+ else None
+ )
+ existing_rows = _read_existing_rows_by_first_pk(
+ existing_delta_table, first_pk, first_components, current_partition
+ )
# For composite PKs the IN filter is a superset — narrow to exact matches.
if len(present_pks) > 1 and existing_rows.num_rows > 0:
@@ -259,7 +275,6 @@ def _enrich_cdc_rows(
# For SCD2 tables, keep only "current" rows (valid_to IS NULL) so we
# enrich with the most recent state rather than a historical one.
- history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else None
if (
history is not None
and existing_rows.num_rows > 0
@@ -1664,7 +1679,11 @@ def _process_message_reported(
# The lane stamped the batch with its table's validity columns, so the batch names them.
history = history_naming_of(pa_table.column_names)
scd2_writer = Scd2DeltaWriter(
- delta_table_ref, valid_from_column=history.valid_from, valid_to_column=history.valid_to
+ delta_table_ref,
+ valid_from_column=history.valid_from,
+ valid_to_column=history.valid_to,
+ partition_column=history.partition_column,
+ current_partition=CURRENT_HISTORY_PARTITION,
)
delta_table = async_to_sync(scd2_writer.write)(
data=pa_table,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 87e047858c38..cbe95f9d26f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_OP_COLUMN,
CDC_SEQ_COLUMN,
HISTORY_NAMING,
+ HISTORY_PARTITION_COLUMN,
LEGACY_HISTORY_NAMING,
TOAST_OMITTED_COLUMN,
HistoryNaming,
@@ -1739,22 +1740,35 @@ class TestEnrichCdcRows:
assert result.column("name").to_pylist() == ["renamed", "two"]
assert TOAST_OMITTED_COLUMN not in result.column_names
- @parameterized.expand([("legacy_table", LEGACY_HISTORY_NAMING), ("new_table", HISTORY_NAMING)])
- def test_scd2_fill_uses_current_row_not_history(self, _name: str, history: HistoryNaming) -> None:
+ @parameterized.expand(
+ [
+ ("legacy_table", LEGACY_HISTORY_NAMING, False),
+ ("new_table", HISTORY_NAMING, False),
+ # The read keeps to the current partition, which must still hold the open row.
+ ("partitioned_new_table", HISTORY_NAMING, True),
+ ]
+ )
+ def test_scd2_fill_uses_current_row_not_history(
+ self, _name: str, history: HistoryNaming, partitioned: bool
+ ) -> None:
with tempfile.TemporaryDirectory() as path:
ts = pa.timestamp("us", tz="UTC")
+ existing = pa.table(
+ {
+ "id": pa.array([1, 1], pa.int64()),
+ "name": pa.array(["v1", "v2"], pa.string()),
+ "big": pa.array(["old-toast", "current-toast"], pa.string()),
+ history.valid_from: pa.array([0, 1_000_000], ts),
+ history.valid_to: pa.array([1_000_000, None], ts),
+ }
+ )
+ if partitioned:
+ existing = existing.append_column(HISTORY_PARTITION_COLUMN, pa.array(["1970-01", "current"]))
write_deltalake(
path,
- pa.table(
- {
- "id": pa.array([1, 1], pa.int64()),
- "name": pa.array(["v1", "v2"], pa.string()),
- "big": pa.array(["old-toast", "current-toast"], pa.string()),
- history.valid_from: pa.array([0, 1_000_000], ts),
- history.valid_to: pa.array([1_000_000, None], ts),
- }
- ),
+ existing,
mode="overwrite",
+ partition_by=[HISTORY_PARTITION_COLUMN] if partitioned else None,
)
# The lane stamps the batch with its table's validity columns before the loader sees it.
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index ccd357b0bd17..5b91764d3d28 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_SEQ_COLUMN,
CDC_TIMESTAMP_COLUMN,
DELETED_AT_COLUMN,
+ HISTORY_NAMING,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
@@ -756,7 +757,13 @@ class TestSchemaInferenceMode(BaseTest):
class TestHistoryHiddenColumns(SimpleTestCase):
def test_names_match_the_columns_capture_writes(self) -> None:
- assert HISTORY_HIDDEN_COLUMNS == {CDC_OP_COLUMN, CDC_SEQ_COLUMN, CDC_TIMESTAMP_COLUMN, DELETED_AT_COLUMN}
+ assert HISTORY_HIDDEN_COLUMNS == {
+ CDC_OP_COLUMN,
+ CDC_SEQ_COLUMN,
+ CDC_TIMESTAMP_COLUMN,
+ DELETED_AT_COLUMN,
+ HISTORY_NAMING.partition_column,
+ }
class TestGetHogqlFieldForColumn(SimpleTestCase):