Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): write incremental history once per sync with deltalite (#128621)

Daniel RC4fac6848cd44
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/history_upsert.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/history_upsert.py
new file mode 100644
index 000000000000..fcb9aa539c02
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/history_upsert.py
@@ -0,0 +1,505 @@
+"""SCD Type 2 history written once per sync: `stage` appends each batch, `finalize` writes the sync.
+
+A history write per batch rewrites every `current` file that holds a changed key, so its cost grows
+with the table on every batch. Here each batch only appends to a staging table. `finalize` hands the
+staged rows to deltalite's `history_upsert`, which opens new versions, closes the versions they
+replace, and commits both in one Delta commit.
+
+`history_upsert` drops every row that is not newer than its key's open version, so a call with input
+it already applied changes nothing. A crash after a commit and before the staging cleanup is
+therefore harmless: the re-run applies nothing twice.
+"""
+
+import math
+import time
+import shutil
+import asyncio
+import tempfile
+import functools
+import contextlib
+import dataclasses
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from django.conf import settings
+
+import numpy as np
+import pyarrow as pa
+import deltalake
+import pyarrow.compute as pc
+import pyarrow.dataset as ds
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ CURRENT_HISTORY_PARTITION,
+ HistoryNaming,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
+ align_incoming_decimals_to_delta,
+ normalize_column_name,
+ pyarrow_schema_from_arrow_exportable,
+ realign_decimal_buffers,
+ relax_batch_nullability,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.evolution import evolve_delta_schema
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import VACUUM_RETENTION
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governor
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
+ DELTA_TABLE_PROPERTIES,
+ ensure_table_properties,
+ execute_with_conflict_retry,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import (
+ DeltaWriter,
+ _deltalite_write_stats,
+)
+
+if TYPE_CHECKING:
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
+
+STAGED_BATCH_COLUMN = "_ph_staged_batch"
+# Append order, `staging version << 32 | row`: breaks ties on key and order value, across runs too.
+STAGED_ROW_COLUMN = "_ph_staged_row"
+_STAGING_COLUMNS = (STAGED_BATCH_COLUMN, STAGED_ROW_COLUMN)
+# deltalite refuses a source above this, counted with its key map.
+_SOURCE_GUARD_BYTES = 2 * 1024 * 1024 * 1024
+# The staged size is an estimate, so one call takes at most half of deltalite's source guard.
+MAX_CALL_BYTES = _SOURCE_GUARD_BYTES // 2
+# Each chunk rewrites the `current` files that hold its keys, so chunks stay as large as memory allows.
+CHUNK_TARGET_BYTES = 512 * 1024 * 1024
+# Rows per scanned batch, so one decoded batch of wide rows stays small next to the call budget.
+_SCAN_BATCH_ROWS = 16_384
+_MAX_OPEN_CHUNK_FILES = 256
+# deltalite's source guard also counts its copies of the key, order and tie-break columns (about twice their
+# size) and a key-map entry plus two row indexes per row.
+_INDEX_BYTES_PER_ROW = 48 + 2 * 4
+_SUMMED_STATS = (
+ "rows_in",
+ "rows_deduped",
+ "rows_reread",
+ "rows_backwards",
+ "versions_opened",
+ "versions_closed",
+ "chain_rows_closed",
+ "files_removed",
+ "files_added",
+ "bytes_read",
+ "bytes_written",
+)
+
+
+class HistoryChunkTooLargeError(ValueError):
+ """One key chunk of a sync is above deltalite's source limit or half the free local disk.
+
+ Chunks split keys, not a key's versions, so only a key with too many versions causes it.
+ """
+
+
+class _SpillOutOfDisk(Exception):
+ """A spill pass brought the free local disk below its floor."""
+
+
+@frozen
+class HistoryFinalizeResult:
+ staged_rows: int
+ calls: int
+ rows_in: int = 0
+ rows_deduped: int = 0
+ rows_reread: int = 0
+ rows_backwards: int = 0
+ versions_opened: int = 0
+ versions_closed: int = 0
+ chain_rows_closed: int = 0
+ files_removed: int = 0
+ files_added: int = 0
+ bytes_read: int = 0
+ bytes_written: int = 0
+
+
+async def stage(
+ staging: "DeltaTableRef",
+ data: pa.Table,
+ *,
+ key_columns: Sequence[str],
+ naming: HistoryNaming,
+ run_uuid: str,
+ batch_index: int,
+) -> None:
+ """Stage one batch. Each row needs the key columns and `valid_from`, which also orders a key's versions.
+
+ The commit carries the batch's `run_uuid` and `batch_index`, so the load path's redelivery
+ check finds it on the staging table. A batch staged twice still makes one version per row.
+ """
+ staging_table = await staging.get_delta_table()
+ if data.num_rows == 0 and staging_table is not None:
+ return
+ key_columns = _normalized_keys(key_columns)
+ data = relax_batch_nullability(realign_decimal_buffers(data))
+ missing = [c for c in (*key_columns, naming.valid_from) if c not in data.column_names]
+ if missing:
+ raise ValueError(f"History batch is missing column(s) {missing}")
+ taken = [c for c in _STAGING_COLUMNS if c in data.column_names]
+ if taken:
+ raise ValueError(f"History batch has column(s) {taken}, which the staging table writes itself")
+ data = data.drop_columns([c for c in (naming.valid_to, naming.partition_column) if c in data.column_names])
+ # Creating the table commits version 0, so the first append is version 1.
+ first_row = (staging_table.version() + 1 if staging_table is not None else 1) << 32
+ data = data.append_column(
+ STAGED_BATCH_COLUMN, pa.repeat(pa.scalar(f"{run_uuid}:{batch_index}"), data.num_rows)
+ ).append_column(STAGED_ROW_COLUMN, pa.array(np.arange(first_row, first_row + data.num_rows, dtype=np.int64)))
+
+ if staging_table is None:
+ staging_table = await asyncio.to_thread(
+ deltalake.DeltaTable.create,
+ table_uri=await staging.get_table_uri(),
+ schema=data.schema,
+ storage_options=staging.get_storage_options(),
+ configuration=DELTA_TABLE_PROPERTIES,
+ )
+ staging.adopt_created_table(staging_table)
+ # An empty first batch still leaves a staging table, which the load path reads the schema from.
+ if data.num_rows == 0:
+ return
+ else:
+ # A new column joins as nullable, because the rows staged before it do not have it.
+ staging_table = await evolve_delta_schema(staging_table, data.schema)
+ data = align_incoming_decimals_to_delta(data, staging_table.schema())
+ await asyncio.to_thread(
+ deltalake.write_deltalake,
+ table_or_uri=staging_table,
+ data=data,
+ mode="append",
+ schema_mode="merge",
+ commit_properties=deltalake.CommitProperties(
+ custom_metadata={"run_uuid": run_uuid, "batch_index": str(batch_index)}
+ ),
+ )
+
+
+async def finalize(
+ table: "DeltaTableRef",
+ staging: "DeltaTableRef",
+ *,
+ key_columns: Sequence[str],
+ naming: HistoryNaming,
+ commit_metadata: dict[str, str] | None = None,
+ max_call_bytes: int = MAX_CALL_BYTES,
+ chunk_target_bytes: int = CHUNK_TARGET_BYTES,
+) -> HistoryFinalizeResult | None:
+ """Write every staged row into the history table, then delete the staged rows. None when nothing is staged.
+
+ Staged rows that fit one call go in one commit. Larger syncs split by a hash of the key, one commit
+ per chunk. Chunks hold disjoint keys, so a crash between two chunk commits leaves each key either
+ fully applied or not at all, and the re-run applies only the rest. Only the last call carries
+ `commit_metadata`, so a tagged commit means the whole sync is in the table. A last call that keeps no
+ row makes no commit, so its tag is missing although the sync is applied.
+
+ Chunks spill to local disk in passes. Each pass reads the staging table again and stops at half the disk
+ that was free when it started, then runs again with half the chunks, so a large sync needs more reads,
+ not more disk.
+ """
+ key_columns = _normalized_keys(key_columns)
+ partition_column = naming.partition_column
+ if partition_column is None:
+ raise ValueError(f"{table.resource_name} has no history partition column, which a history upsert needs")
+ staging_table = await staging.get_delta_table()
+ if staging_table is None:
+ return None
+ add_actions = pa.table(staging_table.get_add_actions(flatten=True))
+ if add_actions.num_rows == 0 or pc.sum(add_actions["num_records"]).as_py() == 0:
+ return None
+
+ staged_schema = pyarrow_schema_from_arrow_exportable(staging_table.schema())
+ history_table = await _history_table(table, staged_schema, naming, partition_column)
+ staging_dataset = staging_table.to_pyarrow_dataset()
+ # The batch tags are only needed for the cleanup, so they never reach the upsert.
+ columns = [name for name in staging_dataset.schema.names if name != STAGED_BATCH_COLUMN]
+ narrow_columns = list(dict.fromkeys([*key_columns, naming.valid_from, STAGED_ROW_COLUMN]))
+ call_bytes, batch_tags = await asyncio.to_thread(_measure, staging_dataset, columns, narrow_columns)
+ # One handle for every chunk, so each call does not load the table's log and checkpoint again.
+ handle = await asyncio.to_thread(_open_deltalite, await table.get_table_uri(), table.get_storage_options())
+
+ totals = dict.fromkeys(_SUMMED_STATS, 0)
+ calls = staged_rows = 0
+
+ async def _apply(load: Callable[[], pa.Table], footprint: int, chunks: int, *, last: bool) -> None:
+ nonlocal calls, staged_rows
+ stats, rows = await _history_upsert(
+ table,
+ handle,
+ history_table,
+ load,
+ footprint,
+ key_columns=key_columns,
+ naming=naming,
+ partition_column=partition_column,
+ commit_metadata=commit_metadata if last else None,
+ chunk=calls,
+ chunks=chunks,
+ )
+ staged_rows += rows
+ calls += 1
+ for name in _SUMMED_STATS:
+ totals[name] += int(getattr(stats, name))
+
+ with tempfile.TemporaryDirectory(prefix="history-finalize-") as directory:
+ if call_bytes <= max_call_bytes:
+ await _apply(functools.partial(_read_staged, staging_dataset, columns), call_bytes, 1, last=True)
+ else:
+ chunks = math.ceil(call_bytes / chunk_target_bytes)
+ spill_directory = Path(directory)
+ per_pass = max(1, min(_MAX_OPEN_CHUNK_FILES, _free_bytes(spill_directory) // 2 // chunk_target_bytes))
+ first = 0
+ while first < chunks:
+ buckets = range(first, min(chunks, first + per_pass))
+ # A chunk can be far above the target, and other spills share the disk, so the pass watches
+ # the free disk itself and stops at half of what was free when it started.
+ floor = _free_bytes(spill_directory) // 2
+ try:
+ spilled = await asyncio.to_thread(
+ _spill_chunks,
+ handle,
+ history_table.schema(),
+ staging_dataset,
+ columns,
+ key_columns,
+ narrow_columns,
+ chunks,
+ buckets,
+ spill_directory,
+ floor,
+ )
+ except _SpillOutOfDisk:
+ for path in spill_directory.iterdir():
+ path.unlink()
+ if len(buckets) == 1:
+ raise HistoryChunkTooLargeError(
+ f"{table.resource_name}: a key chunk of this sync needs more than half the free "
+ f"local disk ({floor} bytes)"
+ )
+ per_pass = len(buckets) // 2
+ continue
+ for index, (path, footprint) in enumerate(spilled):
+ last = buckets.stop == chunks and index == len(spilled) - 1
+ await _apply(functools.partial(_read_chunk, path), footprint, chunks, last=last)
+ path.unlink()
+ first = buckets.stop
+
+ # Only after every chunk committed: a crash before this point leaves the rows staged for the re-run.
+ await DeltaWriter(staging).delete_where(STAGED_BATCH_COLUMN, batch_tags)
+ await _vacuum_staging(staging)
+ refreshed = await table.get_delta_table()
+ if refreshed is not None:
+ await ensure_table_properties(refreshed, table.logger)
+ result = HistoryFinalizeResult(staged_rows=staged_rows, calls=calls, **totals)
+ await table.logger.ainfo("history_finalize", call_bytes=call_bytes, **dataclasses.asdict(result))
+ return result
+
+
+async def _history_table(
+ table: "DeltaTableRef", staged_schema: pa.Schema, naming: HistoryNaming, partition_column: str
+) -> deltalake.DeltaTable:
+ """The history table, created or evolved to the staged columns plus `valid_to` and the partition column."""
+ fields = [field for field in staged_schema if field.name not in _STAGING_COLUMNS]
+ valid_from_type = staged_schema.field(naming.valid_from).type
+ schema = pa.schema([*fields, pa.field(naming.valid_to, valid_from_type), pa.field(partition_column, pa.string())])
+ history_table = await table.get_delta_table()
+ if history_table is None:
+ history_table = await asyncio.to_thread(
+ deltalake.DeltaTable.create,
+ table_uri=await table.get_table_uri(),
+ schema=schema,
+ mode="ignore",
+ partition_by=[partition_column],
+ storage_options=table.get_storage_options(),
+ configuration=DELTA_TABLE_PROPERTIES,
+ )
+ table.adopt_created_table(history_table)
+ return history_table
+ return await evolve_delta_schema(history_table, schema)
+
+
+def _measure(dataset: ds.Dataset, columns: list[str], narrow_columns: list[str]) -> tuple[int, set[str]]:
+ """What one call with every staged row counts against deltalite's source guard, and the staged batch tags.
+
+ One batch is in memory at a time. Parquet stores repeated and null values in a few bytes, so only
+ decoded batches give that size.
+ """
+ footprint = 0
+ tags: set[str] = set()
+ for batch in _scanner(dataset, [*columns, STAGED_BATCH_COLUMN]).to_batches():
+ tag_column = batch.column(STAGED_BATCH_COLUMN)
+ tags.update(str(tag) for tag in pc.unique(tag_column).drop_null().to_pylist())
+ footprint += _footprint(batch, narrow_columns) - tag_column.nbytes
+ return footprint, tags
+
+
+def _free_bytes(directory: Path) -> int:
+ return shutil.disk_usage(directory).free
+
+
+def _footprint(batch: pa.RecordBatch, narrow_columns: list[str]) -> int:
+ narrow = sum(batch.column(name).nbytes for name in narrow_columns)
+ return batch.nbytes + 2 * narrow + batch.num_rows * _INDEX_BYTES_PER_ROW
+
+
+def _read_staged(dataset: ds.Dataset, columns: list[str]) -> pa.Table:
+ return _scanner(dataset, columns).to_table()
+
+
+def _open_deltalite(uri: str, storage_options: dict[str, str]) -> Any:
+ import deltalite # noqa: PLC0415 - keeps deltalite off the import path of processes that never write with it
+
+ return deltalite.DeltaLiteTable.open(uri, storage_options)
+
+
+def _spill_chunks(
+ handle: Any,
+ history_schema: Any,
+ dataset: ds.Dataset,
+ columns: list[str],
+ key_columns: Sequence[str],
+ narrow_columns: list[str],
+ chunks: int,
+ buckets: range,
+ directory: Path,
+ free_floor: int,
+) -> list[tuple[Path, int]]:
+ """Write the staged rows of the key buckets in `buckets` into one local Arrow file per bucket.
+
+ The rows are aligned to the history table's decimal types first, so the history table hashes the keys
+ the upsert matches. It hashes them after it casts them to its own key types, so two staged values that
+ become one table key land in the same bucket. Returns the files that received rows, in bucket order,
+ each with its call footprint. Stops once the free disk drops below `free_floor`, or a bucket's
+ footprint passes deltalite's source limit.
+ """
+ paths = {bucket: directory / f"chunk-{bucket}.arrow" for bucket in buckets}
+ footprints = dict.fromkeys(buckets, 0)
+ staged_schema = pa.schema([dataset.schema.field(name) for name in columns])
+ schema = align_incoming_decimals_to_delta(staged_schema.empty_table(), history_schema).schema
+ options = pa.ipc.IpcWriteOptions(compression="zstd")
+ with contextlib.ExitStack() as stack:
+ writers = {
+ bucket: stack.enter_context(pa.ipc.new_file(str(path), schema, options=options))
+ for bucket, path in paths.items()
+ }
+ for staged in _scanner(dataset, columns).to_batches():
+ if staged.num_rows == 0:
+ continue
+ aligned = align_incoming_decimals_to_delta(pa.Table.from_batches([staged]), history_schema)
+ for batch in aligned.to_batches():
+ ids = handle.key_buckets(batch, list(key_columns), chunks)
+ order = pc.sort_indices(ids)
+ sorted_batch = batch.take(order)
+ starts = np.searchsorted(ids.take(order).to_numpy(), np.arange(chunks + 1))
+ for bucket, writer in writers.items():
+ count = int(starts[bucket + 1] - starts[bucket])
+ if count:
+ rows = sorted_batch.slice(int(starts[bucket]), count)
+ writer.write_batch(rows)
+ footprints[bucket] += _footprint(rows, narrow_columns)
+ if footprints[bucket] > _SOURCE_GUARD_BYTES:
+ raise HistoryChunkTooLargeError(
+ f"A key chunk of this sync holds over deltalite's {_SOURCE_GUARD_BYTES} byte "
+ "source limit, so one key has too many versions"
+ )
+ if _free_bytes(directory) < free_floor:
+ raise _SpillOutOfDisk()
+ return [(paths[bucket], footprints[bucket]) for bucket in buckets if footprints[bucket]]
+
+
+def _read_chunk(path: Path) -> pa.Table:
+ with pa.OSFile(str(path), "rb") as source:
+ return pa.ipc.open_file(source).read_all()
+
+
+def _scanner(dataset: ds.Dataset, columns: list[str]) -> ds.Scanner:
+ return dataset.scanner(
+ columns=columns, batch_size=_SCAN_BATCH_ROWS, batch_readahead=1, fragment_readahead=1, use_threads=False
+ )
+
+
+def _normalized_keys(key_columns: Sequence[str]) -> list[str]:
+ if not key_columns:
+ raise ValueError("A history table needs a primary key")
+ return [normalize_column_name(key) for key in key_columns]
+
+
+async def _history_upsert(
+ table: "DeltaTableRef",
+ handle: Any,
+ history_table: deltalake.DeltaTable,
+ load: Callable[[], pa.Table],
+ footprint: int,
+ *,
+ key_columns: Sequence[str],
+ naming: HistoryNaming,
+ partition_column: str,
+ commit_metadata: dict[str, str] | None,
+ chunk: int,
+ chunks: int,
+) -> tuple[Any, int]:
+ """Read one call's rows inside its governor admission and write them. Returns the stats and the row count."""
+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
+ # Rows are read only once admitted, so a finalize waiting for room holds none. The closed versions
+ # deltalite writes are at most one per source row, so they count with the source.
+ async with get_governor().admit(source_bytes=2 * footprint, n_partitions=1) as adm:
+ data = align_incoming_decimals_to_delta(await asyncio.to_thread(load), history_table.schema())
+
+ def _upsert() -> Any:
+ return handle.history_upsert(
+ data,
+ list(key_columns),
+ naming.valid_from,
+ naming.valid_from,
+ naming.valid_to,
+ partition_column,
+ CURRENT_HISTORY_PARTITION,
+ STAGED_ROW_COLUMN,
+ commit_metadata=commit_metadata,
+ max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,
+ **adm.upsert_kwargs,
+ )
+
+ started = time.perf_counter()
+ stats = await asyncio.to_thread(_upsert)
+ duration_s = time.perf_counter() - started
+ rows = data.num_rows
+
+ committed = bool(stats.committed)
+ if committed:
+ table.note_deltalite_commit(stats.version)
+ with contextlib.suppress(Exception):
+ await table.logger.ainfo(
+ "history upsert: committed" if committed else "history upsert: nothing to write",
+ chunk=chunk,
+ chunks=chunks,
+ duration_ms=round(duration_s * 1000),
+ governor_mode=adm.mode,
+ governor_predicted_peak_mb=adm.predicted_peak_mb,
+ governor_budget_mb=adm.budget_mb,
+ governor_capacity_exceeded=adm.capacity_exceeded,
+ governor_wait_ms=adm.wait_ms,
+ governor_peak_rss_mb=adm.rss.peak_mb if adm.rss else None,
+ **_deltalite_write_stats(stats),
+ )
+ return stats, rows
+
+
+async def _vacuum_staging(staging: "DeltaTableRef") -> None:
+ # The delete only tombstones the rows. Vacuum drops their files past retention, so staging doesn't grow.
+ staged = await staging.get_delta_table()
+ if staged is None:
+ return
+ await execute_with_conflict_retry(
+ staged,
+ lambda: staged.vacuum(
+ retention_hours=int(VACUUM_RETENTION.total_seconds() // 3600),
+ enforce_retention_duration=False,
+ dry_run=False,
+ ),
+ "vacuum_history_staging",
+ staging.logger,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_history_upsert.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_history_upsert.py
new file mode 100644
index 000000000000..5d0bc214b475
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_history_upsert.py
@@ -0,0 +1,292 @@
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+from unittest.mock import patch
+
+import pyarrow as pa
+import deltalake
+
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_NAMING
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta import history_upsert
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.history_upsert import (
+ HistoryChunkTooLargeError,
+ HistoryFinalizeResult,
+ finalize,
+ stage,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import (
+ make_local_table_ref,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import (
+ DeltaWriter,
+ commit_matches,
+)
+
+_TS = pa.timestamp("us", tz="UTC")
+_VALID_FROM = HISTORY_NAMING.valid_from
+_VALID_TO = HISTORY_NAMING.valid_to
+_PARTITION = str(HISTORY_NAMING.partition_column)
+
+
+def _at(month: int, day: int = 1) -> datetime:
+ return datetime(2026, month, day, tzinfo=UTC)
+
+
+class _Tables:
+ def __init__(self, tmp_path: Path, keys: list[str] | None = None) -> None:
+ self.path = tmp_path / "history"
+ self.history = make_local_table_ref(str(self.path))
+ self.staging = make_local_table_ref(str(tmp_path / "staging"))
+ self.keys = keys or ["id"]
+
+ async def stage(self, batch: pa.Table, run_uuid: str, batch_index: int) -> None:
+ await stage(
+ self.staging,
+ batch,
+ key_columns=self.keys,
+ naming=HISTORY_NAMING,
+ run_uuid=run_uuid,
+ batch_index=batch_index,
+ )
+
+ async def finalize(self, **kwargs: Any) -> HistoryFinalizeResult | None:
+ return await finalize(self.history, self.staging, key_columns=self.keys, naming=HISTORY_NAMING, **kwargs)
+
+ async def run(self, run_uuid: str, *batches: pa.Table, **kwargs: Any) -> HistoryFinalizeResult | None:
+ for index, batch in enumerate(batches):
+ await self.stage(batch, run_uuid, index)
+ return await self.finalize(**kwargs)
+
+ def rows(self) -> list[tuple[Any, ...]]:
+ rows = deltalake.DeltaTable(str(self.path)).to_pyarrow_table().to_pylist()
+ return sorted(
+ ((r["id"], r["name"], r[_VALID_FROM], r[_VALID_TO], r[_PARTITION]) for r in rows),
+ key=lambda r: (r[0] is None, r[0] or 0, r[2]),
+ )
+
+ def version(self) -> int:
+ return deltalake.DeltaTable(str(self.path)).version()
+
+
+def _batch(rows: list[tuple[Any, str, datetime]]) -> pa.Table:
+ return pa.table(
+ {
+ "id": pa.array([row[0] for row in rows], type=pa.int64()),
+ "name": pa.array([row[1] for row in rows]),
+ _VALID_FROM: pa.array([row[2] for row in rows], type=_TS),
+ }
+ )
+
+
+def _keys(count: int, name: str, at: datetime) -> pa.Table:
+ return _batch([(key, name, at) for key in range(count)])
+
+
+_TINY_CHUNKS = {"max_call_bytes": 1, "chunk_target_bytes": 256}
+
+
+def _disk_filled_by_chunk_files(*, times: int | None) -> Any:
+ """A free-disk reading that is 0 while a chunk file exists, for the first `times` such readings."""
+ full_readings = 0
+
+ def _free_bytes(directory: Path) -> int:
+ nonlocal full_readings
+ if any(directory.iterdir()) and (times is None or full_readings < times):
+ full_readings += 1
+ return 0
+ return 10**12
+
+ return _free_bytes
+
+
+class TestHistoryUpsert:
+ @pytest.mark.asyncio
+ async def test_a_sync_closes_changed_keys_and_skips_rows_that_are_not_newer(self, tmp_path: Path) -> None:
+ tables = _Tables(tmp_path)
+ await tables.run("run-1", _batch([(1, "a", _at(1)), (2, "b", _at(1)), (3, "c", _at(2))]))
+
+ result = await tables.run(
+ "run-2",
+ _batch([(1, "a2", _at(3)), (2, "b", _at(1))]),
+ # Key 2 comes back at its open cursor (a lookback re-read), key 3 with a cursor that went backwards.
+ _batch([(1, "a3", _at(4, 10)), (2, "b", _at(1)), (3, "stale", _at(1)), (4, "d", _at(4))]),
+ )
+
+ assert tables.rows() == [
+ (1, "a", _at(1), _at(3), "2026-03"),
+ (1, "a2", _at(3), _at(4, 10), "2026-04"),
+ (1, "a3", _at(4, 10), None, "current"),
+ (2, "b", _at(1), None, "current"),
+ (3, "c", _at(2), None, "current"),
+ (4, "d", _at(4), None, "current"),
+ ]
+ assert result is not None
+ assert (result.staged_rows, result.rows_deduped, result.rows_reread, result.rows_backwards) == (6, 1, 1, 1)
+ assert (result.versions_closed, result.chain_rows_closed) == (1, 1)
+
+ @pytest.mark.parametrize("same_run", [True, False])
+ @pytest.mark.asyncio
+ async def test_the_last_staged_row_wins_a_tie_on_key_and_cursor(self, tmp_path: Path, same_run: bool) -> None:
+ tables = _Tables(tmp_path)
+ # Across runs: a run whose finalize never ran leaves its rows staged, and the next run stages from batch 0.
+ await tables.stage(_batch([(1, "stale", _at(3))]), "run-2", 5)
+ await tables.stage(_batch([(1, "fresh", _at(3))]), "run-2" if same_run else "run-3", 6 if same_run else 0)
+ await tables.finalize()
+
+ assert [row[1] for row in tables.rows()] == ["fresh"]
+
+ @pytest.mark.asyncio
+ async def test_a_null_key_part_matches_its_open_version(self, tmp_path: Path) -> None:
+ tables = _Tables(tmp_path, keys=["id", "name"])
+ await tables.run("run-1", _batch([(None, "x", _at(1))]))
+ await tables.run("run-2", _batch([(None, "x", _at(2))]))
+
+ assert tables.rows() == [
+ (None, "x", _at(1), _at(2), "2026-02"),
+ (None, "x", _at(2), None, "current"),
+ ]
+
+ @pytest.mark.asyncio
+ async def test_a_required_column_that_arrives_later_keeps_the_earlier_rows(self, tmp_path: Path) -> None:
+ tables = _Tables(tmp_path)
+ later = _batch([(2, "b", _at(1))])
+ later = later.append_column(pa.field("plan", pa.string(), nullable=False), pa.array(["pro"]))
+ await tables.run("run-1", _batch([(1, "a", _at(1))]), later)
+
+ assert [row[0] for row in tables.rows()] == [1, 2]
+
+ @pytest.mark.asyncio
+ async def test_a_source_column_named_like_staging_metadata_is_refused(self, tmp_path: Path) -> None:
+ batch = _batch([(1, "a", _at(1))]).append_column("_ph_staged_row", pa.array([5]))
+
+ with pytest.raises(ValueError, match="_ph_staged_row"):
+ await _Tables(tmp_path).stage(batch, "run", 0)
+
+ @pytest.mark.asyncio
+ async def test_nothing_staged_writes_nothing(self, tmp_path: Path) -> None:
+ tables = _Tables(tmp_path)
+ assert await tables.finalize() is None
+ # The load path reads the staging table's schema after every batch, an empty first one too.
+ await tables.stage(_batch([(1, "a", _at(1))]).slice(0, 0), "run-0", 0)
+ assert await tables.staging.get_delta_table() is not None
+ assert await tables.finalize() is None
+ await tables.run("run-1", _batch([(1, "a", _at(1))]))
+
+ assert await tables.finalize() is None
+
+ @pytest.mark.parametrize(
+ "name_width,chunk_kwargs,files_per_pass,disk_fills",
+ [
+ (1, _TINY_CHUNKS, 256, False),
+ # An open-file cap of two spills the sync in several passes.
+ (1, _TINY_CHUNKS, 2, False),
+ # A disk that fills during a pass runs it again with fewer chunks.
+ (1, _TINY_CHUNKS, 256, True),
+ # Parquet stores the repeated names in a few bytes, so only their decoded size crosses the limit.
+ (1000, {"max_call_bytes": 20_000, "chunk_target_bytes": 10_000}, 256, False),
+ # Narrow rows decode under the limit, and only deltalite's key copies and per-row index entries cross it.
+ (1, {"max_call_bytes": 3_000, "chunk_target_bytes": 1_500}, 256, False),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_a_chunked_finalize_writes_the_same_history_as_one_call(
+ self, tmp_path: Path, name_width: int, chunk_kwargs: dict[str, int], files_per_pass: int, disk_fills: bool
+ ) -> None:
+ one_call, chunked = _Tables(tmp_path / "one"), _Tables(tmp_path / "chunked")
+ syncs = [
+ [_keys(40, "a" * name_width, _at(1))],
+ [_keys(40, "b" * name_width, _at(2)), _keys(40, "c" * name_width, _at(3)), _keys(40, "a", _at(1))],
+ ]
+ tag = {"run_uuid": "run-1", "batch_index": "2"}
+ for index, batches in enumerate(syncs):
+ last_sync = index == len(syncs) - 1
+ await one_call.run(f"run-{index}", *batches, commit_metadata=tag if last_sync else None)
+ with (
+ patch.object(history_upsert, "_MAX_OPEN_CHUNK_FILES", files_per_pass),
+ patch.object(
+ history_upsert,
+ "_free_bytes",
+ _disk_filled_by_chunk_files(times=1) if disk_fills else lambda _directory: 10**12,
+ ),
+ ):
+ result = await chunked.run(
+ f"run-{index}", *batches, commit_metadata=tag if last_sync else None, **chunk_kwargs
+ )
+ assert result is not None and result.calls > 1
+
+ assert chunked.rows() == one_call.rows()
+ for tables in (one_call, chunked):
+ history = deltalake.DeltaTable(str(tables.path))
+ tagged = [commit["version"] for commit in history.history() if commit_matches(commit, tag)]
+ assert tagged == [history.version()]
+
+ @pytest.mark.parametrize(
+ "limit",
+ [
+ patch.object(history_upsert, "_SOURCE_GUARD_BYTES", 1),
+ # Every pass fills the disk, down to a pass of one chunk.
+ patch.object(history_upsert, "_free_bytes", _disk_filled_by_chunk_files(times=None)),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_a_chunk_above_a_limit_is_refused_and_stays_staged(self, tmp_path: Path, limit: Any) -> None:
+ tables = _Tables(tmp_path)
+ await tables.stage(_keys(40, "a", _at(1)), "run-1", 0)
+
+ with limit:
+ with pytest.raises(HistoryChunkTooLargeError):
+ await tables.finalize(**_TINY_CHUNKS)
+
+ assert (await tables.finalize()) is not None
+ assert len(tables.rows()) == 40
+
+ @pytest.mark.parametrize("chunk_kwargs", [{}, _TINY_CHUNKS])
+ @pytest.mark.asyncio
+ async def test_a_rerun_after_a_crash_past_the_commit_changes_nothing(
+ self, tmp_path: Path, chunk_kwargs: dict[str, int]
+ ) -> None:
+ tables = _Tables(tmp_path)
+ await tables.run("run-1", _keys(20, "a", _at(1)))
+ await tables.stage(_keys(20, "b", _at(2)), "run-2", 0)
+ with patch.object(DeltaWriter, "delete_where", side_effect=RuntimeError("crash")):
+ with pytest.raises(RuntimeError):
+ await tables.finalize(**chunk_kwargs)
+ rows, version = tables.rows(), tables.version()
+
+ rerun = await tables.finalize(**chunk_kwargs)
+
+ assert (tables.rows(), tables.version()) == (rows, version)
+ assert rerun is not None and rerun.rows_reread == 20
+ assert await tables.finalize(**chunk_kwargs) is None
+
+ @pytest.mark.asyncio
+ async def test_a_rerun_after_a_crash_between_chunk_commits_finishes_the_rest_once(self, tmp_path: Path) -> None:
+ tables = _Tables(tmp_path)
+ await tables.run("run-1", _keys(40, "a", _at(1)))
+ await tables.stage(_keys(40, "b", _at(2)), "run-2", 0)
+ real_upsert = history_upsert._history_upsert
+ calls = 0
+
+ async def _crash_on_second_chunk(*args: Any, **kwargs: Any) -> Any:
+ nonlocal calls
+ calls += 1
+ if calls == 2:
+ raise RuntimeError("crash")
+ return await real_upsert(*args, **kwargs)
+
+ tag = {"run_uuid": "run-2", "batch_index": "0"}
+ with patch.object(history_upsert, "_history_upsert", side_effect=_crash_on_second_chunk):
+ with pytest.raises(RuntimeError):
+ await tables.finalize(commit_metadata=tag, **_TINY_CHUNKS)
+ await tables.finalize(commit_metadata=tag, **_TINY_CHUNKS)
+
+ table = deltalake.DeltaTable(str(tables.path))
+ assert [commit["version"] for commit in table.history() if commit_matches(commit, tag)] == [table.version()]
+ history = tables.rows()
+ assert len(history) == 80
+ assert [row for row in history if row[0] == 7] == [
+ (7, "a", _at(1), _at(2), "2026-02"),
+ (7, "b", _at(2), None, "current"),
+ ]