Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(deltalite): add history_upsert for SCD2 history tables (#128450)

Daniel RC328aa36e118a
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 0d455a194cb5..83e0d88c3aa7 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -4110,6 +4110,7 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tracing",
+ "twox-hash",
"url",
"uuid",
]
diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.toml
index 198b803903e8..ffb83a440bf6 100644
--- a/rust/deltalite/core/Cargo.toml
+++ b/rust/deltalite/core/Cargo.toml
@@ -56,6 +56,8 @@ indexmap = "2"
# Already linked by `deltalake`; decodes Add paths into the form the snapshot reports.
percent-encoding = "2"
sqlparser = "0.61"
+# Already linked by `parquet` (bloom filters); fixed hash for `history::key_buckets`.
+twox-hash = { version = "2", default-features = false, features = ["xxhash64"] }
async-trait = { workspace = true }
bytes = { workspace = true }
diff --git a/rust/deltalite/core/src/handle.rs b/rust/deltalite/core/src/handle.rs
index ded4753160bd..a861ab990e84 100644
--- a/rust/deltalite/core/src/handle.rs
+++ b/rust/deltalite/core/src/handle.rs
@@ -31,6 +31,7 @@ use metrics::counter;
use serde_json::Value;
use crate::errors::{Error, Result};
+use crate::history::{history_upsert_with_state, HistoryUpsertOptions, HistoryUpsertStats};
use crate::limits::{env_switch, switch_setting};
use crate::logprobe::{check_anchor, commit_path, probe_log, trust_window, LogAnchor, LogProbe};
use crate::prefetch::CheckpointCache;
@@ -415,6 +416,89 @@ impl TableHandle {
}
}
+ /// Run one history upsert (see [`crate::history`]), retrying conflicts against a
+ /// refreshed snapshot. A re-planned attempt reads the open rows again, so a concurrent
+ /// commit cannot make it close a row twice. Unlike [`TableHandle::upsert`], a
+ /// concurrent schema or protocol change is planned against too: every attempt checks
+ /// the refreshed table again, and a history write copies existing rows whole instead of
+ /// replacing them with source values, so the result equals running after that commit.
+ pub async fn history_upsert(
+ &mut self,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: HistoryUpsertOptions,
+ multipart: MultipartConfig,
+ ) -> Result<HistoryUpsertStats> {
+ let result = self
+ .history_upsert_with_retries(source_batches, source_schema, opts, multipart)
+ .await;
+ self.prefetch.clear();
+ result
+ }
+
+ async fn history_upsert_with_retries(
+ &mut self,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: HistoryUpsertOptions,
+ multipart: MultipartConfig,
+ ) -> Result<HistoryUpsertStats> {
+ let mut open_ms = 0u64;
+ let t = Instant::now();
+ self.refresh().await?;
+ open_ms += t.elapsed().as_millis() as u64;
+
+ let mut attempt = 0usize;
+ loop {
+ let view = wrap_multipart_view(self.table.clone(), multipart, self.commit_probe());
+ match history_upsert_with_state(
+ &view,
+ source_batches.clone(),
+ source_schema.clone(),
+ opts.clone(),
+ &mut self.relax_cache,
+ )
+ .await
+ {
+ Err(Error::Conflict(_)) if attempt < CONFLICT_RETRIES => {
+ attempt += 1;
+ tokio::time::sleep(std::time::Duration::from_millis(50 * attempt as u64)).await;
+ let t = Instant::now();
+ self.refresh().await?;
+ open_ms += t.elapsed().as_millis() as u64;
+ }
+ Ok((mut stats, state)) => {
+ let t = Instant::now();
+ if let Some(state) = state {
+ if self.adopt_commit_snapshot {
+ self.table.state = Some(state);
+ self.prefetch.clear();
+ self.list_on_next_refresh = self.on_checkpoint_boundary();
+ } else {
+ self.refresh().await?;
+ }
+ }
+ open_ms += t.elapsed().as_millis() as u64;
+ stats.open_ms = open_ms;
+ stats.initial_open_ms = self.take_initial_open_ms();
+ return Ok(stats);
+ }
+ Err(e) => return Err(e),
+ }
+ }
+ }
+
+ /// Bucket ids for splitting a history source by key; see [`crate::history::key_buckets`].
+ /// Uses the loaded snapshot's schema, without a refresh.
+ pub fn key_buckets(
+ &self,
+ batches: &[RecordBatch],
+ key_columns: &[String],
+ n: u32,
+ ) -> Result<arrow_array::UInt32Array> {
+ crate::history::key_buckets(&self.table, batches, key_columns, n)
+ }
+
/// Compact small files (see [`crate::compact`]) against a refreshed snapshot, then
/// adopt the committed state so later writes through this handle see it.
pub async fn compact(
diff --git a/rust/deltalite/core/src/history.rs b/rust/deltalite/core/src/history.rs
new file mode 100644
index 000000000000..ba99eef4c64d
--- /dev/null
+++ b/rust/deltalite/core/src/history.rs
@@ -0,0 +1,1654 @@
+//! Incremental SCD2 history upsert.
+//!
+//! The table holds one row per version of a key. Open versions (`valid_to` null) live in
+//! the `current` partition; a closed version lives in the `%Y-%m` (UTC) month partition of
+//! its `valid_to`. [`history_upsert`] applies a source batch of versions in one data commit:
+//!
+//! 1. Rows with the same (key, order value) collapse to the one with the highest tie-break
+//! value.
+//! 2. Only rows strictly newer than the key's open row are kept. A row equal to it is a
+//! re-read, an older one is a backwards row; both are dropped and counted.
+//! 3. The kept rows of a key form a chain: each closes at the next kept row's
+//! `valid_from`, and the newest stays open in `current`.
+//! 4. When a key keeps a row and has an open row, the open row closes at the first kept
+//! row's `valid_from` and moves to its month partition.
+//!
+//! Keys are null-safe: a null key part matches a null key part. A null order value or a
+//! null `valid_from` is an error. Closed rows are only ever appended, so earlier closed
+//! versions of a key in the same month stay. A source that keeps nothing (a re-run of the
+//! same input) writes nothing and makes no commit.
+//!
+//! The machinery is the upsert's (see [`crate::upsert`]): the same listing, stats pruning,
+//! budgets, writer and commit. Two things differ. The probe reads the order column next to
+//! the keys, so a file that holds only re-read keys is not rewritten, and it reads every
+//! candidate file in full, because the open order of every source key must be known
+//! before the kept rows are chosen. The probe cache is not used, because its entries
+//! record keys only.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Instant;
+
+use arrow_array::cast::AsArray;
+use arrow_array::{
+ new_null_array, Array, ArrayRef, BooleanArray, RecordBatch, StringArray, UInt32Array,
+};
+use arrow_cast::{cast_with_options, CastOptions};
+use arrow_ord::sort::sort_to_indices;
+use arrow_row::{RowConverter, Rows, SortField};
+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
+use arrow_select::concat::concat;
+use arrow_select::filter::{filter, filter_record_batch};
+use arrow_select::interleave::interleave_record_batch;
+use arrow_select::take::{take, take_record_batch};
+use chrono::{DateTime, NaiveDate, Utc};
+use deltalake::kernel::{Action, Add};
+use deltalake::table::config::TablePropertiesExt;
+use deltalake::table::state::DeltaTableState;
+use deltalake::writer::RecordBatchWriter;
+use deltalake::{DeltaTable, ObjectStore};
+use futures::{StreamExt, TryStreamExt};
+use metrics::{counter, histogram};
+use parquet::arrow::ProjectionMask;
+use parquet::file::metadata::ParquetMetaData;
+use serde_json::Value;
+use tokio::sync::{mpsc, Semaphore};
+use tracing::info;
+
+use crate::errors::{Error, Result};
+use crate::limits::{
+ check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, ProcessLimits,
+ SourceFootprint,
+};
+use crate::schema::cast_to_schema;
+use crate::upsert::{
+ add_partition_column, byte_bounded_batch_rows, columns_needing_relax,
+ commit_partition_overwrite, ensure_supported_table, json_cmp, list_files_by_partition,
+ max_row_group_fetch_bytes, min_max_json, open_builder, open_rewrite_stream,
+ relax_columns_to_nullable, resolve_target_file_size, roll_writer, source_slice_rows,
+ BudgetPermit, Budgets, RelaxCache, TargetFile, DEFAULT_MAX_FETCH_BYTES,
+ INITIAL_PROBE_ESTIMATE_BYTES,
+};
+use crate::writer::StreamingWriter;
+
+/// Seed of the [`key_buckets`] hash. Part of the API: changing it moves every key to a
+/// different bucket.
+pub const KEY_BUCKET_SEED: u64 = 0x6465_6c74_616c_6974;
+
+#[derive(Clone)]
+pub struct HistoryUpsertOptions {
+ /// Columns that identify a key, in order. Compared null-safely.
+ pub key_columns: Vec<String>,
+ /// Column that orders the versions of a key. Must not be null.
+ pub order_column: String,
+ /// Column with the instant a version starts. Must not be null. It is the `valid_to`
+ /// of the version it replaces.
+ pub valid_from_column: String,
+ /// Column with the instant a version ends, null while it is open. Set by the
+ /// operation; the source must not carry it.
+ pub valid_to_column: String,
+ /// The table's only partition column (string). Set by the operation; the source must
+ /// not carry it.
+ pub partition_column: String,
+ /// Partition value of the open versions.
+ pub current_partition: String,
+ /// Source column that decides between rows with the same (key, order value): the
+ /// highest value wins. Must have a fixed-width type, such as an integer or a timestamp.
+ /// Dropped before writing when the table has no such column.
+ pub tie_break_column: String,
+ pub commit_metadata: Option<HashMap<String, Value>>,
+ pub probe_concurrency: usize,
+ pub max_parallel_files: usize,
+ pub max_buffered_bytes: usize,
+ pub max_fetch_bytes: usize,
+ pub read_ahead_row_groups: bool,
+ pub read_batch_size: usize,
+ /// `None` uses the table's `delta.targetFileSize`.
+ pub target_file_size: Option<usize>,
+ pub max_row_group_bytes: Option<usize>,
+ /// Ceiling for the source-size guard; `None` resolves from the environment, `Some(0)`
+ /// disables. See [`crate::limits::resolve_max_source_bytes`].
+ pub max_source_bytes: Option<usize>,
+ pub limits: Arc<ProcessLimits>,
+}
+
+impl Default for HistoryUpsertOptions {
+ fn default() -> Self {
+ Self {
+ key_columns: Vec::new(),
+ order_column: String::new(),
+ valid_from_column: String::new(),
+ valid_to_column: String::new(),
+ partition_column: String::new(),
+ current_partition: String::new(),
+ tie_break_column: String::new(),
+ commit_metadata: None,
+ probe_concurrency: 8,
+ max_parallel_files: 4,
+ max_buffered_bytes: 64 * 1024 * 1024,
+ max_fetch_bytes: DEFAULT_MAX_FETCH_BYTES,
+ read_ahead_row_groups: false,
+ read_batch_size: 8192,
+ target_file_size: None,
+ max_row_group_bytes: None,
+ max_source_bytes: None,
+ limits: ProcessLimits::global().clone(),
+ }
+ }
+}
+
+#[derive(Default, Debug, Clone)]
+pub struct HistoryUpsertStats {
+ /// The committed table version, or the version the call read when it made no commit.
+ pub version: i64,
+ /// Whether the call made a data commit. False when the source kept no row.
+ pub committed: bool,
+ pub rows_in: usize,
+ /// Source rows dropped because another row had the same (key, order value) and a
+ /// higher tie-break value.
+ pub rows_deduped: usize,
+ /// Source rows whose order value equals the key's open row.
+ pub rows_reread: usize,
+ /// Source rows whose order value is lower than the key's open row.
+ pub rows_backwards: usize,
+ /// Open versions written (one per key that kept a row).
+ pub versions_opened: usize,
+ /// Existing open rows closed and moved to their month partition.
+ pub versions_closed: usize,
+ /// Kept source rows written already closed (every kept row but the newest of a key).
+ pub chain_rows_closed: usize,
+ /// Open rows copied unchanged into new files of the current partition.
+ pub rows_copied: usize,
+ pub files_removed: usize,
+ pub files_added: usize,
+ /// Current-partition files left untouched.
+ pub files_carried_over: usize,
+ /// Current-partition files whose key and order columns the probe read.
+ pub files_probed: usize,
+ /// Stored bytes of the rewritten files (the probe's narrow reads are not counted).
+ pub bytes_read: u64,
+ pub bytes_written: u64,
+ pub plan_ms: u64,
+ pub probe_ms: u64,
+ pub rewrite_ms: u64,
+ pub commit_ms: u64,
+ pub columns_relaxed: usize,
+ pub open_ms: u64,
+ pub initial_open_ms: u64,
+ pub ingest_ms: u64,
+ pub relax_ms: u64,
+ pub maintenance_ms: u64,
+}
+
+/// Apply `source_batches` to the history table `table`. See the module docs.
+pub async fn history_upsert(
+ table: &DeltaTable,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: HistoryUpsertOptions,
+) -> Result<HistoryUpsertStats> {
+ history_upsert_with_state(
+ table,
+ source_batches,
+ source_schema,
+ opts,
+ &mut RelaxCache::default(),
+ )
+ .await
+ .map(|(stats, _)| stats)
+}
+
+/// [`history_upsert`] that also returns the table state after the call, for a handle to
+/// adopt. `None` when the table did not move.
+pub async fn history_upsert_with_state(
+ table: &DeltaTable,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: HistoryUpsertOptions,
+ relax_cache: &mut RelaxCache,
+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {
+ let started = Instant::now();
+ let result = history_with_relax(table, source_batches, source_schema, opts, relax_cache).await;
+ histogram!("deltalite_history_upsert_duration_seconds").record(started.elapsed().as_secs_f64());
+ match &result {
+ Ok((stats, _)) => {
+ let outcome = if stats.committed { "ok" } else { "noop" };
+ counter!("deltalite_history_upserts_total", "outcome" => outcome).increment(1);
+ counter!("deltalite_history_versions_closed_total")
+ .increment((stats.versions_closed + stats.chain_rows_closed) as u64);
+ counter!("deltalite_history_versions_opened_total")
+ .increment(stats.versions_opened as u64);
+ }
+ Err(e) => {
+ counter!(
+ "deltalite_history_upserts_total",
+ "outcome" => "error",
+ "error_kind" => e.kind()
+ )
+ .increment(1);
+ }
+ }
+ result
+}
+
+async fn history_with_relax(
+ table: &DeltaTable,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: HistoryUpsertOptions,
+ relax_cache: &mut RelaxCache,
+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {
+ // Before the relax, so an unsupported table or source never makes a metadata commit.
+ ensure_history_table(table, &opts)?;
+ validate_source(table, &source_batches, &source_schema, &opts)?;
+ let relax_started = Instant::now();
+ let mut relax =
+ columns_needing_relax(table, &source_batches, &source_schema, relax_cache).await?;
+ // The source never carries the partition column, but every written row sets it.
+ relax.retain(|c| c != &opts.partition_column);
+ if relax.is_empty() {
+ let relax_ms = relax_started.elapsed().as_millis() as u64;
+ let (mut stats, state) =
+ history_inner(table, source_batches, source_schema, &opts, false).await?;
+ stats.relax_ms = relax_ms;
+ if stats.committed {
+ if let Ok(committed) = u64::try_from(stats.version) {
+ relax_cache.advance_own_commit(committed);
+ }
+ }
+ return Ok((stats, state));
+ }
+
+ // Plan first, so a call that keeps no row makes no commit at all, not even the relax.
+ let (planned, _) = history_inner(
+ table,
+ source_batches.clone(),
+ source_schema.clone(),
+ &opts,
+ true,
+ )
+ .await?;
+ if planned.versions_opened == 0 {
+ return Ok((planned, None));
+ }
+ relax_columns_to_nullable(table, &relax).await?;
+ let mut fresh = table.clone();
+ fresh.update_incremental(None).await?;
+ let relax_ms = relax_started.elapsed().as_millis() as u64;
+ let (mut stats, state) =
+ history_inner(&fresh, source_batches, source_schema, &opts, false).await?;
+ stats.columns_relaxed = relax.len();
+ stats.relax_ms = relax_ms;
+ // A concurrent commit can still make the data a no-op after the relax landed.
+ Ok((stats, state.or(fresh.state)))
+}
+
+fn validate_source(
+ table: &DeltaTable,
+ source_batches: &[RecordBatch],
+ source_schema: &SchemaRef,
+ opts: &HistoryUpsertOptions,
+) -> Result<()> {
+ // Every check below reads the declared schema, and the planner indexes columns by it.
+ if let Some(i) = source_batches
+ .iter()
+ .position(|b| b.schema().fields() != source_schema.fields())
+ {
+ return Err(Error::SchemaMismatch(format!(
+ "source batch {i} does not match the declared source schema"
+ )));
+ }
+ let table_schema = RecordBatchWriter::for_table(table)?.arrow_schema();
+ Columns::resolve(&table_schema, source_schema, opts)?;
+ for (role, column) in [
+ ("order", &opts.order_column),
+ ("valid_from", &opts.valid_from_column),
+ ] {
+ let idx = source_schema.index_of(column)?;
+ let nulls: usize = source_batches
+ .iter()
+ .map(|b| b.column(idx).logical_null_count())
+ .sum();
+ if nulls > 0 {
+ return Err(Error::Generic(format!(
+ "{role} column '{column}' holds {nulls} null value(s); every history row needs one"
+ )));
+ }
+ }
+ Ok(())
+}
+
+fn ensure_history_table(table: &DeltaTable, opts: &HistoryUpsertOptions) -> Result<()> {
+ ensure_supported_table(table)?;
+ let snapshot = table.snapshot()?;
+ let partition_columns = snapshot.metadata().partition_columns();
+ if partition_columns.len() != 1 || partition_columns[0] != opts.partition_column {
+ return Err(Error::Unsupported(format!(
+ "history_upsert needs a table partitioned by exactly '{}', table is partitioned \
+ by {partition_columns:?}",
+ opts.partition_column
+ )));
+ }
+ Ok(())
+}
+
+struct Columns {
+ key_idx: Vec<usize>,
+ order_idx: usize,
+ valid_from_idx: usize,
+ valid_to_idx: usize,
+ partition_idx: usize,
+ tie_source_idx: usize,
+ /// The tie-break column is not a table column, so it is dropped before writing.
+ strip_tie: bool,
+}
+
+impl Columns {
+ fn resolve(
+ table_schema: &SchemaRef,
+ source_schema: &SchemaRef,
+ opts: &HistoryUpsertOptions,
+ ) -> Result<Self> {
+ if opts.key_columns.is_empty() {
+ return Err(Error::Generic(
+ "key_columns must not be empty for a history upsert".into(),
+ ));
+ }
+ if is_month_name(&opts.current_partition) || opts.current_partition.is_empty() {
+ return Err(Error::Generic(format!(
+ "current_partition '{}' must be non-empty and must not look like a month \
+ partition (YYYY-MM)",
+ opts.current_partition
+ )));
+ }
+ let set_by_operation = [&opts.valid_to_column, &opts.partition_column];
+ let table_idx = |name: &str, role: &str| {
+ table_schema.index_of(name).map_err(|_| {
+ Error::SchemaMismatch(format!("{role} column '{name}' not found in table schema"))
+ })
+ };
+ let source_column = |name: &str, role: &str| -> Result<usize> {
+ if set_by_operation.iter().any(|c| c.as_str() == name) {
+ return Err(Error::Generic(format!(
+ "'{name}' is set by history_upsert and cannot be the {role} column"
+ )));
+ }
+ source_schema.index_of(name).map_err(|_| {
+ Error::SchemaMismatch(format!("{role} column '{name}' is missing from the source"))
+ })?;
+ table_idx(name, role)
+ };
+
+ let mut key_idx = Vec::with_capacity(opts.key_columns.len());
+ for k in &opts.key_columns {
+ key_idx.push(source_column(k, "key")?);
+ }
+ let order_idx = source_column(&opts.order_column, "order")?;
+ let valid_from_idx = source_column(&opts.valid_from_column, "valid_from")?;
+ let valid_to_idx = table_idx(&opts.valid_to_column, "valid_to")?;
+ let partition_idx = table_idx(&opts.partition_column, "partition")?;
+
+ for c in set_by_operation {
+ if source_schema.index_of(c).is_ok() {
+ return Err(Error::SchemaMismatch(format!(
+ "source carries column '{c}', which history_upsert sets itself"
+ )));
+ }
+ }
+ if set_by_operation.contains(&&opts.tie_break_column) {
+ return Err(Error::Generic(format!(
+ "tie-break column '{}' is set by history_upsert",
+ opts.tie_break_column
+ )));
+ }
+ let tie_source_idx = source_schema
+ .index_of(&opts.tie_break_column)
+ .map_err(|_| {
+ Error::SchemaMismatch(format!(
+ "tie-break column '{}' is missing from the source",
+ opts.tie_break_column
+ ))
+ })?;
+ // Fixed width only: the size guard counts in-memory bytes, and a dictionary or
+ // view column can encode to many times that when every row is row-encoded.
+ let tie_type = source_schema.field(tie_source_idx).data_type();
+ if !tie_type.is_primitive() {
+ return Err(Error::SchemaMismatch(format!(
+ "tie-break column '{}' must have a fixed-width type such as an integer or a \
+ timestamp, source has {tie_type}",
+ opts.tie_break_column
+ )));
+ }
+ let strip_tie = table_schema.index_of(&opts.tie_break_column).is_err();
+
+ let extra: Vec<&String> = source_schema
+ .fields()
+ .iter()
+ .map(|f| f.name())
+ .filter(|n| **n != opts.tie_break_column && table_schema.index_of(n).is_err())
+ .collect();
+ if !extra.is_empty() {
+ return Err(Error::SchemaMismatch(format!(
+ "source has columns not present in the table schema: {extra:?}. \
+ Additive schema evolution must run before history_upsert."
+ )));
+ }
+
+ if !matches!(
+ table_schema.field(partition_idx).data_type(),
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ ) {
+ return Err(Error::Unsupported(format!(
+ "history partition column '{}' must be a string column, table has {}",
+ opts.partition_column,
+ table_schema.field(partition_idx).data_type()
+ )));
+ }
+ let valid_to_type = table_schema.field(valid_to_idx).data_type();
+ if !matches!(
+ valid_to_type,
+ DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64
+ ) {
+ return Err(Error::SchemaMismatch(format!(
+ "valid_to column '{}' must be a date or timestamp column, table has {valid_to_type}",
+ opts.valid_to_column
+ )));
+ }
+ Ok(Self {
+ key_idx,
+ order_idx,
+ valid_from_idx,
+ valid_to_idx,
+ partition_idx,
+ tie_source_idx,
+ strip_tie,
+ })
+ }
+}
+
+enum FirstKeyRange {
+ NoValues,
+ /// Min and max of the non-null values, comparable against Add-action stats.
+ Range(Value, Value),
+ Unknown,
+}
+
+/// The source, encoded and deduped. Shared read-only by every probe and reader task.
+struct SourcePlan {
+ /// Source batches cast to the table schema, with `valid_to` and the partition column
+ /// null until the rows are written.
+ batches: Vec<RecordBatch>,
+ offsets: Vec<usize>,
+ key_converter: RowConverter,
+ order_converter: RowConverter,
+ /// Order value of every source row, as order-preserving bytes.
+ orders: Rows,
+ /// Every source row's `valid_from`, cast to the table's `valid_to` type.
+ valid_from: ArrayRef,
+ /// One global row per (key, order value), sorted by key, then order value.
+ deduped: Vec<u32>,
+ /// Range of `deduped` that holds each key, in key order. A key's index here is its id.
+ groups: Vec<(u32, u32)>,
+ by_key: HashMap<Box<[u8]>, u32>,
+ first_key: FirstKeyRange,
+ first_key_has_null: bool,
+ rows_in: usize,
+}
+
+impl SourcePlan {
+ fn build(
+ source_batches: Vec<RecordBatch>,
+ table_schema: &SchemaRef,
+ cols: &Columns,
+ opts: &HistoryUpsertOptions,
+ ) -> Result<Self> {
+ let work_schema = nullable_schema(table_schema);
+ let mut batches = Vec::with_capacity(source_batches.len());
+ let mut ties = Vec::with_capacity(source_batches.len());
+ let mut offsets = Vec::with_capacity(source_batches.len());
+ let mut rows_in = 0usize;
+ for b in source_batches {
+ ties.push(b.column(cols.tie_source_idx).clone());
+ let b = if cols.strip_tie {
+ let keep: Vec<usize> = (0..b.num_columns())
+ .filter(|i| *i != cols.tie_source_idx)
+ .collect();
+ b.project(&keep)?
+ } else {
+ b
+ };
+ offsets.push(rows_in);
+ rows_in += b.num_rows();
+ batches.push(cast_to_schema(&b, &work_schema)?);
+ }
+ if rows_in > u32::MAX as usize {
+ return Err(Error::Generic(format!(
+ "source has {rows_in} rows, more than a history upsert supports"
+ )));
+ }
+
+ // Guard before the narrow columns are copied out, and count those copies: the
+ // key, order, tie-break and `valid_from` arrays plus the key encodings.
+ let column_bytes = |idx: usize| -> usize {
+ batches
+ .iter()
+ .map(|b| b.column(idx).get_array_memory_size())
+ .sum()
+ };
+ let tie_bytes: usize = ties.iter().map(|t| t.get_array_memory_size()).sum();
+ let narrow_bytes = cols.key_idx.iter().map(|i| column_bytes(*i)).sum::<usize>() * 2
+ + column_bytes(cols.order_idx)
+ + column_bytes(cols.valid_from_idx)
+ + tie_bytes;
+ let footprint = SourceFootprint {
+ source_bytes: batches
+ .iter()
+ .map(RecordBatch::get_array_memory_size)
+ .sum::<usize>()
+ + tie_bytes,
+ pk_set_bytes: estimate_pk_set_bytes(narrow_bytes, rows_in)
+ .saturating_add(rows_in.saturating_mul(2 * size_of::<u32>())),
+ };
+ check_source_size(&footprint, resolve_max_source_bytes(opts.max_source_bytes))?;
+
+ let column = |idx: usize| -> Result<ArrayRef> {
+ let parts: Vec<&dyn Array> = batches.iter().map(|b| b.column(idx).as_ref()).collect();
+ if parts.is_empty() {
+ return Ok(new_null_array(table_schema.field(idx).data_type(), 0));
+ }
+ Ok(concat(&parts)?)
+ };
+ let keys: Vec<ArrayRef> = cols
+ .key_idx
+ .iter()
+ .map(|i| column(*i))
+ .collect::<Result<_>>()?;
+ let order = column(cols.order_idx)?;
+ let tie: ArrayRef = match ties.len() {
+ 0 => new_null_array(&DataType::Null, 0),
+ _ => concat(&ties.iter().map(|t| t.as_ref()).collect::<Vec<_>>())?,
+ };
+
+ let valid_from = column(cols.valid_from_idx)?;
+ let valid_from = cast_strict(
+ &valid_from,
+ table_schema.field(cols.valid_to_idx).data_type(),
+ )?;
+
+ let key_converter = RowConverter::new(
+ keys.iter()
+ .map(|k| SortField::new(k.data_type().clone()))
+ .collect(),
+ )?;
+ let order_converter = RowConverter::new(vec![SortField::new(order.data_type().clone())])?;
+ let tie_converter = RowConverter::new(vec![SortField::new(tie.data_type().clone())])?;
+ let key_rows = key_converter.convert_columns(&keys)?;
+ let orders = order_converter.convert_columns(std::slice::from_ref(&order))?;
+ let tie_rows = tie_converter.convert_columns(std::slice::from_ref(&tie))?;
+
+ let first_key_has_null = keys[0].null_count() > 0;
+ let first_key = if keys[0].null_count() == keys[0].len() {
+ FirstKeyRange::NoValues
+ } else {
+ match min_max_json(keys[0].as_ref()) {
+ Some((min, max)) => FirstKeyRange::Range(min, max),
+ None => FirstKeyRange::Unknown,
+ }
+ };
+ drop(keys);
+
+ // A stable sort keeps input order among rows equal on all three, so the last
+ // staged row of a (key, order value) still wins a tie on the tie-break value.
+ let mut sorted: Vec<u32> = (0..rows_in as u32).collect();
+ sorted.sort_by(|&a, &b| {
+ let (a, b) = (a as usize, b as usize);
+ key_rows
+ .row(a)
+ .as_ref()
+ .cmp(key_rows.row(b).as_ref())
+ .then_with(|| orders.row(a).as_ref().cmp(orders.row(b).as_ref()))
+ .then_with(|| tie_rows.row(a).as_ref().cmp(tie_rows.row(b).as_ref()))
+ });
+ drop(tie_rows);
+
+ let mut deduped = Vec::with_capacity(sorted.len());
+ let mut groups = Vec::new();
+ let mut by_key = HashMap::new();
+ let mut i = 0usize;
+ while i < sorted.len() {
+ let key = key_rows.row(sorted[i] as usize);
+ let mut end = i + 1;
+ while end < sorted.len() && key_rows.row(sorted[end] as usize).as_ref() == key.as_ref()
+ {
+ end += 1;
+ }
+ let start = deduped.len() as u32;
+ for t in i..end {
+ let last_of_order = t + 1 == end
+ || orders.row(sorted[t] as usize).as_ref()
+ != orders.row(sorted[t + 1] as usize).as_ref();
+ if last_of_order {
+ deduped.push(sorted[t]);
+ }
+ }
+ by_key.insert(Box::<[u8]>::from(key.as_ref()), groups.len() as u32);
+ groups.push((start, deduped.len() as u32));
+ i = end;
+ }
+
+ Ok(Self {
+ batches,
+ offsets,
+ key_converter,
+ order_converter,
+ orders,
+ valid_from,
+ deduped,
+ groups,
+ by_key,
+ first_key,
+ first_key_has_null,
+ rows_in,
+ })
+ }
+
+ fn order(&self, row: u32) -> &[u8] {
+ self.orders.row(row as usize).data()
+ }
+
+ fn newest_order(&self, group: u32) -> &[u8] {
+ let (_, end) = self.groups[group as usize];
+ self.order(self.deduped[end as usize - 1])
+ }
+
+ fn locate(&self, row: u32) -> (usize, usize) {
+ let row = row as usize;
+ let batch = self.offsets.partition_point(|start| *start <= row) - 1;
+ (batch, row - self.offsets[batch])
+ }
+
+ /// Choose the kept rows of every key against the open order values the probe found.
+ fn decide(&self, open: &[Option<Box<[u8]>>]) -> Decision {
+ let mut d = Decision {
+ closing: vec![None; self.groups.len()],
+ writes: Vec::new(),
+ rows_reread: 0,
+ rows_backwards: 0,
+ versions_opened: 0,
+ chain_rows_closed: 0,
+ };
+ for (group, &(start, end)) in self.groups.iter().enumerate() {
+ let rows = &self.deduped[start as usize..end as usize];
+ let kept_from = match &open[group] {
+ None => 0,
+ Some(open) => {
+ let kept_from = rows.partition_point(|r| self.order(*r) <= open.as_ref());
+ for r in &rows[..kept_from] {
+ if self.order(*r) == open.as_ref() {
+ d.rows_reread += 1;
+ } else {
+ d.rows_backwards += 1;
+ }
+ }
+ kept_from
+ }
+ };
+ let kept = &rows[kept_from..];
+ let Some(first) = kept.first() else {
+ continue;
+ };
+ d.versions_opened += 1;
+ d.chain_rows_closed += kept.len() - 1;
+ if open[group].is_some() {
+ d.closing[group] = Some(*first);
+ }
+ for (t, r) in kept.iter().enumerate() {
+ d.writes.push((*r, kept.get(t + 1).copied()));
+ }
+ }
+ d
+ }
+}
+
+/// A kept source row and the next kept row of its key, whose `valid_from` closes it
+/// (`None` for the newest, which stays open).
+type KeptRow = (u32, Option<u32>);
+
+struct Decision {
+ /// Per key: the source row whose `valid_from` closes the key's open row.
+ closing: Vec<Option<u32>>,
+ /// Kept source rows in write order.
+ writes: Vec<KeptRow>,
+ rows_reread: usize,
+ rows_backwards: usize,
+ versions_opened: usize,
+ chain_rows_closed: usize,
+}
+
+/// The table schema with every column nullable. Source batches are held in it until they
+/// are written: the operation sets `valid_to` and the partition column later, and a
+/// column the table still marks non-nullable may hold nulls until the relax commits.
+/// Written batches are rebuilt in the table schema, which checks nullability again.
+fn nullable_schema(table_schema: &SchemaRef) -> SchemaRef {
+ let fields: Vec<Field> = table_schema
+ .fields()
+ .iter()
+ .map(|f| f.as_ref().clone().with_nullable(true))
+ .collect();
+ Arc::new(Schema::new_with_metadata(
+ fields,
+ table_schema.metadata().clone(),
+ ))
+}
+
+fn cast_strict(array: &ArrayRef, to: &DataType) -> Result<ArrayRef> {
+ if array.data_type() == to {
+ return Ok(array.clone());
+ }
+ let opts = CastOptions {
+ safe: false,
+ ..Default::default()
+ };
+ Ok(cast_with_options(array, to, &opts)?)
+}
+
+/// `%Y-%m` of each value in UTC, null for null. Must equal pyarrow's
+/// `pc.strftime(values, "%Y-%m")` for UTC timestamps, which is how callers name the
+/// month partitions they read.
+pub fn format_months(values: &dyn Array) -> Result<StringArray> {
+ let raw = |to: &DataType| arrow_cast::cast(&arrow_array::make_array(values.to_data()), to);
+ let instants: Vec<Option<DateTime<Utc>>> = match values.data_type() {
+ DataType::Timestamp(unit, _) => {
+ let raw = raw(&DataType::Int64)?;
+ let raw = raw
+ .as_any()
+ .downcast_ref::<arrow_array::Int64Array>()
+ .ok_or_else(|| Error::Generic("timestamp did not cast to Int64".into()))?;
+ raw.iter()
+ .map(|v| {
+ v.map(|v| {
+ match unit {
+ TimeUnit::Second => DateTime::from_timestamp(v, 0),
+ TimeUnit::Millisecond => DateTime::from_timestamp_millis(v),
+ TimeUnit::Microsecond => DateTime::from_timestamp_micros(v),
+ TimeUnit::Nanosecond => Some(DateTime::from_timestamp_nanos(v)),
+ }
+ .ok_or_else(|| out_of_range(v))
+ })
+ .transpose()
+ })
+ .collect::<Result<_>>()?
+ }
+ DataType::Date32 => {
+ let raw = raw(&DataType::Int32)?;
+ let raw = raw
+ .as_any()
+ .downcast_ref::<arrow_array::Int32Array>()
+ .ok_or_else(|| Error::Generic("date did not cast to Int32".into()))?;
+ raw.iter()
+ .map(|v| {
+ v.map(|days| {
+ NaiveDate::from_num_days_from_ce_opt(days.saturating_add(719_163))
+ .map(|d| d.and_time(chrono::NaiveTime::MIN).and_utc())
+ .ok_or_else(|| out_of_range(i64::from(days)))
+ })
+ .transpose()
+ })
+ .collect::<Result<_>>()?
+ }
+ DataType::Date64 => {
+ let raw = raw(&DataType::Int64)?;
+ let raw = raw
+ .as_any()
+ .downcast_ref::<arrow_array::Int64Array>()
+ .ok_or_else(|| Error::Generic("date did not cast to Int64".into()))?;
+ raw.iter()
+ .map(|v| {
+ v.map(|ms| DateTime::from_timestamp_millis(ms).ok_or_else(|| out_of_range(ms)))
+ .transpose()
+ })
+ .collect::<Result<_>>()?
+ }
+ other => {
+ return Err(Error::SchemaMismatch(format!(
+ "cannot name a month partition from a {other} value"
+ )))
+ }
+ };
+ Ok(instants
+ .into_iter()
+ .map(|dt| dt.map(|dt| dt.format("%Y-%m").to_string()))
+ .collect())
+}
+
+fn is_month_name(value: &str) -> bool {
+ let b = value.as_bytes();
+ b.len() == 7 && b[4] == b'-' && b[..4].iter().chain(&b[5..]).all(u8::is_ascii_digit)
+}
+
+fn out_of_range(v: i64) -> Error {
+ Error::Generic(format!(
+ "valid_to value {v} is outside the supported date range"
+ ))
+}
+
+/// Set `valid_to` and the partition column of `batch` (in the table's column order).
+/// Row `i` closes at the `valid_from` of source row `closes[i]` into that value's month;
+/// a null entry leaves it open in `current`.
+fn stamp_validity(
+ batch: &RecordBatch,
+ closes: &UInt32Array,
+ plan: &SourcePlan,
+ cols: &Columns,
+ table_schema: &SchemaRef,
+ current: &str,
+) -> Result<RecordBatch> {
+ let valid_to = take(plan.valid_from.as_ref(), closes, None)?;
+ let months = format_months(valid_to.as_ref())?;
+ let partition: StringArray = months.iter().map(|m| Some(m.unwrap_or(current))).collect();
+ let partition: ArrayRef = Arc::new(partition);
+ let partition = cast_strict(
+ &partition,
+ table_schema.field(cols.partition_idx).data_type(),
+ )?;
+
+ let mut columns = batch.columns().to_vec();
+ columns[cols.valid_to_idx] = valid_to;
+ columns[cols.partition_idx] = partition;
+ Ok(RecordBatch::try_new(table_schema.clone(), columns)?)
+}
+
+/// Whether Add-action stats prove that `f` holds no source key. Null key parts match, so
+/// a file is safe from a null source key only when its first key column has no nulls.
+fn file_cannot_hold_source_key(f: &TargetFile, first_key: &str, plan: &SourcePlan) -> bool {
+ let Some(stats) = &f.stats else { return false };
+ let Ok(parsed) = serde_json::from_str::<Value>(stats) else {
+ return false;
+ };
+ let records = parsed.get("numRecords").and_then(Value::as_i64);
+ if records == Some(0) {
+ return true;
+ }
+ let nulls = parsed
+ .get("nullCount")
+ .and_then(|n| n.get(first_key))
+ .and_then(Value::as_i64);
+ if plan.first_key_has_null && nulls != Some(0) {
+ return false;
+ }
+ if records.is_some() && nulls == records {
+ return true;
+ }
+ match &plan.first_key {
+ FirstKeyRange::NoValues => true,
+ FirstKeyRange::Unknown => false,
+ FirstKeyRange::Range(src_min, src_max) => {
+ let file_min = parsed.get("minValues").and_then(|m| m.get(first_key));
+ let file_max = parsed.get("maxValues").and_then(|m| m.get(first_key));
+ let (Some(file_min), Some(file_max)) = (file_min, file_max) else {
+ return false;
+ };
+ matches!(json_cmp(file_max, src_min), Some(std::cmp::Ordering::Less))
+ || matches!(
+ json_cmp(file_min, src_max),
+ Some(std::cmp::Ordering::Greater)
+ )
+ }
+ }
+}
+
+/// The key and order columns of `batch` in the table's types. A column the batch does not
+/// have reads as null, which a null key part of the source matches.
+fn key_and_order_columns(
+ batch: &RecordBatch,
+ plan_types: &[(String, DataType)],
+) -> Result<Vec<ArrayRef>> {
+ plan_types
+ .iter()
+ .map(|(name, ty)| match batch.column_by_name(name) {
+ Some(c) => cast_strict(c, ty),
+ None => Ok(new_null_array(ty, batch.num_rows())),
+ })
+ .collect()
+}
+
+/// What the probe found in one file: the source keys it holds with their open order
+/// values, and whether any of them has a newer source row.
+struct FileProbe {
+ file: TargetFile,
+ seen: Vec<(u32, Box<[u8]>)>,
+ newer: bool,
+}
+
+/// Read the key and order columns of `f` and record every open row whose key the source
+/// holds. Reads the whole file: the open order of every such key is needed later.
+#[allow(clippy::too_many_arguments)]
+async fn probe_file(
+ store: &Arc<dyn ObjectStore>,
+ f: TargetFile,
+ plan: &SourcePlan,
+ columns: &[(String, DataType)],
+ key_count: usize,
+ opts: &HistoryUpsertOptions,
+ budgets: &Budgets,
+) -> Result<(FileProbe, Arc<ParquetMetaData>)> {
+ let (builder, held_fetch) = open_builder(store, &f, budgets).await?;
+ let metadata = builder.metadata().clone();
+ let file_schema = builder.schema().clone();
+ let mut projection: Vec<usize> = columns
+ .iter()
+ .filter_map(|(name, _)| file_schema.index_of(name).ok())
+ .collect();
+ projection.sort_unstable();
+ projection.dedup();
+
+ let mut probe = FileProbe {
+ file: f,
+ seen: Vec::new(),
+ newer: false,
+ };
+ let mut note = |batch: &RecordBatch| -> Result<()> {
+ let cols = key_and_order_columns(batch, columns)?;
+ let keys = plan.key_converter.convert_columns(&cols[..key_count])?;
+ let orders = plan.order_converter.convert_columns(&cols[key_count..])?;
+ for i in 0..batch.num_rows() {
+ if let Some(&group) = plan.by_key.get(keys.row(i).as_ref()) {
+ let order = orders.row(i);
+ probe.newer = probe.newer || plan.newest_order(group) > order.as_ref();
+ probe.seen.push((group, order.as_ref().into()));
+ }
+ }
+ Ok(())
+ };
+
+ if projection.is_empty() {
+ // Every row of the file has null keys and a null order value, so one row decides it.
+ if metadata.file_metadata().num_rows() > 0 {
+ note(&RecordBatch::try_new_with_options(
+ Arc::new(Schema::empty()),
+ vec![],
+ &arrow_array::RecordBatchOptions::new().with_row_count(Some(1)),
+ )?)?;
+ }
+ return Ok((probe, metadata));
+ }
+
+ let fetch = match held_fetch {
+ Some(fetch) => fetch,
+ None => {
+ budgets
+ .acquire_fetch(max_row_group_fetch_bytes(&metadata, Some(&projection)))
+ .await?
+ }
+ };
+ let mask = ProjectionMask::roots(builder.parquet_schema(), projection);
+ let batch_rows = byte_bounded_batch_rows(
+ &metadata,
+ INITIAL_PROBE_ESTIMATE_BYTES,
+ opts.read_batch_size,
+ );
+ let mut stream = builder
+ .with_projection(mask)
+ .with_batch_size(batch_rows)
+ .build()?;
+ let mut estimate_bytes = INITIAL_PROBE_ESTIMATE_BYTES;
+ loop {
+ let mut permit = budgets.acquire_bytes(estimate_bytes).await?;
+ let Some(batch) = stream.try_next().await? else {
+ break;
+ };
+ let actual = batch.get_array_memory_size();
+ budgets.top_up(&mut permit, actual).await?;
+ estimate_bytes = actual.max(16 * 1024);
+ note(&batch)?;
+ }
+ drop(stream);
+ drop(fetch);
+ Ok((probe, metadata))
+}
+
+/// A decoded batch on its way to the writers, split by destination. The permit is
+/// released once both parts are in the write buffers.
+struct WriteMessage {
+ current: Option<RecordBatch>,
+ closed: Option<RecordBatch>,
+ _permit: BudgetPermit,
+}
+
+struct CloseOutcome {
+ rows_copied: usize,
+ versions_closed: usize,
+ bytes_read: u64,
+ remove: Action,
+}
+
+/// Stream one current-partition file to the writer, closing the open rows of keys that
+/// kept a newer source row. Every batch carries its budget permits to the writer.
+#[allow(clippy::too_many_arguments)]
+async fn close_file(
+ store: Arc<dyn ObjectStore>,
+ f: TargetFile,
+ plan: Arc<SourcePlan>,
+ closing: Arc<Vec<Option<u32>>>,
+ cols: Arc<Columns>,
+ table_schema: SchemaRef,
+ opts: Arc<HistoryUpsertOptions>,
+ budgets: Budgets,
+ tx: mpsc::UnboundedSender<WriteMessage>,
+) -> Result<CloseOutcome> {
+ let (mut stream, fetch) = open_rewrite_stream(
+ &store,
+ &f,
+ &budgets,
+ opts.read_ahead_row_groups,
+ opts.read_batch_size,
+ )
+ .await?;
+ let mut rows_copied = 0usize;
+ let mut versions_closed = 0usize;
+ let mut estimate_bytes = crate::upsert::INITIAL_DECODE_ESTIMATE_BYTES;
+ loop {
+ let mut permit = budgets.acquire_bytes(estimate_bytes).await?;
+ let Some(batch) = stream.try_next().await? else {
+ break;
+ };
+ let actual = batch.get_array_memory_size();
+ budgets.top_up(&mut permit, actual).await?;
+ estimate_bytes = actual.max(64 * 1024);
+
+ let batch = add_partition_column(&batch, &opts.partition_column, &opts.current_partition)?;
+ let batch = cast_to_schema(&batch, &table_schema)?;
+ let key_cols: Vec<ArrayRef> = cols
+ .key_idx
+ .iter()
+ .map(|i| batch.column(*i).clone())
+ .collect();
+ let keys = plan.key_converter.convert_columns(&key_cols)?;
+ let closes: UInt32Array = (0..batch.num_rows())
+ .map(|i| {
+ plan.by_key
+ .get(keys.row(i).as_ref())
+ .and_then(|group| closing[*group as usize])
+ })
+ .collect();
+ let closed = closes.len() - closes.null_count();
+ versions_closed += closed;
+ rows_copied += batch.num_rows() - closed;
+ let message = if closed == 0 {
+ WriteMessage {
+ current: Some(batch),
+ closed: None,
+ _permit: permit,
+ }
+ } else {
+ let is_closed: BooleanArray = (0..closes.len())
+ .map(|i| Some(closes.is_valid(i)))
+ .collect();
+ let stays_open: BooleanArray =
+ (0..closes.len()).map(|i| Some(closes.is_null(i))).collect();
+ let current = filter_record_batch(&batch, &stays_open)?;
+ let closed_rows = stamp_validity(
+ &filter_record_batch(&batch, &is_closed)?,
+ filter(&closes, &is_closed)?.as_primitive(),
+ &plan,
+ &cols,
+ &table_schema,
+ &opts.current_partition,
+ )?;
+ WriteMessage {
+ current: (current.num_rows() > 0).then_some(current),
+ closed: Some(closed_rows),
+ _permit: permit,
+ }
+ };
+ tx.send(message)
+ .map_err(|_| Error::Generic("writer task ended before its readers".into()))?;
+ }
+ drop(stream);
+ drop(fetch);
+ Ok(CloseOutcome {
+ rows_copied,
+ versions_closed,
+ bytes_read: f.size,
+ remove: Action::Remove(f.remove),
+ })
+}
+
+/// Most month files the closed-row writer keeps open. Each open file holds its own
+/// buffers, so closes that span many months would otherwise grow memory per month.
+const MAX_OPEN_MONTH_FILES: usize = 32;
+
+/// Write closed rows one month at a time, flushing every open month file before the
+/// writer would hold more than [`MAX_OPEN_MONTH_FILES`].
+async fn write_closed(
+ writer: &mut StreamingWriter,
+ batch: &RecordBatch,
+ partition_idx: usize,
+ adds: &mut Vec<Add>,
+ target_file_size: usize,
+ max_row_group_bytes: Option<usize>,
+) -> Result<()> {
+ let order = sort_to_indices(batch.column(partition_idx), None, None)?;
+ let sorted = take_record_batch(batch, &order)?;
+ let runs = arrow_ord::partition::partition(&[sorted.column(partition_idx).clone()])?;
+ for run in runs.ranges() {
+ let month = sorted.slice(run.start, run.end - run.start);
+ if writer.open_files() >= MAX_OPEN_MONTH_FILES && !writer.holds_partition_of(&month)? {
+ adds.extend(writer.flush().await?);
+ }
+ writer.write(month)?;
+ roll_writer(writer, adds, target_file_size, max_row_group_bytes).await?;
+ }
+ Ok(())
+}
+
+/// Writes chunks of kept source rows, stamped with their `valid_to` and partition.
+struct SourceWriter<'a> {
+ plan: &'a SourcePlan,
+ cols: &'a Columns,
+ table_schema: &'a SchemaRef,
+ opts: &'a HistoryUpsertOptions,
+ target_file_size: usize,
+}
+
+impl SourceWriter<'_> {
+ fn stamped(&self, rows: &[KeptRow]) -> Result<RecordBatch> {
+ // Hand interleave only the batches this chunk reads; its cost grows with the
+ // number of batches it is given.
+ let mut batch_slots: HashMap<usize, usize> = HashMap::new();
+ let mut batch_refs: Vec<&RecordBatch> = Vec::new();
+ let located: Vec<(usize, usize)> = rows
+ .iter()
+ .map(|(r, _)| {
+ let (batch, row) = self.plan.locate(*r);
+ let slot = *batch_slots.entry(batch).or_insert_with(|| {
+ batch_refs.push(&self.plan.batches[batch]);
+ batch_refs.len() - 1
+ });
+ (slot, row)
+ })
+ .collect();
+ let closes: UInt32Array = rows.iter().map(|(_, next)| *next).collect();
+ stamp_validity(
+ &interleave_record_batch(&batch_refs, &located)?,
+ &closes,
+ self.plan,
+ self.cols,
+ self.table_schema,
+ &self.opts.current_partition,
+ )
+ }
+
+ /// Write newest kept rows, which all stay open in the current partition.
+ async fn write_open(
+ &self,
+ rows: &[KeptRow],
+ writer: &mut StreamingWriter,
+ adds: &mut Vec<Add>,
+ ) -> Result<()> {
+ let selected = self.stamped(rows)?;
+ let max_row_group_bytes = self.opts.max_row_group_bytes;
+ let slice_rows =
+ source_slice_rows(&selected, self.opts.read_batch_size, max_row_group_bytes);
+ let mut offset = 0usize;
+ while offset < selected.num_rows() {
+ let n = (selected.num_rows() - offset).min(slice_rows);
+ writer.write(selected.slice(offset, n))?;
+ offset += n;
+ roll_writer(writer, adds, self.target_file_size, max_row_group_bytes).await?;
+ }
+ Ok(())
+ }
+}
+
+async fn history_inner(
+ table: &DeltaTable,
+ source_batches: Vec<RecordBatch>,
+ source_schema: SchemaRef,
+ opts: &HistoryUpsertOptions,
+ plan_only: bool,
+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {
+ // Again here: after a relax the snapshot is reloaded, and a concurrent commit may have
+ // enabled a feature that makes a blind rewrite unsafe.
+ ensure_history_table(table, opts)?;
+ let snapshot = table.snapshot()?;
+ let table_schema: SchemaRef = RecordBatchWriter::for_table(table)?.arrow_schema();
+ let cols = Columns::resolve(&table_schema, &source_schema, opts)?;
+ let target_file_size = resolve_target_file_size(
+ opts.target_file_size,
+ snapshot.table_config().target_file_size().get() as usize,
+ );
+ let read_version = snapshot.version() as i64;
+
+ let plan_started = Instant::now();
+ let plan = SourcePlan::build(source_batches, &table_schema, &cols, opts)?;
+ let mut stats = HistoryUpsertStats {
+ version: read_version,
+ rows_in: plan.rows_in,
+ rows_deduped: plan.rows_in - plan.deduped.len(),
+ ..Default::default()
+ };
+ if plan.groups.is_empty() {
+ stats.plan_ms = plan_started.elapsed().as_millis() as u64;
+ return Ok((stats, None));
+ }
+
+ let files = list_files_by_partition(
+ table,
+ Some(&opts.partition_column),
+ &[opts.current_partition.as_str()],
+ )
+ .await?
+ .into_iter()
+ .next()
+ .unwrap_or_default();
+ let (candidates, pruned): (Vec<TargetFile>, Vec<TargetFile>) = files
+ .into_iter()
+ .partition(|f| !file_cannot_hold_source_key(f, &opts.key_columns[0], &plan));
+ stats.files_carried_over = pruned.len();
+ stats.plan_ms = plan_started.elapsed().as_millis() as u64;
+
+ let budgets = Budgets::new(
+ opts.max_buffered_bytes,
+ opts.max_fetch_bytes,
+ opts.limits.clone(),
+ );
+ let store: Arc<dyn ObjectStore> = table.object_store();
+
+ // Held from the probe on, as an upsert's partition worker holds it.
+ let _partition_permit = opts.limits.acquire_partition().await?;
+ let probe_started = Instant::now();
+ let probe_columns: Vec<(String, DataType)> = cols
+ .key_idx
+ .iter()
+ .chain(std::iter::once(&cols.order_idx))
+ .map(|i| {
+ let f = table_schema.field(*i);
+ (f.name().clone(), f.data_type().clone())
+ })
+ .collect();
+ stats.files_probed = candidates.len();
+ let key_count = cols.key_idx.len();
+ let mut open: Vec<Option<Box<[u8]>>> = vec![None; plan.groups.len()];
+ let mut probed: Vec<(TargetFile, Vec<u32>)> = Vec::new();
+ {
+ let results =
+ futures::stream::iter(
+ candidates.into_iter().map(|f| {
+ let store = store.clone();
+ let (plan, probe_columns, budgets) = (&plan, &probe_columns, &budgets);
+ async move {
+ probe_file(&store, f, plan, probe_columns, key_count, opts, budgets).await
+ }
+ }),
+ )
+ .buffered(opts.probe_concurrency.max(1));
+ futures::pin_mut!(results);
+ let retained_footer_limit = opts.max_parallel_files.max(1);
+ let mut retained = 0usize;
+ while let Some((mut probe, metadata)) = results.try_next().await? {
+ if probe.newer && retained < retained_footer_limit {
+ probe.file.metadata = Some(metadata);
+ retained += 1;
+ }
+ let mut groups = Vec::with_capacity(probe.seen.len());
+ for (group, order) in probe.seen {
+ let slot = &mut open[group as usize];
+ if slot.as_ref().is_none_or(|o| order > *o) {
+ *slot = Some(order);
+ }
+ groups.push(group);
+ }
+ groups.sort_unstable();
+ groups.dedup();
+ probed.push((probe.file, groups));
+ }
+ }
+ stats.probe_ms = probe_started.elapsed().as_millis() as u64;
+
+ let decision = plan.decide(&open);
+ stats.rows_reread = decision.rows_reread;
+ stats.rows_backwards = decision.rows_backwards;
+ stats.versions_opened = decision.versions_opened;
+ stats.chain_rows_closed = decision.chain_rows_closed;
+ if decision.writes.is_empty() || plan_only {
+ // Nothing kept means nothing closes either: the table already holds this input.
+ stats.files_carried_over += probed.len();
+ return Ok((stats, None));
+ }
+
+ let mut rewrite = Vec::new();
+ for (file, groups) in probed {
+ if groups
+ .iter()
+ .any(|group| decision.closing[*group as usize].is_some())
+ {
+ rewrite.push(file);
+ } else {
+ stats.files_carried_over += 1;
+ }
+ }
+
+ // Open rows and closed rows go through separate writers, so a flush of closed rows
+ // never cuts a current-partition file short.
+ let rewrite_started = Instant::now();
+ let plan = Arc::new(plan);
+ let closing = Arc::new(decision.closing);
+ let cols = Arc::new(cols);
+ let opts_arc = Arc::new(opts.clone());
+ let current_writer = StreamingWriter::for_table(table)?;
+ let closed_writer = StreamingWriter::for_table(table)?;
+ let (tx, mut rx) = mpsc::unbounded_channel::<WriteMessage>();
+ let max_row_group_bytes = opts.max_row_group_bytes;
+ let partition_idx = cols.partition_idx;
+ let writer_task = tokio::spawn(async move {
+ let (mut current_writer, mut closed_writer) = (current_writer, closed_writer);
+ let mut adds = Vec::new();
+ while let Some(message) = rx.recv().await {
+ if let Some(batch) = message.current {
+ current_writer.write(batch)?;
+ roll_writer(
+ &mut current_writer,
+ &mut adds,
+ target_file_size,
+ max_row_group_bytes,
+ )
+ .await?;
+ }
+ if let Some(batch) = message.closed {
+ write_closed(
+ &mut closed_writer,
+ &batch,
+ partition_idx,
+ &mut adds,
+ target_file_size,
+ max_row_group_bytes,
+ )
+ .await?;
+ }
+ }
+ Ok::<_, Error>((current_writer, closed_writer, adds))
+ });
+
+ // Same reader discipline as the upsert: per-call then process-wide file permits, and
+ // readers queued behind a failed one stop instead of reading.
+ let file_sem = Arc::new(Semaphore::new(opts.max_parallel_files.max(1)));
+ let reader_failed = Arc::new(AtomicBool::new(false));
+ let mut readers = Vec::with_capacity(rewrite.len());
+ for f in rewrite {
+ let sem = file_sem.clone();
+ let failed = reader_failed.clone();
+ let (store, plan, closing, cols) =
+ (store.clone(), plan.clone(), closing.clone(), cols.clone());
+ let (schema, opts, budgets, tx) = (
+ table_schema.clone(),
+ opts_arc.clone(),
+ budgets.clone(),
+ tx.clone(),
+ );
+ readers.push(tokio::spawn(async move {
+ let _local = sem
+ .acquire_owned()
+ .await
+ .map_err(|_| Error::Generic("file semaphore closed".into()))?;
+ if failed.load(Ordering::Relaxed) {
+ return Ok(None);
+ }
+ let outcome = async {
+ let _global = opts.limits.acquire_file().await?;
+ close_file(store, f, plan, closing, cols, schema, opts, budgets, tx).await
+ }
+ .await;
+ if outcome.is_err() {
+ failed.store(true, Ordering::Relaxed);
+ }
+ outcome.map(Some)
+ }));
+ }
+ drop(tx);
+
+ let mut removes: Vec<Action> = Vec::new();
+ let mut reader_err: Option<Error> = None;
+ for h in readers {
+ match h
+ .await
+ .map_err(|e| Error::Generic(format!("file worker panicked: {e}")))?
+ {
+ Ok(Some(o)) => {
+ stats.rows_copied += o.rows_copied;
+ stats.versions_closed += o.versions_closed;
+ stats.bytes_read += o.bytes_read;
+ removes.push(o.remove);
+ }
+ Ok(None) => {}
+ Err(e) => reader_err = reader_err.or(Some(e)),
+ }
+ }
+ let (mut current_writer, mut closed_writer, mut adds) = writer_task
+ .await
+ .map_err(|e| Error::Generic(format!("writer task panicked: {e}")))??;
+ if let Some(e) = reader_err {
+ return Err(e);
+ }
+
+ let (opens, mut chain): (Vec<KeptRow>, Vec<KeptRow>) = decision
+ .writes
+ .into_iter()
+ .partition(|(_, next)| next.is_none());
+ let source = SourceWriter {
+ plan: &plan,
+ cols: &cols,
+ table_schema: &table_schema,
+ opts,
+ target_file_size,
+ };
+ for chunk in opens.chunks(opts.read_batch_size.max(1)) {
+ source
+ .write_open(chunk, &mut current_writer, &mut adds)
+ .await?;
+ }
+ // Sorted by month, so the closed-row writer moves through months in order instead of
+ // reopening them; `write_closed` keeps the open month files under the limit.
+ let next_rows: UInt32Array = chain.iter().map(|(_, next)| *next).collect();
+ let months = format_months(take(plan.valid_from.as_ref(), &next_rows, None)?.as_ref())?;
+ let mut by_month: Vec<usize> = (0..chain.len()).collect();
+ by_month.sort_by_key(|i| months.value(*i));
+ chain = by_month.iter().map(|i| chain[*i]).collect();
+ for chunk in chain.chunks(opts.read_batch_size.max(1)) {
+ write_closed(
+ &mut closed_writer,
+ &source.stamped(chunk)?,
+ partition_idx,
+ &mut adds,
+ target_file_size,
+ max_row_group_bytes,
+ )
+ .await?;
+ }
+ adds.extend(current_writer.flush().await?);
+ adds.extend(closed_writer.flush().await?);
+ stats.rewrite_ms = rewrite_started.elapsed().as_millis() as u64;
+ stats.files_added = adds.len();
+ stats.files_removed = removes.len();
+ stats.bytes_written = adds.iter().map(|a| a.size.max(0) as u64).sum();
+
+ let mut actions = removes;
+ actions.extend(adds.into_iter().map(Action::Add));
+ // Zero commit retries: without DataFusion, delta-rs does not check a `Write` against
+ // a concurrent commit that only adds files, so a rebased commit could open a key a
+ // second time. A stale commit fails as a conflict instead, and the handle re-plans.
+ let committed = commit_partition_overwrite(
+ table,
+ snapshot,
+ actions,
+ std::slice::from_ref(&opts.partition_column),
+ 0,
+ opts.commit_metadata.clone(),
+ )
+ .await?;
+ stats.commit_ms = committed.commit_ms;
+ stats.maintenance_ms = committed.maintenance_ms;
+ stats.committed = true;
+ stats.version = committed.version;
+ info!(
+ version = stats.version,
+ rows_in = stats.rows_in,
+ versions_opened = stats.versions_opened,
+ versions_closed = stats.versions_closed,
+ chain_rows_closed = stats.chain_rows_closed,
+ rows_reread = stats.rows_reread,
+ rows_backwards = stats.rows_backwards,
+ files_added = stats.files_added,
+ files_removed = stats.files_removed,
+ "history upsert committed"
+ );
+ Ok((stats, Some(committed.state)))
+}
+
+/// A bucket in `0..n` for every row of `batches`, from the values of `key_columns`.
+///
+/// The key columns are cast to their types in `table`, the cast [`history_upsert`] matches
+/// keys after, so source values that become one table key always share a bucket. Each key
+/// is then encoded with arrow-row (null-safe, so null key parts hash alike) and hashed with
+/// xxHash64 seeded with [`KEY_BUCKET_SEED`], reduced modulo `n`. The ids are the same in
+/// every process that runs one deltalite build. arrow-row may change its encoding between
+/// arrow releases, so do not persist the ids.
+pub fn key_buckets(
+ table: &DeltaTable,
+ batches: &[RecordBatch],
+ key_columns: &[String],
+ n: u32,
+) -> Result<UInt32Array> {
+ if n == 0 {
+ return Err(Error::Generic("key_buckets needs n >= 1".into()));
+ }
+ if key_columns.is_empty() {
+ return Err(Error::Generic(
+ "key_buckets needs at least one key column".into(),
+ ));
+ }
+ let table_schema = RecordBatchWriter::for_table(table)?.arrow_schema();
+ let key_types = key_columns
+ .iter()
+ .map(|k| {
+ Ok(table_schema
+ .field_with_name(k)
+ .map_err(|_| {
+ Error::SchemaMismatch(format!("key column '{k}' not found in table schema"))
+ })?
+ .data_type()
+ .clone())
+ })
+ .collect::<Result<Vec<DataType>>>()?;
+ let converter = RowConverter::new(key_types.iter().cloned().map(SortField::new).collect())?;
+ let mut out = Vec::with_capacity(batches.iter().map(RecordBatch::num_rows).sum());
+ for batch in batches {
+ let cols: Vec<ArrayRef> = key_columns
+ .iter()
+ .zip(&key_types)
+ .map(|(k, ty)| {
+ let col = batch.column_by_name(k).ok_or_else(|| {
+ Error::SchemaMismatch(format!("key column '{k}' is missing from the data"))
+ })?;
+ cast_strict(col, ty)
+ })
+ .collect::<Result<_>>()?;
+ let rows = converter.convert_columns(&cols)?;
+ out.extend(rows.iter().map(|row| {
+ (twox_hash::XxHash64::oneshot(KEY_BUCKET_SEED, row.as_ref()) % u64::from(n)) as u32
+ }));
+ }
+ Ok(UInt32Array::from(out))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow_array::Int64Array;
+ use deltalake::kernel::{DataType as KernelType, StructField};
+ use deltalake::operations::create::CreateBuilder;
+
+ #[tokio::test]
+ async fn the_write_checks_the_snapshot_it_writes_against() {
+ let dir = tempfile::tempdir().unwrap();
+ let table = CreateBuilder::new()
+ .with_location(dir.path().to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new("p", KernelType::STRING, false),
+ ])
+ .with_partition_columns(vec!["p".to_string()])
+ .with_configuration(vec![("delta.appendOnly", Some("true"))])
+ .await
+ .unwrap();
+ let source = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, true)]));
+ let opts = HistoryUpsertOptions {
+ key_columns: vec!["v".into()],
+ order_column: "v".into(),
+ valid_from_column: "v".into(),
+ valid_to_column: "v".into(),
+ partition_column: "p".into(),
+ current_partition: "current".into(),
+ tie_break_column: "v".into(),
+ ..Default::default()
+ };
+ let err = history_inner(
+ &table,
+ vec![RecordBatch::new_empty(source.clone())],
+ source,
+ &opts,
+ false,
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::Unsupported(_)), "{err}");
+ }
+
+ #[tokio::test]
+ async fn closed_rows_hold_at_most_the_month_file_limit_open() {
+ let dir = tempfile::tempdir().unwrap();
+ let table = CreateBuilder::new()
+ .with_location(dir.path().to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new("p", KernelType::STRING, false),
+ ])
+ .with_partition_columns(vec!["p".to_string()])
+ .await
+ .unwrap();
+ let schema = RecordBatchWriter::for_table(&table).unwrap().arrow_schema();
+ let batch = |months: &[String]| {
+ RecordBatch::try_new(
+ schema.clone(),
+ vec![
+ Arc::new(Int64Array::from_iter_values(0..months.len() as i64)),
+ Arc::new(StringArray::from_iter_values(months.iter().rev())),
+ ],
+ )
+ .unwrap()
+ };
+ let months: Vec<String> = (0..MAX_OPEN_MONTH_FILES)
+ .map(|i| format!("{}-{:02}", 2000 + i / 12, i % 12 + 1))
+ .collect();
+ let mut writer = StreamingWriter::for_table(&table).unwrap();
+ let mut adds = Vec::new();
+ let mut write = async |months: &[String]| {
+ write_closed(&mut writer, &batch(months), 1, &mut adds, usize::MAX, None)
+ .await
+ .unwrap();
+ (adds.len(), writer.open_files())
+ };
+
+ assert_eq!(write(&months).await, (0, MAX_OPEN_MONTH_FILES));
+ assert_eq!(
+ write(&months[..1]).await,
+ (0, MAX_OPEN_MONTH_FILES),
+ "a month already open needs no flush"
+ );
+ assert_eq!(
+ write(&["1999-12".to_string()]).await,
+ (MAX_OPEN_MONTH_FILES, 1)
+ );
+ }
+}
diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rs
index ae1c9f405bcd..f8531bc3851a 100644
--- a/rust/deltalite/core/src/lib.rs
+++ b/rust/deltalite/core/src/lib.rs
@@ -33,6 +33,7 @@ pub mod compact;
pub mod deadline;
pub mod errors;
pub mod handle;
+pub mod history;
pub mod limits;
mod logprobe;
pub mod pkset;
@@ -50,6 +51,9 @@ mod writer;
pub use compact::{compact, CompactOptions, CompactStats};
pub use errors::{Error, Result};
pub use handle::{LiveFile, TableHandle};
+pub use history::{
+ history_upsert, key_buckets, HistoryUpsertOptions, HistoryUpsertStats, KEY_BUCKET_SEED,
+};
pub use limits::{Budget, BudgetSnapshot, LimitsSnapshot, ProcessLimits};
pub use prefetch::CheckpointCache;
pub use probecache::{ProbeCache, ProbeCacheStats};
diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rs
index 1f6c7ffd9009..9511f49d5e97 100644
--- a/rust/deltalite/core/src/upsert.rs
+++ b/rust/deltalite/core/src/upsert.rs
@@ -47,7 +47,7 @@ use deltalake::{DeltaTable, ObjectStore, PartitionFilter, PartitionValue, Path};
use futures::{StreamExt, TryStreamExt};
use metrics::{counter, histogram};
use parquet::arrow::arrow_reader::ArrowReaderMetadata;
-use parquet::arrow::async_reader::ParquetObjectReader;
+use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStream};
use parquet::arrow::ParquetRecordBatchStreamBuilder;
use parquet::arrow::ProjectionMask;
use parquet::file::metadata::ParquetMetaData;
@@ -76,7 +76,7 @@ const WHOLE_TABLE: &str = "__deltalite_whole_table__";
/// the observed batch size after the first decode.
pub(crate) const INITIAL_DECODE_ESTIMATE_BYTES: usize = 4 * 1024 * 1024;
/// First-iteration pre-decode reservation for narrow PK-column probe batches.
-const INITIAL_PROBE_ESTIMATE_BYTES: usize = 256 * 1024;
+pub(crate) const INITIAL_PROBE_ESTIMATE_BYTES: usize = 256 * 1024;
/// Default per-call fetch budget. Readers of ordinary files (a few MB per row group)
/// never wait on it at the default `max_parallel_files`; readers of files whose single
@@ -407,7 +407,7 @@ impl RelaxCache {
/// committed. Valid only when the commit landed directly on the verified version
/// (`committed == verified + 1`): a bigger jump means another writer's actions are
/// part of the new version, so the memo stays behind and the next upsert rescans.
- fn advance_own_commit(&mut self, committed: u64) {
+ pub(crate) fn advance_own_commit(&mut self, committed: u64) {
if self.version.is_some_and(|v| committed == v + 1) {
self.version = Some(committed);
}
@@ -855,7 +855,7 @@ async fn upsert_with_relax(
/// (present with nulls, or absent so the cast will null-pad it) or in the table's own
/// files (Add-action `nullCount` stats). Only positive evidence relaxes: files without
/// stats are left alone, so a clean table's schema is never touched.
-async fn columns_needing_relax(
+pub(crate) async fn columns_needing_relax(
table: &DeltaTable,
source_batches: &[RecordBatch],
source_schema: &Schema,
@@ -936,7 +936,10 @@ async fn columns_needing_relax(
/// the upsert that follows plans against the relaxed schema; concurrent writers see a
/// MetadataChanged conflict and re-plan (or fall back), exactly as for any other
/// schema evolution.
-async fn relax_columns_to_nullable(table: &DeltaTable, columns: &[String]) -> Result<()> {
+pub(crate) async fn relax_columns_to_nullable(
+ table: &DeltaTable,
+ columns: &[String],
+) -> Result<()> {
let snapshot = table.snapshot()?;
let schema = snapshot.schema();
let relaxed = StructType::try_new(schema.fields().map(|f| {
@@ -1169,8 +1172,69 @@ async fn upsert_inner(
let rewrite_ms = rewrite_started.elapsed().as_millis() as u64;
// --- Commit ----------------------------------------------------------------------
+ let committed = commit_partition_overwrite(
+ table,
+ snapshot,
+ actions,
+ &partition_columns,
+ opts.commit_max_retries,
+ opts.commit_metadata.clone(),
+ )
+ .await?;
+
+ // Only a landed commit makes the written files live and the removed files dead. Before
+ // it, a written file may be an orphan that no snapshot ever names.
+ if let Some(cache) = &opts.probe_cache {
+ cache.remove_all(removed_keys.iter().map(String::as_str));
+ let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;
+ for (key, keys) in written_keys {
+ keys.store(key, signature);
+ }
+ }
+
+ stats.plan_ms = plan_ms;
+ stats.rewrite_ms = rewrite_ms;
+ stats.commit_ms = committed.commit_ms;
+ stats.maintenance_ms = committed.maintenance_ms;
+ stats.version = committed.version;
+ tracing::Span::current().record("version", stats.version);
+ info!(
+ version = stats.version,
+ partitions = stats.partitions_touched,
+ files_added = stats.files_added,
+ files_removed = stats.files_removed,
+ files_carried_over = stats.files_carried_over,
+ rows_updated = stats.rows_updated,
+ rows_inserted = stats.rows_inserted,
+ rows_copied = stats.rows_copied,
+ plan_ms = stats.plan_ms,
+ rewrite_ms = stats.rewrite_ms,
+ commit_ms = stats.commit_ms,
+ "upsert committed"
+ );
+ Ok((stats, committed.state))
+}
+
+/// A landed [`commit_partition_overwrite`].
+pub(crate) struct Committed {
+ pub(crate) version: i64,
+ pub(crate) state: DeltaTableState,
+ pub(crate) commit_ms: u64,
+ pub(crate) maintenance_ms: u64,
+}
+
+/// Commit a rewrite's Remove and Add actions as one partition-overwrite `Write`, then run
+/// log maintenance on a checkpoint boundary.
+pub(crate) async fn commit_partition_overwrite(
+ table: &DeltaTable,
+ snapshot: &DeltaTableState,
+ actions: Vec<Action>,
+ partition_columns: &[String],
+ max_retries: usize,
+ commit_metadata: Option<HashMap<String, Value>>,
+) -> Result<Committed> {
let commit_started = Instant::now();
- let predicate = partition_col.as_ref().map(|c| {
+ let predicate = partition_columns.first().map(|c| {
let vals: Vec<String> = actions
.iter()
.filter_map(|a| match a {
@@ -1189,7 +1253,7 @@ async fn upsert_inner(
partition_by: if partition_columns.is_empty() {
None
} else {
- Some(partition_columns.clone())
+ Some(partition_columns.to_vec())
},
predicate,
};
@@ -1200,10 +1264,10 @@ async fn upsert_inner(
// caller back to the delta-rs MERGE for a write that has in fact happened. They run
// best-effort below instead.
let mut props = CommitProperties::default()
- .with_max_retries(opts.commit_max_retries)
+ .with_max_retries(max_retries)
.with_create_checkpoint(false)
.with_cleanup_expired_logs(Some(false));
- if let Some(md) = opts.commit_metadata.clone() {
+ if let Some(md) = commit_metadata {
props = props.with_metadata(md);
}
@@ -1216,43 +1280,19 @@ async fn upsert_inner(
.await?;
let commit_ms = commit_started.elapsed().as_millis() as u64;
- // Only a landed commit makes the written files live and the removed files dead. Before
- // it, a written file may be an orphan that no snapshot ever names.
- if let Some(cache) = &opts.probe_cache {
- cache.remove_all(removed_keys.iter().map(String::as_str));
- let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;
- for (key, keys) in written_keys {
- keys.store(key, signature);
- }
- }
-
+ let mut maintenance_ms = 0;
if (finalized.version() + 1) % checkpoint_interval == 0 {
let maintenance_started = Instant::now();
best_effort_log_maintenance(table, finalized.version(), cleanup_enabled).await;
- stats.maintenance_ms = maintenance_started.elapsed().as_millis() as u64;
- }
-
- stats.plan_ms = plan_ms;
- stats.rewrite_ms = rewrite_ms;
- stats.commit_ms = commit_ms;
- stats.version = i64::try_from(finalized.version())
- .map_err(|_| Error::Generic("committed version overflows i64".into()))?;
- tracing::Span::current().record("version", stats.version);
- info!(
- version = stats.version,
- partitions = stats.partitions_touched,
- files_added = stats.files_added,
- files_removed = stats.files_removed,
- files_carried_over = stats.files_carried_over,
- rows_updated = stats.rows_updated,
- rows_inserted = stats.rows_inserted,
- rows_copied = stats.rows_copied,
- plan_ms = stats.plan_ms,
- rewrite_ms = stats.rewrite_ms,
- commit_ms = stats.commit_ms,
- "upsert committed"
- );
- Ok((stats, finalized.snapshot))
+ maintenance_ms = maintenance_started.elapsed().as_millis() as u64;
+ }
+ Ok(Committed {
+ version: i64::try_from(finalized.version())
+ .map_err(|_| Error::Generic("committed version overflows i64".into()))?,
+ state: finalized.snapshot,
+ commit_ms,
+ maintenance_ms,
+ })
}
/// Checkpoint and expired-log cleanup after a durable commit, tolerating failure: the
@@ -1297,7 +1337,7 @@ pub(crate) async fn best_effort_log_maintenance(
/// Explicit argument > table property (already folded with delta-rs's default by the
/// caller). Zero/absent explicit values fall through to the table's value.
-fn resolve_target_file_size(explicit: Option<usize>, table_value: usize) -> usize {
+pub(crate) fn resolve_target_file_size(explicit: Option<usize>, table_value: usize) -> usize {
match explicit {
Some(v) if v > 0 => v,
_ => table_value,
@@ -1569,7 +1609,7 @@ fn source_pk_range(source: &PartitionSource, col_idx: usize) -> Result<Option<(V
///
/// Each list keeps the snapshot's file order, as the per-partition filter did. Where
/// the index cannot reproduce the filter's equality exactly, this falls back to it.
-async fn list_files_by_partition(
+pub(crate) async fn list_files_by_partition(
table: &DeltaTable,
partition_col: Option<&str>,
values: &[&str],
@@ -1832,7 +1872,7 @@ fn file_cannot_match(
|| matches!(json_cmp(fmin, src_max), Some(std::cmp::Ordering::Greater))
}
-fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {
+pub(crate) fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {
match (a, b) {
(Value::String(x), Value::String(y)) => Some(x.cmp(y)),
(Value::Number(x), Value::Number(y)) => x.as_f64()?.partial_cmp(&y.as_f64()?),
@@ -1840,7 +1880,7 @@ fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {
}
}
-fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {
+pub(crate) fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {
use arrow_array::cast::AsArray;
use arrow_array::types::*;
@@ -1919,7 +1959,11 @@ fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {
/// read back from storage is missing the column that the writer needs in order to route
/// the rows back to the same partition. Getting this wrong would silently misroute
/// rewritten rows.
-fn add_partition_column(batch: &RecordBatch, name: &str, value: &str) -> Result<RecordBatch> {
+pub(crate) fn add_partition_column(
+ batch: &RecordBatch,
+ name: &str,
+ value: &str,
+) -> Result<RecordBatch> {
let n = batch.num_rows();
let arr: Arc<dyn Array> = Arc::new(StringArray::from(vec![value; n]));
let mut fields: Vec<Field> = batch
@@ -2302,24 +2346,16 @@ struct FileOutcome {
remove: Action,
}
-/// Stream one existing file, drop replaced rows, and send surviving batches to the
-/// partition's writer task. Every batch sent carries byte-budget permits acquired
-/// *before* the send; the writer releases them only after `write()` has copied the
-/// batch into its buffer, so queued + in-write survivor data across the whole process
-/// never exceeds the budgets.
-#[allow(clippy::too_many_arguments)]
-async fn filter_file(
- store: Arc<dyn ObjectStore>,
- f: TargetFile,
- pkset: Arc<PkSet>,
- table_schema: SchemaRef,
- partition_col: Option<String>,
- partition_value: String,
- opts: Arc<UpsertOptions>,
- budgets: Budgets,
- tx: mpsc::UnboundedSender<(RecordBatch, BudgetPermit)>,
-) -> Result<FileOutcome> {
- let (mut builder, held_fetch) = open_builder(&store, &f, &budgets).await?;
+/// Open a full-width stream over `f` for a rewrite, holding the fetch permit for the
+/// compressed bytes the reader will hold. The permit must outlive the stream.
+pub(crate) async fn open_rewrite_stream(
+ store: &Arc<dyn ObjectStore>,
+ f: &TargetFile,
+ budgets: &Budgets,
+ read_ahead_row_groups: bool,
+ read_batch_size: usize,
+) -> Result<(ParquetRecordBatchStream<ParquetObjectReader>, FetchPermit)> {
+ let (mut builder, held_fetch) = open_builder(store, f, budgets).await?;
// Reserve the compressed bytes the reader will hold before it fetches any data.
// Taken once per file and before any decode budget (see `Budgets::acquire_fetch`).
let fetch = match held_fetch {
@@ -2327,7 +2363,7 @@ async fn filter_file(
None => {
let meta = builder.metadata().clone();
let pair = max_row_group_pair_fetch_bytes(&meta);
- if opts.read_ahead_row_groups
+ if read_ahead_row_groups
&& meta.num_row_groups() > 1
&& pair <= budgets.fetch_cap_bytes()
{
@@ -2349,9 +2385,36 @@ async fn filter_file(
let batch_rows = byte_bounded_batch_rows(
builder.metadata(),
INITIAL_DECODE_ESTIMATE_BYTES,
- opts.read_batch_size,
+ read_batch_size,
);
- let mut stream = builder.with_batch_size(batch_rows).build()?;
+ Ok((builder.with_batch_size(batch_rows).build()?, fetch))
+}
+
+/// Stream one existing file, drop replaced rows, and send surviving batches to the
+/// partition's writer task. Every batch sent carries byte-budget permits acquired
+/// *before* the send; the writer releases them only after `write()` has copied the
+/// batch into its buffer, so queued + in-write survivor data across the whole process
+/// never exceeds the budgets.
+#[allow(clippy::too_many_arguments)]
+async fn filter_file(
+ store: Arc<dyn ObjectStore>,
+ f: TargetFile,
+ pkset: Arc<PkSet>,
+ table_schema: SchemaRef,
+ partition_col: Option<String>,
+ partition_value: String,
+ opts: Arc<UpsertOptions>,
+ budgets: Budgets,
+ tx: mpsc::UnboundedSender<(RecordBatch, BudgetPermit)>,
+) -> Result<FileOutcome> {
+ let (mut stream, fetch) = open_rewrite_stream(
+ &store,
+ &f,
+ &budgets,
+ opts.read_ahead_row_groups,
+ opts.read_batch_size,
+ )
+ .await?;
let mut rows_updated = 0usize;
let mut rows_copied = 0usize;
@@ -2731,7 +2794,7 @@ async fn rewrite_partition(
/// source write holds about as many decoded bytes as a survivor batch, and a row group
/// passes the cap by at most that much on either path. Otherwise one `read_batch_size`
/// slice of wide rows goes in one write and closes as one row group, far past the cap.
-fn source_slice_rows(
+pub(crate) fn source_slice_rows(
batch: &RecordBatch,
read_batch_size: usize,
max_row_group_bytes: Option<usize>,
@@ -2747,7 +2810,7 @@ fn source_slice_rows(
}
/// Close the open file at `target_file_size`, else its row group at `max_row_group_bytes`.
-async fn roll_writer(
+pub(crate) async fn roll_writer(
writer: &mut StreamingWriter,
adds: &mut Vec<Add>,
target_file_size: usize,
diff --git a/rust/deltalite/core/src/writer.rs b/rust/deltalite/core/src/writer.rs
index b5c458742def..9780fe191aa7 100644
--- a/rust/deltalite/core/src/writer.rs
+++ b/rust/deltalite/core/src/writer.rs
@@ -174,6 +174,26 @@ impl StreamingWriter {
self
}
+ /// Number of files open for writing, one per partition value written since the last
+ /// flush.
+ pub(crate) fn open_files(&self) -> usize {
+ self.open.len()
+ }
+
+ /// Whether a file is open for the partition of the first row of `values`.
+ pub(crate) fn holds_partition_of(&self, values: &RecordBatch) -> Result<bool> {
+ if values.num_rows() == 0 || self.partition_columns.is_empty() {
+ return Ok(!self.open.is_empty());
+ }
+ let schema = values.schema();
+ let mut arrays: Vec<ArrayRef> = Vec::with_capacity(self.partition_columns.len());
+ for c in &self.partition_columns {
+ arrays.push(values.column(schema.index_of(c)?).clone());
+ }
+ let values = self.partition_values_at(&arrays, 0)?;
+ Ok(self.open.contains_key(&hive_partition_path(&values)))
+ }
+
/// Bytes held across all open files; the rollover measure.
pub(crate) fn buffer_len(&self) -> usize {
self.open.values().map(OpenFile::buffer_len).sum()
diff --git a/rust/deltalite/core/tests/fixtures/month_vectors.json b/rust/deltalite/core/tests/fixtures/month_vectors.json
new file mode 100644
index 000000000000..988acc0108b2
--- /dev/null
+++ b/rust/deltalite/core/tests/fixtures/month_vectors.json
@@ -0,0 +1,11 @@
+[
+ {"valid_to": "2026-01-31T23:59:59.999999Z", "month": "2026-01"},
+ {"valid_to": "2026-02-01T00:00:00Z", "month": "2026-02"},
+ {"valid_to": "2024-02-29T12:00:00Z", "month": "2024-02"},
+ {"valid_to": "2025-12-31T23:59:59Z", "month": "2025-12"},
+ {"valid_to": "2026-01-01T00:00:00Z", "month": "2026-01"},
+ {"valid_to": "1970-01-01T00:00:00Z", "month": "1970-01"},
+ {"valid_to": "1969-12-31T23:59:59.999999Z", "month": "1969-12"},
+ {"valid_to": "2000-03-01T00:00:00.000001Z", "month": "2000-03"},
+ {"valid_to": "2099-11-30T23:00:00Z", "month": "2099-11"}
+]
diff --git a/rust/deltalite/core/tests/history_upsert.rs b/rust/deltalite/core/tests/history_upsert.rs
new file mode 100644
index 000000000000..a880e687e055
--- /dev/null
+++ b/rust/deltalite/core/tests/history_upsert.rs
@@ -0,0 +1,1122 @@
+//! `history_upsert` against real tables: the SCD2 rules, what it rewrites, and what it
+//! refuses. The Python suite (`python/tests/test_history.py`) runs the same rules against
+//! a reference model over random inputs.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};
+use std::sync::{Arc, LazyLock, Mutex};
+
+use arrow_array::types::Int32Type;
+use arrow_array::{
+ Array, DictionaryArray, Int32Array, Int64Array, RecordBatch, StringArray,
+ TimestampMicrosecondArray,
+};
+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
+use bytes::Bytes;
+use chrono::DateTime;
+use deltalake::kernel::{DataType as KernelType, PrimitiveType, StructField};
+use deltalake::logstore::{
+ default_logstore, logstore_factories, object_store_factories, LogStore, LogStoreFactory,
+ ObjectStoreFactory, StorageConfig,
+};
+use deltalake::operations::create::CreateBuilder;
+use deltalake::DeltaResult;
+use deltalake::Path;
+use deltalite_core::errors::Error;
+use deltalite_core::handle::TableHandle;
+use deltalite_core::history::{
+ format_months, history_upsert, HistoryUpsertOptions, HistoryUpsertStats,
+};
+use deltalite_core::limits::ProcessLimits;
+use deltalite_core::table::MultipartConfig;
+use futures::stream::BoxStream;
+use object_store::local::LocalFileSystem;
+use object_store::{
+ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
+ ObjectStoreExt, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions,
+};
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+
+/// One source row: (id, order value, valid_from, payload, tie-break).
+type Src<'a> = (Option<&'a str>, i64, &'a str, &'a str, i64);
+
+/// One live table row: (partition, id, order value, valid_from, valid_to, payload).
+type Live = (String, Option<String>, i64, i64, Option<i64>, String);
+
+fn ts(s: &str) -> i64 {
+ DateTime::parse_from_rfc3339(s).unwrap().timestamp_micros()
+}
+
+fn utc() -> DataType {
+ DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()))
+}
+
+fn source_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Utf8, true),
+ Field::new("v", DataType::Int64, true),
+ Field::new("valid_from", utc(), true),
+ Field::new("payload", DataType::Utf8, true),
+ Field::new("seq", DataType::Int64, true),
+ ]))
+}
+
+fn source(rows: &[Src]) -> RecordBatch {
+ RecordBatch::try_new(
+ source_schema(),
+ vec![
+ Arc::new(StringArray::from(
+ rows.iter().map(|r| r.0).collect::<Vec<_>>(),
+ )),
+ Arc::new(Int64Array::from(
+ rows.iter().map(|r| r.1).collect::<Vec<_>>(),
+ )),
+ Arc::new(
+ TimestampMicrosecondArray::from(rows.iter().map(|r| ts(r.2)).collect::<Vec<_>>())
+ .with_timezone("UTC"),
+ ),
+ Arc::new(StringArray::from(
+ rows.iter().map(|r| r.3).collect::<Vec<_>>(),
+ )),
+ Arc::new(Int64Array::from(
+ rows.iter().map(|r| r.4).collect::<Vec<_>>(),
+ )),
+ ],
+ )
+ .unwrap()
+}
+
+async fn create(dir: &std::path::Path, partition_by: &[&str]) -> TableHandle {
+ CreateBuilder::new()
+ .with_location(dir.to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("id", KernelType::STRING, true),
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new(
+ "valid_from",
+ KernelType::Primitive(PrimitiveType::Timestamp),
+ true,
+ ),
+ StructField::new(
+ "valid_to",
+ KernelType::Primitive(PrimitiveType::Timestamp),
+ true,
+ ),
+ StructField::new("payload", KernelType::STRING, true),
+ StructField::new("p", KernelType::STRING, false),
+ StructField::new("q", KernelType::STRING, true),
+ ])
+ .with_partition_columns(partition_by.iter().map(|c| c.to_string()))
+ .await
+ .unwrap();
+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())
+ .await
+ .unwrap()
+}
+
+fn opts() -> HistoryUpsertOptions {
+ HistoryUpsertOptions {
+ key_columns: vec!["id".into()],
+ order_column: "v".into(),
+ valid_from_column: "valid_from".into(),
+ valid_to_column: "valid_to".into(),
+ partition_column: "p".into(),
+ current_partition: "current".into(),
+ tie_break_column: "seq".into(),
+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),
+ ..Default::default()
+ }
+}
+
+async fn apply_with(
+ handle: &mut TableHandle,
+ rows: &[Src<'_>],
+ opts: HistoryUpsertOptions,
+) -> deltalite_core::Result<HistoryUpsertStats> {
+ handle
+ .history_upsert(
+ vec![source(rows)],
+ source_schema(),
+ opts,
+ MultipartConfig::default(),
+ )
+ .await
+}
+
+async fn apply(handle: &mut TableHandle, rows: &[Src<'_>]) -> HistoryUpsertStats {
+ apply_with(handle, rows, opts()).await.unwrap()
+}
+
+/// Every live row, sorted.
+async fn read(handle: &TableHandle) -> Vec<Live> {
+ let mut out = Vec::new();
+ for f in handle.files().unwrap() {
+ let bytes = handle
+ .table()
+ .object_store()
+ .get(&Path::parse(&f.path).unwrap())
+ .await
+ .unwrap()
+ .bytes()
+ .await
+ .unwrap();
+ let partition = f.partition_values["p"].clone().unwrap();
+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)
+ .unwrap()
+ .build()
+ .unwrap()
+ {
+ let batch = batch.unwrap();
+ let col = |name: &str| batch.column_by_name(name).unwrap().clone();
+ let (id, v, from, to, payload) = (
+ col("id"),
+ col("v"),
+ col("valid_from"),
+ col("valid_to"),
+ col("payload"),
+ );
+ let id = id.as_any().downcast_ref::<StringArray>().unwrap();
+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();
+ let from = from
+ .as_any()
+ .downcast_ref::<TimestampMicrosecondArray>()
+ .unwrap();
+ let to = to
+ .as_any()
+ .downcast_ref::<TimestampMicrosecondArray>()
+ .unwrap();
+ let payload = payload.as_any().downcast_ref::<StringArray>().unwrap();
+ for i in 0..batch.num_rows() {
+ out.push((
+ partition.clone(),
+ id.is_valid(i).then(|| id.value(i).to_string()),
+ v.value(i),
+ from.value(i),
+ to.is_valid(i).then(|| to.value(i)),
+ payload.value(i).to_string(),
+ ));
+ }
+ }
+ }
+ out.sort();
+ out
+}
+
+fn open(id: Option<&str>, v: i64, from: &str, payload: &str) -> Live {
+ (
+ "current".into(),
+ id.map(str::to_string),
+ v,
+ ts(from),
+ None,
+ payload.into(),
+ )
+}
+
+fn closed(month: &str, id: Option<&str>, v: i64, from: &str, to: &str, payload: &str) -> Live {
+ (
+ month.into(),
+ id.map(str::to_string),
+ v,
+ ts(from),
+ Some(ts(to)),
+ payload.into(),
+ )
+}
+
+fn sorted(mut rows: Vec<Live>) -> Vec<Live> {
+ rows.sort();
+ rows
+}
+
+#[tokio::test]
+async fn applies_each_rule_against_the_open_row() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ let first = apply(
+ &mut h,
+ &[
+ (Some("changed"), 10, "2026-01-10T00:00:00Z", "c1", 1),
+ (Some("reread"), 10, "2026-01-10T00:00:00Z", "r1", 2),
+ (Some("backwards"), 10, "2026-01-10T00:00:00Z", "b1", 3),
+ (Some("mixed"), 10, "2026-01-10T00:00:00Z", "m1", 4),
+ ],
+ )
+ .await;
+ assert_eq!(
+ (
+ first.versions_opened,
+ first.versions_closed,
+ first.files_removed
+ ),
+ (4, 0, 0),
+ "{first:?}"
+ );
+
+ let second = apply(
+ &mut h,
+ &[
+ (Some("changed"), 20, "2026-02-03T00:00:00Z", "c2", 10),
+ (Some("reread"), 10, "2026-01-10T00:00:00Z", "r1-again", 11),
+ (Some("backwards"), 5, "2026-01-05T00:00:00Z", "b0", 12),
+ (Some("mixed"), 7, "2026-01-07T00:00:00Z", "m0", 13),
+ (Some("mixed"), 30, "2026-03-01T00:00:00Z", "m3", 14),
+ (Some("new"), 1, "2026-03-02T00:00:00Z", "n1", 15),
+ ],
+ )
+ .await;
+ assert_eq!(
+ (
+ second.rows_in,
+ second.rows_reread,
+ second.rows_backwards,
+ second.versions_opened,
+ second.versions_closed,
+ second.chain_rows_closed,
+ second.rows_copied,
+ ),
+ (6, 1, 2, 3, 2, 0, 2),
+ "{second:?}"
+ );
+ assert_eq!(
+ read(&h).await,
+ sorted(vec![
+ closed(
+ "2026-02",
+ Some("changed"),
+ 10,
+ "2026-01-10T00:00:00Z",
+ "2026-02-03T00:00:00Z",
+ "c1"
+ ),
+ open(Some("changed"), 20, "2026-02-03T00:00:00Z", "c2"),
+ open(Some("reread"), 10, "2026-01-10T00:00:00Z", "r1"),
+ open(Some("backwards"), 10, "2026-01-10T00:00:00Z", "b1"),
+ closed(
+ "2026-03",
+ Some("mixed"),
+ 10,
+ "2026-01-10T00:00:00Z",
+ "2026-03-01T00:00:00Z",
+ "m1"
+ ),
+ open(Some("mixed"), 30, "2026-03-01T00:00:00Z", "m3"),
+ open(Some("new"), 1, "2026-03-02T00:00:00Z", "n1"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn chains_versions_in_one_call_and_a_tie_takes_the_highest_tie_break() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ let stats = apply(
+ &mut h,
+ &[
+ (Some("a"), 3, "2026-03-01T00:00:00Z", "third", 1),
+ (Some("a"), 2, "2026-02-01T00:00:00Z", "second-wins", 9),
+ (Some("a"), 1, "2026-01-01T00:00:00Z", "first", 1),
+ (Some("a"), 2, "2026-02-01T00:00:00Z", "second-loses", 5),
+ ],
+ )
+ .await;
+ assert_eq!(
+ (
+ stats.rows_deduped,
+ stats.versions_opened,
+ stats.chain_rows_closed
+ ),
+ (1, 1, 2),
+ "{stats:?}"
+ );
+ assert_eq!(
+ read(&h).await,
+ sorted(vec![
+ closed(
+ "2026-02",
+ Some("a"),
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-02-01T00:00:00Z",
+ "first"
+ ),
+ closed(
+ "2026-03",
+ Some("a"),
+ 2,
+ "2026-02-01T00:00:00Z",
+ "2026-03-01T00:00:00Z",
+ "second-wins"
+ ),
+ open(Some("a"), 3, "2026-03-01T00:00:00Z", "third"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn a_null_key_matches_its_open_row() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ apply(&mut h, &[(None, 1, "2026-01-01T00:00:00Z", "n1", 1)]).await;
+ let stats = apply(&mut h, &[(None, 2, "2026-02-01T00:00:00Z", "n2", 2)]).await;
+ assert_eq!(stats.versions_closed, 1, "{stats:?}");
+ assert_eq!(
+ read(&h).await,
+ sorted(vec![
+ closed(
+ "2026-02",
+ None,
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-02-01T00:00:00Z",
+ "n1"
+ ),
+ open(None, 2, "2026-02-01T00:00:00Z", "n2"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn a_rerun_of_the_same_input_makes_no_commit() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ let input = [
+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),
+ (Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 2),
+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 3),
+ ];
+ let first = apply(&mut h, &input).await;
+ let rows = read(&h).await;
+ let files = h.files().unwrap();
+
+ let rerun = apply(&mut h, &input).await;
+ assert!(first.committed && !rerun.committed, "{rerun:?}");
+ assert_eq!(rerun.version, first.version);
+ assert_eq!(
+ (
+ rerun.rows_reread,
+ rerun.rows_backwards,
+ rerun.files_added,
+ rerun.files_removed,
+ rerun.files_carried_over
+ ),
+ (2, 1, 0, 0, 1),
+ "{rerun:?}"
+ );
+ assert_eq!(h.version(), first.version);
+ assert_eq!(h.files().unwrap(), files);
+ assert_eq!(read(&h).await, rows);
+}
+
+#[tokio::test]
+async fn closed_rows_land_in_the_utc_month_of_valid_to() {
+ let vectors: Vec<serde_json::Value> =
+ serde_json::from_str(include_str!("fixtures/month_vectors.json")).unwrap();
+ let instants: Vec<i64> = vectors
+ .iter()
+ .map(|v| ts(v["valid_to"].as_str().unwrap()))
+ .collect();
+ let months =
+ format_months(&TimestampMicrosecondArray::from(instants.clone()).with_timezone("UTC"))
+ .unwrap();
+ for (v, m) in vectors.iter().zip(months.iter()) {
+ assert_eq!(m, v["month"].as_str(), "{v}");
+ }
+
+ // Each key opens before 1969 and closes at one vector instant.
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ let ids: Vec<String> = (0..vectors.len()).map(|i| format!("k{i}")).collect();
+ let opens: Vec<Src> = ids
+ .iter()
+ .map(|id| (Some(id.as_str()), 1, "1960-01-01T00:00:00Z", "old", 1))
+ .collect();
+ apply(&mut h, &opens).await;
+ let closes: Vec<Src> = ids
+ .iter()
+ .zip(&vectors)
+ .map(|(id, v)| {
+ (
+ Some(id.as_str()),
+ 2,
+ v["valid_to"].as_str().unwrap(),
+ "new",
+ 2,
+ )
+ })
+ .collect();
+ apply(&mut h, &closes).await;
+ let closed_months: HashMap<String, String> = read(&h)
+ .await
+ .into_iter()
+ .filter(|row| row.4.is_some())
+ .map(|row| (row.1.unwrap(), row.0))
+ .collect();
+ for (id, v) in ids.iter().zip(&vectors) {
+ assert_eq!(closed_months[id], v["month"].as_str().unwrap(), "{v}");
+ }
+}
+
+#[tokio::test]
+async fn earlier_closed_versions_in_the_same_month_stay() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;
+ apply(&mut h, &[(Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 2)]).await;
+ apply(&mut h, &[(Some("a"), 3, "2026-02-15T00:00:00Z", "a3", 3)]).await;
+ assert_eq!(
+ read(&h).await,
+ sorted(vec![
+ closed(
+ "2026-02",
+ Some("a"),
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-02-01T00:00:00Z",
+ "a1"
+ ),
+ closed(
+ "2026-02",
+ Some("a"),
+ 2,
+ "2026-02-01T00:00:00Z",
+ "2026-02-15T00:00:00Z",
+ "a2"
+ ),
+ open(Some("a"), 3, "2026-02-15T00:00:00Z", "a3"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn a_file_holding_only_reread_keys_is_not_rewritten() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;
+ apply(&mut h, &[(Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2)]).await;
+ let stats = apply(
+ &mut h,
+ &[
+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1-again", 3),
+ (Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 4),
+ ],
+ )
+ .await;
+ assert_eq!(
+ (
+ stats.files_probed,
+ stats.files_removed,
+ stats.files_carried_over,
+ stats.rows_reread
+ ),
+ (2, 1, 1, 1),
+ "{stats:?}"
+ );
+}
+
+#[tokio::test]
+async fn refuses_input_it_cannot_apply() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut history = create(dir.path(), &["p"]).await;
+ apply(
+ &mut history,
+ &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a", 1)],
+ )
+ .await;
+ // A null order value, and a null hidden in the dictionary of a dictionary-encoded
+ // `valid_from`, which a plain null count does not see.
+ let valid = source(&[(Some("a"), 2, "2026-02-01T00:00:00Z", "a", 2)]);
+ let null_dictionary_value: Arc<dyn Array> = Arc::new(
+ DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(vec![0]),
+ Arc::new(TimestampMicrosecondArray::from(vec![None::<i64>]).with_timezone("UTC")),
+ )
+ .unwrap(),
+ );
+ let null_order: Arc<dyn Array> = Arc::new(Int64Array::from(vec![None::<i64>]));
+ // A dictionary or string tie-break can expand far past its in-memory size when every
+ // row is encoded, which the source-size guard would not see.
+ let dictionary_tie: Arc<dyn Array> = Arc::new(
+ DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(vec![0]),
+ Arc::new(StringArray::from(vec!["x".repeat(1024)])),
+ )
+ .unwrap(),
+ );
+ let string_tie: Arc<dyn Array> = Arc::new(StringArray::from(vec!["1"]));
+ for (case, column, value, kind) in [
+ ("null order", 1, null_order, "generic"),
+ (
+ "null dictionary valid_from",
+ 2,
+ null_dictionary_value,
+ "generic",
+ ),
+ ("dictionary tie-break", 4, dictionary_tie, "schema_mismatch"),
+ ("string tie-break", 4, string_tie, "schema_mismatch"),
+ ] {
+ let mut fields: Vec<Field> = source_schema()
+ .fields()
+ .iter()
+ .map(|f| f.as_ref().clone())
+ .collect();
+ fields[column] = Field::new(fields[column].name(), value.data_type().clone(), true);
+ let schema: SchemaRef = Arc::new(Schema::new(fields));
+ let mut cols = valid.columns().to_vec();
+ cols[column] = value;
+ let batch = RecordBatch::try_new(schema.clone(), cols).unwrap();
+ let err = history
+ .history_upsert(vec![batch], schema, opts(), MultipartConfig::default())
+ .await
+ .unwrap_err();
+ assert_eq!(err.kind(), kind, "{case}: {err}");
+ }
+
+ // A batch whose columns differ from the declared schema, here a string tie-break
+ // declared as an integer, is refused before any column is read.
+ let mut cols = valid.columns().to_vec();
+ cols[4] = Arc::new(StringArray::from(vec!["1"]));
+ let mut fields: Vec<Field> = source_schema()
+ .fields()
+ .iter()
+ .map(|f| f.as_ref().clone())
+ .collect();
+ fields[4] = Field::new("seq", DataType::Utf8, true);
+ let undeclared = RecordBatch::try_new(Arc::new(Schema::new(fields)), cols).unwrap();
+ let err = history
+ .history_upsert(
+ vec![undeclared],
+ source_schema(),
+ opts(),
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(err.kind(), "schema_mismatch", "{err}");
+
+ for (case, partition_by) in [("unpartitioned", vec![]), ("other column", vec!["q"])] {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &partition_by).await;
+ let err = apply_with(
+ &mut h,
+ &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a", 1)],
+ opts(),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::Unsupported(_)), "{case}: {err}");
+ }
+
+ let month_named = HistoryUpsertOptions {
+ current_partition: "2026-01".into(),
+ ..opts()
+ };
+ let err = apply_with(
+ &mut history,
+ &[(Some("a"), 3, "2026-03-01T00:00:00Z", "a", 3)],
+ month_named,
+ )
+ .await
+ .unwrap_err();
+ assert!(
+ matches!(err, Error::Generic(ref m) if m.contains("month")),
+ "{err}"
+ );
+}
+
+#[tokio::test]
+async fn a_concurrent_commit_on_the_same_file_is_retried_against_the_new_state() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ apply(
+ &mut h,
+ &[
+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),
+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2),
+ ],
+ )
+ .await;
+
+ // A stale snapshot plans to remove the file another commit already removed.
+ let stale = h.table().clone();
+ apply(&mut h, &[(Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 3)]).await;
+ let err = history_upsert(
+ &stale,
+ vec![source(&[(Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 4)])],
+ source_schema(),
+ opts(),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::Conflict(_)), "{err}");
+
+ // A concurrent commit that only adds files must not let a stale plan open a key twice.
+ let stale = h.table().clone();
+ apply(&mut h, &[(Some("c"), 1, "2026-02-01T00:00:00Z", "c1", 5)]).await;
+ let err = history_upsert(
+ &stale,
+ vec![source(&[(Some("c"), 2, "2026-02-02T00:00:00Z", "c2", 6)])],
+ source_schema(),
+ opts(),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::Conflict(_)), "{err}");
+
+ h.refresh().await.unwrap();
+ assert_eq!(
+ read(&h).await,
+ sorted(vec![
+ closed(
+ "2026-02",
+ Some("a"),
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-02-01T00:00:00Z",
+ "a1"
+ ),
+ open(Some("a"), 2, "2026-02-01T00:00:00Z", "a2"),
+ open(Some("b"), 1, "2026-01-01T00:00:00Z", "b1"),
+ open(Some("c"), 1, "2026-02-01T00:00:00Z", "c1"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn a_handle_plans_again_when_another_commit_lands_before_its_own() {
+ register_race_scheme();
+ let dir = tempfile::tempdir().unwrap();
+ let mut plain = create(dir.path(), &["p"]).await;
+ apply(
+ &mut plain,
+ &[
+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),
+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2),
+ ],
+ )
+ .await;
+
+ // The competing commit lands after the racing call planned, just before its commit
+ // file is written, so the racing commit finds its version taken.
+ let path = dir.path().to_str().unwrap().to_string();
+ let raced = Arc::new(AtomicBool::new(false));
+ let (other_path, other_raced) = (path.clone(), raced.clone());
+ arm_race(
+ dir.path(),
+ Box::new(move || {
+ Box::pin(async move {
+ let mut other = TableHandle::open(other_path, HashMap::new()).await.unwrap();
+ apply(
+ &mut other,
+ &[(Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 3)],
+ )
+ .await;
+ other_raced.store(true, AtomicOrdering::SeqCst);
+ })
+ }),
+ );
+ let mut racing = TableHandle::open(format!("dlrace://{path}"), HashMap::new())
+ .await
+ .unwrap();
+ let stats = apply(
+ &mut racing,
+ &[(Some("a"), 2, "2026-03-01T00:00:00Z", "a2", 4)],
+ )
+ .await;
+
+ assert!(raced.load(AtomicOrdering::SeqCst));
+ assert_eq!((stats.version, stats.versions_closed), (3, 1), "{stats:?}");
+ plain.refresh().await.unwrap();
+ assert_eq!(
+ read(&plain).await,
+ sorted(vec![
+ closed(
+ "2026-03",
+ Some("a"),
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-03-01T00:00:00Z",
+ "a1"
+ ),
+ open(Some("a"), 2, "2026-03-01T00:00:00Z", "a2"),
+ closed(
+ "2026-02",
+ Some("b"),
+ 1,
+ "2026-01-01T00:00:00Z",
+ "2026-02-01T00:00:00Z",
+ "b1"
+ ),
+ open(Some("b"), 2, "2026-02-01T00:00:00Z", "b2"),
+ ])
+ );
+}
+
+#[tokio::test]
+async fn a_call_that_keeps_no_row_makes_no_relax_commit() {
+ let dir = tempfile::tempdir().unwrap();
+ CreateBuilder::new()
+ .with_location(dir.path().to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("id", KernelType::STRING, true),
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new(
+ "valid_from",
+ KernelType::Primitive(PrimitiveType::Timestamp),
+ true,
+ ),
+ StructField::new(
+ "valid_to",
+ KernelType::Primitive(PrimitiveType::Timestamp),
+ true,
+ ),
+ StructField::new("payload", KernelType::STRING, false),
+ StructField::new("p", KernelType::STRING, false),
+ ])
+ .with_partition_columns(vec!["p".to_string()])
+ .await
+ .unwrap();
+ let mut h = TableHandle::open(dir.path().to_str().unwrap().into(), HashMap::new())
+ .await
+ .unwrap();
+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;
+ let with_null_payload = |v: i64, at: &str| {
+ let batch = source(&[(Some("a"), v, at, "", 2)]);
+ let mut cols = batch.columns().to_vec();
+ cols[3] = Arc::new(StringArray::from(vec![None::<&str>]));
+ RecordBatch::try_new(source_schema(), cols).unwrap()
+ };
+ let version = h.version();
+
+ // A re-read with a null in the non-nullable column: nothing is kept, nothing commits.
+ let reread = h
+ .history_upsert(
+ vec![with_null_payload(1, "2026-01-01T00:00:00Z")],
+ source_schema(),
+ opts(),
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ (reread.committed, reread.columns_relaxed),
+ (false, 0),
+ "{reread:?}"
+ );
+ assert_eq!(h.version(), version);
+ assert!(h
+ .schema_json()
+ .unwrap()
+ .contains(r#""name":"payload","type":"string","nullable":false"#));
+
+ // A kept row with the same null relaxes the column and lands.
+ let kept = h
+ .history_upsert(
+ vec![with_null_payload(2, "2026-02-01T00:00:00Z")],
+ source_schema(),
+ opts(),
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ (kept.committed, kept.columns_relaxed),
+ (true, 1),
+ "{kept:?}"
+ );
+ assert_eq!(kept.version, version + 2);
+}
+
+#[tokio::test]
+async fn closes_many_rows_under_tiny_budgets_and_returns_every_permit() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut h = create(dir.path(), &["p"]).await;
+ let ids: Vec<String> = (0..400).map(|i| format!("k{i:04}")).collect();
+ for chunk in ids.chunks(100) {
+ let rows: Vec<Src> = chunk
+ .iter()
+ .map(|id| (Some(id.as_str()), 1, "2026-01-01T00:00:00Z", "old", 1))
+ .collect();
+ apply(&mut h, &rows).await;
+ }
+
+ let limits = Arc::new(ProcessLimits::with_fetch(1, 1, 1024, 1024));
+ let tiny = HistoryUpsertOptions {
+ max_parallel_files: 4,
+ probe_concurrency: 4,
+ max_buffered_bytes: 1024,
+ max_fetch_bytes: 1024,
+ read_batch_size: 16,
+ target_file_size: Some(4 * 1024),
+ limits: limits.clone(),
+ ..opts()
+ };
+ // The closes land in 50 months, more than the closed-row writer keeps open at once.
+ let instants: Vec<String> = (0..ids.len())
+ .map(|i| format!("{}-{:02}-15T00:00:00Z", 2026 + i % 50 / 12, i % 50 % 12 + 1))
+ .collect();
+ let rows: Vec<Src> = ids
+ .iter()
+ .zip(&instants)
+ .map(|(id, at)| (Some(id.as_str()), 2, at.as_str(), "new", 2))
+ .collect();
+ let stats = apply_with(&mut h, &rows, tiny).await.unwrap();
+ assert_eq!(
+ (
+ stats.files_removed,
+ stats.versions_closed,
+ stats.versions_opened
+ ),
+ (4, 400, 400),
+ "{stats:?}"
+ );
+ let snapshot = limits.snapshot();
+ for budget in [
+ snapshot.partitions,
+ snapshot.files,
+ snapshot.buffer_kb,
+ snapshot.fetch_kb,
+ ] {
+ assert_eq!(budget.available, budget.total, "{snapshot:?}");
+ }
+ let live = read(&h).await;
+ assert_eq!(live.len(), 800);
+ let mut per_month: HashMap<String, usize> = HashMap::new();
+ for row in live.iter().filter(|r| r.0 != "current") {
+ *per_month.entry(row.0.clone()).or_default() += 1;
+ }
+ assert_eq!(per_month.len(), 50);
+ assert!(per_month.values().all(|n| *n == 8), "{per_month:?}");
+}
+
+#[tokio::test]
+async fn key_bucket_ids_are_fixed_and_follow_the_table_key_types() {
+ let dir = tempfile::tempdir().unwrap();
+ let h = create(dir.path(), &["p"]).await;
+ let keys = ["id".to_string(), "v".to_string()];
+ let batch = |v: Arc<dyn Array>| {
+ let ids = StringArray::from(vec![
+ Some("a"),
+ Some("b"),
+ None,
+ Some("a"),
+ Some("customer-42"),
+ ]);
+ RecordBatch::try_new(
+ Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Utf8, true),
+ Field::new("v", v.data_type().clone(), true),
+ ])),
+ vec![Arc::new(ids), v],
+ )
+ .unwrap()
+ };
+ let typed = batch(Arc::new(Int64Array::from(vec![
+ Some(1),
+ Some(1),
+ Some(1),
+ None,
+ Some(7),
+ ])));
+ let buckets = h
+ .key_buckets(std::slice::from_ref(&typed), &keys, 16)
+ .unwrap();
+ assert_eq!(buckets.values().to_vec(), vec![4, 12, 0, 6, 6]);
+
+ // The same key gets the same bucket in a later call with other rows around it.
+ let again = h
+ .key_buckets(&[typed.slice(3, 2), typed.slice(0, 3)], &keys, 16)
+ .unwrap();
+ assert_eq!(
+ again.values().to_vec(),
+ [&buckets.values()[3..], &buckets.values()[..3]].concat()
+ );
+
+ // Strings that cast to the same table key share its bucket, as they share the key.
+ let strings = batch(Arc::new(StringArray::from(vec![
+ Some("01"),
+ Some("1"),
+ Some("001"),
+ None,
+ Some("07"),
+ ])));
+ assert_eq!(
+ h.key_buckets(&[strings], &keys, 16)
+ .unwrap()
+ .values()
+ .to_vec(),
+ buckets.values().to_vec()
+ );
+}
+
+// ---- a store that lands a competing commit just before a commit file is written ------
+
+type Race = Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>;
+
+/// Armed races by table path prefix; each fires once, on the first commit file written
+/// under its prefix.
+static RACES: LazyLock<Mutex<Vec<(String, Race)>>> = LazyLock::new(Default::default);
+
+fn arm_race(dir: &std::path::Path, race: Race) {
+ let prefix = Path::from(dir.to_string_lossy().as_ref()).to_string();
+ RACES.lock().unwrap().push((prefix, race));
+}
+
+fn take_race(location: &Path) -> Option<Race> {
+ let mut races = RACES.lock().unwrap();
+ let i = races
+ .iter()
+ .position(|(prefix, _)| location.as_ref().starts_with(prefix.as_str()))?;
+ Some(races.swap_remove(i).1)
+}
+
+#[derive(Debug)]
+struct RaceStore {
+ inner: Arc<dyn ObjectStore>,
+}
+
+impl std::fmt::Display for RaceStore {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "RaceStore({})", self.inner)
+ }
+}
+
+#[async_trait::async_trait]
+impl ObjectStore for RaceStore {
+ async fn put_opts(
+ &self,
+ location: &Path,
+ payload: PutPayload,
+ opts: PutOptions,
+ ) -> object_store::Result<PutResult> {
+ let is_commit = matches!(opts.mode, PutMode::Create)
+ && location.as_ref().contains("_delta_log/")
+ && location.as_ref().ends_with(".json");
+ if is_commit {
+ if let Some(race) = take_race(location) {
+ race().await;
+ }
+ }
+ self.inner.put_opts(location, payload, opts).await
+ }
+
+ async fn put_multipart_opts(
+ &self,
+ location: &Path,
+ opts: PutMultipartOptions,
+ ) -> object_store::Result<Box<dyn MultipartUpload>> {
+ self.inner.put_multipart_opts(location, opts).await
+ }
+
+ async fn get_opts(
+ &self,
+ location: &Path,
+ options: GetOptions,
+ ) -> object_store::Result<GetResult> {
+ self.inner.get_opts(location, options).await
+ }
+
+ async fn get_ranges(
+ &self,
+ location: &Path,
+ ranges: &[std::ops::Range<u64>],
+ ) -> object_store::Result<Vec<Bytes>> {
+ self.inner.get_ranges(location, ranges).await
+ }
+
+ fn delete_stream(
+ &self,
+ locations: BoxStream<'static, object_store::Result<Path>>,
+ ) -> BoxStream<'static, object_store::Result<Path>> {
+ self.inner.delete_stream(locations)
+ }
+
+ // Sorted: cloud stores list lexicographically and delta-kernel relies on it.
+ fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result<ObjectMeta>> {
+ sorted_listing(self.inner.list(prefix))
+ }
+
+ fn list_with_offset(
+ &self,
+ prefix: Option<&Path>,
+ offset: &Path,
+ ) -> BoxStream<'static, object_store::Result<ObjectMeta>> {
+ sorted_listing(self.inner.list_with_offset(prefix, offset))
+ }
+
+ async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result<ListResult> {
+ self.inner.list_with_delimiter(prefix).await
+ }
+
+ async fn copy_opts(
+ &self,
+ from: &Path,
+ to: &Path,
+ options: CopyOptions,
+ ) -> object_store::Result<()> {
+ self.inner.copy_opts(from, to, options).await
+ }
+
+ async fn rename_opts(
+ &self,
+ from: &Path,
+ to: &Path,
+ options: RenameOptions,
+ ) -> object_store::Result<()> {
+ self.inner.rename_opts(from, to, options).await
+ }
+}
+
+fn sorted_listing(
+ stream: BoxStream<'static, object_store::Result<ObjectMeta>>,
+) -> BoxStream<'static, object_store::Result<ObjectMeta>> {
+ use futures::{StreamExt, TryStreamExt};
+ futures::stream::once(async move {
+ let mut items: Vec<ObjectMeta> = stream.try_collect().await?;
+ items.sort_by(|a, b| a.location.cmp(&b.location));
+ Ok::<_, object_store::Error>(futures::stream::iter(items.into_iter().map(Ok)))
+ })
+ .try_flatten()
+ .boxed()
+}
+
+#[derive(Debug, Default)]
+struct RaceFactory;
+
+impl ObjectStoreFactory for RaceFactory {
+ fn parse_url_opts(
+ &self,
+ url: &url::Url,
+ _config: &StorageConfig,
+ ) -> DeltaResult<(Arc<dyn ObjectStore>, Path)> {
+ let store = RaceStore {
+ inner: Arc::new(LocalFileSystem::new()),
+ };
+ Ok((Arc::new(store), Path::from_url_path(url.path())?))
+ }
+}
+
+impl LogStoreFactory for RaceFactory {
+ fn with_options(
+ &self,
+ prefixed_store: Arc<dyn ObjectStore>,
+ root_store: Arc<dyn ObjectStore>,
+ location: &url::Url,
+ options: &StorageConfig,
+ ) -> DeltaResult<Arc<dyn LogStore>> {
+ Ok(default_logstore(
+ prefixed_store,
+ root_store,
+ location,
+ options,
+ ))
+ }
+}
+
+fn register_race_scheme() {
+ let scheme = url::Url::parse("dlrace://").expect("static url");
+ object_store_factories().insert(scheme.clone(), Arc::new(RaceFactory));
+ logstore_factories().insert(scheme, Arc::new(RaceFactory));
+}
diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md
index bcca01c9761a..7f47d75d81f9 100644
--- a/rust/deltalite/python/README.md
+++ b/rust/deltalite/python/README.md
@@ -102,6 +102,8 @@ rejected (raising `DeltaLiteError`) rather than silently double-inserted.
| `DeltaLiteTable.is_deltatable(uri, storage_options=None)` | `True` if a Delta table exists at `uri`. |
| `DeltaLiteTable.table_exists(uri, storage_options=None)` | `True` if the log at `uri` holds a commit or a checkpoint, the same check as `DeltaTable.is_deltatable`. Storage errors raise; they never read as `False`. |
| `.upsert(data, primary_keys, partition_key=None, **opts)` | Insert-or-replace `data` by key. Returns `UpsertStats`. See knobs below. |
+| `.history_upsert(data, key_columns, order_column, valid_from_column, valid_to_column, partition_column, current_partition, tie_break_column, **opts)` | Apply versions to an SCD2 history table in one data commit. Returns `HistoryUpsertStats`. See "History upsert" below. |
+| `.key_buckets(data, key_columns, n)` | Bucket ids for splitting a `history_upsert` source by key. See "History upsert" below. |
| `.compact(**opts)` | Small-file compaction, the replacement for `DeltaTable.optimize.compact`. Returns a stats dict. See "Compaction" below. |
| `.restore(version, *, commit_metadata=None)` | delta-rs's restore to `version`, in one commit, with the GIL released. Returns `numRemovedFile`, `numRestoredFile` and the committed `version`. The handle moves onto the restored table. |
| `.version()` | Current table version (`int`). |
@@ -139,6 +141,40 @@ summed over partitions, part of `rewrite_ms`), `commit_ms`
non-zero only on checkpoint-boundary commits). `columns_relaxed` counts
non-nullable columns flipped to nullable before the write.
+### History upsert
+
+`history_upsert` writes an SCD2 history table: one row per version of a key.
+Open versions (`valid_to_column` null) live in the `current_partition` value of `partition_column`.
+A closed version lives in the partition named by its `valid_to_column` as `%Y-%m` in UTC, which is what `pyarrow.compute.strftime(valid_to, "%Y-%m")` returns for a UTC timestamp.
+The table must be partitioned by exactly `partition_column`, a string column, or the call raises `DeltaLiteUnsupportedTableError`.
+
+Per key (`key_columns`, compared null-safely: a null key part matches a null key part):
+
+1. Rows with the same `order_column` value collapse to the one with the highest `tie_break_column` value. On an equal tie-break value, the later row wins.
+2. Only rows with an order value strictly greater than the key's open row are kept. Equal rows count as `rows_reread`, lower rows as `rows_backwards`.
+3. The kept rows form a chain: each closes at the next kept row's `valid_from_column`, and the newest stays open.
+4. When a key keeps a row and has an open row, the open row closes at the first kept row's `valid_from_column` and moves to its month partition.
+
+The source must not carry `valid_to_column` or `partition_column`, since the operation sets them.
+A null order value or a null `valid_from_column` raises `DeltaLiteError`.
+`tie_break_column` must have a fixed-width type, such as an integer or a timestamp, and is written only when the table has that column.
+Closed rows are only appended, so earlier closed versions of a key stay.
+A source that keeps no row, such as a re-run of the same input, makes no commit and returns `committed=False`, so a re-run after a crash is safe. That includes the nullability relax: it commits only when the call also writes rows.
+
+Each call reads the key and order columns of every current-partition file that its stats do not rule out, so a file that holds only re-read keys is not rewritten.
+The probe cache is not used.
+The knobs are the `upsert` ones that apply: `commit_metadata`, `max_parallel_files`, `max_buffered_bytes`, `max_fetch_bytes`, `probe_concurrency`, `read_batch_size`, `target_file_size`, `max_row_group_bytes`, `max_source_bytes`, `read_ahead_row_groups` and the multipart and ranged-read settings.
+`max_parallel_partitions` is accepted and has no effect, because only the current partition is rewritten.
+A commit that finds a newer table version is never rebased onto it, since delta-rs does not check a concurrent commit that only adds files: the call plans again against the new version, up to five times, then raises `DeltaLiteCommitConflictError`.
+
+`HistoryUpsertStats` counts `rows_in`, `rows_deduped`, `rows_reread`, `rows_backwards`, `versions_opened`, `versions_closed` (open rows closed), `chain_rows_closed` (kept rows written closed), `rows_copied`, `files_added`, `files_removed`, `files_carried_over`, `files_probed`, `bytes_read` (stored size of the rewritten files) and `bytes_written`, with `version`, `committed` and the `upsert` timings.
+
+`DeltaLiteTable.key_buckets(data, key_columns, n)` returns a `pyarrow.UInt32Array` with a bucket id in `0..n` for every row, for splitting a source that is too large for one call into calls on disjoint keys.
+The key columns are first cast to the table's types, as `history_upsert` casts them before it matches keys, so source values that become one table key always share a bucket.
+Each key is then encoded with arrow-row (null-safe) and hashed with xxHash64 under a fixed seed, then reduced modulo `n`.
+The ids are the same in every process that runs one deltalite build, for keys of the same Arrow types.
+arrow-row may change its encoding between arrow releases, so do not persist the ids.
+
### Compaction
`compact` plans bins exactly as delta-rs's `optimize.compact` does, from the
@@ -338,7 +374,8 @@ labels only): `deltalite_upserts_total` (`outcome`, `prune_strategy`,
`deltalite.upsert` and asserts identical logical content — not by version
equality.
- **Not supported:** tables with deletion vectors or column mapping (detected
- and raised as `DeltaLiteUnsupportedTableError`), and SCD2 merges.
+ and raised as `DeltaLiteUnsupportedTableError`). SCD2 history tables are
+ written with `history_upsert`, not with `upsert`.
- deltalite rejects duplicate source primary keys that `MERGE` silently
double-inserts — check pre-existing data if you migrate an existing pipeline.
diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs
index f46792891f55..a75769260bfe 100644
--- a/rust/deltalite/python/src/lib.rs
+++ b/rust/deltalite/python/src/lib.rs
@@ -14,6 +14,7 @@ use arrow_schema::SchemaRef;
use deltalake::writer::RecordBatchWriter;
use deltalite_core::errors::Error;
use deltalite_core::handle::TableHandle;
+use deltalite_core::history::HistoryUpsertOptions;
use deltalite_core::limits::{Budget, ProcessLimits};
use deltalite_core::probecache::ProbeCache;
use deltalite_core::schema::import_column;
@@ -238,6 +239,119 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {
}
}
+/// Counters describing what one `history_upsert` did.
+#[pyclass(module = "deltalite", skip_from_py_object)]
+#[derive(Clone, Debug, Default)]
+pub struct HistoryUpsertStats {
+ #[pyo3(get)]
+ pub version: i64,
+ #[pyo3(get)]
+ pub committed: bool,
+ #[pyo3(get)]
+ pub rows_in: usize,
+ #[pyo3(get)]
+ pub rows_deduped: usize,
+ #[pyo3(get)]
+ pub rows_reread: usize,
+ #[pyo3(get)]
+ pub rows_backwards: usize,
+ #[pyo3(get)]
+ pub versions_opened: usize,
+ #[pyo3(get)]
+ pub versions_closed: usize,
+ #[pyo3(get)]
+ pub chain_rows_closed: usize,
+ #[pyo3(get)]
+ pub rows_copied: usize,
+ #[pyo3(get)]
+ pub files_removed: usize,
+ #[pyo3(get)]
+ pub files_added: usize,
+ #[pyo3(get)]
+ pub files_carried_over: usize,
+ #[pyo3(get)]
+ pub files_probed: usize,
+ #[pyo3(get)]
+ pub bytes_read: u64,
+ #[pyo3(get)]
+ pub bytes_written: u64,
+ #[pyo3(get)]
+ pub plan_ms: u64,
+ #[pyo3(get)]
+ pub probe_ms: u64,
+ #[pyo3(get)]
+ pub rewrite_ms: u64,
+ #[pyo3(get)]
+ pub commit_ms: u64,
+ #[pyo3(get)]
+ pub columns_relaxed: usize,
+ #[pyo3(get)]
+ pub open_ms: u64,
+ #[pyo3(get)]
+ pub initial_open_ms: u64,
+ #[pyo3(get)]
+ pub ingest_ms: u64,
+ #[pyo3(get)]
+ pub relax_ms: u64,
+ #[pyo3(get)]
+ pub maintenance_ms: u64,
+}
+
+#[pymethods]
+impl HistoryUpsertStats {
+ fn __repr__(&self) -> String {
+ format!(
+ "HistoryUpsertStats(version={}, committed={}, rows_in={}, rows_deduped={}, \
+ rows_reread={}, rows_backwards={}, versions_opened={}, versions_closed={}, \
+ chain_rows_closed={}, files_added={}, files_removed={})",
+ self.version,
+ self.committed,
+ self.rows_in,
+ self.rows_deduped,
+ self.rows_reread,
+ self.rows_backwards,
+ self.versions_opened,
+ self.versions_closed,
+ self.chain_rows_closed,
+ self.files_added,
+ self.files_removed
+ )
+ }
+}
+
+impl From<deltalite_core::history::HistoryUpsertStats> for HistoryUpsertStats {
+ fn from(s: deltalite_core::history::HistoryUpsertStats) -> Self {
+ Self {
+ version: s.version,
+ committed: s.committed,
+ rows_in: s.rows_in,
+ rows_deduped: s.rows_deduped,
+ rows_reread: s.rows_reread,
+ rows_backwards: s.rows_backwards,
+ versions_opened: s.versions_opened,
+ versions_closed: s.versions_closed,
+ chain_rows_closed: s.chain_rows_closed,
+ rows_copied: s.rows_copied,
+ files_removed: s.files_removed,
+ files_added: s.files_added,
+ files_carried_over: s.files_carried_over,
+ files_probed: s.files_probed,
+ bytes_read: s.bytes_read,
+ bytes_written: s.bytes_written,
+ plan_ms: s.plan_ms,
+ probe_ms: s.probe_ms,
+ rewrite_ms: s.rewrite_ms,
+ commit_ms: s.commit_ms,
+ columns_relaxed: s.columns_relaxed,
+ open_ms: s.open_ms,
+ initial_open_ms: s.initial_open_ms,
+ ingest_ms: s.ingest_ms,
+ relax_ms: s.relax_ms,
+ maintenance_ms: s.maintenance_ms,
+ }
+ }
+}
+
/// A handle on a Delta table for deltalite writes, plus snapshot reads (version, table
/// id, schema, live files) served from the loaded state so a writer does not need a
/// second delta-rs `DeltaTable` open for them. `history` is the one read that goes back
@@ -527,6 +641,127 @@ impl DeltaLiteTable {
Ok(stats)
}
+ /// Apply a batch of versions to an SCD2 history table in one data commit. See the
+ /// README's "History upsert" section for the rules, refusals and knobs.
+ /// `max_parallel_partitions` is accepted and has no effect.
+ #[pyo3(signature = (
+ data,
+ key_columns,
+ order_column,
+ valid_from_column,
+ valid_to_column,
+ partition_column,
+ current_partition,
+ tie_break_column,
+ *,
+ commit_metadata = None,
+ max_parallel_partitions = 1,
+ max_parallel_files = 4,
+ max_buffered_bytes = 67108864,
+ max_fetch_bytes = 134217728,
+ probe_concurrency = 8,
+ read_batch_size = 8192,
+ target_file_size = None,
+ max_row_group_bytes = None,
+ max_source_bytes = None,
+ multipart_threshold = None,
+ multipart_part_size = None,
+ multipart_concurrency = None,
+ range_get_part_size = None,
+ range_get_concurrency = None,
+ read_ahead_row_groups = false,
+ ))]
+ #[allow(clippy::too_many_arguments)]
+ fn history_upsert(
+ &mut self,
+ py: Python<'_>,
+ data: &Bound<'_, PyAny>,
+ key_columns: Vec<String>,
+ order_column: String,
+ valid_from_column: String,
+ valid_to_column: String,
+ partition_column: String,
+ current_partition: String,
+ tie_break_column: String,
+ commit_metadata: Option<HashMap<String, String>>,
+ max_parallel_partitions: usize,
+ max_parallel_files: usize,
+ max_buffered_bytes: usize,
+ max_fetch_bytes: usize,
+ probe_concurrency: usize,
+ read_batch_size: usize,
+ target_file_size: Option<usize>,
+ max_row_group_bytes: Option<usize>,
+ max_source_bytes: Option<usize>,
+ multipart_threshold: Option<usize>,
+ multipart_part_size: Option<usize>,
+ multipart_concurrency: Option<usize>,
+ range_get_part_size: Option<usize>,
+ range_get_concurrency: Option<usize>,
+ read_ahead_row_groups: bool,
+ ) -> PyResult<HistoryUpsertStats> {
+ let _ = max_parallel_partitions;
+ let ingest_started = std::time::Instant::now();
+ let (schema, batches) = read_pyarrow(data)?;
+ let ingest_ms = ingest_started.elapsed().as_millis() as u64;
+
+ let opts = HistoryUpsertOptions {
+ key_columns,
+ order_column,
+ valid_from_column,
+ valid_to_column,
+ partition_column,
+ current_partition,
+ tie_break_column,
+ commit_metadata: commit_metadata.map(|m| {
+ m.into_iter()
+ .map(|(k, v)| (k, Value::String(v)))
+ .collect::<HashMap<String, Value>>()
+ }),
+ probe_concurrency,
+ max_parallel_files,
+ max_buffered_bytes,
+ max_fetch_bytes,
+ read_ahead_row_groups,
+ read_batch_size,
+ target_file_size,
+ max_row_group_bytes,
+ max_source_bytes,
+ limits: ProcessLimits::global().clone(),
+ };
+ let multipart = MultipartConfig::resolve(
+ multipart_threshold,
+ multipart_part_size,
+ multipart_concurrency,
+ range_get_part_size,
+ range_get_concurrency,
+ );
+ let handle = &mut self.handle;
+ let stats = py
+ .detach(|| runtime().block_on(handle.history_upsert(batches, schema, opts, multipart)))
+ .map_err(to_py_err)?;
+ let mut stats: HistoryUpsertStats = stats.into();
+ stats.ingest_ms = ingest_ms;
+ Ok(stats)
+ }
+
+ /// A null-safe bucket id in `0..n` (`pyarrow.UInt32Array`) for every row of `data`,
+ /// from `key_columns` cast to this table's types. See the README.
+ fn key_buckets(
+ &mut self,
+ py: Python<'_>,
+ data: &Bound<'_, PyAny>,
+ key_columns: Vec<String>,
+ n: u32,
+ ) -> PyResult<Py<PyAny>> {
+ let (_, batches) = read_pyarrow(data)?;
+ let handle = &mut self.handle;
+ let buckets = py
+ .detach(|| handle.key_buckets(&batches, &key_columns, n))
+ .map_err(to_py_err)?;
+ Ok(buckets.to_data().to_pyarrow(py)?.unbind())
+ }
+
/// Compact small files the way `DeltaTable.optimize.compact` does, streaming each bin
/// one file and one byte-bounded batch at a time (see `deltalite_core::compact`).
///
@@ -772,6 +1007,7 @@ fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;
m.add_function(wrap_pyfunction!(limits_snapshot, m)?)?;
m.add_class::<UpsertStats>()?;
+ m.add_class::<HistoryUpsertStats>()?;
m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;
m.add(
"DeltaLiteCommitConflictError",
diff --git a/rust/deltalite/python/tests/test_history.py b/rust/deltalite/python/tests/test_history.py
new file mode 100644
index 000000000000..74f070181d72
--- /dev/null
+++ b/rust/deltalite/python/tests/test_history.py
@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import json
+import random
+import dataclasses
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+import pyarrow as pa
+import deltalake
+import deltalite
+import pyarrow.compute as pc
+
+CURRENT = "current"
+TS = pa.timestamp("us", tz="UTC")
+MONTH_VECTORS = json.loads(
+ (Path(__file__).resolve().parents[2] / "core" / "tests" / "fixtures" / "month_vectors.json").read_text()
+)
+TABLE_SCHEMA = pa.schema(
+ [
+ pa.field("id", pa.string()),
+ pa.field("n", pa.int64()),
+ pa.field("ts", TS),
+ pa.field("payload", pa.string()),
+ pa.field("valid_to", TS),
+ pa.field("p", pa.string(), nullable=False),
+ ]
+)
+SOURCE_SCHEMA = pa.schema(
+ [
+ pa.field("id", pa.string()),
+ pa.field("n", pa.int64()),
+ pa.field("ts", TS),
+ pa.field("payload", pa.string()),
+ pa.field("seq", pa.int64()),
+ ]
+)
+
+
+# The product names month partitions with `pc.strftime(valid_to, "%Y-%m")`.
+def month(instant: datetime) -> str:
+ return pc.strftime(pa.array([instant], TS), format="%Y-%m")[0].as_py()
+
+
[email protected](frozen=True)
+class Src:
+ id: str | None
+ n: int | None
+ ts: datetime
+ payload: str
+ seq: int
+
+ @property
+ def key(self) -> tuple[str | None, int | None]:
+ return (self.id, self.n)
+
+
[email protected](frozen=True)
+class Row:
+ p: str
+ id: str | None
+ n: int | None
+ ts: datetime
+ valid_to: datetime | None
+ payload: str
+
+
[email protected](frozen=True)
+class Counts:
+ committed: bool
+ rows_reread: int
+ rows_backwards: int
+ versions_opened: int
+ versions_closed: int
+ chain_rows_closed: int
+
+
+def model_apply(rows: list[Row], source: list[Src]) -> tuple[list[Row], Counts]:
+ open_rows = {(r.id, r.n): r for r in rows if r.p == CURRENT}
+ best: dict[tuple, Src] = {}
+ for s in source:
+ k = (s.key, s.ts)
+ if k not in best or s.seq >= best[k].seq:
+ best[k] = s
+ by_key: dict[tuple, list[Src]] = {}
+ for s in sorted(best.values(), key=lambda s: s.ts):
+ by_key.setdefault(s.key, []).append(s)
+
+ out = [r for r in rows if r.p != CURRENT]
+ reread = backwards = opened = closed = chained = 0
+ for key, versions in by_key.items():
+ old = open_rows.get(key)
+ kept = [s for s in versions if old is None or s.ts > old.ts]
+ if old is not None:
+ reread += sum(s.ts == old.ts for s in versions)
+ backwards += sum(s.ts < old.ts for s in versions)
+ if not kept:
+ continue
+ if old is not None:
+ open_rows.pop(key)
+ out.append(dataclasses.replace(old, valid_to=kept[0].ts, p=month(kept[0].ts)))
+ closed += 1
+ for this, following in zip(kept, kept[1:]):
+ out.append(Row(month(following.ts), this.id, this.n, this.ts, following.ts, this.payload))
+ chained += 1
+ last = kept[-1]
+ out.append(Row(CURRENT, last.id, last.n, last.ts, None, last.payload))
+ opened += 1
+ out.extend(open_rows.values())
+ return sorted(out, key=_order), Counts(opened > 0, reread, backwards, opened, closed, chained)
+
+
+def _order(r: Row) -> tuple:
+ return (r.p, r.id or "", r.id is None, r.n or 0, r.n is None, r.ts, r.valid_to or r.ts, r.payload)
+
+
+def read(uri: str) -> list[Row]:
+ t = deltalake.DeltaTable(uri).to_pyarrow_table()
+ rows = [Row(d["p"], d["id"], d["n"], d["ts"], d["valid_to"], d["payload"]) for d in t.to_pylist()]
+ return sorted(rows, key=_order)
+
+
+def source_table(rows: list[Src]) -> pa.Table:
+ return pa.table(
+ {
+ "id": pa.array([r.id for r in rows], pa.string()),
+ "n": pa.array([r.n for r in rows], pa.int64()),
+ "ts": pa.array([r.ts for r in rows], TS),
+ "payload": pa.array([r.payload for r in rows], pa.string()),
+ "seq": pa.array([r.seq for r in rows], pa.int64()),
+ },
+ schema=SOURCE_SCHEMA,
+ )
+
+
+def create(tmp_path: Path, partition_by: list[str] | None = None) -> str:
+ uri = str(tmp_path / "history")
+ deltalake.DeltaTable.create(uri, schema=TABLE_SCHEMA, partition_by=["p"] if partition_by is None else partition_by)
+ return uri
+
+
+def split_into_batches(table: pa.Table, rng: random.Random) -> pa.Table:
+ cuts = sorted(rng.randint(0, table.num_rows) for _ in range(rng.randint(0, 4)))
+ bounds = [0, *cuts, table.num_rows]
+ batches = [b for start, end in zip(bounds, bounds[1:]) for b in table.slice(start, end - start).to_batches()]
+ if rng.random() < 0.5:
+ batches.insert(rng.randint(0, len(batches)), pa.RecordBatch.from_pylist([], schema=SOURCE_SCHEMA))
+ return pa.Table.from_batches(batches, schema=SOURCE_SCHEMA)
+
+
+def history_upsert(
+ uri: str, rows: list[Src], split_rng: random.Random | None = None, **kwargs
+) -> deltalite.HistoryUpsertStats:
+ data = source_table(rows)
+ if split_rng is not None:
+ data = split_into_batches(data, split_rng)
+ return deltalite.DeltaLiteTable.open(uri).history_upsert(
+ data, ["id", "n"], "ts", "ts", "valid_to", "p", CURRENT, "seq", **kwargs
+ )
+
+
+# Instants on both sides of month boundaries, so chains and closes cross partitions.
+INSTANTS = sorted(
+ {datetime.fromisoformat(v["valid_to"].replace("Z", "+00:00")) for v in MONTH_VECTORS}
+ | {datetime(2026, 1, 1, tzinfo=UTC) + timedelta(days=13 * i, hours=i) for i in range(10)}
+)
+
+
[email protected]("seed", range(60))
+def test_matches_the_reference_model_over_random_calls(tmp_path, seed):
+ rng = random.Random(seed)
+ uri = create(tmp_path)
+ keys = [(rng.choice(["a", "b", "c", None]), rng.choice([1, 2, None])) for _ in range(rng.randint(1, 8))]
+ small_files = rng.random() < 0.5
+ # Its own generator, so the batch cuts do not change the rows each seed generates.
+ split_rng = random.Random(f"split-{seed}")
+ expected: list[Row] = []
+ seq = 0
+ for call in range(rng.randint(2, 5)):
+ source = []
+ for _ in range(rng.randint(0, 25)):
+ seq += 1
+ key = rng.choice(keys)
+ source.append(Src(key[0], key[1], rng.choice(INSTANTS), f"s{seed}-c{call}-{seq}", rng.randint(0, 10**6)))
+ expected, counts = model_apply(expected, source)
+ stats = history_upsert(
+ uri,
+ source,
+ split_rng,
+ **({"target_file_size": 2048, "read_batch_size": 3} if small_files else {}),
+ )
+ got = Counts(
+ stats.committed,
+ stats.rows_reread,
+ stats.rows_backwards,
+ stats.versions_opened,
+ stats.versions_closed,
+ stats.chain_rows_closed,
+ )
+ assert got == counts, f"call {call}: {stats!r}"
+ assert read(uri) == expected, f"call {call}"
+
+
+def test_month_vectors_match_the_product_month_and_the_written_partitions(tmp_path):
+ instants = [datetime.fromisoformat(v["valid_to"].replace("Z", "+00:00")) for v in MONTH_VECTORS]
+ assert [month(i) for i in instants] == [v["month"] for v in MONTH_VECTORS]
+
+ uri = create(tmp_path)
+ opened_at = datetime(1960, 1, 1, tzinfo=UTC)
+ history_upsert(uri, [Src(f"k{i}", 1, opened_at, "old", 1) for i in range(len(instants))])
+ history_upsert(uri, [Src(f"k{i}", 1, instant, "new", 2) for i, instant in enumerate(instants)])
+ closed = {r.id: r.p for r in read(uri) if r.valid_to is not None}
+ assert closed == {f"k{i}": v["month"] for i, v in enumerate(MONTH_VECTORS)}
+
+
[email protected]("partition_by", [[], ["id"]])
+def test_refuses_a_table_not_partitioned_by_the_history_column(tmp_path, partition_by):
+ uri = create(tmp_path, partition_by)
+ with pytest.raises(deltalite.DeltaLiteUnsupportedTableError):
+ history_upsert(uri, [Src("a", 1, INSTANTS[0], "a", 1)])
+
+
+def test_key_buckets_are_null_safe_and_independent_of_batching(tmp_path):
+ table = deltalite.DeltaLiteTable.open(create(tmp_path))
+ data = pa.table(
+ {
+ "id": pa.array(["a", "b", None, "a", "customer-42"], pa.string()),
+ "n": pa.array([1, 1, 1, None, 7], pa.int64()),
+ }
+ )
+ buckets = table.key_buckets(data, ["id", "n"], 16)
+ assert isinstance(buckets, pa.UInt32Array)
+ # Pinned in the Rust suite too, so an encoding change shows up on the upgrade that makes it.
+ assert buckets.to_pylist() == [4, 12, 0, 6, 6]
+ rechunked = pa.Table.from_batches(data.slice(3).to_batches() + data.slice(0, 3).to_batches())
+ assert table.key_buckets(rechunked, ["id", "n"], 16).to_pylist() == [6, 6, 4, 12, 0]
+
+
+def test_calls_split_by_key_bucket_equal_one_call_when_source_keys_need_a_cast(tmp_path):
+ # "1", "01" and "001" are one integer key in the table, so they must share a bucket.
+ def source(n_values: list[str]) -> pa.Table:
+ rows = len(n_values)
+ return pa.table(
+ {
+ "id": pa.array(["k"] * 3 + [f"other-{i}" for i in range(rows - 3)], pa.string()),
+ "n": pa.array(n_values, pa.string()),
+ "ts": pa.array(INSTANTS[:rows], TS),
+ "payload": pa.array([f"p{i}" for i in range(rows)], pa.string()),
+ "seq": pa.array(range(rows), pa.int64()),
+ }
+ )
+
+ data = source(["1", "01", "001", *(str(i) for i in range(2, 12))])
+ whole_uri, split_uri = create(tmp_path / "whole"), create(tmp_path / "split")
+ args = (["id", "n"], "ts", "ts", "valid_to", "p", CURRENT, "seq")
+ deltalite.DeltaLiteTable.open(whole_uri).history_upsert(data, *args)
+
+ split = deltalite.DeltaLiteTable.open(split_uri)
+ buckets = split.key_buckets(data, ["id", "n"], 8)
+ assert len(set(buckets.to_pylist()[:3])) == 1
+ # Any call order must give the same table; descending is as good as any.
+ for bucket in sorted(set(buckets.to_pylist()), reverse=True):
+ split.history_upsert(data.filter(pc.equal(buckets, bucket)), *args)
+
+ assert read(split_uri) == read(whole_uri)