:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(deltalite): add history_upsert for SCD2 history tables (#128450)
Daniel RC328aa36e118a
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 0d455a194cb5..83e0d88c3aa7 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -4110,6 +4110,7 @@ dependencies = ["thiserror 2.0.18","tokio","tracing",+ "twox-hash","url","uuid",]diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.tomlindex 198b803903e8..ffb83a440bf6 100644--- a/rust/deltalite/core/Cargo.toml+++ b/rust/deltalite/core/Cargo.toml@@ -56,6 +56,8 @@ indexmap = "2"# Already linked by `deltalake`; decodes Add paths into the form the snapshot reports.percent-encoding = "2"sqlparser = "0.61"+# Already linked by `parquet` (bloom filters); fixed hash for `history::key_buckets`.+twox-hash = { version = "2", default-features = false, features = ["xxhash64"] }async-trait = { workspace = true }bytes = { workspace = true }diff --git a/rust/deltalite/core/src/handle.rs b/rust/deltalite/core/src/handle.rsindex ded4753160bd..a861ab990e84 100644--- a/rust/deltalite/core/src/handle.rs+++ b/rust/deltalite/core/src/handle.rs@@ -31,6 +31,7 @@ use metrics::counter;use serde_json::Value;use crate::errors::{Error, Result};+use crate::history::{history_upsert_with_state, HistoryUpsertOptions, HistoryUpsertStats};use crate::limits::{env_switch, switch_setting};use crate::logprobe::{check_anchor, commit_path, probe_log, trust_window, LogAnchor, LogProbe};use crate::prefetch::CheckpointCache;@@ -415,6 +416,89 @@ impl TableHandle {}}+ /// Run one history upsert (see [`crate::history`]), retrying conflicts against a+ /// refreshed snapshot. A re-planned attempt reads the open rows again, so a concurrent+ /// commit cannot make it close a row twice. Unlike [`TableHandle::upsert`], a+ /// concurrent schema or protocol change is planned against too: every attempt checks+ /// the refreshed table again, and a history write copies existing rows whole instead of+ /// replacing them with source values, so the result equals running after that commit.+ pub async fn history_upsert(+ &mut self,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: HistoryUpsertOptions,+ multipart: MultipartConfig,+ ) -> Result<HistoryUpsertStats> {+ let result = self+ .history_upsert_with_retries(source_batches, source_schema, opts, multipart)+ .await;+ self.prefetch.clear();+ result+ }++ async fn history_upsert_with_retries(+ &mut self,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: HistoryUpsertOptions,+ multipart: MultipartConfig,+ ) -> Result<HistoryUpsertStats> {+ let mut open_ms = 0u64;+ let t = Instant::now();+ self.refresh().await?;+ open_ms += t.elapsed().as_millis() as u64;++ let mut attempt = 0usize;+ loop {+ let view = wrap_multipart_view(self.table.clone(), multipart, self.commit_probe());+ match history_upsert_with_state(+ &view,+ source_batches.clone(),+ source_schema.clone(),+ opts.clone(),+ &mut self.relax_cache,+ )+ .await+ {+ Err(Error::Conflict(_)) if attempt < CONFLICT_RETRIES => {+ attempt += 1;+ tokio::time::sleep(std::time::Duration::from_millis(50 * attempt as u64)).await;+ let t = Instant::now();+ self.refresh().await?;+ open_ms += t.elapsed().as_millis() as u64;+ }+ Ok((mut stats, state)) => {+ let t = Instant::now();+ if let Some(state) = state {+ if self.adopt_commit_snapshot {+ self.table.state = Some(state);+ self.prefetch.clear();+ self.list_on_next_refresh = self.on_checkpoint_boundary();+ } else {+ self.refresh().await?;+ }+ }+ open_ms += t.elapsed().as_millis() as u64;+ stats.open_ms = open_ms;+ stats.initial_open_ms = self.take_initial_open_ms();+ return Ok(stats);+ }+ Err(e) => return Err(e),+ }+ }+ }++ /// Bucket ids for splitting a history source by key; see [`crate::history::key_buckets`].+ /// Uses the loaded snapshot's schema, without a refresh.+ pub fn key_buckets(+ &self,+ batches: &[RecordBatch],+ key_columns: &[String],+ n: u32,+ ) -> Result<arrow_array::UInt32Array> {+ crate::history::key_buckets(&self.table, batches, key_columns, n)+ }+/// Compact small files (see [`crate::compact`]) against a refreshed snapshot, then/// adopt the committed state so later writes through this handle see it.pub async fn compact(diff --git a/rust/deltalite/core/src/history.rs b/rust/deltalite/core/src/history.rsnew file mode 100644index 000000000000..ba99eef4c64d--- /dev/null+++ b/rust/deltalite/core/src/history.rs@@ -0,0 +1,1654 @@+//! Incremental SCD2 history upsert.+//!+//! The table holds one row per version of a key. Open versions (`valid_to` null) live in+//! the `current` partition; a closed version lives in the `%Y-%m` (UTC) month partition of+//! its `valid_to`. [`history_upsert`] applies a source batch of versions in one data commit:+//!+//! 1. Rows with the same (key, order value) collapse to the one with the highest tie-break+//! value.+//! 2. Only rows strictly newer than the key's open row are kept. A row equal to it is a+//! re-read, an older one is a backwards row; both are dropped and counted.+//! 3. The kept rows of a key form a chain: each closes at the next kept row's+//! `valid_from`, and the newest stays open in `current`.+//! 4. When a key keeps a row and has an open row, the open row closes at the first kept+//! row's `valid_from` and moves to its month partition.+//!+//! Keys are null-safe: a null key part matches a null key part. A null order value or a+//! null `valid_from` is an error. Closed rows are only ever appended, so earlier closed+//! versions of a key in the same month stay. A source that keeps nothing (a re-run of the+//! same input) writes nothing and makes no commit.+//!+//! The machinery is the upsert's (see [`crate::upsert`]): the same listing, stats pruning,+//! budgets, writer and commit. Two things differ. The probe reads the order column next to+//! the keys, so a file that holds only re-read keys is not rewritten, and it reads every+//! candidate file in full, because the open order of every source key must be known+//! before the kept rows are chosen. The probe cache is not used, because its entries+//! record keys only.++use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Instant;++use arrow_array::cast::AsArray;+use arrow_array::{+ new_null_array, Array, ArrayRef, BooleanArray, RecordBatch, StringArray, UInt32Array,+};+use arrow_cast::{cast_with_options, CastOptions};+use arrow_ord::sort::sort_to_indices;+use arrow_row::{RowConverter, Rows, SortField};+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};+use arrow_select::concat::concat;+use arrow_select::filter::{filter, filter_record_batch};+use arrow_select::interleave::interleave_record_batch;+use arrow_select::take::{take, take_record_batch};+use chrono::{DateTime, NaiveDate, Utc};+use deltalake::kernel::{Action, Add};+use deltalake::table::config::TablePropertiesExt;+use deltalake::table::state::DeltaTableState;+use deltalake::writer::RecordBatchWriter;+use deltalake::{DeltaTable, ObjectStore};+use futures::{StreamExt, TryStreamExt};+use metrics::{counter, histogram};+use parquet::arrow::ProjectionMask;+use parquet::file::metadata::ParquetMetaData;+use serde_json::Value;+use tokio::sync::{mpsc, Semaphore};+use tracing::info;++use crate::errors::{Error, Result};+use crate::limits::{+ check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, ProcessLimits,+ SourceFootprint,+};+use crate::schema::cast_to_schema;+use crate::upsert::{+ add_partition_column, byte_bounded_batch_rows, columns_needing_relax,+ commit_partition_overwrite, ensure_supported_table, json_cmp, list_files_by_partition,+ max_row_group_fetch_bytes, min_max_json, open_builder, open_rewrite_stream,+ relax_columns_to_nullable, resolve_target_file_size, roll_writer, source_slice_rows,+ BudgetPermit, Budgets, RelaxCache, TargetFile, DEFAULT_MAX_FETCH_BYTES,+ INITIAL_PROBE_ESTIMATE_BYTES,+};+use crate::writer::StreamingWriter;++/// Seed of the [`key_buckets`] hash. Part of the API: changing it moves every key to a+/// different bucket.+pub const KEY_BUCKET_SEED: u64 = 0x6465_6c74_616c_6974;++#[derive(Clone)]+pub struct HistoryUpsertOptions {+ /// Columns that identify a key, in order. Compared null-safely.+ pub key_columns: Vec<String>,+ /// Column that orders the versions of a key. Must not be null.+ pub order_column: String,+ /// Column with the instant a version starts. Must not be null. It is the `valid_to`+ /// of the version it replaces.+ pub valid_from_column: String,+ /// Column with the instant a version ends, null while it is open. Set by the+ /// operation; the source must not carry it.+ pub valid_to_column: String,+ /// The table's only partition column (string). Set by the operation; the source must+ /// not carry it.+ pub partition_column: String,+ /// Partition value of the open versions.+ pub current_partition: String,+ /// Source column that decides between rows with the same (key, order value): the+ /// highest value wins. Must have a fixed-width type, such as an integer or a timestamp.+ /// Dropped before writing when the table has no such column.+ pub tie_break_column: String,+ pub commit_metadata: Option<HashMap<String, Value>>,+ pub probe_concurrency: usize,+ pub max_parallel_files: usize,+ pub max_buffered_bytes: usize,+ pub max_fetch_bytes: usize,+ pub read_ahead_row_groups: bool,+ pub read_batch_size: usize,+ /// `None` uses the table's `delta.targetFileSize`.+ pub target_file_size: Option<usize>,+ pub max_row_group_bytes: Option<usize>,+ /// Ceiling for the source-size guard; `None` resolves from the environment, `Some(0)`+ /// disables. See [`crate::limits::resolve_max_source_bytes`].+ pub max_source_bytes: Option<usize>,+ pub limits: Arc<ProcessLimits>,+}++impl Default for HistoryUpsertOptions {+ fn default() -> Self {+ Self {+ key_columns: Vec::new(),+ order_column: String::new(),+ valid_from_column: String::new(),+ valid_to_column: String::new(),+ partition_column: String::new(),+ current_partition: String::new(),+ tie_break_column: String::new(),+ commit_metadata: None,+ probe_concurrency: 8,+ max_parallel_files: 4,+ max_buffered_bytes: 64 * 1024 * 1024,+ max_fetch_bytes: DEFAULT_MAX_FETCH_BYTES,+ read_ahead_row_groups: false,+ read_batch_size: 8192,+ target_file_size: None,+ max_row_group_bytes: None,+ max_source_bytes: None,+ limits: ProcessLimits::global().clone(),+ }+ }+}++#[derive(Default, Debug, Clone)]+pub struct HistoryUpsertStats {+ /// The committed table version, or the version the call read when it made no commit.+ pub version: i64,+ /// Whether the call made a data commit. False when the source kept no row.+ pub committed: bool,+ pub rows_in: usize,+ /// Source rows dropped because another row had the same (key, order value) and a+ /// higher tie-break value.+ pub rows_deduped: usize,+ /// Source rows whose order value equals the key's open row.+ pub rows_reread: usize,+ /// Source rows whose order value is lower than the key's open row.+ pub rows_backwards: usize,+ /// Open versions written (one per key that kept a row).+ pub versions_opened: usize,+ /// Existing open rows closed and moved to their month partition.+ pub versions_closed: usize,+ /// Kept source rows written already closed (every kept row but the newest of a key).+ pub chain_rows_closed: usize,+ /// Open rows copied unchanged into new files of the current partition.+ pub rows_copied: usize,+ pub files_removed: usize,+ pub files_added: usize,+ /// Current-partition files left untouched.+ pub files_carried_over: usize,+ /// Current-partition files whose key and order columns the probe read.+ pub files_probed: usize,+ /// Stored bytes of the rewritten files (the probe's narrow reads are not counted).+ pub bytes_read: u64,+ pub bytes_written: u64,+ pub plan_ms: u64,+ pub probe_ms: u64,+ pub rewrite_ms: u64,+ pub commit_ms: u64,+ pub columns_relaxed: usize,+ pub open_ms: u64,+ pub initial_open_ms: u64,+ pub ingest_ms: u64,+ pub relax_ms: u64,+ pub maintenance_ms: u64,+}++/// Apply `source_batches` to the history table `table`. See the module docs.+pub async fn history_upsert(+ table: &DeltaTable,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: HistoryUpsertOptions,+) -> Result<HistoryUpsertStats> {+ history_upsert_with_state(+ table,+ source_batches,+ source_schema,+ opts,+ &mut RelaxCache::default(),+ )+ .await+ .map(|(stats, _)| stats)+}++/// [`history_upsert`] that also returns the table state after the call, for a handle to+/// adopt. `None` when the table did not move.+pub async fn history_upsert_with_state(+ table: &DeltaTable,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: HistoryUpsertOptions,+ relax_cache: &mut RelaxCache,+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {+ let started = Instant::now();+ let result = history_with_relax(table, source_batches, source_schema, opts, relax_cache).await;+ histogram!("deltalite_history_upsert_duration_seconds").record(started.elapsed().as_secs_f64());+ match &result {+ Ok((stats, _)) => {+ let outcome = if stats.committed { "ok" } else { "noop" };+ counter!("deltalite_history_upserts_total", "outcome" => outcome).increment(1);+ counter!("deltalite_history_versions_closed_total")+ .increment((stats.versions_closed + stats.chain_rows_closed) as u64);+ counter!("deltalite_history_versions_opened_total")+ .increment(stats.versions_opened as u64);+ }+ Err(e) => {+ counter!(+ "deltalite_history_upserts_total",+ "outcome" => "error",+ "error_kind" => e.kind()+ )+ .increment(1);+ }+ }+ result+}++async fn history_with_relax(+ table: &DeltaTable,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: HistoryUpsertOptions,+ relax_cache: &mut RelaxCache,+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {+ // Before the relax, so an unsupported table or source never makes a metadata commit.+ ensure_history_table(table, &opts)?;+ validate_source(table, &source_batches, &source_schema, &opts)?;+ let relax_started = Instant::now();+ let mut relax =+ columns_needing_relax(table, &source_batches, &source_schema, relax_cache).await?;+ // The source never carries the partition column, but every written row sets it.+ relax.retain(|c| c != &opts.partition_column);+ if relax.is_empty() {+ let relax_ms = relax_started.elapsed().as_millis() as u64;+ let (mut stats, state) =+ history_inner(table, source_batches, source_schema, &opts, false).await?;+ stats.relax_ms = relax_ms;+ if stats.committed {+ if let Ok(committed) = u64::try_from(stats.version) {+ relax_cache.advance_own_commit(committed);+ }+ }+ return Ok((stats, state));+ }++ // Plan first, so a call that keeps no row makes no commit at all, not even the relax.+ let (planned, _) = history_inner(+ table,+ source_batches.clone(),+ source_schema.clone(),+ &opts,+ true,+ )+ .await?;+ if planned.versions_opened == 0 {+ return Ok((planned, None));+ }+ relax_columns_to_nullable(table, &relax).await?;+ let mut fresh = table.clone();+ fresh.update_incremental(None).await?;+ let relax_ms = relax_started.elapsed().as_millis() as u64;+ let (mut stats, state) =+ history_inner(&fresh, source_batches, source_schema, &opts, false).await?;+ stats.columns_relaxed = relax.len();+ stats.relax_ms = relax_ms;+ // A concurrent commit can still make the data a no-op after the relax landed.+ Ok((stats, state.or(fresh.state)))+}++fn validate_source(+ table: &DeltaTable,+ source_batches: &[RecordBatch],+ source_schema: &SchemaRef,+ opts: &HistoryUpsertOptions,+) -> Result<()> {+ // Every check below reads the declared schema, and the planner indexes columns by it.+ if let Some(i) = source_batches+ .iter()+ .position(|b| b.schema().fields() != source_schema.fields())+ {+ return Err(Error::SchemaMismatch(format!(+ "source batch {i} does not match the declared source schema"+ )));+ }+ let table_schema = RecordBatchWriter::for_table(table)?.arrow_schema();+ Columns::resolve(&table_schema, source_schema, opts)?;+ for (role, column) in [+ ("order", &opts.order_column),+ ("valid_from", &opts.valid_from_column),+ ] {+ let idx = source_schema.index_of(column)?;+ let nulls: usize = source_batches+ .iter()+ .map(|b| b.column(idx).logical_null_count())+ .sum();+ if nulls > 0 {+ return Err(Error::Generic(format!(+ "{role} column '{column}' holds {nulls} null value(s); every history row needs one"+ )));+ }+ }+ Ok(())+}++fn ensure_history_table(table: &DeltaTable, opts: &HistoryUpsertOptions) -> Result<()> {+ ensure_supported_table(table)?;+ let snapshot = table.snapshot()?;+ let partition_columns = snapshot.metadata().partition_columns();+ if partition_columns.len() != 1 || partition_columns[0] != opts.partition_column {+ return Err(Error::Unsupported(format!(+ "history_upsert needs a table partitioned by exactly '{}', table is partitioned \+ by {partition_columns:?}",+ opts.partition_column+ )));+ }+ Ok(())+}++struct Columns {+ key_idx: Vec<usize>,+ order_idx: usize,+ valid_from_idx: usize,+ valid_to_idx: usize,+ partition_idx: usize,+ tie_source_idx: usize,+ /// The tie-break column is not a table column, so it is dropped before writing.+ strip_tie: bool,+}++impl Columns {+ fn resolve(+ table_schema: &SchemaRef,+ source_schema: &SchemaRef,+ opts: &HistoryUpsertOptions,+ ) -> Result<Self> {+ if opts.key_columns.is_empty() {+ return Err(Error::Generic(+ "key_columns must not be empty for a history upsert".into(),+ ));+ }+ if is_month_name(&opts.current_partition) || opts.current_partition.is_empty() {+ return Err(Error::Generic(format!(+ "current_partition '{}' must be non-empty and must not look like a month \+ partition (YYYY-MM)",+ opts.current_partition+ )));+ }+ let set_by_operation = [&opts.valid_to_column, &opts.partition_column];+ let table_idx = |name: &str, role: &str| {+ table_schema.index_of(name).map_err(|_| {+ Error::SchemaMismatch(format!("{role} column '{name}' not found in table schema"))+ })+ };+ let source_column = |name: &str, role: &str| -> Result<usize> {+ if set_by_operation.iter().any(|c| c.as_str() == name) {+ return Err(Error::Generic(format!(+ "'{name}' is set by history_upsert and cannot be the {role} column"+ )));+ }+ source_schema.index_of(name).map_err(|_| {+ Error::SchemaMismatch(format!("{role} column '{name}' is missing from the source"))+ })?;+ table_idx(name, role)+ };++ let mut key_idx = Vec::with_capacity(opts.key_columns.len());+ for k in &opts.key_columns {+ key_idx.push(source_column(k, "key")?);+ }+ let order_idx = source_column(&opts.order_column, "order")?;+ let valid_from_idx = source_column(&opts.valid_from_column, "valid_from")?;+ let valid_to_idx = table_idx(&opts.valid_to_column, "valid_to")?;+ let partition_idx = table_idx(&opts.partition_column, "partition")?;++ for c in set_by_operation {+ if source_schema.index_of(c).is_ok() {+ return Err(Error::SchemaMismatch(format!(+ "source carries column '{c}', which history_upsert sets itself"+ )));+ }+ }+ if set_by_operation.contains(&&opts.tie_break_column) {+ return Err(Error::Generic(format!(+ "tie-break column '{}' is set by history_upsert",+ opts.tie_break_column+ )));+ }+ let tie_source_idx = source_schema+ .index_of(&opts.tie_break_column)+ .map_err(|_| {+ Error::SchemaMismatch(format!(+ "tie-break column '{}' is missing from the source",+ opts.tie_break_column+ ))+ })?;+ // Fixed width only: the size guard counts in-memory bytes, and a dictionary or+ // view column can encode to many times that when every row is row-encoded.+ let tie_type = source_schema.field(tie_source_idx).data_type();+ if !tie_type.is_primitive() {+ return Err(Error::SchemaMismatch(format!(+ "tie-break column '{}' must have a fixed-width type such as an integer or a \+ timestamp, source has {tie_type}",+ opts.tie_break_column+ )));+ }+ let strip_tie = table_schema.index_of(&opts.tie_break_column).is_err();++ let extra: Vec<&String> = source_schema+ .fields()+ .iter()+ .map(|f| f.name())+ .filter(|n| **n != opts.tie_break_column && table_schema.index_of(n).is_err())+ .collect();+ if !extra.is_empty() {+ return Err(Error::SchemaMismatch(format!(+ "source has columns not present in the table schema: {extra:?}. \+ Additive schema evolution must run before history_upsert."+ )));+ }++ if !matches!(+ table_schema.field(partition_idx).data_type(),+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View+ ) {+ return Err(Error::Unsupported(format!(+ "history partition column '{}' must be a string column, table has {}",+ opts.partition_column,+ table_schema.field(partition_idx).data_type()+ )));+ }+ let valid_to_type = table_schema.field(valid_to_idx).data_type();+ if !matches!(+ valid_to_type,+ DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64+ ) {+ return Err(Error::SchemaMismatch(format!(+ "valid_to column '{}' must be a date or timestamp column, table has {valid_to_type}",+ opts.valid_to_column+ )));+ }+ Ok(Self {+ key_idx,+ order_idx,+ valid_from_idx,+ valid_to_idx,+ partition_idx,+ tie_source_idx,+ strip_tie,+ })+ }+}++enum FirstKeyRange {+ NoValues,+ /// Min and max of the non-null values, comparable against Add-action stats.+ Range(Value, Value),+ Unknown,+}++/// The source, encoded and deduped. Shared read-only by every probe and reader task.+struct SourcePlan {+ /// Source batches cast to the table schema, with `valid_to` and the partition column+ /// null until the rows are written.+ batches: Vec<RecordBatch>,+ offsets: Vec<usize>,+ key_converter: RowConverter,+ order_converter: RowConverter,+ /// Order value of every source row, as order-preserving bytes.+ orders: Rows,+ /// Every source row's `valid_from`, cast to the table's `valid_to` type.+ valid_from: ArrayRef,+ /// One global row per (key, order value), sorted by key, then order value.+ deduped: Vec<u32>,+ /// Range of `deduped` that holds each key, in key order. A key's index here is its id.+ groups: Vec<(u32, u32)>,+ by_key: HashMap<Box<[u8]>, u32>,+ first_key: FirstKeyRange,+ first_key_has_null: bool,+ rows_in: usize,+}++impl SourcePlan {+ fn build(+ source_batches: Vec<RecordBatch>,+ table_schema: &SchemaRef,+ cols: &Columns,+ opts: &HistoryUpsertOptions,+ ) -> Result<Self> {+ let work_schema = nullable_schema(table_schema);+ let mut batches = Vec::with_capacity(source_batches.len());+ let mut ties = Vec::with_capacity(source_batches.len());+ let mut offsets = Vec::with_capacity(source_batches.len());+ let mut rows_in = 0usize;+ for b in source_batches {+ ties.push(b.column(cols.tie_source_idx).clone());+ let b = if cols.strip_tie {+ let keep: Vec<usize> = (0..b.num_columns())+ .filter(|i| *i != cols.tie_source_idx)+ .collect();+ b.project(&keep)?+ } else {+ b+ };+ offsets.push(rows_in);+ rows_in += b.num_rows();+ batches.push(cast_to_schema(&b, &work_schema)?);+ }+ if rows_in > u32::MAX as usize {+ return Err(Error::Generic(format!(+ "source has {rows_in} rows, more than a history upsert supports"+ )));+ }++ // Guard before the narrow columns are copied out, and count those copies: the+ // key, order, tie-break and `valid_from` arrays plus the key encodings.+ let column_bytes = |idx: usize| -> usize {+ batches+ .iter()+ .map(|b| b.column(idx).get_array_memory_size())+ .sum()+ };+ let tie_bytes: usize = ties.iter().map(|t| t.get_array_memory_size()).sum();+ let narrow_bytes = cols.key_idx.iter().map(|i| column_bytes(*i)).sum::<usize>() * 2+ + column_bytes(cols.order_idx)+ + column_bytes(cols.valid_from_idx)+ + tie_bytes;+ let footprint = SourceFootprint {+ source_bytes: batches+ .iter()+ .map(RecordBatch::get_array_memory_size)+ .sum::<usize>()+ + tie_bytes,+ pk_set_bytes: estimate_pk_set_bytes(narrow_bytes, rows_in)+ .saturating_add(rows_in.saturating_mul(2 * size_of::<u32>())),+ };+ check_source_size(&footprint, resolve_max_source_bytes(opts.max_source_bytes))?;++ let column = |idx: usize| -> Result<ArrayRef> {+ let parts: Vec<&dyn Array> = batches.iter().map(|b| b.column(idx).as_ref()).collect();+ if parts.is_empty() {+ return Ok(new_null_array(table_schema.field(idx).data_type(), 0));+ }+ Ok(concat(&parts)?)+ };+ let keys: Vec<ArrayRef> = cols+ .key_idx+ .iter()+ .map(|i| column(*i))+ .collect::<Result<_>>()?;+ let order = column(cols.order_idx)?;+ let tie: ArrayRef = match ties.len() {+ 0 => new_null_array(&DataType::Null, 0),+ _ => concat(&ties.iter().map(|t| t.as_ref()).collect::<Vec<_>>())?,+ };++ let valid_from = column(cols.valid_from_idx)?;+ let valid_from = cast_strict(+ &valid_from,+ table_schema.field(cols.valid_to_idx).data_type(),+ )?;++ let key_converter = RowConverter::new(+ keys.iter()+ .map(|k| SortField::new(k.data_type().clone()))+ .collect(),+ )?;+ let order_converter = RowConverter::new(vec![SortField::new(order.data_type().clone())])?;+ let tie_converter = RowConverter::new(vec![SortField::new(tie.data_type().clone())])?;+ let key_rows = key_converter.convert_columns(&keys)?;+ let orders = order_converter.convert_columns(std::slice::from_ref(&order))?;+ let tie_rows = tie_converter.convert_columns(std::slice::from_ref(&tie))?;++ let first_key_has_null = keys[0].null_count() > 0;+ let first_key = if keys[0].null_count() == keys[0].len() {+ FirstKeyRange::NoValues+ } else {+ match min_max_json(keys[0].as_ref()) {+ Some((min, max)) => FirstKeyRange::Range(min, max),+ None => FirstKeyRange::Unknown,+ }+ };+ drop(keys);++ // A stable sort keeps input order among rows equal on all three, so the last+ // staged row of a (key, order value) still wins a tie on the tie-break value.+ let mut sorted: Vec<u32> = (0..rows_in as u32).collect();+ sorted.sort_by(|&a, &b| {+ let (a, b) = (a as usize, b as usize);+ key_rows+ .row(a)+ .as_ref()+ .cmp(key_rows.row(b).as_ref())+ .then_with(|| orders.row(a).as_ref().cmp(orders.row(b).as_ref()))+ .then_with(|| tie_rows.row(a).as_ref().cmp(tie_rows.row(b).as_ref()))+ });+ drop(tie_rows);++ let mut deduped = Vec::with_capacity(sorted.len());+ let mut groups = Vec::new();+ let mut by_key = HashMap::new();+ let mut i = 0usize;+ while i < sorted.len() {+ let key = key_rows.row(sorted[i] as usize);+ let mut end = i + 1;+ while end < sorted.len() && key_rows.row(sorted[end] as usize).as_ref() == key.as_ref()+ {+ end += 1;+ }+ let start = deduped.len() as u32;+ for t in i..end {+ let last_of_order = t + 1 == end+ || orders.row(sorted[t] as usize).as_ref()+ != orders.row(sorted[t + 1] as usize).as_ref();+ if last_of_order {+ deduped.push(sorted[t]);+ }+ }+ by_key.insert(Box::<[u8]>::from(key.as_ref()), groups.len() as u32);+ groups.push((start, deduped.len() as u32));+ i = end;+ }++ Ok(Self {+ batches,+ offsets,+ key_converter,+ order_converter,+ orders,+ valid_from,+ deduped,+ groups,+ by_key,+ first_key,+ first_key_has_null,+ rows_in,+ })+ }++ fn order(&self, row: u32) -> &[u8] {+ self.orders.row(row as usize).data()+ }++ fn newest_order(&self, group: u32) -> &[u8] {+ let (_, end) = self.groups[group as usize];+ self.order(self.deduped[end as usize - 1])+ }++ fn locate(&self, row: u32) -> (usize, usize) {+ let row = row as usize;+ let batch = self.offsets.partition_point(|start| *start <= row) - 1;+ (batch, row - self.offsets[batch])+ }++ /// Choose the kept rows of every key against the open order values the probe found.+ fn decide(&self, open: &[Option<Box<[u8]>>]) -> Decision {+ let mut d = Decision {+ closing: vec![None; self.groups.len()],+ writes: Vec::new(),+ rows_reread: 0,+ rows_backwards: 0,+ versions_opened: 0,+ chain_rows_closed: 0,+ };+ for (group, &(start, end)) in self.groups.iter().enumerate() {+ let rows = &self.deduped[start as usize..end as usize];+ let kept_from = match &open[group] {+ None => 0,+ Some(open) => {+ let kept_from = rows.partition_point(|r| self.order(*r) <= open.as_ref());+ for r in &rows[..kept_from] {+ if self.order(*r) == open.as_ref() {+ d.rows_reread += 1;+ } else {+ d.rows_backwards += 1;+ }+ }+ kept_from+ }+ };+ let kept = &rows[kept_from..];+ let Some(first) = kept.first() else {+ continue;+ };+ d.versions_opened += 1;+ d.chain_rows_closed += kept.len() - 1;+ if open[group].is_some() {+ d.closing[group] = Some(*first);+ }+ for (t, r) in kept.iter().enumerate() {+ d.writes.push((*r, kept.get(t + 1).copied()));+ }+ }+ d+ }+}++/// A kept source row and the next kept row of its key, whose `valid_from` closes it+/// (`None` for the newest, which stays open).+type KeptRow = (u32, Option<u32>);++struct Decision {+ /// Per key: the source row whose `valid_from` closes the key's open row.+ closing: Vec<Option<u32>>,+ /// Kept source rows in write order.+ writes: Vec<KeptRow>,+ rows_reread: usize,+ rows_backwards: usize,+ versions_opened: usize,+ chain_rows_closed: usize,+}++/// The table schema with every column nullable. Source batches are held in it until they+/// are written: the operation sets `valid_to` and the partition column later, and a+/// column the table still marks non-nullable may hold nulls until the relax commits.+/// Written batches are rebuilt in the table schema, which checks nullability again.+fn nullable_schema(table_schema: &SchemaRef) -> SchemaRef {+ let fields: Vec<Field> = table_schema+ .fields()+ .iter()+ .map(|f| f.as_ref().clone().with_nullable(true))+ .collect();+ Arc::new(Schema::new_with_metadata(+ fields,+ table_schema.metadata().clone(),+ ))+}++fn cast_strict(array: &ArrayRef, to: &DataType) -> Result<ArrayRef> {+ if array.data_type() == to {+ return Ok(array.clone());+ }+ let opts = CastOptions {+ safe: false,+ ..Default::default()+ };+ Ok(cast_with_options(array, to, &opts)?)+}++/// `%Y-%m` of each value in UTC, null for null. Must equal pyarrow's+/// `pc.strftime(values, "%Y-%m")` for UTC timestamps, which is how callers name the+/// month partitions they read.+pub fn format_months(values: &dyn Array) -> Result<StringArray> {+ let raw = |to: &DataType| arrow_cast::cast(&arrow_array::make_array(values.to_data()), to);+ let instants: Vec<Option<DateTime<Utc>>> = match values.data_type() {+ DataType::Timestamp(unit, _) => {+ let raw = raw(&DataType::Int64)?;+ let raw = raw+ .as_any()+ .downcast_ref::<arrow_array::Int64Array>()+ .ok_or_else(|| Error::Generic("timestamp did not cast to Int64".into()))?;+ raw.iter()+ .map(|v| {+ v.map(|v| {+ match unit {+ TimeUnit::Second => DateTime::from_timestamp(v, 0),+ TimeUnit::Millisecond => DateTime::from_timestamp_millis(v),+ TimeUnit::Microsecond => DateTime::from_timestamp_micros(v),+ TimeUnit::Nanosecond => Some(DateTime::from_timestamp_nanos(v)),+ }+ .ok_or_else(|| out_of_range(v))+ })+ .transpose()+ })+ .collect::<Result<_>>()?+ }+ DataType::Date32 => {+ let raw = raw(&DataType::Int32)?;+ let raw = raw+ .as_any()+ .downcast_ref::<arrow_array::Int32Array>()+ .ok_or_else(|| Error::Generic("date did not cast to Int32".into()))?;+ raw.iter()+ .map(|v| {+ v.map(|days| {+ NaiveDate::from_num_days_from_ce_opt(days.saturating_add(719_163))+ .map(|d| d.and_time(chrono::NaiveTime::MIN).and_utc())+ .ok_or_else(|| out_of_range(i64::from(days)))+ })+ .transpose()+ })+ .collect::<Result<_>>()?+ }+ DataType::Date64 => {+ let raw = raw(&DataType::Int64)?;+ let raw = raw+ .as_any()+ .downcast_ref::<arrow_array::Int64Array>()+ .ok_or_else(|| Error::Generic("date did not cast to Int64".into()))?;+ raw.iter()+ .map(|v| {+ v.map(|ms| DateTime::from_timestamp_millis(ms).ok_or_else(|| out_of_range(ms)))+ .transpose()+ })+ .collect::<Result<_>>()?+ }+ other => {+ return Err(Error::SchemaMismatch(format!(+ "cannot name a month partition from a {other} value"+ )))+ }+ };+ Ok(instants+ .into_iter()+ .map(|dt| dt.map(|dt| dt.format("%Y-%m").to_string()))+ .collect())+}++fn is_month_name(value: &str) -> bool {+ let b = value.as_bytes();+ b.len() == 7 && b[4] == b'-' && b[..4].iter().chain(&b[5..]).all(u8::is_ascii_digit)+}++fn out_of_range(v: i64) -> Error {+ Error::Generic(format!(+ "valid_to value {v} is outside the supported date range"+ ))+}++/// Set `valid_to` and the partition column of `batch` (in the table's column order).+/// Row `i` closes at the `valid_from` of source row `closes[i]` into that value's month;+/// a null entry leaves it open in `current`.+fn stamp_validity(+ batch: &RecordBatch,+ closes: &UInt32Array,+ plan: &SourcePlan,+ cols: &Columns,+ table_schema: &SchemaRef,+ current: &str,+) -> Result<RecordBatch> {+ let valid_to = take(plan.valid_from.as_ref(), closes, None)?;+ let months = format_months(valid_to.as_ref())?;+ let partition: StringArray = months.iter().map(|m| Some(m.unwrap_or(current))).collect();+ let partition: ArrayRef = Arc::new(partition);+ let partition = cast_strict(+ &partition,+ table_schema.field(cols.partition_idx).data_type(),+ )?;++ let mut columns = batch.columns().to_vec();+ columns[cols.valid_to_idx] = valid_to;+ columns[cols.partition_idx] = partition;+ Ok(RecordBatch::try_new(table_schema.clone(), columns)?)+}++/// Whether Add-action stats prove that `f` holds no source key. Null key parts match, so+/// a file is safe from a null source key only when its first key column has no nulls.+fn file_cannot_hold_source_key(f: &TargetFile, first_key: &str, plan: &SourcePlan) -> bool {+ let Some(stats) = &f.stats else { return false };+ let Ok(parsed) = serde_json::from_str::<Value>(stats) else {+ return false;+ };+ let records = parsed.get("numRecords").and_then(Value::as_i64);+ if records == Some(0) {+ return true;+ }+ let nulls = parsed+ .get("nullCount")+ .and_then(|n| n.get(first_key))+ .and_then(Value::as_i64);+ if plan.first_key_has_null && nulls != Some(0) {+ return false;+ }+ if records.is_some() && nulls == records {+ return true;+ }+ match &plan.first_key {+ FirstKeyRange::NoValues => true,+ FirstKeyRange::Unknown => false,+ FirstKeyRange::Range(src_min, src_max) => {+ let file_min = parsed.get("minValues").and_then(|m| m.get(first_key));+ let file_max = parsed.get("maxValues").and_then(|m| m.get(first_key));+ let (Some(file_min), Some(file_max)) = (file_min, file_max) else {+ return false;+ };+ matches!(json_cmp(file_max, src_min), Some(std::cmp::Ordering::Less))+ || matches!(+ json_cmp(file_min, src_max),+ Some(std::cmp::Ordering::Greater)+ )+ }+ }+}++/// The key and order columns of `batch` in the table's types. A column the batch does not+/// have reads as null, which a null key part of the source matches.+fn key_and_order_columns(+ batch: &RecordBatch,+ plan_types: &[(String, DataType)],+) -> Result<Vec<ArrayRef>> {+ plan_types+ .iter()+ .map(|(name, ty)| match batch.column_by_name(name) {+ Some(c) => cast_strict(c, ty),+ None => Ok(new_null_array(ty, batch.num_rows())),+ })+ .collect()+}++/// What the probe found in one file: the source keys it holds with their open order+/// values, and whether any of them has a newer source row.+struct FileProbe {+ file: TargetFile,+ seen: Vec<(u32, Box<[u8]>)>,+ newer: bool,+}++/// Read the key and order columns of `f` and record every open row whose key the source+/// holds. Reads the whole file: the open order of every such key is needed later.+#[allow(clippy::too_many_arguments)]+async fn probe_file(+ store: &Arc<dyn ObjectStore>,+ f: TargetFile,+ plan: &SourcePlan,+ columns: &[(String, DataType)],+ key_count: usize,+ opts: &HistoryUpsertOptions,+ budgets: &Budgets,+) -> Result<(FileProbe, Arc<ParquetMetaData>)> {+ let (builder, held_fetch) = open_builder(store, &f, budgets).await?;+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema().clone();+ let mut projection: Vec<usize> = columns+ .iter()+ .filter_map(|(name, _)| file_schema.index_of(name).ok())+ .collect();+ projection.sort_unstable();+ projection.dedup();++ let mut probe = FileProbe {+ file: f,+ seen: Vec::new(),+ newer: false,+ };+ let mut note = |batch: &RecordBatch| -> Result<()> {+ let cols = key_and_order_columns(batch, columns)?;+ let keys = plan.key_converter.convert_columns(&cols[..key_count])?;+ let orders = plan.order_converter.convert_columns(&cols[key_count..])?;+ for i in 0..batch.num_rows() {+ if let Some(&group) = plan.by_key.get(keys.row(i).as_ref()) {+ let order = orders.row(i);+ probe.newer = probe.newer || plan.newest_order(group) > order.as_ref();+ probe.seen.push((group, order.as_ref().into()));+ }+ }+ Ok(())+ };++ if projection.is_empty() {+ // Every row of the file has null keys and a null order value, so one row decides it.+ if metadata.file_metadata().num_rows() > 0 {+ note(&RecordBatch::try_new_with_options(+ Arc::new(Schema::empty()),+ vec![],+ &arrow_array::RecordBatchOptions::new().with_row_count(Some(1)),+ )?)?;+ }+ return Ok((probe, metadata));+ }++ let fetch = match held_fetch {+ Some(fetch) => fetch,+ None => {+ budgets+ .acquire_fetch(max_row_group_fetch_bytes(&metadata, Some(&projection)))+ .await?+ }+ };+ let mask = ProjectionMask::roots(builder.parquet_schema(), projection);+ let batch_rows = byte_bounded_batch_rows(+ &metadata,+ INITIAL_PROBE_ESTIMATE_BYTES,+ opts.read_batch_size,+ );+ let mut stream = builder+ .with_projection(mask)+ .with_batch_size(batch_rows)+ .build()?;+ let mut estimate_bytes = INITIAL_PROBE_ESTIMATE_BYTES;+ loop {+ let mut permit = budgets.acquire_bytes(estimate_bytes).await?;+ let Some(batch) = stream.try_next().await? else {+ break;+ };+ let actual = batch.get_array_memory_size();+ budgets.top_up(&mut permit, actual).await?;+ estimate_bytes = actual.max(16 * 1024);+ note(&batch)?;+ }+ drop(stream);+ drop(fetch);+ Ok((probe, metadata))+}++/// A decoded batch on its way to the writers, split by destination. The permit is+/// released once both parts are in the write buffers.+struct WriteMessage {+ current: Option<RecordBatch>,+ closed: Option<RecordBatch>,+ _permit: BudgetPermit,+}++struct CloseOutcome {+ rows_copied: usize,+ versions_closed: usize,+ bytes_read: u64,+ remove: Action,+}++/// Stream one current-partition file to the writer, closing the open rows of keys that+/// kept a newer source row. Every batch carries its budget permits to the writer.+#[allow(clippy::too_many_arguments)]+async fn close_file(+ store: Arc<dyn ObjectStore>,+ f: TargetFile,+ plan: Arc<SourcePlan>,+ closing: Arc<Vec<Option<u32>>>,+ cols: Arc<Columns>,+ table_schema: SchemaRef,+ opts: Arc<HistoryUpsertOptions>,+ budgets: Budgets,+ tx: mpsc::UnboundedSender<WriteMessage>,+) -> Result<CloseOutcome> {+ let (mut stream, fetch) = open_rewrite_stream(+ &store,+ &f,+ &budgets,+ opts.read_ahead_row_groups,+ opts.read_batch_size,+ )+ .await?;+ let mut rows_copied = 0usize;+ let mut versions_closed = 0usize;+ let mut estimate_bytes = crate::upsert::INITIAL_DECODE_ESTIMATE_BYTES;+ loop {+ let mut permit = budgets.acquire_bytes(estimate_bytes).await?;+ let Some(batch) = stream.try_next().await? else {+ break;+ };+ let actual = batch.get_array_memory_size();+ budgets.top_up(&mut permit, actual).await?;+ estimate_bytes = actual.max(64 * 1024);++ let batch = add_partition_column(&batch, &opts.partition_column, &opts.current_partition)?;+ let batch = cast_to_schema(&batch, &table_schema)?;+ let key_cols: Vec<ArrayRef> = cols+ .key_idx+ .iter()+ .map(|i| batch.column(*i).clone())+ .collect();+ let keys = plan.key_converter.convert_columns(&key_cols)?;+ let closes: UInt32Array = (0..batch.num_rows())+ .map(|i| {+ plan.by_key+ .get(keys.row(i).as_ref())+ .and_then(|group| closing[*group as usize])+ })+ .collect();+ let closed = closes.len() - closes.null_count();+ versions_closed += closed;+ rows_copied += batch.num_rows() - closed;+ let message = if closed == 0 {+ WriteMessage {+ current: Some(batch),+ closed: None,+ _permit: permit,+ }+ } else {+ let is_closed: BooleanArray = (0..closes.len())+ .map(|i| Some(closes.is_valid(i)))+ .collect();+ let stays_open: BooleanArray =+ (0..closes.len()).map(|i| Some(closes.is_null(i))).collect();+ let current = filter_record_batch(&batch, &stays_open)?;+ let closed_rows = stamp_validity(+ &filter_record_batch(&batch, &is_closed)?,+ filter(&closes, &is_closed)?.as_primitive(),+ &plan,+ &cols,+ &table_schema,+ &opts.current_partition,+ )?;+ WriteMessage {+ current: (current.num_rows() > 0).then_some(current),+ closed: Some(closed_rows),+ _permit: permit,+ }+ };+ tx.send(message)+ .map_err(|_| Error::Generic("writer task ended before its readers".into()))?;+ }+ drop(stream);+ drop(fetch);+ Ok(CloseOutcome {+ rows_copied,+ versions_closed,+ bytes_read: f.size,+ remove: Action::Remove(f.remove),+ })+}++/// Most month files the closed-row writer keeps open. Each open file holds its own+/// buffers, so closes that span many months would otherwise grow memory per month.+const MAX_OPEN_MONTH_FILES: usize = 32;++/// Write closed rows one month at a time, flushing every open month file before the+/// writer would hold more than [`MAX_OPEN_MONTH_FILES`].+async fn write_closed(+ writer: &mut StreamingWriter,+ batch: &RecordBatch,+ partition_idx: usize,+ adds: &mut Vec<Add>,+ target_file_size: usize,+ max_row_group_bytes: Option<usize>,+) -> Result<()> {+ let order = sort_to_indices(batch.column(partition_idx), None, None)?;+ let sorted = take_record_batch(batch, &order)?;+ let runs = arrow_ord::partition::partition(&[sorted.column(partition_idx).clone()])?;+ for run in runs.ranges() {+ let month = sorted.slice(run.start, run.end - run.start);+ if writer.open_files() >= MAX_OPEN_MONTH_FILES && !writer.holds_partition_of(&month)? {+ adds.extend(writer.flush().await?);+ }+ writer.write(month)?;+ roll_writer(writer, adds, target_file_size, max_row_group_bytes).await?;+ }+ Ok(())+}++/// Writes chunks of kept source rows, stamped with their `valid_to` and partition.+struct SourceWriter<'a> {+ plan: &'a SourcePlan,+ cols: &'a Columns,+ table_schema: &'a SchemaRef,+ opts: &'a HistoryUpsertOptions,+ target_file_size: usize,+}++impl SourceWriter<'_> {+ fn stamped(&self, rows: &[KeptRow]) -> Result<RecordBatch> {+ // Hand interleave only the batches this chunk reads; its cost grows with the+ // number of batches it is given.+ let mut batch_slots: HashMap<usize, usize> = HashMap::new();+ let mut batch_refs: Vec<&RecordBatch> = Vec::new();+ let located: Vec<(usize, usize)> = rows+ .iter()+ .map(|(r, _)| {+ let (batch, row) = self.plan.locate(*r);+ let slot = *batch_slots.entry(batch).or_insert_with(|| {+ batch_refs.push(&self.plan.batches[batch]);+ batch_refs.len() - 1+ });+ (slot, row)+ })+ .collect();+ let closes: UInt32Array = rows.iter().map(|(_, next)| *next).collect();+ stamp_validity(+ &interleave_record_batch(&batch_refs, &located)?,+ &closes,+ self.plan,+ self.cols,+ self.table_schema,+ &self.opts.current_partition,+ )+ }++ /// Write newest kept rows, which all stay open in the current partition.+ async fn write_open(+ &self,+ rows: &[KeptRow],+ writer: &mut StreamingWriter,+ adds: &mut Vec<Add>,+ ) -> Result<()> {+ let selected = self.stamped(rows)?;+ let max_row_group_bytes = self.opts.max_row_group_bytes;+ let slice_rows =+ source_slice_rows(&selected, self.opts.read_batch_size, max_row_group_bytes);+ let mut offset = 0usize;+ while offset < selected.num_rows() {+ let n = (selected.num_rows() - offset).min(slice_rows);+ writer.write(selected.slice(offset, n))?;+ offset += n;+ roll_writer(writer, adds, self.target_file_size, max_row_group_bytes).await?;+ }+ Ok(())+ }+}++async fn history_inner(+ table: &DeltaTable,+ source_batches: Vec<RecordBatch>,+ source_schema: SchemaRef,+ opts: &HistoryUpsertOptions,+ plan_only: bool,+) -> Result<(HistoryUpsertStats, Option<DeltaTableState>)> {+ // Again here: after a relax the snapshot is reloaded, and a concurrent commit may have+ // enabled a feature that makes a blind rewrite unsafe.+ ensure_history_table(table, opts)?;+ let snapshot = table.snapshot()?;+ let table_schema: SchemaRef = RecordBatchWriter::for_table(table)?.arrow_schema();+ let cols = Columns::resolve(&table_schema, &source_schema, opts)?;+ let target_file_size = resolve_target_file_size(+ opts.target_file_size,+ snapshot.table_config().target_file_size().get() as usize,+ );+ let read_version = snapshot.version() as i64;++ let plan_started = Instant::now();+ let plan = SourcePlan::build(source_batches, &table_schema, &cols, opts)?;+ let mut stats = HistoryUpsertStats {+ version: read_version,+ rows_in: plan.rows_in,+ rows_deduped: plan.rows_in - plan.deduped.len(),+ ..Default::default()+ };+ if plan.groups.is_empty() {+ stats.plan_ms = plan_started.elapsed().as_millis() as u64;+ return Ok((stats, None));+ }++ let files = list_files_by_partition(+ table,+ Some(&opts.partition_column),+ &[opts.current_partition.as_str()],+ )+ .await?+ .into_iter()+ .next()+ .unwrap_or_default();+ let (candidates, pruned): (Vec<TargetFile>, Vec<TargetFile>) = files+ .into_iter()+ .partition(|f| !file_cannot_hold_source_key(f, &opts.key_columns[0], &plan));+ stats.files_carried_over = pruned.len();+ stats.plan_ms = plan_started.elapsed().as_millis() as u64;++ let budgets = Budgets::new(+ opts.max_buffered_bytes,+ opts.max_fetch_bytes,+ opts.limits.clone(),+ );+ let store: Arc<dyn ObjectStore> = table.object_store();++ // Held from the probe on, as an upsert's partition worker holds it.+ let _partition_permit = opts.limits.acquire_partition().await?;+ let probe_started = Instant::now();+ let probe_columns: Vec<(String, DataType)> = cols+ .key_idx+ .iter()+ .chain(std::iter::once(&cols.order_idx))+ .map(|i| {+ let f = table_schema.field(*i);+ (f.name().clone(), f.data_type().clone())+ })+ .collect();+ stats.files_probed = candidates.len();+ let key_count = cols.key_idx.len();+ let mut open: Vec<Option<Box<[u8]>>> = vec![None; plan.groups.len()];+ let mut probed: Vec<(TargetFile, Vec<u32>)> = Vec::new();+ {+ let results =+ futures::stream::iter(+ candidates.into_iter().map(|f| {+ let store = store.clone();+ let (plan, probe_columns, budgets) = (&plan, &probe_columns, &budgets);+ async move {+ probe_file(&store, f, plan, probe_columns, key_count, opts, budgets).await+ }+ }),+ )+ .buffered(opts.probe_concurrency.max(1));+ futures::pin_mut!(results);+ let retained_footer_limit = opts.max_parallel_files.max(1);+ let mut retained = 0usize;+ while let Some((mut probe, metadata)) = results.try_next().await? {+ if probe.newer && retained < retained_footer_limit {+ probe.file.metadata = Some(metadata);+ retained += 1;+ }+ let mut groups = Vec::with_capacity(probe.seen.len());+ for (group, order) in probe.seen {+ let slot = &mut open[group as usize];+ if slot.as_ref().is_none_or(|o| order > *o) {+ *slot = Some(order);+ }+ groups.push(group);+ }+ groups.sort_unstable();+ groups.dedup();+ probed.push((probe.file, groups));+ }+ }+ stats.probe_ms = probe_started.elapsed().as_millis() as u64;++ let decision = plan.decide(&open);+ stats.rows_reread = decision.rows_reread;+ stats.rows_backwards = decision.rows_backwards;+ stats.versions_opened = decision.versions_opened;+ stats.chain_rows_closed = decision.chain_rows_closed;+ if decision.writes.is_empty() || plan_only {+ // Nothing kept means nothing closes either: the table already holds this input.+ stats.files_carried_over += probed.len();+ return Ok((stats, None));+ }++ let mut rewrite = Vec::new();+ for (file, groups) in probed {+ if groups+ .iter()+ .any(|group| decision.closing[*group as usize].is_some())+ {+ rewrite.push(file);+ } else {+ stats.files_carried_over += 1;+ }+ }++ // Open rows and closed rows go through separate writers, so a flush of closed rows+ // never cuts a current-partition file short.+ let rewrite_started = Instant::now();+ let plan = Arc::new(plan);+ let closing = Arc::new(decision.closing);+ let cols = Arc::new(cols);+ let opts_arc = Arc::new(opts.clone());+ let current_writer = StreamingWriter::for_table(table)?;+ let closed_writer = StreamingWriter::for_table(table)?;+ let (tx, mut rx) = mpsc::unbounded_channel::<WriteMessage>();+ let max_row_group_bytes = opts.max_row_group_bytes;+ let partition_idx = cols.partition_idx;+ let writer_task = tokio::spawn(async move {+ let (mut current_writer, mut closed_writer) = (current_writer, closed_writer);+ let mut adds = Vec::new();+ while let Some(message) = rx.recv().await {+ if let Some(batch) = message.current {+ current_writer.write(batch)?;+ roll_writer(+ &mut current_writer,+ &mut adds,+ target_file_size,+ max_row_group_bytes,+ )+ .await?;+ }+ if let Some(batch) = message.closed {+ write_closed(+ &mut closed_writer,+ &batch,+ partition_idx,+ &mut adds,+ target_file_size,+ max_row_group_bytes,+ )+ .await?;+ }+ }+ Ok::<_, Error>((current_writer, closed_writer, adds))+ });++ // Same reader discipline as the upsert: per-call then process-wide file permits, and+ // readers queued behind a failed one stop instead of reading.+ let file_sem = Arc::new(Semaphore::new(opts.max_parallel_files.max(1)));+ let reader_failed = Arc::new(AtomicBool::new(false));+ let mut readers = Vec::with_capacity(rewrite.len());+ for f in rewrite {+ let sem = file_sem.clone();+ let failed = reader_failed.clone();+ let (store, plan, closing, cols) =+ (store.clone(), plan.clone(), closing.clone(), cols.clone());+ let (schema, opts, budgets, tx) = (+ table_schema.clone(),+ opts_arc.clone(),+ budgets.clone(),+ tx.clone(),+ );+ readers.push(tokio::spawn(async move {+ let _local = sem+ .acquire_owned()+ .await+ .map_err(|_| Error::Generic("file semaphore closed".into()))?;+ if failed.load(Ordering::Relaxed) {+ return Ok(None);+ }+ let outcome = async {+ let _global = opts.limits.acquire_file().await?;+ close_file(store, f, plan, closing, cols, schema, opts, budgets, tx).await+ }+ .await;+ if outcome.is_err() {+ failed.store(true, Ordering::Relaxed);+ }+ outcome.map(Some)+ }));+ }+ drop(tx);++ let mut removes: Vec<Action> = Vec::new();+ let mut reader_err: Option<Error> = None;+ for h in readers {+ match h+ .await+ .map_err(|e| Error::Generic(format!("file worker panicked: {e}")))?+ {+ Ok(Some(o)) => {+ stats.rows_copied += o.rows_copied;+ stats.versions_closed += o.versions_closed;+ stats.bytes_read += o.bytes_read;+ removes.push(o.remove);+ }+ Ok(None) => {}+ Err(e) => reader_err = reader_err.or(Some(e)),+ }+ }+ let (mut current_writer, mut closed_writer, mut adds) = writer_task+ .await+ .map_err(|e| Error::Generic(format!("writer task panicked: {e}")))??;+ if let Some(e) = reader_err {+ return Err(e);+ }++ let (opens, mut chain): (Vec<KeptRow>, Vec<KeptRow>) = decision+ .writes+ .into_iter()+ .partition(|(_, next)| next.is_none());+ let source = SourceWriter {+ plan: &plan,+ cols: &cols,+ table_schema: &table_schema,+ opts,+ target_file_size,+ };+ for chunk in opens.chunks(opts.read_batch_size.max(1)) {+ source+ .write_open(chunk, &mut current_writer, &mut adds)+ .await?;+ }+ // Sorted by month, so the closed-row writer moves through months in order instead of+ // reopening them; `write_closed` keeps the open month files under the limit.+ let next_rows: UInt32Array = chain.iter().map(|(_, next)| *next).collect();+ let months = format_months(take(plan.valid_from.as_ref(), &next_rows, None)?.as_ref())?;+ let mut by_month: Vec<usize> = (0..chain.len()).collect();+ by_month.sort_by_key(|i| months.value(*i));+ chain = by_month.iter().map(|i| chain[*i]).collect();+ for chunk in chain.chunks(opts.read_batch_size.max(1)) {+ write_closed(+ &mut closed_writer,+ &source.stamped(chunk)?,+ partition_idx,+ &mut adds,+ target_file_size,+ max_row_group_bytes,+ )+ .await?;+ }+ adds.extend(current_writer.flush().await?);+ adds.extend(closed_writer.flush().await?);+ stats.rewrite_ms = rewrite_started.elapsed().as_millis() as u64;+ stats.files_added = adds.len();+ stats.files_removed = removes.len();+ stats.bytes_written = adds.iter().map(|a| a.size.max(0) as u64).sum();++ let mut actions = removes;+ actions.extend(adds.into_iter().map(Action::Add));+ // Zero commit retries: without DataFusion, delta-rs does not check a `Write` against+ // a concurrent commit that only adds files, so a rebased commit could open a key a+ // second time. A stale commit fails as a conflict instead, and the handle re-plans.+ let committed = commit_partition_overwrite(+ table,+ snapshot,+ actions,+ std::slice::from_ref(&opts.partition_column),+ 0,+ opts.commit_metadata.clone(),+ )+ .await?;+ stats.commit_ms = committed.commit_ms;+ stats.maintenance_ms = committed.maintenance_ms;+ stats.committed = true;+ stats.version = committed.version;+ info!(+ version = stats.version,+ rows_in = stats.rows_in,+ versions_opened = stats.versions_opened,+ versions_closed = stats.versions_closed,+ chain_rows_closed = stats.chain_rows_closed,+ rows_reread = stats.rows_reread,+ rows_backwards = stats.rows_backwards,+ files_added = stats.files_added,+ files_removed = stats.files_removed,+ "history upsert committed"+ );+ Ok((stats, Some(committed.state)))+}++/// A bucket in `0..n` for every row of `batches`, from the values of `key_columns`.+///+/// The key columns are cast to their types in `table`, the cast [`history_upsert`] matches+/// keys after, so source values that become one table key always share a bucket. Each key+/// is then encoded with arrow-row (null-safe, so null key parts hash alike) and hashed with+/// xxHash64 seeded with [`KEY_BUCKET_SEED`], reduced modulo `n`. The ids are the same in+/// every process that runs one deltalite build. arrow-row may change its encoding between+/// arrow releases, so do not persist the ids.+pub fn key_buckets(+ table: &DeltaTable,+ batches: &[RecordBatch],+ key_columns: &[String],+ n: u32,+) -> Result<UInt32Array> {+ if n == 0 {+ return Err(Error::Generic("key_buckets needs n >= 1".into()));+ }+ if key_columns.is_empty() {+ return Err(Error::Generic(+ "key_buckets needs at least one key column".into(),+ ));+ }+ let table_schema = RecordBatchWriter::for_table(table)?.arrow_schema();+ let key_types = key_columns+ .iter()+ .map(|k| {+ Ok(table_schema+ .field_with_name(k)+ .map_err(|_| {+ Error::SchemaMismatch(format!("key column '{k}' not found in table schema"))+ })?+ .data_type()+ .clone())+ })+ .collect::<Result<Vec<DataType>>>()?;+ let converter = RowConverter::new(key_types.iter().cloned().map(SortField::new).collect())?;+ let mut out = Vec::with_capacity(batches.iter().map(RecordBatch::num_rows).sum());+ for batch in batches {+ let cols: Vec<ArrayRef> = key_columns+ .iter()+ .zip(&key_types)+ .map(|(k, ty)| {+ let col = batch.column_by_name(k).ok_or_else(|| {+ Error::SchemaMismatch(format!("key column '{k}' is missing from the data"))+ })?;+ cast_strict(col, ty)+ })+ .collect::<Result<_>>()?;+ let rows = converter.convert_columns(&cols)?;+ out.extend(rows.iter().map(|row| {+ (twox_hash::XxHash64::oneshot(KEY_BUCKET_SEED, row.as_ref()) % u64::from(n)) as u32+ }));+ }+ Ok(UInt32Array::from(out))+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow_array::Int64Array;+ use deltalake::kernel::{DataType as KernelType, StructField};+ use deltalake::operations::create::CreateBuilder;++ #[tokio::test]+ async fn the_write_checks_the_snapshot_it_writes_against() {+ let dir = tempfile::tempdir().unwrap();+ let table = CreateBuilder::new()+ .with_location(dir.path().to_str().unwrap())+ .with_columns(vec![+ StructField::new("v", KernelType::LONG, true),+ StructField::new("p", KernelType::STRING, false),+ ])+ .with_partition_columns(vec!["p".to_string()])+ .with_configuration(vec![("delta.appendOnly", Some("true"))])+ .await+ .unwrap();+ let source = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, true)]));+ let opts = HistoryUpsertOptions {+ key_columns: vec!["v".into()],+ order_column: "v".into(),+ valid_from_column: "v".into(),+ valid_to_column: "v".into(),+ partition_column: "p".into(),+ current_partition: "current".into(),+ tie_break_column: "v".into(),+ ..Default::default()+ };+ let err = history_inner(+ &table,+ vec![RecordBatch::new_empty(source.clone())],+ source,+ &opts,+ false,+ )+ .await+ .unwrap_err();+ assert!(matches!(err, Error::Unsupported(_)), "{err}");+ }++ #[tokio::test]+ async fn closed_rows_hold_at_most_the_month_file_limit_open() {+ let dir = tempfile::tempdir().unwrap();+ let table = CreateBuilder::new()+ .with_location(dir.path().to_str().unwrap())+ .with_columns(vec![+ StructField::new("v", KernelType::LONG, true),+ StructField::new("p", KernelType::STRING, false),+ ])+ .with_partition_columns(vec!["p".to_string()])+ .await+ .unwrap();+ let schema = RecordBatchWriter::for_table(&table).unwrap().arrow_schema();+ let batch = |months: &[String]| {+ RecordBatch::try_new(+ schema.clone(),+ vec![+ Arc::new(Int64Array::from_iter_values(0..months.len() as i64)),+ Arc::new(StringArray::from_iter_values(months.iter().rev())),+ ],+ )+ .unwrap()+ };+ let months: Vec<String> = (0..MAX_OPEN_MONTH_FILES)+ .map(|i| format!("{}-{:02}", 2000 + i / 12, i % 12 + 1))+ .collect();+ let mut writer = StreamingWriter::for_table(&table).unwrap();+ let mut adds = Vec::new();+ let mut write = async |months: &[String]| {+ write_closed(&mut writer, &batch(months), 1, &mut adds, usize::MAX, None)+ .await+ .unwrap();+ (adds.len(), writer.open_files())+ };++ assert_eq!(write(&months).await, (0, MAX_OPEN_MONTH_FILES));+ assert_eq!(+ write(&months[..1]).await,+ (0, MAX_OPEN_MONTH_FILES),+ "a month already open needs no flush"+ );+ assert_eq!(+ write(&["1999-12".to_string()]).await,+ (MAX_OPEN_MONTH_FILES, 1)+ );+ }+}diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rsindex ae1c9f405bcd..f8531bc3851a 100644--- a/rust/deltalite/core/src/lib.rs+++ b/rust/deltalite/core/src/lib.rs@@ -33,6 +33,7 @@ pub mod compact;pub mod deadline;pub mod errors;pub mod handle;+pub mod history;pub mod limits;mod logprobe;pub mod pkset;@@ -50,6 +51,9 @@ mod writer;pub use compact::{compact, CompactOptions, CompactStats};pub use errors::{Error, Result};pub use handle::{LiveFile, TableHandle};+pub use history::{+ history_upsert, key_buckets, HistoryUpsertOptions, HistoryUpsertStats, KEY_BUCKET_SEED,+};pub use limits::{Budget, BudgetSnapshot, LimitsSnapshot, ProcessLimits};pub use prefetch::CheckpointCache;pub use probecache::{ProbeCache, ProbeCacheStats};diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rsindex 1f6c7ffd9009..9511f49d5e97 100644--- a/rust/deltalite/core/src/upsert.rs+++ b/rust/deltalite/core/src/upsert.rs@@ -47,7 +47,7 @@ use deltalake::{DeltaTable, ObjectStore, PartitionFilter, PartitionValue, Path};use futures::{StreamExt, TryStreamExt};use metrics::{counter, histogram};use parquet::arrow::arrow_reader::ArrowReaderMetadata;-use parquet::arrow::async_reader::ParquetObjectReader;+use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStream};use parquet::arrow::ParquetRecordBatchStreamBuilder;use parquet::arrow::ProjectionMask;use parquet::file::metadata::ParquetMetaData;@@ -76,7 +76,7 @@ const WHOLE_TABLE: &str = "__deltalite_whole_table__";/// the observed batch size after the first decode.pub(crate) const INITIAL_DECODE_ESTIMATE_BYTES: usize = 4 * 1024 * 1024;/// First-iteration pre-decode reservation for narrow PK-column probe batches.-const INITIAL_PROBE_ESTIMATE_BYTES: usize = 256 * 1024;+pub(crate) const INITIAL_PROBE_ESTIMATE_BYTES: usize = 256 * 1024;/// Default per-call fetch budget. Readers of ordinary files (a few MB per row group)/// never wait on it at the default `max_parallel_files`; readers of files whose single@@ -407,7 +407,7 @@ impl RelaxCache {/// committed. Valid only when the commit landed directly on the verified version/// (`committed == verified + 1`): a bigger jump means another writer's actions are/// part of the new version, so the memo stays behind and the next upsert rescans.- fn advance_own_commit(&mut self, committed: u64) {+ pub(crate) fn advance_own_commit(&mut self, committed: u64) {if self.version.is_some_and(|v| committed == v + 1) {self.version = Some(committed);}@@ -855,7 +855,7 @@ async fn upsert_with_relax(/// (present with nulls, or absent so the cast will null-pad it) or in the table's own/// files (Add-action `nullCount` stats). Only positive evidence relaxes: files without/// stats are left alone, so a clean table's schema is never touched.-async fn columns_needing_relax(+pub(crate) async fn columns_needing_relax(table: &DeltaTable,source_batches: &[RecordBatch],source_schema: &Schema,@@ -936,7 +936,10 @@ async fn columns_needing_relax(/// the upsert that follows plans against the relaxed schema; concurrent writers see a/// MetadataChanged conflict and re-plan (or fall back), exactly as for any other/// schema evolution.-async fn relax_columns_to_nullable(table: &DeltaTable, columns: &[String]) -> Result<()> {+pub(crate) async fn relax_columns_to_nullable(+ table: &DeltaTable,+ columns: &[String],+) -> Result<()> {let snapshot = table.snapshot()?;let schema = snapshot.schema();let relaxed = StructType::try_new(schema.fields().map(|f| {@@ -1169,8 +1172,69 @@ async fn upsert_inner(let rewrite_ms = rewrite_started.elapsed().as_millis() as u64;// --- Commit ----------------------------------------------------------------------+ let committed = commit_partition_overwrite(+ table,+ snapshot,+ actions,+ &partition_columns,+ opts.commit_max_retries,+ opts.commit_metadata.clone(),+ )+ .await?;++ // Only a landed commit makes the written files live and the removed files dead. Before+ // it, a written file may be an orphan that no snapshot ever names.+ if let Some(cache) = &opts.probe_cache {+ cache.remove_all(removed_keys.iter().map(String::as_str));+ let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;+ for (key, keys) in written_keys {+ keys.store(key, signature);+ }+ }++ stats.plan_ms = plan_ms;+ stats.rewrite_ms = rewrite_ms;+ stats.commit_ms = committed.commit_ms;+ stats.maintenance_ms = committed.maintenance_ms;+ stats.version = committed.version;+ tracing::Span::current().record("version", stats.version);+ info!(+ version = stats.version,+ partitions = stats.partitions_touched,+ files_added = stats.files_added,+ files_removed = stats.files_removed,+ files_carried_over = stats.files_carried_over,+ rows_updated = stats.rows_updated,+ rows_inserted = stats.rows_inserted,+ rows_copied = stats.rows_copied,+ plan_ms = stats.plan_ms,+ rewrite_ms = stats.rewrite_ms,+ commit_ms = stats.commit_ms,+ "upsert committed"+ );+ Ok((stats, committed.state))+}++/// A landed [`commit_partition_overwrite`].+pub(crate) struct Committed {+ pub(crate) version: i64,+ pub(crate) state: DeltaTableState,+ pub(crate) commit_ms: u64,+ pub(crate) maintenance_ms: u64,+}++/// Commit a rewrite's Remove and Add actions as one partition-overwrite `Write`, then run+/// log maintenance on a checkpoint boundary.+pub(crate) async fn commit_partition_overwrite(+ table: &DeltaTable,+ snapshot: &DeltaTableState,+ actions: Vec<Action>,+ partition_columns: &[String],+ max_retries: usize,+ commit_metadata: Option<HashMap<String, Value>>,+) -> Result<Committed> {let commit_started = Instant::now();- let predicate = partition_col.as_ref().map(|c| {+ let predicate = partition_columns.first().map(|c| {let vals: Vec<String> = actions.iter().filter_map(|a| match a {@@ -1189,7 +1253,7 @@ async fn upsert_inner(partition_by: if partition_columns.is_empty() {None} else {- Some(partition_columns.clone())+ Some(partition_columns.to_vec())},predicate,};@@ -1200,10 +1264,10 @@ async fn upsert_inner(// caller back to the delta-rs MERGE for a write that has in fact happened. They run// best-effort below instead.let mut props = CommitProperties::default()- .with_max_retries(opts.commit_max_retries)+ .with_max_retries(max_retries).with_create_checkpoint(false).with_cleanup_expired_logs(Some(false));- if let Some(md) = opts.commit_metadata.clone() {+ if let Some(md) = commit_metadata {props = props.with_metadata(md);}@@ -1216,43 +1280,19 @@ async fn upsert_inner(.await?;let commit_ms = commit_started.elapsed().as_millis() as u64;- // Only a landed commit makes the written files live and the removed files dead. Before- // it, a written file may be an orphan that no snapshot ever names.- if let Some(cache) = &opts.probe_cache {- cache.remove_all(removed_keys.iter().map(String::as_str));- let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;- for (key, keys) in written_keys {- keys.store(key, signature);- }- }-+ let mut maintenance_ms = 0;if (finalized.version() + 1) % checkpoint_interval == 0 {let maintenance_started = Instant::now();best_effort_log_maintenance(table, finalized.version(), cleanup_enabled).await;- stats.maintenance_ms = maintenance_started.elapsed().as_millis() as u64;- }-- stats.plan_ms = plan_ms;- stats.rewrite_ms = rewrite_ms;- stats.commit_ms = commit_ms;- stats.version = i64::try_from(finalized.version())- .map_err(|_| Error::Generic("committed version overflows i64".into()))?;- tracing::Span::current().record("version", stats.version);- info!(- version = stats.version,- partitions = stats.partitions_touched,- files_added = stats.files_added,- files_removed = stats.files_removed,- files_carried_over = stats.files_carried_over,- rows_updated = stats.rows_updated,- rows_inserted = stats.rows_inserted,- rows_copied = stats.rows_copied,- plan_ms = stats.plan_ms,- rewrite_ms = stats.rewrite_ms,- commit_ms = stats.commit_ms,- "upsert committed"- );- Ok((stats, finalized.snapshot))+ maintenance_ms = maintenance_started.elapsed().as_millis() as u64;+ }+ Ok(Committed {+ version: i64::try_from(finalized.version())+ .map_err(|_| Error::Generic("committed version overflows i64".into()))?,+ state: finalized.snapshot,+ commit_ms,+ maintenance_ms,+ })}/// Checkpoint and expired-log cleanup after a durable commit, tolerating failure: the@@ -1297,7 +1337,7 @@ pub(crate) async fn best_effort_log_maintenance(/// Explicit argument > table property (already folded with delta-rs's default by the/// caller). Zero/absent explicit values fall through to the table's value.-fn resolve_target_file_size(explicit: Option<usize>, table_value: usize) -> usize {+pub(crate) fn resolve_target_file_size(explicit: Option<usize>, table_value: usize) -> usize {match explicit {Some(v) if v > 0 => v,_ => table_value,@@ -1569,7 +1609,7 @@ fn source_pk_range(source: &PartitionSource, col_idx: usize) -> Result<Option<(V////// Each list keeps the snapshot's file order, as the per-partition filter did. Where/// the index cannot reproduce the filter's equality exactly, this falls back to it.-async fn list_files_by_partition(+pub(crate) async fn list_files_by_partition(table: &DeltaTable,partition_col: Option<&str>,values: &[&str],@@ -1832,7 +1872,7 @@ fn file_cannot_match(|| matches!(json_cmp(fmin, src_max), Some(std::cmp::Ordering::Greater))}-fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {+pub(crate) fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {match (a, b) {(Value::String(x), Value::String(y)) => Some(x.cmp(y)),(Value::Number(x), Value::Number(y)) => x.as_f64()?.partial_cmp(&y.as_f64()?),@@ -1840,7 +1880,7 @@ fn json_cmp(a: &Value, b: &Value) -> Option<std::cmp::Ordering> {}}-fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {+pub(crate) fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {use arrow_array::cast::AsArray;use arrow_array::types::*;@@ -1919,7 +1959,11 @@ fn min_max_json(col: &dyn Array) -> Option<(Value, Value)> {/// read back from storage is missing the column that the writer needs in order to route/// the rows back to the same partition. Getting this wrong would silently misroute/// rewritten rows.-fn add_partition_column(batch: &RecordBatch, name: &str, value: &str) -> Result<RecordBatch> {+pub(crate) fn add_partition_column(+ batch: &RecordBatch,+ name: &str,+ value: &str,+) -> Result<RecordBatch> {let n = batch.num_rows();let arr: Arc<dyn Array> = Arc::new(StringArray::from(vec![value; n]));let mut fields: Vec<Field> = batch@@ -2302,24 +2346,16 @@ struct FileOutcome {remove: Action,}-/// Stream one existing file, drop replaced rows, and send surviving batches to the-/// partition's writer task. Every batch sent carries byte-budget permits acquired-/// *before* the send; the writer releases them only after `write()` has copied the-/// batch into its buffer, so queued + in-write survivor data across the whole process-/// never exceeds the budgets.-#[allow(clippy::too_many_arguments)]-async fn filter_file(- store: Arc<dyn ObjectStore>,- f: TargetFile,- pkset: Arc<PkSet>,- table_schema: SchemaRef,- partition_col: Option<String>,- partition_value: String,- opts: Arc<UpsertOptions>,- budgets: Budgets,- tx: mpsc::UnboundedSender<(RecordBatch, BudgetPermit)>,-) -> Result<FileOutcome> {- let (mut builder, held_fetch) = open_builder(&store, &f, &budgets).await?;+/// Open a full-width stream over `f` for a rewrite, holding the fetch permit for the+/// compressed bytes the reader will hold. The permit must outlive the stream.+pub(crate) async fn open_rewrite_stream(+ store: &Arc<dyn ObjectStore>,+ f: &TargetFile,+ budgets: &Budgets,+ read_ahead_row_groups: bool,+ read_batch_size: usize,+) -> Result<(ParquetRecordBatchStream<ParquetObjectReader>, FetchPermit)> {+ let (mut builder, held_fetch) = open_builder(store, f, budgets).await?;// Reserve the compressed bytes the reader will hold before it fetches any data.// Taken once per file and before any decode budget (see `Budgets::acquire_fetch`).let fetch = match held_fetch {@@ -2327,7 +2363,7 @@ async fn filter_file(None => {let meta = builder.metadata().clone();let pair = max_row_group_pair_fetch_bytes(&meta);- if opts.read_ahead_row_groups+ if read_ahead_row_groups&& meta.num_row_groups() > 1&& pair <= budgets.fetch_cap_bytes(){@@ -2349,9 +2385,36 @@ async fn filter_file(let batch_rows = byte_bounded_batch_rows(builder.metadata(),INITIAL_DECODE_ESTIMATE_BYTES,- opts.read_batch_size,+ read_batch_size,);- let mut stream = builder.with_batch_size(batch_rows).build()?;+ Ok((builder.with_batch_size(batch_rows).build()?, fetch))+}++/// Stream one existing file, drop replaced rows, and send surviving batches to the+/// partition's writer task. Every batch sent carries byte-budget permits acquired+/// *before* the send; the writer releases them only after `write()` has copied the+/// batch into its buffer, so queued + in-write survivor data across the whole process+/// never exceeds the budgets.+#[allow(clippy::too_many_arguments)]+async fn filter_file(+ store: Arc<dyn ObjectStore>,+ f: TargetFile,+ pkset: Arc<PkSet>,+ table_schema: SchemaRef,+ partition_col: Option<String>,+ partition_value: String,+ opts: Arc<UpsertOptions>,+ budgets: Budgets,+ tx: mpsc::UnboundedSender<(RecordBatch, BudgetPermit)>,+) -> Result<FileOutcome> {+ let (mut stream, fetch) = open_rewrite_stream(+ &store,+ &f,+ &budgets,+ opts.read_ahead_row_groups,+ opts.read_batch_size,+ )+ .await?;let mut rows_updated = 0usize;let mut rows_copied = 0usize;@@ -2731,7 +2794,7 @@ async fn rewrite_partition(/// source write holds about as many decoded bytes as a survivor batch, and a row group/// passes the cap by at most that much on either path. Otherwise one `read_batch_size`/// slice of wide rows goes in one write and closes as one row group, far past the cap.-fn source_slice_rows(+pub(crate) fn source_slice_rows(batch: &RecordBatch,read_batch_size: usize,max_row_group_bytes: Option<usize>,@@ -2747,7 +2810,7 @@ fn source_slice_rows(}/// Close the open file at `target_file_size`, else its row group at `max_row_group_bytes`.-async fn roll_writer(+pub(crate) async fn roll_writer(writer: &mut StreamingWriter,adds: &mut Vec<Add>,target_file_size: usize,diff --git a/rust/deltalite/core/src/writer.rs b/rust/deltalite/core/src/writer.rsindex b5c458742def..9780fe191aa7 100644--- a/rust/deltalite/core/src/writer.rs+++ b/rust/deltalite/core/src/writer.rs@@ -174,6 +174,26 @@ impl StreamingWriter {self}+ /// Number of files open for writing, one per partition value written since the last+ /// flush.+ pub(crate) fn open_files(&self) -> usize {+ self.open.len()+ }++ /// Whether a file is open for the partition of the first row of `values`.+ pub(crate) fn holds_partition_of(&self, values: &RecordBatch) -> Result<bool> {+ if values.num_rows() == 0 || self.partition_columns.is_empty() {+ return Ok(!self.open.is_empty());+ }+ let schema = values.schema();+ let mut arrays: Vec<ArrayRef> = Vec::with_capacity(self.partition_columns.len());+ for c in &self.partition_columns {+ arrays.push(values.column(schema.index_of(c)?).clone());+ }+ let values = self.partition_values_at(&arrays, 0)?;+ Ok(self.open.contains_key(&hive_partition_path(&values)))+ }+/// Bytes held across all open files; the rollover measure.pub(crate) fn buffer_len(&self) -> usize {self.open.values().map(OpenFile::buffer_len).sum()diff --git a/rust/deltalite/core/tests/fixtures/month_vectors.json b/rust/deltalite/core/tests/fixtures/month_vectors.jsonnew file mode 100644index 000000000000..988acc0108b2--- /dev/null+++ b/rust/deltalite/core/tests/fixtures/month_vectors.json@@ -0,0 +1,11 @@+[+ {"valid_to": "2026-01-31T23:59:59.999999Z", "month": "2026-01"},+ {"valid_to": "2026-02-01T00:00:00Z", "month": "2026-02"},+ {"valid_to": "2024-02-29T12:00:00Z", "month": "2024-02"},+ {"valid_to": "2025-12-31T23:59:59Z", "month": "2025-12"},+ {"valid_to": "2026-01-01T00:00:00Z", "month": "2026-01"},+ {"valid_to": "1970-01-01T00:00:00Z", "month": "1970-01"},+ {"valid_to": "1969-12-31T23:59:59.999999Z", "month": "1969-12"},+ {"valid_to": "2000-03-01T00:00:00.000001Z", "month": "2000-03"},+ {"valid_to": "2099-11-30T23:00:00Z", "month": "2099-11"}+]diff --git a/rust/deltalite/core/tests/history_upsert.rs b/rust/deltalite/core/tests/history_upsert.rsnew file mode 100644index 000000000000..a880e687e055--- /dev/null+++ b/rust/deltalite/core/tests/history_upsert.rs@@ -0,0 +1,1122 @@+//! `history_upsert` against real tables: the SCD2 rules, what it rewrites, and what it+//! refuses. The Python suite (`python/tests/test_history.py`) runs the same rules against+//! a reference model over random inputs.++use std::collections::HashMap;+use std::future::Future;+use std::pin::Pin;+use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};+use std::sync::{Arc, LazyLock, Mutex};++use arrow_array::types::Int32Type;+use arrow_array::{+ Array, DictionaryArray, Int32Array, Int64Array, RecordBatch, StringArray,+ TimestampMicrosecondArray,+};+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};+use bytes::Bytes;+use chrono::DateTime;+use deltalake::kernel::{DataType as KernelType, PrimitiveType, StructField};+use deltalake::logstore::{+ default_logstore, logstore_factories, object_store_factories, LogStore, LogStoreFactory,+ ObjectStoreFactory, StorageConfig,+};+use deltalake::operations::create::CreateBuilder;+use deltalake::DeltaResult;+use deltalake::Path;+use deltalite_core::errors::Error;+use deltalite_core::handle::TableHandle;+use deltalite_core::history::{+ format_months, history_upsert, HistoryUpsertOptions, HistoryUpsertStats,+};+use deltalite_core::limits::ProcessLimits;+use deltalite_core::table::MultipartConfig;+use futures::stream::BoxStream;+use object_store::local::LocalFileSystem;+use object_store::{+ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,+ ObjectStoreExt, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions,+};+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;++/// One source row: (id, order value, valid_from, payload, tie-break).+type Src<'a> = (Option<&'a str>, i64, &'a str, &'a str, i64);++/// One live table row: (partition, id, order value, valid_from, valid_to, payload).+type Live = (String, Option<String>, i64, i64, Option<i64>, String);++fn ts(s: &str) -> i64 {+ DateTime::parse_from_rfc3339(s).unwrap().timestamp_micros()+}++fn utc() -> DataType {+ DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()))+}++fn source_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("id", DataType::Utf8, true),+ Field::new("v", DataType::Int64, true),+ Field::new("valid_from", utc(), true),+ Field::new("payload", DataType::Utf8, true),+ Field::new("seq", DataType::Int64, true),+ ]))+}++fn source(rows: &[Src]) -> RecordBatch {+ RecordBatch::try_new(+ source_schema(),+ vec![+ Arc::new(StringArray::from(+ rows.iter().map(|r| r.0).collect::<Vec<_>>(),+ )),+ Arc::new(Int64Array::from(+ rows.iter().map(|r| r.1).collect::<Vec<_>>(),+ )),+ Arc::new(+ TimestampMicrosecondArray::from(rows.iter().map(|r| ts(r.2)).collect::<Vec<_>>())+ .with_timezone("UTC"),+ ),+ Arc::new(StringArray::from(+ rows.iter().map(|r| r.3).collect::<Vec<_>>(),+ )),+ Arc::new(Int64Array::from(+ rows.iter().map(|r| r.4).collect::<Vec<_>>(),+ )),+ ],+ )+ .unwrap()+}++async fn create(dir: &std::path::Path, partition_by: &[&str]) -> TableHandle {+ CreateBuilder::new()+ .with_location(dir.to_str().unwrap())+ .with_columns(vec![+ StructField::new("id", KernelType::STRING, true),+ StructField::new("v", KernelType::LONG, true),+ StructField::new(+ "valid_from",+ KernelType::Primitive(PrimitiveType::Timestamp),+ true,+ ),+ StructField::new(+ "valid_to",+ KernelType::Primitive(PrimitiveType::Timestamp),+ true,+ ),+ StructField::new("payload", KernelType::STRING, true),+ StructField::new("p", KernelType::STRING, false),+ StructField::new("q", KernelType::STRING, true),+ ])+ .with_partition_columns(partition_by.iter().map(|c| c.to_string()))+ .await+ .unwrap();+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())+ .await+ .unwrap()+}++fn opts() -> HistoryUpsertOptions {+ HistoryUpsertOptions {+ key_columns: vec!["id".into()],+ order_column: "v".into(),+ valid_from_column: "valid_from".into(),+ valid_to_column: "valid_to".into(),+ partition_column: "p".into(),+ current_partition: "current".into(),+ tie_break_column: "seq".into(),+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),+ ..Default::default()+ }+}++async fn apply_with(+ handle: &mut TableHandle,+ rows: &[Src<'_>],+ opts: HistoryUpsertOptions,+) -> deltalite_core::Result<HistoryUpsertStats> {+ handle+ .history_upsert(+ vec![source(rows)],+ source_schema(),+ opts,+ MultipartConfig::default(),+ )+ .await+}++async fn apply(handle: &mut TableHandle, rows: &[Src<'_>]) -> HistoryUpsertStats {+ apply_with(handle, rows, opts()).await.unwrap()+}++/// Every live row, sorted.+async fn read(handle: &TableHandle) -> Vec<Live> {+ let mut out = Vec::new();+ for f in handle.files().unwrap() {+ let bytes = handle+ .table()+ .object_store()+ .get(&Path::parse(&f.path).unwrap())+ .await+ .unwrap()+ .bytes()+ .await+ .unwrap();+ let partition = f.partition_values["p"].clone().unwrap();+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)+ .unwrap()+ .build()+ .unwrap()+ {+ let batch = batch.unwrap();+ let col = |name: &str| batch.column_by_name(name).unwrap().clone();+ let (id, v, from, to, payload) = (+ col("id"),+ col("v"),+ col("valid_from"),+ col("valid_to"),+ col("payload"),+ );+ let id = id.as_any().downcast_ref::<StringArray>().unwrap();+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();+ let from = from+ .as_any()+ .downcast_ref::<TimestampMicrosecondArray>()+ .unwrap();+ let to = to+ .as_any()+ .downcast_ref::<TimestampMicrosecondArray>()+ .unwrap();+ let payload = payload.as_any().downcast_ref::<StringArray>().unwrap();+ for i in 0..batch.num_rows() {+ out.push((+ partition.clone(),+ id.is_valid(i).then(|| id.value(i).to_string()),+ v.value(i),+ from.value(i),+ to.is_valid(i).then(|| to.value(i)),+ payload.value(i).to_string(),+ ));+ }+ }+ }+ out.sort();+ out+}++fn open(id: Option<&str>, v: i64, from: &str, payload: &str) -> Live {+ (+ "current".into(),+ id.map(str::to_string),+ v,+ ts(from),+ None,+ payload.into(),+ )+}++fn closed(month: &str, id: Option<&str>, v: i64, from: &str, to: &str, payload: &str) -> Live {+ (+ month.into(),+ id.map(str::to_string),+ v,+ ts(from),+ Some(ts(to)),+ payload.into(),+ )+}++fn sorted(mut rows: Vec<Live>) -> Vec<Live> {+ rows.sort();+ rows+}++#[tokio::test]+async fn applies_each_rule_against_the_open_row() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ let first = apply(+ &mut h,+ &[+ (Some("changed"), 10, "2026-01-10T00:00:00Z", "c1", 1),+ (Some("reread"), 10, "2026-01-10T00:00:00Z", "r1", 2),+ (Some("backwards"), 10, "2026-01-10T00:00:00Z", "b1", 3),+ (Some("mixed"), 10, "2026-01-10T00:00:00Z", "m1", 4),+ ],+ )+ .await;+ assert_eq!(+ (+ first.versions_opened,+ first.versions_closed,+ first.files_removed+ ),+ (4, 0, 0),+ "{first:?}"+ );++ let second = apply(+ &mut h,+ &[+ (Some("changed"), 20, "2026-02-03T00:00:00Z", "c2", 10),+ (Some("reread"), 10, "2026-01-10T00:00:00Z", "r1-again", 11),+ (Some("backwards"), 5, "2026-01-05T00:00:00Z", "b0", 12),+ (Some("mixed"), 7, "2026-01-07T00:00:00Z", "m0", 13),+ (Some("mixed"), 30, "2026-03-01T00:00:00Z", "m3", 14),+ (Some("new"), 1, "2026-03-02T00:00:00Z", "n1", 15),+ ],+ )+ .await;+ assert_eq!(+ (+ second.rows_in,+ second.rows_reread,+ second.rows_backwards,+ second.versions_opened,+ second.versions_closed,+ second.chain_rows_closed,+ second.rows_copied,+ ),+ (6, 1, 2, 3, 2, 0, 2),+ "{second:?}"+ );+ assert_eq!(+ read(&h).await,+ sorted(vec![+ closed(+ "2026-02",+ Some("changed"),+ 10,+ "2026-01-10T00:00:00Z",+ "2026-02-03T00:00:00Z",+ "c1"+ ),+ open(Some("changed"), 20, "2026-02-03T00:00:00Z", "c2"),+ open(Some("reread"), 10, "2026-01-10T00:00:00Z", "r1"),+ open(Some("backwards"), 10, "2026-01-10T00:00:00Z", "b1"),+ closed(+ "2026-03",+ Some("mixed"),+ 10,+ "2026-01-10T00:00:00Z",+ "2026-03-01T00:00:00Z",+ "m1"+ ),+ open(Some("mixed"), 30, "2026-03-01T00:00:00Z", "m3"),+ open(Some("new"), 1, "2026-03-02T00:00:00Z", "n1"),+ ])+ );+}++#[tokio::test]+async fn chains_versions_in_one_call_and_a_tie_takes_the_highest_tie_break() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ let stats = apply(+ &mut h,+ &[+ (Some("a"), 3, "2026-03-01T00:00:00Z", "third", 1),+ (Some("a"), 2, "2026-02-01T00:00:00Z", "second-wins", 9),+ (Some("a"), 1, "2026-01-01T00:00:00Z", "first", 1),+ (Some("a"), 2, "2026-02-01T00:00:00Z", "second-loses", 5),+ ],+ )+ .await;+ assert_eq!(+ (+ stats.rows_deduped,+ stats.versions_opened,+ stats.chain_rows_closed+ ),+ (1, 1, 2),+ "{stats:?}"+ );+ assert_eq!(+ read(&h).await,+ sorted(vec![+ closed(+ "2026-02",+ Some("a"),+ 1,+ "2026-01-01T00:00:00Z",+ "2026-02-01T00:00:00Z",+ "first"+ ),+ closed(+ "2026-03",+ Some("a"),+ 2,+ "2026-02-01T00:00:00Z",+ "2026-03-01T00:00:00Z",+ "second-wins"+ ),+ open(Some("a"), 3, "2026-03-01T00:00:00Z", "third"),+ ])+ );+}++#[tokio::test]+async fn a_null_key_matches_its_open_row() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ apply(&mut h, &[(None, 1, "2026-01-01T00:00:00Z", "n1", 1)]).await;+ let stats = apply(&mut h, &[(None, 2, "2026-02-01T00:00:00Z", "n2", 2)]).await;+ assert_eq!(stats.versions_closed, 1, "{stats:?}");+ assert_eq!(+ read(&h).await,+ sorted(vec![+ closed(+ "2026-02",+ None,+ 1,+ "2026-01-01T00:00:00Z",+ "2026-02-01T00:00:00Z",+ "n1"+ ),+ open(None, 2, "2026-02-01T00:00:00Z", "n2"),+ ])+ );+}++#[tokio::test]+async fn a_rerun_of_the_same_input_makes_no_commit() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ let input = [+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),+ (Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 2),+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 3),+ ];+ let first = apply(&mut h, &input).await;+ let rows = read(&h).await;+ let files = h.files().unwrap();++ let rerun = apply(&mut h, &input).await;+ assert!(first.committed && !rerun.committed, "{rerun:?}");+ assert_eq!(rerun.version, first.version);+ assert_eq!(+ (+ rerun.rows_reread,+ rerun.rows_backwards,+ rerun.files_added,+ rerun.files_removed,+ rerun.files_carried_over+ ),+ (2, 1, 0, 0, 1),+ "{rerun:?}"+ );+ assert_eq!(h.version(), first.version);+ assert_eq!(h.files().unwrap(), files);+ assert_eq!(read(&h).await, rows);+}++#[tokio::test]+async fn closed_rows_land_in_the_utc_month_of_valid_to() {+ let vectors: Vec<serde_json::Value> =+ serde_json::from_str(include_str!("fixtures/month_vectors.json")).unwrap();+ let instants: Vec<i64> = vectors+ .iter()+ .map(|v| ts(v["valid_to"].as_str().unwrap()))+ .collect();+ let months =+ format_months(&TimestampMicrosecondArray::from(instants.clone()).with_timezone("UTC"))+ .unwrap();+ for (v, m) in vectors.iter().zip(months.iter()) {+ assert_eq!(m, v["month"].as_str(), "{v}");+ }++ // Each key opens before 1969 and closes at one vector instant.+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ let ids: Vec<String> = (0..vectors.len()).map(|i| format!("k{i}")).collect();+ let opens: Vec<Src> = ids+ .iter()+ .map(|id| (Some(id.as_str()), 1, "1960-01-01T00:00:00Z", "old", 1))+ .collect();+ apply(&mut h, &opens).await;+ let closes: Vec<Src> = ids+ .iter()+ .zip(&vectors)+ .map(|(id, v)| {+ (+ Some(id.as_str()),+ 2,+ v["valid_to"].as_str().unwrap(),+ "new",+ 2,+ )+ })+ .collect();+ apply(&mut h, &closes).await;+ let closed_months: HashMap<String, String> = read(&h)+ .await+ .into_iter()+ .filter(|row| row.4.is_some())+ .map(|row| (row.1.unwrap(), row.0))+ .collect();+ for (id, v) in ids.iter().zip(&vectors) {+ assert_eq!(closed_months[id], v["month"].as_str().unwrap(), "{v}");+ }+}++#[tokio::test]+async fn earlier_closed_versions_in_the_same_month_stay() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;+ apply(&mut h, &[(Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 2)]).await;+ apply(&mut h, &[(Some("a"), 3, "2026-02-15T00:00:00Z", "a3", 3)]).await;+ assert_eq!(+ read(&h).await,+ sorted(vec![+ closed(+ "2026-02",+ Some("a"),+ 1,+ "2026-01-01T00:00:00Z",+ "2026-02-01T00:00:00Z",+ "a1"+ ),+ closed(+ "2026-02",+ Some("a"),+ 2,+ "2026-02-01T00:00:00Z",+ "2026-02-15T00:00:00Z",+ "a2"+ ),+ open(Some("a"), 3, "2026-02-15T00:00:00Z", "a3"),+ ])+ );+}++#[tokio::test]+async fn a_file_holding_only_reread_keys_is_not_rewritten() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;+ apply(&mut h, &[(Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2)]).await;+ let stats = apply(+ &mut h,+ &[+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1-again", 3),+ (Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 4),+ ],+ )+ .await;+ assert_eq!(+ (+ stats.files_probed,+ stats.files_removed,+ stats.files_carried_over,+ stats.rows_reread+ ),+ (2, 1, 1, 1),+ "{stats:?}"+ );+}++#[tokio::test]+async fn refuses_input_it_cannot_apply() {+ let dir = tempfile::tempdir().unwrap();+ let mut history = create(dir.path(), &["p"]).await;+ apply(+ &mut history,+ &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a", 1)],+ )+ .await;+ // A null order value, and a null hidden in the dictionary of a dictionary-encoded+ // `valid_from`, which a plain null count does not see.+ let valid = source(&[(Some("a"), 2, "2026-02-01T00:00:00Z", "a", 2)]);+ let null_dictionary_value: Arc<dyn Array> = Arc::new(+ DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0]),+ Arc::new(TimestampMicrosecondArray::from(vec![None::<i64>]).with_timezone("UTC")),+ )+ .unwrap(),+ );+ let null_order: Arc<dyn Array> = Arc::new(Int64Array::from(vec![None::<i64>]));+ // A dictionary or string tie-break can expand far past its in-memory size when every+ // row is encoded, which the source-size guard would not see.+ let dictionary_tie: Arc<dyn Array> = Arc::new(+ DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0]),+ Arc::new(StringArray::from(vec!["x".repeat(1024)])),+ )+ .unwrap(),+ );+ let string_tie: Arc<dyn Array> = Arc::new(StringArray::from(vec!["1"]));+ for (case, column, value, kind) in [+ ("null order", 1, null_order, "generic"),+ (+ "null dictionary valid_from",+ 2,+ null_dictionary_value,+ "generic",+ ),+ ("dictionary tie-break", 4, dictionary_tie, "schema_mismatch"),+ ("string tie-break", 4, string_tie, "schema_mismatch"),+ ] {+ let mut fields: Vec<Field> = source_schema()+ .fields()+ .iter()+ .map(|f| f.as_ref().clone())+ .collect();+ fields[column] = Field::new(fields[column].name(), value.data_type().clone(), true);+ let schema: SchemaRef = Arc::new(Schema::new(fields));+ let mut cols = valid.columns().to_vec();+ cols[column] = value;+ let batch = RecordBatch::try_new(schema.clone(), cols).unwrap();+ let err = history+ .history_upsert(vec![batch], schema, opts(), MultipartConfig::default())+ .await+ .unwrap_err();+ assert_eq!(err.kind(), kind, "{case}: {err}");+ }++ // A batch whose columns differ from the declared schema, here a string tie-break+ // declared as an integer, is refused before any column is read.+ let mut cols = valid.columns().to_vec();+ cols[4] = Arc::new(StringArray::from(vec!["1"]));+ let mut fields: Vec<Field> = source_schema()+ .fields()+ .iter()+ .map(|f| f.as_ref().clone())+ .collect();+ fields[4] = Field::new("seq", DataType::Utf8, true);+ let undeclared = RecordBatch::try_new(Arc::new(Schema::new(fields)), cols).unwrap();+ let err = history+ .history_upsert(+ vec![undeclared],+ source_schema(),+ opts(),+ MultipartConfig::default(),+ )+ .await+ .unwrap_err();+ assert_eq!(err.kind(), "schema_mismatch", "{err}");++ for (case, partition_by) in [("unpartitioned", vec![]), ("other column", vec!["q"])] {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &partition_by).await;+ let err = apply_with(+ &mut h,+ &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a", 1)],+ opts(),+ )+ .await+ .unwrap_err();+ assert!(matches!(err, Error::Unsupported(_)), "{case}: {err}");+ }++ let month_named = HistoryUpsertOptions {+ current_partition: "2026-01".into(),+ ..opts()+ };+ let err = apply_with(+ &mut history,+ &[(Some("a"), 3, "2026-03-01T00:00:00Z", "a", 3)],+ month_named,+ )+ .await+ .unwrap_err();+ assert!(+ matches!(err, Error::Generic(ref m) if m.contains("month")),+ "{err}"+ );+}++#[tokio::test]+async fn a_concurrent_commit_on_the_same_file_is_retried_against_the_new_state() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ apply(+ &mut h,+ &[+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2),+ ],+ )+ .await;++ // A stale snapshot plans to remove the file another commit already removed.+ let stale = h.table().clone();+ apply(&mut h, &[(Some("a"), 2, "2026-02-01T00:00:00Z", "a2", 3)]).await;+ let err = history_upsert(+ &stale,+ vec![source(&[(Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 4)])],+ source_schema(),+ opts(),+ )+ .await+ .unwrap_err();+ assert!(matches!(err, Error::Conflict(_)), "{err}");++ // A concurrent commit that only adds files must not let a stale plan open a key twice.+ let stale = h.table().clone();+ apply(&mut h, &[(Some("c"), 1, "2026-02-01T00:00:00Z", "c1", 5)]).await;+ let err = history_upsert(+ &stale,+ vec![source(&[(Some("c"), 2, "2026-02-02T00:00:00Z", "c2", 6)])],+ source_schema(),+ opts(),+ )+ .await+ .unwrap_err();+ assert!(matches!(err, Error::Conflict(_)), "{err}");++ h.refresh().await.unwrap();+ assert_eq!(+ read(&h).await,+ sorted(vec![+ closed(+ "2026-02",+ Some("a"),+ 1,+ "2026-01-01T00:00:00Z",+ "2026-02-01T00:00:00Z",+ "a1"+ ),+ open(Some("a"), 2, "2026-02-01T00:00:00Z", "a2"),+ open(Some("b"), 1, "2026-01-01T00:00:00Z", "b1"),+ open(Some("c"), 1, "2026-02-01T00:00:00Z", "c1"),+ ])+ );+}++#[tokio::test]+async fn a_handle_plans_again_when_another_commit_lands_before_its_own() {+ register_race_scheme();+ let dir = tempfile::tempdir().unwrap();+ let mut plain = create(dir.path(), &["p"]).await;+ apply(+ &mut plain,+ &[+ (Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1),+ (Some("b"), 1, "2026-01-01T00:00:00Z", "b1", 2),+ ],+ )+ .await;++ // The competing commit lands after the racing call planned, just before its commit+ // file is written, so the racing commit finds its version taken.+ let path = dir.path().to_str().unwrap().to_string();+ let raced = Arc::new(AtomicBool::new(false));+ let (other_path, other_raced) = (path.clone(), raced.clone());+ arm_race(+ dir.path(),+ Box::new(move || {+ Box::pin(async move {+ let mut other = TableHandle::open(other_path, HashMap::new()).await.unwrap();+ apply(+ &mut other,+ &[(Some("b"), 2, "2026-02-01T00:00:00Z", "b2", 3)],+ )+ .await;+ other_raced.store(true, AtomicOrdering::SeqCst);+ })+ }),+ );+ let mut racing = TableHandle::open(format!("dlrace://{path}"), HashMap::new())+ .await+ .unwrap();+ let stats = apply(+ &mut racing,+ &[(Some("a"), 2, "2026-03-01T00:00:00Z", "a2", 4)],+ )+ .await;++ assert!(raced.load(AtomicOrdering::SeqCst));+ assert_eq!((stats.version, stats.versions_closed), (3, 1), "{stats:?}");+ plain.refresh().await.unwrap();+ assert_eq!(+ read(&plain).await,+ sorted(vec![+ closed(+ "2026-03",+ Some("a"),+ 1,+ "2026-01-01T00:00:00Z",+ "2026-03-01T00:00:00Z",+ "a1"+ ),+ open(Some("a"), 2, "2026-03-01T00:00:00Z", "a2"),+ closed(+ "2026-02",+ Some("b"),+ 1,+ "2026-01-01T00:00:00Z",+ "2026-02-01T00:00:00Z",+ "b1"+ ),+ open(Some("b"), 2, "2026-02-01T00:00:00Z", "b2"),+ ])+ );+}++#[tokio::test]+async fn a_call_that_keeps_no_row_makes_no_relax_commit() {+ let dir = tempfile::tempdir().unwrap();+ CreateBuilder::new()+ .with_location(dir.path().to_str().unwrap())+ .with_columns(vec![+ StructField::new("id", KernelType::STRING, true),+ StructField::new("v", KernelType::LONG, true),+ StructField::new(+ "valid_from",+ KernelType::Primitive(PrimitiveType::Timestamp),+ true,+ ),+ StructField::new(+ "valid_to",+ KernelType::Primitive(PrimitiveType::Timestamp),+ true,+ ),+ StructField::new("payload", KernelType::STRING, false),+ StructField::new("p", KernelType::STRING, false),+ ])+ .with_partition_columns(vec!["p".to_string()])+ .await+ .unwrap();+ let mut h = TableHandle::open(dir.path().to_str().unwrap().into(), HashMap::new())+ .await+ .unwrap();+ apply(&mut h, &[(Some("a"), 1, "2026-01-01T00:00:00Z", "a1", 1)]).await;+ let with_null_payload = |v: i64, at: &str| {+ let batch = source(&[(Some("a"), v, at, "", 2)]);+ let mut cols = batch.columns().to_vec();+ cols[3] = Arc::new(StringArray::from(vec![None::<&str>]));+ RecordBatch::try_new(source_schema(), cols).unwrap()+ };+ let version = h.version();++ // A re-read with a null in the non-nullable column: nothing is kept, nothing commits.+ let reread = h+ .history_upsert(+ vec![with_null_payload(1, "2026-01-01T00:00:00Z")],+ source_schema(),+ opts(),+ MultipartConfig::default(),+ )+ .await+ .unwrap();+ assert_eq!(+ (reread.committed, reread.columns_relaxed),+ (false, 0),+ "{reread:?}"+ );+ assert_eq!(h.version(), version);+ assert!(h+ .schema_json()+ .unwrap()+ .contains(r#""name":"payload","type":"string","nullable":false"#));++ // A kept row with the same null relaxes the column and lands.+ let kept = h+ .history_upsert(+ vec![with_null_payload(2, "2026-02-01T00:00:00Z")],+ source_schema(),+ opts(),+ MultipartConfig::default(),+ )+ .await+ .unwrap();+ assert_eq!(+ (kept.committed, kept.columns_relaxed),+ (true, 1),+ "{kept:?}"+ );+ assert_eq!(kept.version, version + 2);+}++#[tokio::test]+async fn closes_many_rows_under_tiny_budgets_and_returns_every_permit() {+ let dir = tempfile::tempdir().unwrap();+ let mut h = create(dir.path(), &["p"]).await;+ let ids: Vec<String> = (0..400).map(|i| format!("k{i:04}")).collect();+ for chunk in ids.chunks(100) {+ let rows: Vec<Src> = chunk+ .iter()+ .map(|id| (Some(id.as_str()), 1, "2026-01-01T00:00:00Z", "old", 1))+ .collect();+ apply(&mut h, &rows).await;+ }++ let limits = Arc::new(ProcessLimits::with_fetch(1, 1, 1024, 1024));+ let tiny = HistoryUpsertOptions {+ max_parallel_files: 4,+ probe_concurrency: 4,+ max_buffered_bytes: 1024,+ max_fetch_bytes: 1024,+ read_batch_size: 16,+ target_file_size: Some(4 * 1024),+ limits: limits.clone(),+ ..opts()+ };+ // The closes land in 50 months, more than the closed-row writer keeps open at once.+ let instants: Vec<String> = (0..ids.len())+ .map(|i| format!("{}-{:02}-15T00:00:00Z", 2026 + i % 50 / 12, i % 50 % 12 + 1))+ .collect();+ let rows: Vec<Src> = ids+ .iter()+ .zip(&instants)+ .map(|(id, at)| (Some(id.as_str()), 2, at.as_str(), "new", 2))+ .collect();+ let stats = apply_with(&mut h, &rows, tiny).await.unwrap();+ assert_eq!(+ (+ stats.files_removed,+ stats.versions_closed,+ stats.versions_opened+ ),+ (4, 400, 400),+ "{stats:?}"+ );+ let snapshot = limits.snapshot();+ for budget in [+ snapshot.partitions,+ snapshot.files,+ snapshot.buffer_kb,+ snapshot.fetch_kb,+ ] {+ assert_eq!(budget.available, budget.total, "{snapshot:?}");+ }+ let live = read(&h).await;+ assert_eq!(live.len(), 800);+ let mut per_month: HashMap<String, usize> = HashMap::new();+ for row in live.iter().filter(|r| r.0 != "current") {+ *per_month.entry(row.0.clone()).or_default() += 1;+ }+ assert_eq!(per_month.len(), 50);+ assert!(per_month.values().all(|n| *n == 8), "{per_month:?}");+}++#[tokio::test]+async fn key_bucket_ids_are_fixed_and_follow_the_table_key_types() {+ let dir = tempfile::tempdir().unwrap();+ let h = create(dir.path(), &["p"]).await;+ let keys = ["id".to_string(), "v".to_string()];+ let batch = |v: Arc<dyn Array>| {+ let ids = StringArray::from(vec![+ Some("a"),+ Some("b"),+ None,+ Some("a"),+ Some("customer-42"),+ ]);+ RecordBatch::try_new(+ Arc::new(Schema::new(vec![+ Field::new("id", DataType::Utf8, true),+ Field::new("v", v.data_type().clone(), true),+ ])),+ vec![Arc::new(ids), v],+ )+ .unwrap()+ };+ let typed = batch(Arc::new(Int64Array::from(vec![+ Some(1),+ Some(1),+ Some(1),+ None,+ Some(7),+ ])));+ let buckets = h+ .key_buckets(std::slice::from_ref(&typed), &keys, 16)+ .unwrap();+ assert_eq!(buckets.values().to_vec(), vec![4, 12, 0, 6, 6]);++ // The same key gets the same bucket in a later call with other rows around it.+ let again = h+ .key_buckets(&[typed.slice(3, 2), typed.slice(0, 3)], &keys, 16)+ .unwrap();+ assert_eq!(+ again.values().to_vec(),+ [&buckets.values()[3..], &buckets.values()[..3]].concat()+ );++ // Strings that cast to the same table key share its bucket, as they share the key.+ let strings = batch(Arc::new(StringArray::from(vec![+ Some("01"),+ Some("1"),+ Some("001"),+ None,+ Some("07"),+ ])));+ assert_eq!(+ h.key_buckets(&[strings], &keys, 16)+ .unwrap()+ .values()+ .to_vec(),+ buckets.values().to_vec()+ );+}++// ---- a store that lands a competing commit just before a commit file is written ------++type Race = Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>;++/// Armed races by table path prefix; each fires once, on the first commit file written+/// under its prefix.+static RACES: LazyLock<Mutex<Vec<(String, Race)>>> = LazyLock::new(Default::default);++fn arm_race(dir: &std::path::Path, race: Race) {+ let prefix = Path::from(dir.to_string_lossy().as_ref()).to_string();+ RACES.lock().unwrap().push((prefix, race));+}++fn take_race(location: &Path) -> Option<Race> {+ let mut races = RACES.lock().unwrap();+ let i = races+ .iter()+ .position(|(prefix, _)| location.as_ref().starts_with(prefix.as_str()))?;+ Some(races.swap_remove(i).1)+}++#[derive(Debug)]+struct RaceStore {+ inner: Arc<dyn ObjectStore>,+}++impl std::fmt::Display for RaceStore {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "RaceStore({})", self.inner)+ }+}++#[async_trait::async_trait]+impl ObjectStore for RaceStore {+ async fn put_opts(+ &self,+ location: &Path,+ payload: PutPayload,+ opts: PutOptions,+ ) -> object_store::Result<PutResult> {+ let is_commit = matches!(opts.mode, PutMode::Create)+ && location.as_ref().contains("_delta_log/")+ && location.as_ref().ends_with(".json");+ if is_commit {+ if let Some(race) = take_race(location) {+ race().await;+ }+ }+ self.inner.put_opts(location, payload, opts).await+ }++ async fn put_multipart_opts(+ &self,+ location: &Path,+ opts: PutMultipartOptions,+ ) -> object_store::Result<Box<dyn MultipartUpload>> {+ self.inner.put_multipart_opts(location, opts).await+ }++ async fn get_opts(+ &self,+ location: &Path,+ options: GetOptions,+ ) -> object_store::Result<GetResult> {+ self.inner.get_opts(location, options).await+ }++ async fn get_ranges(+ &self,+ location: &Path,+ ranges: &[std::ops::Range<u64>],+ ) -> object_store::Result<Vec<Bytes>> {+ self.inner.get_ranges(location, ranges).await+ }++ fn delete_stream(+ &self,+ locations: BoxStream<'static, object_store::Result<Path>>,+ ) -> BoxStream<'static, object_store::Result<Path>> {+ self.inner.delete_stream(locations)+ }++ // Sorted: cloud stores list lexicographically and delta-kernel relies on it.+ fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result<ObjectMeta>> {+ sorted_listing(self.inner.list(prefix))+ }++ fn list_with_offset(+ &self,+ prefix: Option<&Path>,+ offset: &Path,+ ) -> BoxStream<'static, object_store::Result<ObjectMeta>> {+ sorted_listing(self.inner.list_with_offset(prefix, offset))+ }++ async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result<ListResult> {+ self.inner.list_with_delimiter(prefix).await+ }++ async fn copy_opts(+ &self,+ from: &Path,+ to: &Path,+ options: CopyOptions,+ ) -> object_store::Result<()> {+ self.inner.copy_opts(from, to, options).await+ }++ async fn rename_opts(+ &self,+ from: &Path,+ to: &Path,+ options: RenameOptions,+ ) -> object_store::Result<()> {+ self.inner.rename_opts(from, to, options).await+ }+}++fn sorted_listing(+ stream: BoxStream<'static, object_store::Result<ObjectMeta>>,+) -> BoxStream<'static, object_store::Result<ObjectMeta>> {+ use futures::{StreamExt, TryStreamExt};+ futures::stream::once(async move {+ let mut items: Vec<ObjectMeta> = stream.try_collect().await?;+ items.sort_by(|a, b| a.location.cmp(&b.location));+ Ok::<_, object_store::Error>(futures::stream::iter(items.into_iter().map(Ok)))+ })+ .try_flatten()+ .boxed()+}++#[derive(Debug, Default)]+struct RaceFactory;++impl ObjectStoreFactory for RaceFactory {+ fn parse_url_opts(+ &self,+ url: &url::Url,+ _config: &StorageConfig,+ ) -> DeltaResult<(Arc<dyn ObjectStore>, Path)> {+ let store = RaceStore {+ inner: Arc::new(LocalFileSystem::new()),+ };+ Ok((Arc::new(store), Path::from_url_path(url.path())?))+ }+}++impl LogStoreFactory for RaceFactory {+ fn with_options(+ &self,+ prefixed_store: Arc<dyn ObjectStore>,+ root_store: Arc<dyn ObjectStore>,+ location: &url::Url,+ options: &StorageConfig,+ ) -> DeltaResult<Arc<dyn LogStore>> {+ Ok(default_logstore(+ prefixed_store,+ root_store,+ location,+ options,+ ))+ }+}++fn register_race_scheme() {+ let scheme = url::Url::parse("dlrace://").expect("static url");+ object_store_factories().insert(scheme.clone(), Arc::new(RaceFactory));+ logstore_factories().insert(scheme, Arc::new(RaceFactory));+}diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.mdindex bcca01c9761a..7f47d75d81f9 100644--- a/rust/deltalite/python/README.md+++ b/rust/deltalite/python/README.md@@ -102,6 +102,8 @@ rejected (raising `DeltaLiteError`) rather than silently double-inserted.| `DeltaLiteTable.is_deltatable(uri, storage_options=None)` | `True` if a Delta table exists at `uri`. || `DeltaLiteTable.table_exists(uri, storage_options=None)` | `True` if the log at `uri` holds a commit or a checkpoint, the same check as `DeltaTable.is_deltatable`. Storage errors raise; they never read as `False`. || `.upsert(data, primary_keys, partition_key=None, **opts)` | Insert-or-replace `data` by key. Returns `UpsertStats`. See knobs below. |+| `.history_upsert(data, key_columns, order_column, valid_from_column, valid_to_column, partition_column, current_partition, tie_break_column, **opts)` | Apply versions to an SCD2 history table in one data commit. Returns `HistoryUpsertStats`. See "History upsert" below. |+| `.key_buckets(data, key_columns, n)` | Bucket ids for splitting a `history_upsert` source by key. See "History upsert" below. || `.compact(**opts)` | Small-file compaction, the replacement for `DeltaTable.optimize.compact`. Returns a stats dict. See "Compaction" below. || `.restore(version, *, commit_metadata=None)` | delta-rs's restore to `version`, in one commit, with the GIL released. Returns `numRemovedFile`, `numRestoredFile` and the committed `version`. The handle moves onto the restored table. || `.version()` | Current table version (`int`). |@@ -139,6 +141,40 @@ summed over partitions, part of `rewrite_ms`), `commit_ms`non-zero only on checkpoint-boundary commits). `columns_relaxed` countsnon-nullable columns flipped to nullable before the write.+### History upsert++`history_upsert` writes an SCD2 history table: one row per version of a key.+Open versions (`valid_to_column` null) live in the `current_partition` value of `partition_column`.+A closed version lives in the partition named by its `valid_to_column` as `%Y-%m` in UTC, which is what `pyarrow.compute.strftime(valid_to, "%Y-%m")` returns for a UTC timestamp.+The table must be partitioned by exactly `partition_column`, a string column, or the call raises `DeltaLiteUnsupportedTableError`.++Per key (`key_columns`, compared null-safely: a null key part matches a null key part):++1. Rows with the same `order_column` value collapse to the one with the highest `tie_break_column` value. On an equal tie-break value, the later row wins.+2. Only rows with an order value strictly greater than the key's open row are kept. Equal rows count as `rows_reread`, lower rows as `rows_backwards`.+3. The kept rows form a chain: each closes at the next kept row's `valid_from_column`, and the newest stays open.+4. When a key keeps a row and has an open row, the open row closes at the first kept row's `valid_from_column` and moves to its month partition.++The source must not carry `valid_to_column` or `partition_column`, since the operation sets them.+A null order value or a null `valid_from_column` raises `DeltaLiteError`.+`tie_break_column` must have a fixed-width type, such as an integer or a timestamp, and is written only when the table has that column.+Closed rows are only appended, so earlier closed versions of a key stay.+A source that keeps no row, such as a re-run of the same input, makes no commit and returns `committed=False`, so a re-run after a crash is safe. That includes the nullability relax: it commits only when the call also writes rows.++Each call reads the key and order columns of every current-partition file that its stats do not rule out, so a file that holds only re-read keys is not rewritten.+The probe cache is not used.+The knobs are the `upsert` ones that apply: `commit_metadata`, `max_parallel_files`, `max_buffered_bytes`, `max_fetch_bytes`, `probe_concurrency`, `read_batch_size`, `target_file_size`, `max_row_group_bytes`, `max_source_bytes`, `read_ahead_row_groups` and the multipart and ranged-read settings.+`max_parallel_partitions` is accepted and has no effect, because only the current partition is rewritten.+A commit that finds a newer table version is never rebased onto it, since delta-rs does not check a concurrent commit that only adds files: the call plans again against the new version, up to five times, then raises `DeltaLiteCommitConflictError`.++`HistoryUpsertStats` counts `rows_in`, `rows_deduped`, `rows_reread`, `rows_backwards`, `versions_opened`, `versions_closed` (open rows closed), `chain_rows_closed` (kept rows written closed), `rows_copied`, `files_added`, `files_removed`, `files_carried_over`, `files_probed`, `bytes_read` (stored size of the rewritten files) and `bytes_written`, with `version`, `committed` and the `upsert` timings.++`DeltaLiteTable.key_buckets(data, key_columns, n)` returns a `pyarrow.UInt32Array` with a bucket id in `0..n` for every row, for splitting a source that is too large for one call into calls on disjoint keys.+The key columns are first cast to the table's types, as `history_upsert` casts them before it matches keys, so source values that become one table key always share a bucket.+Each key is then encoded with arrow-row (null-safe) and hashed with xxHash64 under a fixed seed, then reduced modulo `n`.+The ids are the same in every process that runs one deltalite build, for keys of the same Arrow types.+arrow-row may change its encoding between arrow releases, so do not persist the ids.+### Compaction`compact` plans bins exactly as delta-rs's `optimize.compact` does, from the@@ -338,7 +374,8 @@ labels only): `deltalite_upserts_total` (`outcome`, `prune_strategy`,`deltalite.upsert` and asserts identical logical content — not by versionequality.- **Not supported:** tables with deletion vectors or column mapping (detected- and raised as `DeltaLiteUnsupportedTableError`), and SCD2 merges.+ and raised as `DeltaLiteUnsupportedTableError`). SCD2 history tables are+ written with `history_upsert`, not with `upsert`.- deltalite rejects duplicate source primary keys that `MERGE` silentlydouble-inserts — check pre-existing data if you migrate an existing pipeline.diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rsindex f46792891f55..a75769260bfe 100644--- a/rust/deltalite/python/src/lib.rs+++ b/rust/deltalite/python/src/lib.rs@@ -14,6 +14,7 @@ use arrow_schema::SchemaRef;use deltalake::writer::RecordBatchWriter;use deltalite_core::errors::Error;use deltalite_core::handle::TableHandle;+use deltalite_core::history::HistoryUpsertOptions;use deltalite_core::limits::{Budget, ProcessLimits};use deltalite_core::probecache::ProbeCache;use deltalite_core::schema::import_column;@@ -238,6 +239,119 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {}}+/// Counters describing what one `history_upsert` did.+#[pyclass(module = "deltalite", skip_from_py_object)]+#[derive(Clone, Debug, Default)]+pub struct HistoryUpsertStats {+ #[pyo3(get)]+ pub version: i64,+ #[pyo3(get)]+ pub committed: bool,+ #[pyo3(get)]+ pub rows_in: usize,+ #[pyo3(get)]+ pub rows_deduped: usize,+ #[pyo3(get)]+ pub rows_reread: usize,+ #[pyo3(get)]+ pub rows_backwards: usize,+ #[pyo3(get)]+ pub versions_opened: usize,+ #[pyo3(get)]+ pub versions_closed: usize,+ #[pyo3(get)]+ pub chain_rows_closed: usize,+ #[pyo3(get)]+ pub rows_copied: usize,+ #[pyo3(get)]+ pub files_removed: usize,+ #[pyo3(get)]+ pub files_added: usize,+ #[pyo3(get)]+ pub files_carried_over: usize,+ #[pyo3(get)]+ pub files_probed: usize,+ #[pyo3(get)]+ pub bytes_read: u64,+ #[pyo3(get)]+ pub bytes_written: u64,+ #[pyo3(get)]+ pub plan_ms: u64,+ #[pyo3(get)]+ pub probe_ms: u64,+ #[pyo3(get)]+ pub rewrite_ms: u64,+ #[pyo3(get)]+ pub commit_ms: u64,+ #[pyo3(get)]+ pub columns_relaxed: usize,+ #[pyo3(get)]+ pub open_ms: u64,+ #[pyo3(get)]+ pub initial_open_ms: u64,+ #[pyo3(get)]+ pub ingest_ms: u64,+ #[pyo3(get)]+ pub relax_ms: u64,+ #[pyo3(get)]+ pub maintenance_ms: u64,+}++#[pymethods]+impl HistoryUpsertStats {+ fn __repr__(&self) -> String {+ format!(+ "HistoryUpsertStats(version={}, committed={}, rows_in={}, rows_deduped={}, \+ rows_reread={}, rows_backwards={}, versions_opened={}, versions_closed={}, \+ chain_rows_closed={}, files_added={}, files_removed={})",+ self.version,+ self.committed,+ self.rows_in,+ self.rows_deduped,+ self.rows_reread,+ self.rows_backwards,+ self.versions_opened,+ self.versions_closed,+ self.chain_rows_closed,+ self.files_added,+ self.files_removed+ )+ }+}++impl From<deltalite_core::history::HistoryUpsertStats> for HistoryUpsertStats {+ fn from(s: deltalite_core::history::HistoryUpsertStats) -> Self {+ Self {+ version: s.version,+ committed: s.committed,+ rows_in: s.rows_in,+ rows_deduped: s.rows_deduped,+ rows_reread: s.rows_reread,+ rows_backwards: s.rows_backwards,+ versions_opened: s.versions_opened,+ versions_closed: s.versions_closed,+ chain_rows_closed: s.chain_rows_closed,+ rows_copied: s.rows_copied,+ files_removed: s.files_removed,+ files_added: s.files_added,+ files_carried_over: s.files_carried_over,+ files_probed: s.files_probed,+ bytes_read: s.bytes_read,+ bytes_written: s.bytes_written,+ plan_ms: s.plan_ms,+ probe_ms: s.probe_ms,+ rewrite_ms: s.rewrite_ms,+ commit_ms: s.commit_ms,+ columns_relaxed: s.columns_relaxed,+ open_ms: s.open_ms,+ initial_open_ms: s.initial_open_ms,+ ingest_ms: s.ingest_ms,+ relax_ms: s.relax_ms,+ maintenance_ms: s.maintenance_ms,+ }+ }+}+/// A handle on a Delta table for deltalite writes, plus snapshot reads (version, table/// id, schema, live files) served from the loaded state so a writer does not need a/// second delta-rs `DeltaTable` open for them. `history` is the one read that goes back@@ -527,6 +641,127 @@ impl DeltaLiteTable {Ok(stats)}+ /// Apply a batch of versions to an SCD2 history table in one data commit. See the+ /// README's "History upsert" section for the rules, refusals and knobs.+ /// `max_parallel_partitions` is accepted and has no effect.+ #[pyo3(signature = (+ data,+ key_columns,+ order_column,+ valid_from_column,+ valid_to_column,+ partition_column,+ current_partition,+ tie_break_column,+ *,+ commit_metadata = None,+ max_parallel_partitions = 1,+ max_parallel_files = 4,+ max_buffered_bytes = 67108864,+ max_fetch_bytes = 134217728,+ probe_concurrency = 8,+ read_batch_size = 8192,+ target_file_size = None,+ max_row_group_bytes = None,+ max_source_bytes = None,+ multipart_threshold = None,+ multipart_part_size = None,+ multipart_concurrency = None,+ range_get_part_size = None,+ range_get_concurrency = None,+ read_ahead_row_groups = false,+ ))]+ #[allow(clippy::too_many_arguments)]+ fn history_upsert(+ &mut self,+ py: Python<'_>,+ data: &Bound<'_, PyAny>,+ key_columns: Vec<String>,+ order_column: String,+ valid_from_column: String,+ valid_to_column: String,+ partition_column: String,+ current_partition: String,+ tie_break_column: String,+ commit_metadata: Option<HashMap<String, String>>,+ max_parallel_partitions: usize,+ max_parallel_files: usize,+ max_buffered_bytes: usize,+ max_fetch_bytes: usize,+ probe_concurrency: usize,+ read_batch_size: usize,+ target_file_size: Option<usize>,+ max_row_group_bytes: Option<usize>,+ max_source_bytes: Option<usize>,+ multipart_threshold: Option<usize>,+ multipart_part_size: Option<usize>,+ multipart_concurrency: Option<usize>,+ range_get_part_size: Option<usize>,+ range_get_concurrency: Option<usize>,+ read_ahead_row_groups: bool,+ ) -> PyResult<HistoryUpsertStats> {+ let _ = max_parallel_partitions;+ let ingest_started = std::time::Instant::now();+ let (schema, batches) = read_pyarrow(data)?;+ let ingest_ms = ingest_started.elapsed().as_millis() as u64;++ let opts = HistoryUpsertOptions {+ key_columns,+ order_column,+ valid_from_column,+ valid_to_column,+ partition_column,+ current_partition,+ tie_break_column,+ commit_metadata: commit_metadata.map(|m| {+ m.into_iter()+ .map(|(k, v)| (k, Value::String(v)))+ .collect::<HashMap<String, Value>>()+ }),+ probe_concurrency,+ max_parallel_files,+ max_buffered_bytes,+ max_fetch_bytes,+ read_ahead_row_groups,+ read_batch_size,+ target_file_size,+ max_row_group_bytes,+ max_source_bytes,+ limits: ProcessLimits::global().clone(),+ };+ let multipart = MultipartConfig::resolve(+ multipart_threshold,+ multipart_part_size,+ multipart_concurrency,+ range_get_part_size,+ range_get_concurrency,+ );+ let handle = &mut self.handle;+ let stats = py+ .detach(|| runtime().block_on(handle.history_upsert(batches, schema, opts, multipart)))+ .map_err(to_py_err)?;+ let mut stats: HistoryUpsertStats = stats.into();+ stats.ingest_ms = ingest_ms;+ Ok(stats)+ }++ /// A null-safe bucket id in `0..n` (`pyarrow.UInt32Array`) for every row of `data`,+ /// from `key_columns` cast to this table's types. See the README.+ fn key_buckets(+ &mut self,+ py: Python<'_>,+ data: &Bound<'_, PyAny>,+ key_columns: Vec<String>,+ n: u32,+ ) -> PyResult<Py<PyAny>> {+ let (_, batches) = read_pyarrow(data)?;+ let handle = &mut self.handle;+ let buckets = py+ .detach(|| handle.key_buckets(&batches, &key_columns, n))+ .map_err(to_py_err)?;+ Ok(buckets.to_data().to_pyarrow(py)?.unbind())+ }+/// Compact small files the way `DeltaTable.optimize.compact` does, streaming each bin/// one file and one byte-bounded batch at a time (see `deltalite_core::compact`).///@@ -772,6 +1007,7 @@ fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;m.add_function(wrap_pyfunction!(limits_snapshot, m)?)?;m.add_class::<UpsertStats>()?;+ m.add_class::<HistoryUpsertStats>()?;m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;m.add("DeltaLiteCommitConflictError",diff --git a/rust/deltalite/python/tests/test_history.py b/rust/deltalite/python/tests/test_history.pynew file mode 100644index 000000000000..74f070181d72--- /dev/null+++ b/rust/deltalite/python/tests/test_history.py@@ -0,0 +1,267 @@+from __future__ import annotations++import json+import random+import dataclasses+from datetime import UTC, datetime, timedelta+from pathlib import Path++import pytest++import pyarrow as pa+import deltalake+import deltalite+import pyarrow.compute as pc++CURRENT = "current"+TS = pa.timestamp("us", tz="UTC")+MONTH_VECTORS = json.loads(+ (Path(__file__).resolve().parents[2] / "core" / "tests" / "fixtures" / "month_vectors.json").read_text()+)+TABLE_SCHEMA = pa.schema(+ [+ pa.field("id", pa.string()),+ pa.field("n", pa.int64()),+ pa.field("ts", TS),+ pa.field("payload", pa.string()),+ pa.field("valid_to", TS),+ pa.field("p", pa.string(), nullable=False),+ ]+)+SOURCE_SCHEMA = pa.schema(+ [+ pa.field("id", pa.string()),+ pa.field("n", pa.int64()),+ pa.field("ts", TS),+ pa.field("payload", pa.string()),+ pa.field("seq", pa.int64()),+ ]+)+++# The product names month partitions with `pc.strftime(valid_to, "%Y-%m")`.+def month(instant: datetime) -> str:+ return pc.strftime(pa.array([instant], TS), format="%Y-%m")[0].as_py()++[email protected](frozen=True)+class Src:+ id: str | None+ n: int | None+ ts: datetime+ payload: str+ seq: int++ @property+ def key(self) -> tuple[str | None, int | None]:+ return (self.id, self.n)++[email protected](frozen=True)+class Row:+ p: str+ id: str | None+ n: int | None+ ts: datetime+ valid_to: datetime | None+ payload: str++[email protected](frozen=True)+class Counts:+ committed: bool+ rows_reread: int+ rows_backwards: int+ versions_opened: int+ versions_closed: int+ chain_rows_closed: int+++def model_apply(rows: list[Row], source: list[Src]) -> tuple[list[Row], Counts]:+ open_rows = {(r.id, r.n): r for r in rows if r.p == CURRENT}+ best: dict[tuple, Src] = {}+ for s in source:+ k = (s.key, s.ts)+ if k not in best or s.seq >= best[k].seq:+ best[k] = s+ by_key: dict[tuple, list[Src]] = {}+ for s in sorted(best.values(), key=lambda s: s.ts):+ by_key.setdefault(s.key, []).append(s)++ out = [r for r in rows if r.p != CURRENT]+ reread = backwards = opened = closed = chained = 0+ for key, versions in by_key.items():+ old = open_rows.get(key)+ kept = [s for s in versions if old is None or s.ts > old.ts]+ if old is not None:+ reread += sum(s.ts == old.ts for s in versions)+ backwards += sum(s.ts < old.ts for s in versions)+ if not kept:+ continue+ if old is not None:+ open_rows.pop(key)+ out.append(dataclasses.replace(old, valid_to=kept[0].ts, p=month(kept[0].ts)))+ closed += 1+ for this, following in zip(kept, kept[1:]):+ out.append(Row(month(following.ts), this.id, this.n, this.ts, following.ts, this.payload))+ chained += 1+ last = kept[-1]+ out.append(Row(CURRENT, last.id, last.n, last.ts, None, last.payload))+ opened += 1+ out.extend(open_rows.values())+ return sorted(out, key=_order), Counts(opened > 0, reread, backwards, opened, closed, chained)+++def _order(r: Row) -> tuple:+ return (r.p, r.id or "", r.id is None, r.n or 0, r.n is None, r.ts, r.valid_to or r.ts, r.payload)+++def read(uri: str) -> list[Row]:+ t = deltalake.DeltaTable(uri).to_pyarrow_table()+ rows = [Row(d["p"], d["id"], d["n"], d["ts"], d["valid_to"], d["payload"]) for d in t.to_pylist()]+ return sorted(rows, key=_order)+++def source_table(rows: list[Src]) -> pa.Table:+ return pa.table(+ {+ "id": pa.array([r.id for r in rows], pa.string()),+ "n": pa.array([r.n for r in rows], pa.int64()),+ "ts": pa.array([r.ts for r in rows], TS),+ "payload": pa.array([r.payload for r in rows], pa.string()),+ "seq": pa.array([r.seq for r in rows], pa.int64()),+ },+ schema=SOURCE_SCHEMA,+ )+++def create(tmp_path: Path, partition_by: list[str] | None = None) -> str:+ uri = str(tmp_path / "history")+ deltalake.DeltaTable.create(uri, schema=TABLE_SCHEMA, partition_by=["p"] if partition_by is None else partition_by)+ return uri+++def split_into_batches(table: pa.Table, rng: random.Random) -> pa.Table:+ cuts = sorted(rng.randint(0, table.num_rows) for _ in range(rng.randint(0, 4)))+ bounds = [0, *cuts, table.num_rows]+ batches = [b for start, end in zip(bounds, bounds[1:]) for b in table.slice(start, end - start).to_batches()]+ if rng.random() < 0.5:+ batches.insert(rng.randint(0, len(batches)), pa.RecordBatch.from_pylist([], schema=SOURCE_SCHEMA))+ return pa.Table.from_batches(batches, schema=SOURCE_SCHEMA)+++def history_upsert(+ uri: str, rows: list[Src], split_rng: random.Random | None = None, **kwargs+) -> deltalite.HistoryUpsertStats:+ data = source_table(rows)+ if split_rng is not None:+ data = split_into_batches(data, split_rng)+ return deltalite.DeltaLiteTable.open(uri).history_upsert(+ data, ["id", "n"], "ts", "ts", "valid_to", "p", CURRENT, "seq", **kwargs+ )+++# Instants on both sides of month boundaries, so chains and closes cross partitions.+INSTANTS = sorted(+ {datetime.fromisoformat(v["valid_to"].replace("Z", "+00:00")) for v in MONTH_VECTORS}+ | {datetime(2026, 1, 1, tzinfo=UTC) + timedelta(days=13 * i, hours=i) for i in range(10)}+)++[email protected]("seed", range(60))+def test_matches_the_reference_model_over_random_calls(tmp_path, seed):+ rng = random.Random(seed)+ uri = create(tmp_path)+ keys = [(rng.choice(["a", "b", "c", None]), rng.choice([1, 2, None])) for _ in range(rng.randint(1, 8))]+ small_files = rng.random() < 0.5+ # Its own generator, so the batch cuts do not change the rows each seed generates.+ split_rng = random.Random(f"split-{seed}")+ expected: list[Row] = []+ seq = 0+ for call in range(rng.randint(2, 5)):+ source = []+ for _ in range(rng.randint(0, 25)):+ seq += 1+ key = rng.choice(keys)+ source.append(Src(key[0], key[1], rng.choice(INSTANTS), f"s{seed}-c{call}-{seq}", rng.randint(0, 10**6)))+ expected, counts = model_apply(expected, source)+ stats = history_upsert(+ uri,+ source,+ split_rng,+ **({"target_file_size": 2048, "read_batch_size": 3} if small_files else {}),+ )+ got = Counts(+ stats.committed,+ stats.rows_reread,+ stats.rows_backwards,+ stats.versions_opened,+ stats.versions_closed,+ stats.chain_rows_closed,+ )+ assert got == counts, f"call {call}: {stats!r}"+ assert read(uri) == expected, f"call {call}"+++def test_month_vectors_match_the_product_month_and_the_written_partitions(tmp_path):+ instants = [datetime.fromisoformat(v["valid_to"].replace("Z", "+00:00")) for v in MONTH_VECTORS]+ assert [month(i) for i in instants] == [v["month"] for v in MONTH_VECTORS]++ uri = create(tmp_path)+ opened_at = datetime(1960, 1, 1, tzinfo=UTC)+ history_upsert(uri, [Src(f"k{i}", 1, opened_at, "old", 1) for i in range(len(instants))])+ history_upsert(uri, [Src(f"k{i}", 1, instant, "new", 2) for i, instant in enumerate(instants)])+ closed = {r.id: r.p for r in read(uri) if r.valid_to is not None}+ assert closed == {f"k{i}": v["month"] for i, v in enumerate(MONTH_VECTORS)}++[email protected]("partition_by", [[], ["id"]])+def test_refuses_a_table_not_partitioned_by_the_history_column(tmp_path, partition_by):+ uri = create(tmp_path, partition_by)+ with pytest.raises(deltalite.DeltaLiteUnsupportedTableError):+ history_upsert(uri, [Src("a", 1, INSTANTS[0], "a", 1)])+++def test_key_buckets_are_null_safe_and_independent_of_batching(tmp_path):+ table = deltalite.DeltaLiteTable.open(create(tmp_path))+ data = pa.table(+ {+ "id": pa.array(["a", "b", None, "a", "customer-42"], pa.string()),+ "n": pa.array([1, 1, 1, None, 7], pa.int64()),+ }+ )+ buckets = table.key_buckets(data, ["id", "n"], 16)+ assert isinstance(buckets, pa.UInt32Array)+ # Pinned in the Rust suite too, so an encoding change shows up on the upgrade that makes it.+ assert buckets.to_pylist() == [4, 12, 0, 6, 6]+ rechunked = pa.Table.from_batches(data.slice(3).to_batches() + data.slice(0, 3).to_batches())+ assert table.key_buckets(rechunked, ["id", "n"], 16).to_pylist() == [6, 6, 4, 12, 0]+++def test_calls_split_by_key_bucket_equal_one_call_when_source_keys_need_a_cast(tmp_path):+ # "1", "01" and "001" are one integer key in the table, so they must share a bucket.+ def source(n_values: list[str]) -> pa.Table:+ rows = len(n_values)+ return pa.table(+ {+ "id": pa.array(["k"] * 3 + [f"other-{i}" for i in range(rows - 3)], pa.string()),+ "n": pa.array(n_values, pa.string()),+ "ts": pa.array(INSTANTS[:rows], TS),+ "payload": pa.array([f"p{i}" for i in range(rows)], pa.string()),+ "seq": pa.array(range(rows), pa.int64()),+ }+ )++ data = source(["1", "01", "001", *(str(i) for i in range(2, 12))])+ whole_uri, split_uri = create(tmp_path / "whole"), create(tmp_path / "split")+ args = (["id", "n"], "ts", "ts", "valid_to", "p", CURRENT, "seq")+ deltalite.DeltaLiteTable.open(whole_uri).history_upsert(data, *args)++ split = deltalite.DeltaLiteTable.open(split_uri)+ buckets = split.key_buckets(data, ["id", "n"], 8)+ assert len(set(buckets.to_pylist()[:3])) == 1+ # Any call order must give the same table; descending is as good as any.+ for bucket in sorted(set(buckets.to_pylist()), reverse=True):+ split.history_upsert(data.filter(pc.equal(buckets, bucket)), *args)++ assert read(split_uri) == read(whole_uri)