Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): keep deltalite upsert rows apart from survivors (#119727)

Tom Owersb7ce1e5d2e49
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a8a37c494b60..92d790a200c8 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -3982,7 +3982,7 @@ dependencies = [
[[package]]
name = "deltalite-python"
-version = "0.1.11"
+version = "0.1.12"
dependencies = [
"arrow",
"arrow-array",
diff --git a/rust/deltalite/README.md b/rust/deltalite/README.md
index e5f3eb33d3ed..8d12d5aaa0fb 100644
--- a/rust/deltalite/README.md
+++ b/rust/deltalite/README.md
@@ -113,6 +113,9 @@ Per-call (arguments to `DeltaLiteTable.upsert`): `max_parallel_partitions` (2),
(128 MiB: compressed row-group bytes readers hold before decoding), `prune_strategy`
(`probe`), `probe_concurrency` (8), `read_batch_size` (8192),
`target_file_size` (table's `delta.targetFileSize`, else delta-rs's 100 MiB),
+`isolate_source_rows` (off: the batch's rows share an output file with the copied
+survivors; on: they get their own files),
+`max_row_group_bytes` (`None`: row groups close only at the encoder's row cap),
`max_source_bytes` (2 GiB guard), `multipart_threshold` (64 MiB) /
`multipart_part_size` (16 MiB), `commit_max_retries` (15).
diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rs
index 63d652324a1d..1cb20e6ddb63 100644
--- a/rust/deltalite/core/src/upsert.rs
+++ b/rust/deltalite/core/src/upsert.rs
@@ -36,7 +36,7 @@ use arrow_select::filter::filter_record_batch;
use arrow_select::take::{take, take_record_batch};
use delta_kernel::expressions::Scalar;
use deltalake::kernel::transaction::{CommitBuilder, CommitProperties};
-use deltalake::kernel::{Action, LogicalFileView, MetadataExt as _, Remove, StructType};
+use deltalake::kernel::{Action, Add, LogicalFileView, MetadataExt as _, Remove, StructType};
use deltalake::protocol::checkpoints::{cleanup_metadata, create_checkpoint};
use deltalake::protocol::{DeltaOperation, SaveMode};
use deltalake::table::config::TablePropertiesExt;
@@ -242,6 +242,23 @@ pub struct UpsertOptions {
/// is the write-buffer flush threshold, so it is also a memory knob worth
/// `max_parallel_partitions x target_file_size` of exposure.
pub target_file_size: Option<usize>,
+ /// Write the source rows (updates and inserts) to their own output files instead of
+ /// appending them to the file that receives the survivors of the rewritten files.
+ ///
+ /// Off, a partition's output holds the copied survivors and the new rows together, so
+ /// the next batch that touches any of those new rows must copy every survivor again.
+ /// On, such a batch rewrites only the small file of new rows. The cost is up to one
+ /// more file per touched partition per upsert. Memory is unchanged: the survivors file
+ /// is closed before the first source row is written, as it is at `target_file_size`.
+ pub isolate_source_rows: bool,
+ /// Close an output row group once its encoded bytes reach this size (`None` closes
+ /// row groups only at the encoder's row cap).
+ ///
+ /// A reader fetches a whole compressed row group before it decodes any of it, so a
+ /// table written as one row group per file makes every later upsert hold whole files
+ /// per reader. The check runs between written batches, so a row group can pass the
+ /// cap by up to one batch.
+ pub max_row_group_bytes: Option<usize>,
/// Ceiling for the source-size guard; `None` resolves from the environment,
/// `Some(0)` disables. See [`crate::limits::resolve_max_source_bytes`].
pub max_source_bytes: Option<usize>,
@@ -264,6 +281,8 @@ impl Default for UpsertOptions {
commit_max_retries: 15,
read_batch_size: 8192,
target_file_size: None,
+ isolate_source_rows: false,
+ max_row_group_bytes: None,
max_source_bytes: None,
limits: ProcessLimits::global().clone(),
}
@@ -2253,6 +2272,7 @@ async fn rewrite_partition(
// in the sequential implementation. The channel is unbounded; memory is bounded by
// the byte-budget permits attached to every message, not by message count.
let (tx, mut rx) = mpsc::unbounded_channel::<(RecordBatch, BudgetPermit)>();
+ let max_row_group_bytes = opts.max_row_group_bytes;
let writer_task = tokio::spawn(async move {
let mut writer = writer;
let mut adds = Vec::new();
@@ -2264,9 +2284,13 @@ async fn rewrite_partition(
// Bound resident memory: the writer holds the open file until flushed, so
// flush on a size threshold instead of once at the end. Each flush yields
// Add actions and starts a new file.
- if writer.buffer_len() >= target_file_size {
- adds.extend(writer.flush().await?);
- }
+ roll_writer(
+ &mut writer,
+ &mut adds,
+ target_file_size,
+ max_row_group_bytes,
+ )
+ .await?;
}
Ok::<_, Error>((writer, adds))
});
@@ -2326,6 +2350,9 @@ async fn rewrite_partition(
if let Some(e) = reader_err {
return Err(e);
}
+ if opts.isolate_source_rows {
+ adds.extend(writer.flush().await?);
+ }
// Source rows are written LAST: an updated row therefore carries the source values,
// which is exactly `when_matched_update_all`. The ordering is semantic, not
@@ -2345,14 +2372,19 @@ async fn rewrite_partition(
// Flushing mid-source is order-safe: it closes the current file and opens
// another, and every source row still lands after every survivor.
let selected = sel.select_batch(b)?;
+ let slice_rows = source_slice_rows(&selected, opts.read_batch_size, max_row_group_bytes);
let mut offset = 0usize;
while offset < selected.num_rows() {
- let n = (selected.num_rows() - offset).min(opts.read_batch_size.max(1));
+ let n = (selected.num_rows() - offset).min(slice_rows);
writer.write(selected.slice(offset, n))?;
offset += n;
- if writer.buffer_len() >= target_file_size {
- adds.extend(writer.flush().await?);
- }
+ roll_writer(
+ &mut writer,
+ &mut adds,
+ target_file_size,
+ max_row_group_bytes,
+ )
+ .await?;
}
}
adds.extend(writer.flush().await?);
@@ -2380,6 +2412,40 @@ async fn rewrite_partition(
})
}
+/// Rows per source write. The row-group cap is checked between writes, so with a cap a
+/// source write holds about as many decoded bytes as a survivor batch, and a row group
+/// passes the cap by at most that much on either path. Otherwise one `read_batch_size`
+/// slice of wide rows goes in one write and closes as one row group, far past the cap.
+fn source_slice_rows(
+ batch: &RecordBatch,
+ read_batch_size: usize,
+ max_row_group_bytes: Option<usize>,
+) -> usize {
+ match max_row_group_bytes {
+ Some(cap) if batch.num_rows() > 0 => {
+ let bytes_per_row = batch.get_array_memory_size().div_ceil(batch.num_rows());
+ let slice_bytes = cap.min(INITIAL_DECODE_ESTIMATE_BYTES);
+ batch_rows_for_bytes_per_row(bytes_per_row, slice_bytes, read_batch_size)
+ }
+ _ => read_batch_size.max(1),
+ }
+}
+
+/// Close the open file at `target_file_size`, else its row group at `max_row_group_bytes`.
+async fn roll_writer(
+ writer: &mut StreamingWriter,
+ adds: &mut Vec<Add>,
+ target_file_size: usize,
+ max_row_group_bytes: Option<usize>,
+) -> Result<()> {
+ if writer.buffer_len() >= target_file_size {
+ adds.extend(writer.flush().await?);
+ } else if max_row_group_bytes.is_some_and(|cap| writer.in_progress_bytes() >= cap) {
+ writer.flush_row_groups()?;
+ }
+ Ok(())
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/rust/deltalite/core/src/writer.rs b/rust/deltalite/core/src/writer.rs
index 7ff3706f990e..b5c458742def 100644
--- a/rust/deltalite/core/src/writer.rs
+++ b/rust/deltalite/core/src/writer.rs
@@ -179,6 +179,14 @@ impl StreamingWriter {
self.open.values().map(OpenFile::buffer_len).sum()
}
+ /// Encoded bytes in the row groups the open files have not closed yet.
+ pub(crate) fn in_progress_bytes(&self) -> usize {
+ self.open
+ .values()
+ .map(|f| f.writer.in_progress_size())
+ .sum()
+ }
+
/// Rows in the row groups the open files have not closed yet.
pub(crate) fn in_progress_rows(&self) -> usize {
self.open
diff --git a/rust/deltalite/core/tests/upsert_layout.rs b/rust/deltalite/core/tests/upsert_layout.rs
new file mode 100644
index 000000000000..b357ed307e51
--- /dev/null
+++ b/rust/deltalite/core/tests/upsert_layout.rs
@@ -0,0 +1,276 @@
+//! Output layout of `upsert`: which rows share an output file, and therefore what the
+//! next upsert has to copy again. Both layouts must hold the same rows.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_array::{Array, Int64Array, RecordBatch, StringArray};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use bytes::Bytes;
+use deltalake::kernel::{DataType as KernelType, StructField};
+use deltalake::operations::create::CreateBuilder;
+use deltalake::writer::{DeltaWriter, RecordBatchWriter};
+use deltalake::Path;
+use deltalite_core::handle::TableHandle;
+use deltalite_core::limits::ProcessLimits;
+use deltalite_core::table::MultipartConfig;
+use deltalite_core::upsert::{UpsertOptions, UpsertStats};
+use object_store::ObjectStoreExt;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+
+const TABLE_ROWS: usize = 1000;
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::Utf8, false),
+ Field::new("v", DataType::Int64, true),
+ Field::new("payload", DataType::Utf8, true),
+ Field::new("p", DataType::Utf8, true),
+ ]))
+}
+
+fn rows(keys: &[String], v: i64) -> RecordBatch {
+ let n = keys.len();
+ let payload: Vec<String> = keys.iter().map(|k| format!("{k}-").repeat(20)).collect();
+ RecordBatch::try_new(
+ schema(),
+ vec![
+ Arc::new(StringArray::from(keys.to_vec())),
+ Arc::new(Int64Array::from(vec![v; n])),
+ Arc::new(StringArray::from(payload)),
+ Arc::new(StringArray::from(vec!["a"; n])),
+ ],
+ )
+ .unwrap()
+}
+
+fn keys(prefix: &str, ids: std::ops::Range<usize>) -> Vec<String> {
+ ids.map(|i| format!("{prefix}{i:06}")).collect()
+}
+
+/// A one-partition table whose rows `old000000..` sit in one file.
+async fn table(dir: &std::path::Path) -> TableHandle {
+ let mut table = CreateBuilder::new()
+ .with_location(dir.to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("pk", KernelType::STRING, false),
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new("payload", KernelType::STRING, true),
+ StructField::new("p", KernelType::STRING, true),
+ ])
+ .with_partition_columns(vec!["p".to_string()])
+ .await
+ .unwrap();
+ let mut w = RecordBatchWriter::for_table(&table).unwrap();
+ w.write(rows(&keys("old", 0..TABLE_ROWS), 0)).await.unwrap();
+ w.flush_and_commit(&mut table).await.unwrap();
+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())
+ .await
+ .unwrap()
+}
+
+fn opts() -> UpsertOptions {
+ UpsertOptions {
+ primary_keys: vec!["pk".to_string()],
+ partition_key: Some("p".to_string()),
+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),
+ ..Default::default()
+ }
+}
+
+async fn upsert_with(
+ handle: &mut TableHandle,
+ keys: &[String],
+ v: i64,
+ opts: UpsertOptions,
+) -> UpsertStats {
+ handle
+ .upsert(
+ vec![rows(keys, v)],
+ schema(),
+ opts,
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap()
+}
+
+async fn upsert(handle: &mut TableHandle, keys: &[String], v: i64, isolate: bool) -> UpsertStats {
+ let opts = UpsertOptions {
+ isolate_source_rows: isolate,
+ ..opts()
+ };
+ upsert_with(handle, keys, v, opts).await
+}
+
+async fn file_bytes(handle: &TableHandle, path: &str) -> Bytes {
+ handle
+ .table()
+ .object_store()
+ .get(&Path::parse(path).unwrap())
+ .await
+ .unwrap()
+ .bytes()
+ .await
+ .unwrap()
+}
+
+/// Every live row as (pk, v, payload), sorted.
+async fn read_rows(handle: &TableHandle) -> Vec<(String, i64, String)> {
+ let mut out = Vec::new();
+ for f in handle.files().unwrap() {
+ let bytes = file_bytes(handle, &f.path).await;
+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)
+ .unwrap()
+ .build()
+ .unwrap()
+ {
+ let batch = batch.unwrap();
+ let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone();
+ let (pk, v, payload) = (col("pk"), col("v"), col("payload"));
+ let pk = pk.as_any().downcast_ref::<StringArray>().unwrap();
+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();
+ let payload = payload.as_any().downcast_ref::<StringArray>().unwrap();
+ for i in 0..batch.num_rows() {
+ out.push((
+ pk.value(i).to_string(),
+ v.value(i),
+ payload.value(i).to_string(),
+ ));
+ }
+ }
+ }
+ out.sort();
+ out
+}
+
+/// The first upsert updates 10 old rows and inserts 10 new ones; the second touches the
+/// case's keys. With the option off, the first upsert writes its 20 rows into the file of
+/// the 990 copied survivors, so any later batch that touches them copies all of those
+/// survivors again. With it on, the 20 rows get their own file.
+#[tokio::test]
+async fn isolate_source_rows_copies_no_more_and_keeps_the_same_rows() {
+ // (case, second upsert keys, rows copied per upsert with the option off, then on)
+ type Case = (&'static str, Vec<String>, (usize, usize), (usize, usize));
+ let cases: [Case; 3] = [
+ (
+ "touches the rows the last upsert wrote",
+ [keys("old", 0..10), keys("new", 0..10)].concat(),
+ (990, 990),
+ (990, 0),
+ ),
+ (
+ "touches only old rows",
+ keys("old", 500..510),
+ (990, 1000),
+ (990, 980),
+ ),
+ (
+ "touches old rows and rows the last upsert wrote",
+ [keys("old", 500..505), keys("new", 0..5)].concat(),
+ (990, 1000),
+ (990, 1000),
+ ),
+ ];
+ let first = [keys("old", 0..10), keys("new", 0..10)].concat();
+ for (case, second, copied_off, copied_on) in cases {
+ let (dir_off, dir_on) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
+ let (mut off, mut on) = (table(dir_off.path()).await, table(dir_on.path()).await);
+
+ let first_off = upsert(&mut off, &first, 1, false).await;
+ let first_on = upsert(&mut on, &first, 1, true).await;
+ assert_eq!(first_off.files_added, 1, "{case}: {first_off:?}");
+ assert_eq!(first_on.files_added, 2, "{case}: {first_on:?}");
+
+ let second_off = upsert(&mut off, &second, 2, false).await;
+ let second_on = upsert(&mut on, &second, 2, true).await;
+ assert_eq!(
+ (first_off.rows_copied, second_off.rows_copied),
+ copied_off,
+ "{case}, option off: {first_off:?} {second_off:?}"
+ );
+ assert_eq!(
+ (first_on.rows_copied, second_on.rows_copied),
+ copied_on,
+ "{case}, option on: {first_on:?} {second_on:?}"
+ );
+ assert_eq!(second_off.rows_updated, second_on.rows_updated, "{case}");
+
+ let (rows_off, rows_on) = (read_rows(&off).await, read_rows(&on).await);
+ assert_eq!(rows_off.len(), TABLE_ROWS + 10, "{case}");
+ assert_eq!(rows_off, rows_on, "{case}");
+ }
+}
+
+/// An upsert that only inserts already writes the new rows apart from every survivor,
+/// so the option must add no file there.
+#[tokio::test]
+async fn isolate_source_rows_adds_no_file_to_an_insert_only_upsert() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut handle = table(dir.path()).await;
+ let stats = upsert(&mut handle, &keys("new", 0..10), 1, true).await;
+ assert_eq!(
+ (stats.files_added, stats.files_removed, stats.rows_copied),
+ (1, 0, 0),
+ "{stats:?}"
+ );
+}
+
+/// Rows per row group of each file the upsert added, with `read_batch_size = 100` so the
+/// writer sees survivor batches of at most 100 rows. A cap of one byte closes a row group
+/// after every survivor batch (an update), and after every source row because the source
+/// path also slices its writes by the cap (an update and an insert).
+#[tokio::test]
+async fn max_row_group_bytes_closes_row_groups_between_batches() {
+ // (case, keys, row groups per added file without a cap, then with a one-byte cap)
+ type Case = (&'static str, Vec<String>, Vec<usize>, Vec<usize>);
+ let cases: [Case; 2] = [
+ (
+ "update: survivors of 10 batches, the first missing 10 rows, then 10 source rows",
+ keys("old", 0..10),
+ vec![1000],
+ [vec![90], vec![100; 9], vec![1; 10]].concat(),
+ ),
+ (
+ "insert: 250 source rows",
+ keys("new", 0..250),
+ vec![250],
+ vec![1; 250],
+ ),
+ ];
+ for (case, batch, uncapped, capped) in cases {
+ let mut got = Vec::new();
+ for cap in [None, Some(1)] {
+ let dir = tempfile::tempdir().unwrap();
+ let mut handle = table(dir.path()).await;
+ let before: Vec<String> = handle
+ .files()
+ .unwrap()
+ .into_iter()
+ .map(|f| f.path)
+ .collect();
+ let o = UpsertOptions {
+ read_batch_size: 100,
+ max_row_group_bytes: cap,
+ ..opts()
+ };
+ upsert_with(&mut handle, &batch, 1, o).await;
+ let mut groups = Vec::new();
+ for f in handle.files().unwrap() {
+ if before.contains(&f.path) {
+ continue;
+ }
+ let meta =
+ ParquetRecordBatchReaderBuilder::try_new(file_bytes(&handle, &f.path).await)
+ .unwrap()
+ .metadata()
+ .clone();
+ groups.extend(meta.row_groups().iter().map(|rg| rg.num_rows() as usize));
+ }
+ got.push((groups, read_rows(&handle).await));
+ }
+ assert_eq!(got[0].0, uncapped, "{case}, no cap");
+ assert_eq!(got[1].0, capped, "{case}, one-byte cap");
+ assert_eq!(got[0].1, got[1].1, "{case}: the cap changed the rows");
+ }
+}
diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.toml
index 66d9501e45c0..2dc94b04c601 100644
--- a/rust/deltalite/python/Cargo.toml
+++ b/rust/deltalite/python/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "deltalite-python"
# Mirrors pyproject.toml in this directory (always identical); bump both together.
-version = "0.1.11"
+version = "0.1.12"
edition = "2021"
# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to
# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).
diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md
index f5d17f167110..08e5d1e9ea46 100644
--- a/rust/deltalite/python/README.md
+++ b/rust/deltalite/python/README.md
@@ -195,6 +195,8 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:
| `probe_concurrency` | `8` | Concurrent statistics/probe reads. |
| `read_batch_size` | `8192` | Row-group read batch size. |
| `target_file_size` | table setting, else 100 MiB | Output file size target. |
+| `isolate_source_rows` | `False` | Write the batch's rows to their own files, apart from the unchanged rows copied out of the rewritten files. A later batch that touches the same rows then rewrites only those files. Costs up to one more file per touched partition. |
+| `max_row_group_bytes` | `None` | Close an output row group at this many encoded bytes. A later upsert's reader fetches one whole row group at a time, so smaller row groups lower its memory. `None` closes row groups only at the encoder's row cap. |
| `max_source_bytes` | `2 GiB` | Oversized-batch guard (`0` disables). |
| `multipart_threshold` / `multipart_part_size` | `64 MiB` / `16 MiB` | Multipart upload thresholds (`0` threshold disables). |
| `commit_max_retries` | `15` | Conditional-put commit retry budget. |
diff --git a/rust/deltalite/python/harness/common.py b/rust/deltalite/python/harness/common.py
index ccb4a4a47e8e..4f5d71a38545 100644
--- a/rust/deltalite/python/harness/common.py
+++ b/rust/deltalite/python/harness/common.py
@@ -234,6 +234,7 @@ def run_scenario(
scenario: Scenario,
tmp: Path | str,
storage_options: dict[str, str] | None = None,
+ upsert_kwargs: dict[str, Any] | None = None,
) -> tuple[str, str]:
"""Apply the same batch sequence to two fresh tables via the two paths.
@@ -267,6 +268,7 @@ def run_scenario(
scenario.partition_key,
md,
storage_options=storage_options,
+ **(upsert_kwargs or {}),
)
return uri_m, uri_u
diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.toml
index 56e53f0efa90..d8ce1097b214 100644
--- a/rust/deltalite/python/pyproject.toml
+++ b/rust/deltalite/python/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "maturin"
[project]
name = "deltalite"
# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.
-version = "0.1.11"
+version = "0.1.12"
description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"
readme = "README.md"
requires-python = ">=3.12"
diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs
index 27e884b5c51c..d5bbdea8bac5 100644
--- a/rust/deltalite/python/src/lib.rs
+++ b/rust/deltalite/python/src/lib.rs
@@ -369,6 +369,14 @@ impl DeltaLiteTable {
/// `max_fetch_bytes` caps the compressed row-group bytes this call's file readers
/// hold between fetching and decoding (`DELTALITE_PROCESS_MAX_FETCH_BYTES` caps it
/// per process). A row group larger than the cap still runs, alone.
+ ///
+ /// `isolate_source_rows=True` writes the batch's rows to their own output files
+ /// instead of the file that receives the copied survivors, so a later batch that
+ /// touches the same rows does not copy those survivors again. Off by default.
+ ///
+ /// `max_row_group_bytes` closes an output row group once its encoded bytes reach the
+ /// value, so a later reader fetches less per row group. `None` (the default) closes
+ /// row groups only at the encoder's row cap.
#[pyo3(signature = (
data,
primary_keys,
@@ -385,6 +393,8 @@ impl DeltaLiteTable {
commit_max_retries = 15,
read_batch_size = 8192,
target_file_size = None,
+ isolate_source_rows = false,
+ max_row_group_bytes = None,
max_source_bytes = None,
multipart_threshold = None,
multipart_part_size = None,
@@ -407,6 +417,8 @@ impl DeltaLiteTable {
commit_max_retries: usize,
read_batch_size: usize,
target_file_size: Option<usize>,
+ isolate_source_rows: bool,
+ max_row_group_bytes: Option<usize>,
max_source_bytes: Option<usize>,
multipart_threshold: Option<usize>,
multipart_part_size: Option<usize>,
@@ -440,6 +452,8 @@ impl DeltaLiteTable {
commit_max_retries,
read_batch_size,
target_file_size,
+ isolate_source_rows,
+ max_row_group_bytes,
max_source_bytes,
// Shared with every other concurrent upsert in this process, so the
// process-level memory bound holds regardless of thread count.
diff --git a/rust/deltalite/python/tests/test_parity.py b/rust/deltalite/python/tests/test_parity.py
index 88a5ed318127..040ad4237082 100644
--- a/rust/deltalite/python/tests/test_parity.py
+++ b/rust/deltalite/python/tests/test_parity.py
@@ -262,10 +262,15 @@ SCENARIOS = [
]
+ "upsert_kwargs",
+ [{}, {"isolate_source_rows": True}, {"max_row_group_bytes": 1}],
+ ids=["default", "isolate_source_rows", "max_row_group_bytes"],
+)
@pytest.mark.parametrize("builder", SCENARIOS, ids=[f.__name__[3:] for f in SCENARIOS])
-def test_parity(builder, tmp_path):
+def test_parity(builder, upsert_kwargs, tmp_path):
scenario = builder()
- uri_m, uri_u = run_scenario(scenario, tmp_path)
+ uri_m, uri_u = run_scenario(scenario, tmp_path, upsert_kwargs=upsert_kwargs)
assert_parity(uri_m, uri_u, scenario.name)