:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): keep deltalite upsert rows apart from survivors (#119727)
Tom Owersb7ce1e5d2e49
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex a8a37c494b60..92d790a200c8 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -3982,7 +3982,7 @@ dependencies = [[[package]]name = "deltalite-python"-version = "0.1.11"+version = "0.1.12"dependencies = ["arrow","arrow-array",diff --git a/rust/deltalite/README.md b/rust/deltalite/README.mdindex e5f3eb33d3ed..8d12d5aaa0fb 100644--- a/rust/deltalite/README.md+++ b/rust/deltalite/README.md@@ -113,6 +113,9 @@ Per-call (arguments to `DeltaLiteTable.upsert`): `max_parallel_partitions` (2),(128 MiB: compressed row-group bytes readers hold before decoding), `prune_strategy`(`probe`), `probe_concurrency` (8), `read_batch_size` (8192),`target_file_size` (table's `delta.targetFileSize`, else delta-rs's 100 MiB),+`isolate_source_rows` (off: the batch's rows share an output file with the copied+survivors; on: they get their own files),+`max_row_group_bytes` (`None`: row groups close only at the encoder's row cap),`max_source_bytes` (2 GiB guard), `multipart_threshold` (64 MiB) /`multipart_part_size` (16 MiB), `commit_max_retries` (15).diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rsindex 63d652324a1d..1cb20e6ddb63 100644--- a/rust/deltalite/core/src/upsert.rs+++ b/rust/deltalite/core/src/upsert.rs@@ -36,7 +36,7 @@ use arrow_select::filter::filter_record_batch;use arrow_select::take::{take, take_record_batch};use delta_kernel::expressions::Scalar;use deltalake::kernel::transaction::{CommitBuilder, CommitProperties};-use deltalake::kernel::{Action, LogicalFileView, MetadataExt as _, Remove, StructType};+use deltalake::kernel::{Action, Add, LogicalFileView, MetadataExt as _, Remove, StructType};use deltalake::protocol::checkpoints::{cleanup_metadata, create_checkpoint};use deltalake::protocol::{DeltaOperation, SaveMode};use deltalake::table::config::TablePropertiesExt;@@ -242,6 +242,23 @@ pub struct UpsertOptions {/// is the write-buffer flush threshold, so it is also a memory knob worth/// `max_parallel_partitions x target_file_size` of exposure.pub target_file_size: Option<usize>,+ /// Write the source rows (updates and inserts) to their own output files instead of+ /// appending them to the file that receives the survivors of the rewritten files.+ ///+ /// Off, a partition's output holds the copied survivors and the new rows together, so+ /// the next batch that touches any of those new rows must copy every survivor again.+ /// On, such a batch rewrites only the small file of new rows. The cost is up to one+ /// more file per touched partition per upsert. Memory is unchanged: the survivors file+ /// is closed before the first source row is written, as it is at `target_file_size`.+ pub isolate_source_rows: bool,+ /// Close an output row group once its encoded bytes reach this size (`None` closes+ /// row groups only at the encoder's row cap).+ ///+ /// A reader fetches a whole compressed row group before it decodes any of it, so a+ /// table written as one row group per file makes every later upsert hold whole files+ /// per reader. The check runs between written batches, so a row group can pass the+ /// cap by up to one batch.+ pub max_row_group_bytes: Option<usize>,/// Ceiling for the source-size guard; `None` resolves from the environment,/// `Some(0)` disables. See [`crate::limits::resolve_max_source_bytes`].pub max_source_bytes: Option<usize>,@@ -264,6 +281,8 @@ impl Default for UpsertOptions {commit_max_retries: 15,read_batch_size: 8192,target_file_size: None,+ isolate_source_rows: false,+ max_row_group_bytes: None,max_source_bytes: None,limits: ProcessLimits::global().clone(),}@@ -2253,6 +2272,7 @@ async fn rewrite_partition(// in the sequential implementation. The channel is unbounded; memory is bounded by// the byte-budget permits attached to every message, not by message count.let (tx, mut rx) = mpsc::unbounded_channel::<(RecordBatch, BudgetPermit)>();+ let max_row_group_bytes = opts.max_row_group_bytes;let writer_task = tokio::spawn(async move {let mut writer = writer;let mut adds = Vec::new();@@ -2264,9 +2284,13 @@ async fn rewrite_partition(// Bound resident memory: the writer holds the open file until flushed, so// flush on a size threshold instead of once at the end. Each flush yields// Add actions and starts a new file.- if writer.buffer_len() >= target_file_size {- adds.extend(writer.flush().await?);- }+ roll_writer(+ &mut writer,+ &mut adds,+ target_file_size,+ max_row_group_bytes,+ )+ .await?;}Ok::<_, Error>((writer, adds))});@@ -2326,6 +2350,9 @@ async fn rewrite_partition(if let Some(e) = reader_err {return Err(e);}+ if opts.isolate_source_rows {+ adds.extend(writer.flush().await?);+ }// Source rows are written LAST: an updated row therefore carries the source values,// which is exactly `when_matched_update_all`. The ordering is semantic, not@@ -2345,14 +2372,19 @@ async fn rewrite_partition(// Flushing mid-source is order-safe: it closes the current file and opens// another, and every source row still lands after every survivor.let selected = sel.select_batch(b)?;+ let slice_rows = source_slice_rows(&selected, opts.read_batch_size, max_row_group_bytes);let mut offset = 0usize;while offset < selected.num_rows() {- let n = (selected.num_rows() - offset).min(opts.read_batch_size.max(1));+ let n = (selected.num_rows() - offset).min(slice_rows);writer.write(selected.slice(offset, n))?;offset += n;- if writer.buffer_len() >= target_file_size {- adds.extend(writer.flush().await?);- }+ roll_writer(+ &mut writer,+ &mut adds,+ target_file_size,+ max_row_group_bytes,+ )+ .await?;}}adds.extend(writer.flush().await?);@@ -2380,6 +2412,40 @@ async fn rewrite_partition(})}+/// Rows per source write. The row-group cap is checked between writes, so with a cap a+/// source write holds about as many decoded bytes as a survivor batch, and a row group+/// passes the cap by at most that much on either path. Otherwise one `read_batch_size`+/// slice of wide rows goes in one write and closes as one row group, far past the cap.+fn source_slice_rows(+ batch: &RecordBatch,+ read_batch_size: usize,+ max_row_group_bytes: Option<usize>,+) -> usize {+ match max_row_group_bytes {+ Some(cap) if batch.num_rows() > 0 => {+ let bytes_per_row = batch.get_array_memory_size().div_ceil(batch.num_rows());+ let slice_bytes = cap.min(INITIAL_DECODE_ESTIMATE_BYTES);+ batch_rows_for_bytes_per_row(bytes_per_row, slice_bytes, read_batch_size)+ }+ _ => read_batch_size.max(1),+ }+}++/// Close the open file at `target_file_size`, else its row group at `max_row_group_bytes`.+async fn roll_writer(+ writer: &mut StreamingWriter,+ adds: &mut Vec<Add>,+ target_file_size: usize,+ max_row_group_bytes: Option<usize>,+) -> Result<()> {+ if writer.buffer_len() >= target_file_size {+ adds.extend(writer.flush().await?);+ } else if max_row_group_bytes.is_some_and(|cap| writer.in_progress_bytes() >= cap) {+ writer.flush_row_groups()?;+ }+ Ok(())+}+#[cfg(test)]mod tests {use super::*;diff --git a/rust/deltalite/core/src/writer.rs b/rust/deltalite/core/src/writer.rsindex 7ff3706f990e..b5c458742def 100644--- a/rust/deltalite/core/src/writer.rs+++ b/rust/deltalite/core/src/writer.rs@@ -179,6 +179,14 @@ impl StreamingWriter {self.open.values().map(OpenFile::buffer_len).sum()}+ /// Encoded bytes in the row groups the open files have not closed yet.+ pub(crate) fn in_progress_bytes(&self) -> usize {+ self.open+ .values()+ .map(|f| f.writer.in_progress_size())+ .sum()+ }+/// Rows in the row groups the open files have not closed yet.pub(crate) fn in_progress_rows(&self) -> usize {self.opendiff --git a/rust/deltalite/core/tests/upsert_layout.rs b/rust/deltalite/core/tests/upsert_layout.rsnew file mode 100644index 000000000000..b357ed307e51--- /dev/null+++ b/rust/deltalite/core/tests/upsert_layout.rs@@ -0,0 +1,276 @@+//! Output layout of `upsert`: which rows share an output file, and therefore what the+//! next upsert has to copy again. Both layouts must hold the same rows.++use std::collections::HashMap;+use std::sync::Arc;++use arrow_array::{Array, Int64Array, RecordBatch, StringArray};+use arrow_schema::{DataType, Field, Schema, SchemaRef};+use bytes::Bytes;+use deltalake::kernel::{DataType as KernelType, StructField};+use deltalake::operations::create::CreateBuilder;+use deltalake::writer::{DeltaWriter, RecordBatchWriter};+use deltalake::Path;+use deltalite_core::handle::TableHandle;+use deltalite_core::limits::ProcessLimits;+use deltalite_core::table::MultipartConfig;+use deltalite_core::upsert::{UpsertOptions, UpsertStats};+use object_store::ObjectStoreExt;+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;++const TABLE_ROWS: usize = 1000;++fn schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("pk", DataType::Utf8, false),+ Field::new("v", DataType::Int64, true),+ Field::new("payload", DataType::Utf8, true),+ Field::new("p", DataType::Utf8, true),+ ]))+}++fn rows(keys: &[String], v: i64) -> RecordBatch {+ let n = keys.len();+ let payload: Vec<String> = keys.iter().map(|k| format!("{k}-").repeat(20)).collect();+ RecordBatch::try_new(+ schema(),+ vec![+ Arc::new(StringArray::from(keys.to_vec())),+ Arc::new(Int64Array::from(vec![v; n])),+ Arc::new(StringArray::from(payload)),+ Arc::new(StringArray::from(vec!["a"; n])),+ ],+ )+ .unwrap()+}++fn keys(prefix: &str, ids: std::ops::Range<usize>) -> Vec<String> {+ ids.map(|i| format!("{prefix}{i:06}")).collect()+}++/// A one-partition table whose rows `old000000..` sit in one file.+async fn table(dir: &std::path::Path) -> TableHandle {+ let mut table = CreateBuilder::new()+ .with_location(dir.to_str().unwrap())+ .with_columns(vec![+ StructField::new("pk", KernelType::STRING, false),+ StructField::new("v", KernelType::LONG, true),+ StructField::new("payload", KernelType::STRING, true),+ StructField::new("p", KernelType::STRING, true),+ ])+ .with_partition_columns(vec!["p".to_string()])+ .await+ .unwrap();+ let mut w = RecordBatchWriter::for_table(&table).unwrap();+ w.write(rows(&keys("old", 0..TABLE_ROWS), 0)).await.unwrap();+ w.flush_and_commit(&mut table).await.unwrap();+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())+ .await+ .unwrap()+}++fn opts() -> UpsertOptions {+ UpsertOptions {+ primary_keys: vec!["pk".to_string()],+ partition_key: Some("p".to_string()),+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),+ ..Default::default()+ }+}++async fn upsert_with(+ handle: &mut TableHandle,+ keys: &[String],+ v: i64,+ opts: UpsertOptions,+) -> UpsertStats {+ handle+ .upsert(+ vec![rows(keys, v)],+ schema(),+ opts,+ MultipartConfig::default(),+ )+ .await+ .unwrap()+}++async fn upsert(handle: &mut TableHandle, keys: &[String], v: i64, isolate: bool) -> UpsertStats {+ let opts = UpsertOptions {+ isolate_source_rows: isolate,+ ..opts()+ };+ upsert_with(handle, keys, v, opts).await+}++async fn file_bytes(handle: &TableHandle, path: &str) -> Bytes {+ handle+ .table()+ .object_store()+ .get(&Path::parse(path).unwrap())+ .await+ .unwrap()+ .bytes()+ .await+ .unwrap()+}++/// Every live row as (pk, v, payload), sorted.+async fn read_rows(handle: &TableHandle) -> Vec<(String, i64, String)> {+ let mut out = Vec::new();+ for f in handle.files().unwrap() {+ let bytes = file_bytes(handle, &f.path).await;+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)+ .unwrap()+ .build()+ .unwrap()+ {+ let batch = batch.unwrap();+ let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone();+ let (pk, v, payload) = (col("pk"), col("v"), col("payload"));+ let pk = pk.as_any().downcast_ref::<StringArray>().unwrap();+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();+ let payload = payload.as_any().downcast_ref::<StringArray>().unwrap();+ for i in 0..batch.num_rows() {+ out.push((+ pk.value(i).to_string(),+ v.value(i),+ payload.value(i).to_string(),+ ));+ }+ }+ }+ out.sort();+ out+}++/// The first upsert updates 10 old rows and inserts 10 new ones; the second touches the+/// case's keys. With the option off, the first upsert writes its 20 rows into the file of+/// the 990 copied survivors, so any later batch that touches them copies all of those+/// survivors again. With it on, the 20 rows get their own file.+#[tokio::test]+async fn isolate_source_rows_copies_no_more_and_keeps_the_same_rows() {+ // (case, second upsert keys, rows copied per upsert with the option off, then on)+ type Case = (&'static str, Vec<String>, (usize, usize), (usize, usize));+ let cases: [Case; 3] = [+ (+ "touches the rows the last upsert wrote",+ [keys("old", 0..10), keys("new", 0..10)].concat(),+ (990, 990),+ (990, 0),+ ),+ (+ "touches only old rows",+ keys("old", 500..510),+ (990, 1000),+ (990, 980),+ ),+ (+ "touches old rows and rows the last upsert wrote",+ [keys("old", 500..505), keys("new", 0..5)].concat(),+ (990, 1000),+ (990, 1000),+ ),+ ];+ let first = [keys("old", 0..10), keys("new", 0..10)].concat();+ for (case, second, copied_off, copied_on) in cases {+ let (dir_off, dir_on) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());+ let (mut off, mut on) = (table(dir_off.path()).await, table(dir_on.path()).await);++ let first_off = upsert(&mut off, &first, 1, false).await;+ let first_on = upsert(&mut on, &first, 1, true).await;+ assert_eq!(first_off.files_added, 1, "{case}: {first_off:?}");+ assert_eq!(first_on.files_added, 2, "{case}: {first_on:?}");++ let second_off = upsert(&mut off, &second, 2, false).await;+ let second_on = upsert(&mut on, &second, 2, true).await;+ assert_eq!(+ (first_off.rows_copied, second_off.rows_copied),+ copied_off,+ "{case}, option off: {first_off:?} {second_off:?}"+ );+ assert_eq!(+ (first_on.rows_copied, second_on.rows_copied),+ copied_on,+ "{case}, option on: {first_on:?} {second_on:?}"+ );+ assert_eq!(second_off.rows_updated, second_on.rows_updated, "{case}");++ let (rows_off, rows_on) = (read_rows(&off).await, read_rows(&on).await);+ assert_eq!(rows_off.len(), TABLE_ROWS + 10, "{case}");+ assert_eq!(rows_off, rows_on, "{case}");+ }+}++/// An upsert that only inserts already writes the new rows apart from every survivor,+/// so the option must add no file there.+#[tokio::test]+async fn isolate_source_rows_adds_no_file_to_an_insert_only_upsert() {+ let dir = tempfile::tempdir().unwrap();+ let mut handle = table(dir.path()).await;+ let stats = upsert(&mut handle, &keys("new", 0..10), 1, true).await;+ assert_eq!(+ (stats.files_added, stats.files_removed, stats.rows_copied),+ (1, 0, 0),+ "{stats:?}"+ );+}++/// Rows per row group of each file the upsert added, with `read_batch_size = 100` so the+/// writer sees survivor batches of at most 100 rows. A cap of one byte closes a row group+/// after every survivor batch (an update), and after every source row because the source+/// path also slices its writes by the cap (an update and an insert).+#[tokio::test]+async fn max_row_group_bytes_closes_row_groups_between_batches() {+ // (case, keys, row groups per added file without a cap, then with a one-byte cap)+ type Case = (&'static str, Vec<String>, Vec<usize>, Vec<usize>);+ let cases: [Case; 2] = [+ (+ "update: survivors of 10 batches, the first missing 10 rows, then 10 source rows",+ keys("old", 0..10),+ vec![1000],+ [vec![90], vec![100; 9], vec![1; 10]].concat(),+ ),+ (+ "insert: 250 source rows",+ keys("new", 0..250),+ vec![250],+ vec![1; 250],+ ),+ ];+ for (case, batch, uncapped, capped) in cases {+ let mut got = Vec::new();+ for cap in [None, Some(1)] {+ let dir = tempfile::tempdir().unwrap();+ let mut handle = table(dir.path()).await;+ let before: Vec<String> = handle+ .files()+ .unwrap()+ .into_iter()+ .map(|f| f.path)+ .collect();+ let o = UpsertOptions {+ read_batch_size: 100,+ max_row_group_bytes: cap,+ ..opts()+ };+ upsert_with(&mut handle, &batch, 1, o).await;+ let mut groups = Vec::new();+ for f in handle.files().unwrap() {+ if before.contains(&f.path) {+ continue;+ }+ let meta =+ ParquetRecordBatchReaderBuilder::try_new(file_bytes(&handle, &f.path).await)+ .unwrap()+ .metadata()+ .clone();+ groups.extend(meta.row_groups().iter().map(|rg| rg.num_rows() as usize));+ }+ got.push((groups, read_rows(&handle).await));+ }+ assert_eq!(got[0].0, uncapped, "{case}, no cap");+ assert_eq!(got[1].0, capped, "{case}, one-byte cap");+ assert_eq!(got[0].1, got[1].1, "{case}: the cap changed the rows");+ }+}diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.tomlindex 66d9501e45c0..2dc94b04c601 100644--- a/rust/deltalite/python/Cargo.toml+++ b/rust/deltalite/python/Cargo.toml@@ -1,7 +1,7 @@[package]name = "deltalite-python"# Mirrors pyproject.toml in this directory (always identical); bump both together.-version = "0.1.11"+version = "0.1.12"edition = "2021"# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.mdindex f5d17f167110..08e5d1e9ea46 100644--- a/rust/deltalite/python/README.md+++ b/rust/deltalite/python/README.md@@ -195,6 +195,8 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:| `probe_concurrency` | `8` | Concurrent statistics/probe reads. || `read_batch_size` | `8192` | Row-group read batch size. || `target_file_size` | table setting, else 100 MiB | Output file size target. |+| `isolate_source_rows` | `False` | Write the batch's rows to their own files, apart from the unchanged rows copied out of the rewritten files. A later batch that touches the same rows then rewrites only those files. Costs up to one more file per touched partition. |+| `max_row_group_bytes` | `None` | Close an output row group at this many encoded bytes. A later upsert's reader fetches one whole row group at a time, so smaller row groups lower its memory. `None` closes row groups only at the encoder's row cap. || `max_source_bytes` | `2 GiB` | Oversized-batch guard (`0` disables). || `multipart_threshold` / `multipart_part_size` | `64 MiB` / `16 MiB` | Multipart upload thresholds (`0` threshold disables). || `commit_max_retries` | `15` | Conditional-put commit retry budget. |diff --git a/rust/deltalite/python/harness/common.py b/rust/deltalite/python/harness/common.pyindex ccb4a4a47e8e..4f5d71a38545 100644--- a/rust/deltalite/python/harness/common.py+++ b/rust/deltalite/python/harness/common.py@@ -234,6 +234,7 @@ def run_scenario(scenario: Scenario,tmp: Path | str,storage_options: dict[str, str] | None = None,+ upsert_kwargs: dict[str, Any] | None = None,) -> tuple[str, str]:"""Apply the same batch sequence to two fresh tables via the two paths.@@ -267,6 +268,7 @@ def run_scenario(scenario.partition_key,md,storage_options=storage_options,+ **(upsert_kwargs or {}),)return uri_m, uri_udiff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.tomlindex 56e53f0efa90..d8ce1097b214 100644--- a/rust/deltalite/python/pyproject.toml+++ b/rust/deltalite/python/pyproject.toml@@ -10,7 +10,7 @@ build-backend = "maturin"[project]name = "deltalite"# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.-version = "0.1.11"+version = "0.1.12"description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"readme = "README.md"requires-python = ">=3.12"diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rsindex 27e884b5c51c..d5bbdea8bac5 100644--- a/rust/deltalite/python/src/lib.rs+++ b/rust/deltalite/python/src/lib.rs@@ -369,6 +369,14 @@ impl DeltaLiteTable {/// `max_fetch_bytes` caps the compressed row-group bytes this call's file readers/// hold between fetching and decoding (`DELTALITE_PROCESS_MAX_FETCH_BYTES` caps it/// per process). A row group larger than the cap still runs, alone.+ ///+ /// `isolate_source_rows=True` writes the batch's rows to their own output files+ /// instead of the file that receives the copied survivors, so a later batch that+ /// touches the same rows does not copy those survivors again. Off by default.+ ///+ /// `max_row_group_bytes` closes an output row group once its encoded bytes reach the+ /// value, so a later reader fetches less per row group. `None` (the default) closes+ /// row groups only at the encoder's row cap.#[pyo3(signature = (data,primary_keys,@@ -385,6 +393,8 @@ impl DeltaLiteTable {commit_max_retries = 15,read_batch_size = 8192,target_file_size = None,+ isolate_source_rows = false,+ max_row_group_bytes = None,max_source_bytes = None,multipart_threshold = None,multipart_part_size = None,@@ -407,6 +417,8 @@ impl DeltaLiteTable {commit_max_retries: usize,read_batch_size: usize,target_file_size: Option<usize>,+ isolate_source_rows: bool,+ max_row_group_bytes: Option<usize>,max_source_bytes: Option<usize>,multipart_threshold: Option<usize>,multipart_part_size: Option<usize>,@@ -440,6 +452,8 @@ impl DeltaLiteTable {commit_max_retries,read_batch_size,target_file_size,+ isolate_source_rows,+ max_row_group_bytes,max_source_bytes,// Shared with every other concurrent upsert in this process, so the// process-level memory bound holds regardless of thread count.diff --git a/rust/deltalite/python/tests/test_parity.py b/rust/deltalite/python/tests/test_parity.pyindex 88a5ed318127..040ad4237082 100644--- a/rust/deltalite/python/tests/test_parity.py+++ b/rust/deltalite/python/tests/test_parity.py@@ -262,10 +262,15 @@ SCENARIOS = []+ "upsert_kwargs",+ [{}, {"isolate_source_rows": True}, {"max_row_group_bytes": 1}],+ ids=["default", "isolate_source_rows", "max_row_group_bytes"],+)@pytest.mark.parametrize("builder", SCENARIOS, ids=[f.__name__[3:] for f in SCENARIOS])-def test_parity(builder, tmp_path):+def test_parity(builder, upsert_kwargs, tmp_path):scenario = builder()- uri_m, uri_u = run_scenario(scenario, tmp_path)+ uri_m, uri_u = run_scenario(scenario, tmp_path, upsert_kwargs=upsert_kwargs)assert_parity(uri_m, uri_u, scenario.name)