:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(data-warehouse): cache deltalite probe keys per data file (#121060)
Tom Owers45bfdc1bfb27
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 536a2822885a..3d30f7b574b0 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -3973,6 +3973,7 @@ dependencies = ["metrics","object_store","parquet",+ "percent-encoding","serde_json","sqlparser","tempfile",@@ -3985,7 +3986,7 @@ dependencies = [[[package]]name = "deltalite-python"-version = "0.1.13"+version = "0.1.14"dependencies = ["arrow","arrow-array",diff --git a/rust/deltalite/README.md b/rust/deltalite/README.mdindex 8d12d5aaa0fb..56d46eadc35a 100644--- a/rust/deltalite/README.md+++ b/rust/deltalite/README.md@@ -117,7 +117,9 @@ Per-call (arguments to `DeltaLiteTable.upsert`): `max_parallel_partitions` (2),survivors; on: they get their own files),`max_row_group_bytes` (`None`: row groups close only at the encoder's row cap),`max_source_bytes` (2 GiB guard), `multipart_threshold` (64 MiB) /-`multipart_part_size` (16 MiB), `commit_max_retries` (15).+`multipart_part_size` (16 MiB), `commit_max_retries` (15), `use_probe_cache`+(on: the probe decides a file whose PK hashes the process-wide probe cache holds+without reading it; see `python/README.md`, "Probe cache").Process-global (environment, enforced on top of the per-call knobs so that~15 concurrent upsert threads in one Temporal worker cannot multiply the@@ -125,7 +127,8 @@ budgets): `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS` (8),`DELTALITE_PROCESS_MAX_PARALLEL_FILES` (16),`DELTALITE_PROCESS_MAX_BUFFERED_BYTES` (256 MiB),`DELTALITE_PROCESS_MAX_FETCH_BYTES` (256 MiB), `DELTALITE_MAX_SOURCE_BYTES`,-`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`.+`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`,+`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache).Kill switches for the request savings on the log and on small files (set to `0`to restore the earlier requests): `DELTALITE_PROBE_REFRESH`,@@ -142,7 +145,7 @@ suggests knob values from concurrency + pod memory, e.g.Metrics (`metrics` crate, static labels only): `deltalite_upserts_total`(`outcome`, `prune_strategy`, `error_kind`), `deltalite_upsert_duration_seconds`,-`deltalite_files_{added,removed,carried_over,probed}_total`,+`deltalite_files_{added,removed,carried_over,probed,probe_cached}_total`,`deltalite_rows_{updated,inserted,copied}_total`.## Pre-rollout conditions (from the spike verdict, still open)diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.tomlindex eb61951a67fe..268408c77f37 100644--- a/rust/deltalite/core/Cargo.toml+++ b/rust/deltalite/core/Cargo.toml@@ -53,6 +53,8 @@ arrow-ord = { version = "58" }delta_kernel = { package = "buoyant_kernel", version = "=0.22.2", default-features = false }chrono = { workspace = true }indexmap = "2"+# Already linked by `deltalake`; decodes Add paths into the form the snapshot reports.+percent-encoding = "2"sqlparser = "0.61"async-trait = { workspace = true }diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rsindex 15f93945c8ae..898caba68fda 100644--- a/rust/deltalite/core/src/lib.rs+++ b/rust/deltalite/core/src/lib.rs@@ -36,6 +36,7 @@ pub mod limits;mod logprobe;pub mod pkset;pub mod prefetch;+pub mod probecache;pub mod schema;mod smallfile;mod stats;@@ -49,6 +50,7 @@ pub use errors::{Error, Result};pub use handle::{LiveFile, TableHandle};pub use limits::ProcessLimits;pub use prefetch::CheckpointCache;+pub use probecache::{ProbeCache, ProbeCacheStats};pub use table::{open_table, open_table_multipart, open_table_prefetched, wrap_multipart, MultipartConfig,};diff --git a/rust/deltalite/core/src/pkset.rs b/rust/deltalite/core/src/pkset.rsindex 84750ee3647b..f0243c4be7ce 100644--- a/rust/deltalite/core/src/pkset.rs+++ b/rust/deltalite/core/src/pkset.rs@@ -20,6 +20,7 @@ use arrow_row::{RowConverter, SortField};use arrow_schema::SchemaRef;use crate::errors::{Error, Result};+use crate::probecache::hash_key;/// A hash set of primary-key tuples with SQL NULL semantics (NULL never matches).pub struct PkSet {@@ -124,6 +125,69 @@ impl PkSet {Ok((0..num_rows).any(|i| mask[i] && self.keys.contains(rows.row(i).as_ref())))}+ /// [`Self::contains_any_columns`] that also appends the hash of every matchable row+ /// to `hashes` (see [`crate::probecache`]). It reads every row only when `hashes` is+ /// given; without it, it stops at the first match.+ pub fn probe_columns(+ &self,+ cols: &[Arc<dyn Array>],+ num_rows: usize,+ hashes: Option<&mut Vec<u64>>,+ ) -> Result<bool> {+ let Some(hashes) = hashes else {+ return self.contains_any_columns(cols, num_rows);+ };+ if num_rows == 0 {+ return Ok(false);+ }+ let mask = matchable_mask(cols, num_rows);+ let rows = self+ .converter+ .convert_columns(cols)+ .map_err(|e| Error::Generic(format!("row encode: {e}")))?;+ let mut hit = false;+ for (i, matchable) in mask.iter().enumerate() {+ if !matchable {+ continue;+ }+ let key = rows.row(i);+ hashes.push(hash_key(key.as_ref()));+ hit = hit || self.keys.contains(key.as_ref());+ }+ Ok(hit)+ }++ /// Append the hash of every matchable row of the given PK columns to `hashes`.+ pub fn hash_columns(+ &self,+ cols: &[Arc<dyn Array>],+ num_rows: usize,+ hashes: &mut Vec<u64>,+ ) -> Result<()> {+ if num_rows == 0 {+ return Ok(());+ }+ let mask = matchable_mask(cols, num_rows);+ let rows = self+ .converter+ .convert_columns(cols)+ .map_err(|e| Error::Generic(format!("row encode: {e}")))?;+ hashes.extend(+ (0..num_rows)+ .filter(|i| mask[*i])+ .map(|i| hash_key(rows.row(i).as_ref())),+ );+ Ok(())+ }++ /// The sorted, deduplicated hashes of the keys in the set.+ pub fn sorted_key_hashes(&self) -> Vec<u64> {+ let mut out: Vec<u64> = self.keys.iter().map(|k| hash_key(k)).collect();+ out.sort_unstable();+ out.dedup();+ out+ }+/// For each row of `batch`, whether it is replaced by a source row.pub fn contains_rows(&self, batch: &RecordBatch) -> Result<Vec<bool>> {let cols = self.pk_columns(batch);diff --git a/rust/deltalite/core/src/probecache.rs b/rust/deltalite/core/src/probecache.rsnew file mode 100644index 000000000000..1eec08786ecd--- /dev/null+++ b/rust/deltalite/core/src/probecache.rs@@ -0,0 +1,585 @@+//! Process-wide cache of the primary-key hashes of data files, for the upsert probe.+//!+//! The probe decides, for each candidate file of a touched partition, whether the file+//! holds any key of the batch. Without a cache that costs a footer read and a PK-column+//! read per file on every upsert, so on object storage the probe grows with the number of+//! files in the table, not with the batch. A Delta data file never changes after it is+//! written, so the keys a file holds can be kept for as long as the process lives.+//!+//! Each entry is the sorted, deduplicated 64-bit hashes of the file's matchable PK tuples+//! (no NULL component), encoded exactly as [`crate::pkset::PkSet`] encodes them. A file is+//! a candidate when its hashes and the batch's hashes have a common value. A matching key+//! always gives equal hashes, so the cache never skips a file that holds a key. A hash+//! collision can only add a file to the rewrite set, and a rewrite of a file without+//! matches copies every row unchanged, so the table content stays correct.+//!+//! Entries are keyed by the table URL plus the URL-decoded file path, and carry a+//! signature of the PK columns and their types. A PK type change after schema evolution+//! changes the encoding, so an entry with another signature counts as a miss. An upsert+//! evicts the entries of the files it removes after its commit lands; files that another+//! writer removes are never asked for again and age out of the LRU order.++use std::collections::hash_map::DefaultHasher;+use std::collections::{BTreeMap, HashMap};+use std::hash::{Hash, Hasher};+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex, OnceLock};++use arrow_schema::DataType;++/// Environment variable for the byte cap of the process-wide cache; `0` turns it off.+pub const PROBE_CACHE_BYTES_ENV: &str = "DELTALITE_PROBE_CACHE_BYTES";+/// Default byte cap of the process-wide cache.+pub const DEFAULT_PROBE_CACHE_BYTES: usize = 128 * 1024 * 1024;++/// Fixed bookkeeping cost charged per entry on top of its key and hashes.+const ENTRY_OVERHEAD_BYTES: usize = 128;+/// Reservations for in-flight key collection grow in steps of this size, so a collector+/// touches the shared counter once per step instead of once per batch.+const RESERVE_STEP_BYTES: usize = 64 * 1024;++/// Counters and sizes of a [`ProbeCache`].+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct ProbeCacheStats {+ pub entries: usize,+ pub bytes: usize,+ pub max_bytes: usize,+ /// Lookups that found an entry with the right signature.+ pub hits: u64,+ /// Lookups that found no usable entry.+ pub misses: u64,+ /// Entries stored, by a probe or by an upsert that wrote the file.+ pub inserts: u64,+ /// Entries dropped to stay under the byte cap.+ pub evictions: u64,+}++struct Entry {+ signature: u64,+ hashes: Arc<[u64]>,+ tick: u64,+ bytes: usize,+}++#[derive(Default)]+struct Inner {+ map: HashMap<String, Entry>,+ /// Least recently used first.+ order: BTreeMap<u64, String>,+ tick: u64,+ bytes: usize,+}++impl Inner {+ fn remove(&mut self, key: &str) -> bool {+ match self.map.remove(key) {+ Some(e) => {+ self.order.remove(&e.tick);+ self.bytes -= e.bytes;+ true+ }+ None => false,+ }+ }++ fn evict_to(&mut self, max_bytes: usize) -> u64 {+ let mut evicted = 0;+ while self.bytes > max_bytes {+ let Some((_, key)) = self.order.pop_first() else {+ break;+ };+ if let Some(e) = self.map.remove(&key) {+ self.bytes -= e.bytes;+ evicted += 1;+ }+ }+ evicted+ }+}++/// A byte-capped LRU map from data file to the PK hashes the file holds.+pub struct ProbeCache {+ inner: Mutex<Inner>,+ max_bytes: AtomicUsize,+ /// Bytes reserved by collectors that are still reading or writing a file. Kept apart+ /// from `inner.bytes` and capped at a quarter of `max_bytes`, so the memory of the+ /// cache feature stays under 1.25 x `max_bytes` whatever the concurrency.+ collecting: AtomicUsize,+ hits: AtomicU64,+ misses: AtomicU64,+ inserts: AtomicU64,+ evictions: AtomicU64,+}++impl std::fmt::Debug for ProbeCache {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("ProbeCache")+ .field("stats", &self.stats())+ .finish()+ }+}++impl ProbeCache {+ /// A cache that holds at most `max_bytes`; `0` stores nothing.+ pub fn new(max_bytes: usize) -> Self {+ Self {+ inner: Mutex::new(Inner::default()),+ max_bytes: AtomicUsize::new(max_bytes),+ collecting: AtomicUsize::new(0),+ hits: AtomicU64::new(0),+ misses: AtomicU64::new(0),+ inserts: AtomicU64::new(0),+ evictions: AtomicU64::new(0),+ }+ }++ /// The process-wide cache, sized from [`PROBE_CACHE_BYTES_ENV`] on first use.+ pub fn global() -> &'static Arc<ProbeCache> {+ static GLOBAL: OnceLock<Arc<ProbeCache>> = OnceLock::new();+ GLOBAL.get_or_init(|| Arc::new(ProbeCache::new(max_bytes_from_env())))+ }++ pub fn max_bytes(&self) -> usize {+ self.max_bytes.load(Ordering::Relaxed)+ }++ /// Change the byte cap, evicting least recently used entries down to it.+ pub fn set_max_bytes(&self, max_bytes: usize) {+ self.max_bytes.store(max_bytes, Ordering::Relaxed);+ let evicted = self.lock().evict_to(max_bytes);+ self.evictions.fetch_add(evicted, Ordering::Relaxed);+ }++ pub fn enabled(&self) -> bool {+ self.max_bytes() > 0+ }++ /// Largest entry the cache accepts. A file with more keys than this is probed on every+ /// upsert, so that one huge file cannot push every other entry out.+ pub fn max_entry_bytes(&self) -> usize {+ self.max_bytes() / 8+ }++ pub fn clear(&self) {+ let mut inner = self.lock();+ inner.map.clear();+ inner.order.clear();+ inner.bytes = 0;+ }++ pub fn stats(&self) -> ProbeCacheStats {+ let inner = self.lock();+ ProbeCacheStats {+ entries: inner.map.len(),+ bytes: inner.bytes,+ max_bytes: self.max_bytes(),+ hits: self.hits.load(Ordering::Relaxed),+ misses: self.misses.load(Ordering::Relaxed),+ inserts: self.inserts.load(Ordering::Relaxed),+ evictions: self.evictions.load(Ordering::Relaxed),+ }+ }++ fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {+ // A panic while the lock is held leaves the map consistent (every mutation is a+ // few map operations), so a poisoned lock is still safe to use.+ self.inner.lock().unwrap_or_else(|e| e.into_inner())+ }++ /// The sorted key hashes of `key`, when an entry with `signature` exists.+ pub(crate) fn get(&self, key: &str, signature: u64) -> Option<Arc<[u64]>> {+ if !self.enabled() {+ return None;+ }+ let mut inner = self.lock();+ inner.tick += 1;+ let tick = inner.tick;+ let found = match inner.map.get_mut(key) {+ Some(e) if e.signature == signature => {+ let old = e.tick;+ e.tick = tick;+ Some((old, e.hashes.clone()))+ }+ _ => None,+ };+ match found {+ Some((old, hashes)) => {+ inner.order.remove(&old);+ inner.order.insert(tick, key.to_string());+ drop(inner);+ self.hits.fetch_add(1, Ordering::Relaxed);+ Some(hashes)+ }+ None => {+ drop(inner);+ self.misses.fetch_add(1, Ordering::Relaxed);+ None+ }+ }+ }++ /// Store the key hashes of a file. `hashes` need not be sorted or unique.+ pub(crate) fn insert(&self, key: String, signature: u64, mut hashes: Vec<u64>) {+ if !self.enabled() {+ return;+ }+ hashes.sort_unstable();+ hashes.dedup();+ // The key is held twice: once in `map` and once in `order`.+ let bytes = hashes.len() * 8 + 2 * key.len() + ENTRY_OVERHEAD_BYTES;+ let mut inner = self.lock();+ // Read the cap under the lock: `set_max_bytes` evicts under it too, so a cap lowered+ // concurrently is never missed by this insertion.+ let max_bytes = self.max_bytes();+ if max_bytes == 0 || bytes > max_bytes / 8 {+ return;+ }+ inner.remove(&key);+ inner.tick += 1;+ let tick = inner.tick;+ inner.order.insert(tick, key.clone());+ inner.map.insert(+ key,+ Entry {+ signature,+ hashes: hashes.into(),+ tick,+ bytes,+ },+ );+ inner.bytes += bytes;+ let evicted = inner.evict_to(max_bytes);+ drop(inner);+ self.inserts.fetch_add(1, Ordering::Relaxed);+ self.evictions.fetch_add(evicted, Ordering::Relaxed);+ }++ /// Drop the entries of files that a commit removed from the table.+ pub(crate) fn remove_all<'a>(&self, keys: impl IntoIterator<Item = &'a str>) {+ let mut inner = self.lock();+ for k in keys {+ inner.remove(k);+ }+ }++ /// Reserve `bytes` more for in-flight collection. Never waits: when the pool is full+ /// the collector gives up and its file is simply not cached.+ fn try_reserve(&self, bytes: usize) -> bool {+ let limit = self.max_bytes() / 4;+ self.collecting+ .fetch_update(Ordering::AcqRel, Ordering::Acquire, |cur| {+ let next = cur.checked_add(bytes)?;+ (next <= limit).then_some(next)+ })+ .is_ok()+ }++ fn release(&self, bytes: usize) {+ self.collecting.fetch_sub(bytes, Ordering::AcqRel);+ }++ /// A collector for the key hashes of one file; `None` when the cache is off.+ pub(crate) fn collector(self: &Arc<Self>) -> Option<KeyCollector> {+ self.enabled().then(|| KeyCollector {+ cache: self.clone(),+ hashes: Vec::new(),+ reserved: 0,+ abandoned: false,+ })+ }+}++/// Collects the key hashes of one file while it is read or written. It holds a share of+/// the cache's in-flight pool for the hashes it keeps, and gives up (keeping nothing)+/// once the file would make an entry larger than the cache accepts, or the pool is full.+pub(crate) struct KeyCollector {+ cache: Arc<ProbeCache>,+ hashes: Vec<u64>,+ reserved: usize,+ abandoned: bool,+}++impl KeyCollector {+ /// The buffer to append the hashes of `additional` more rows to, or `None` when the+ /// collector gave up.+ pub(crate) fn reserve(&mut self, additional: usize) -> Option<&mut Vec<u64>> {+ if self.abandoned {+ return None;+ }+ let need_len = self.hashes.len() + additional;+ let max_entry = self.cache.max_entry_bytes();+ if need_len * 8 > max_entry {+ self.abandon();+ return None;+ }+ if need_len > self.hashes.capacity() {+ // Grow the buffer here, geometrically and with an exact capacity, so that the+ // pool is charged for the allocation itself and not just the rows in it.+ let target_len = need_len+ .max(self.hashes.capacity() * 2)+ .max(RESERVE_STEP_BYTES.min(max_entry / 8) / 8)+ .min(max_entry / 8);+ let target_bytes = target_len * 8;+ if target_bytes > self.reserved {+ let step = target_bytes - self.reserved;+ if !self.cache.try_reserve(step) {+ self.abandon();+ return None;+ }+ self.reserved += step;+ }+ self.hashes.reserve_exact(target_len - self.hashes.len());+ }+ Some(&mut self.hashes)+ }++ fn abandon(&mut self) {+ self.abandoned = true;+ self.hashes = Vec::new();+ self.cache.release(self.reserved);+ self.reserved = 0;+ }++ /// What was collected, still holding its share of the pool; `None` if it gave up.+ pub(crate) fn finish(mut self) -> Option<CollectedKeys> {+ if self.abandoned {+ return None;+ }+ let reserved = std::mem::take(&mut self.reserved);+ Some(CollectedKeys {+ cache: self.cache.clone(),+ hashes: std::mem::take(&mut self.hashes),+ reserved,+ })+ }++ /// Store what was collected as the entry for `key`. Does nothing if it gave up.+ pub(crate) fn store(self, key: String, signature: u64) {+ if let Some(keys) = self.finish() {+ keys.store(key, signature);+ }+ }+}++impl Drop for KeyCollector {+ fn drop(&mut self) {+ self.cache.release(self.reserved);+ }+}++/// The finished key hashes of one file, waiting to be stored (for a written file, until+/// its commit lands).+pub(crate) struct CollectedKeys {+ cache: Arc<ProbeCache>,+ hashes: Vec<u64>,+ reserved: usize,+}++impl CollectedKeys {+ pub(crate) fn store(mut self, key: String, signature: u64) {+ let hashes = std::mem::take(&mut self.hashes);+ self.cache.insert(key, signature, hashes);+ }+}++impl Drop for CollectedKeys {+ fn drop(&mut self) {+ self.cache.release(self.reserved);+ }+}++fn max_bytes_from_env() -> usize {+ match std::env::var(PROBE_CACHE_BYTES_ENV) {+ Ok(v) => match v.trim().parse::<usize>() {+ Ok(n) => n,+ Err(_) => {+ tracing::warn!(var = PROBE_CACHE_BYTES_ENV, value = %v, "invalid value, using default");+ DEFAULT_PROBE_CACHE_BYTES+ }+ },+ Err(_) => DEFAULT_PROBE_CACHE_BYTES,+ }+}++/// Cache key of a data file: the table URL and the URL-decoded path relative to it.+pub(crate) fn file_key(table_url: &str, decoded_path: &str) -> String {+ let mut key = String::with_capacity(table_url.len() + 1 + decoded_path.len());+ key.push_str(table_url);+ key.push('\n');+ key.push_str(decoded_path);+ key+}++/// Signature of the key encoding: the PK columns, their table types, and the partition+/// column (a PK on the partition column takes its value from the file's path).+pub(crate) fn key_signature(+ primary_keys: &[String],+ pk_types: &[DataType],+ partition_col: Option<&str>,+) -> u64 {+ let mut h = DefaultHasher::new();+ primary_keys.hash(&mut h);+ for t in pk_types {+ format!("{t:?}").hash(&mut h);+ }+ partition_col.hash(&mut h);+ h.finish()+}++/// Hash of one row-encoded PK tuple.+pub(crate) fn hash_key(encoded: &[u8]) -> u64 {+ let mut h = DefaultHasher::new();+ h.write(encoded);+ h.finish()+}++/// Whether two sorted slices share a value.+pub(crate) fn sorted_intersect(a: &[u64], b: &[u64]) -> bool {+ let (small, large) = if a.len() <= b.len() { (a, b) } else { (b, a) };+ if small.is_empty() {+ return false;+ }+ // Binary search wins when one side is much smaller; a merge walk otherwise.+ let log_large = usize::BITS - large.len().leading_zeros();+ if small.len().saturating_mul(log_large as usize) < small.len() + large.len() {+ return small.iter().any(|v| large.binary_search(v).is_ok());+ }+ let (mut i, mut j) = (0, 0);+ while i < small.len() && j < large.len() {+ match small[i].cmp(&large[j]) {+ std::cmp::Ordering::Less => i += 1,+ std::cmp::Ordering::Greater => j += 1,+ std::cmp::Ordering::Equal => return true,+ }+ }+ false+}++#[cfg(test)]+mod tests {+ use super::*;++ fn entry_bytes(n_hashes: usize, key: &str) -> usize {+ n_hashes * 8 + 2 * key.len() + ENTRY_OVERHEAD_BYTES+ }++ #[test]+ fn get_needs_the_same_signature() {+ let c = ProbeCache::new(1 << 20);+ c.insert("t\na".into(), 1, vec![3, 1, 2, 3]);+ assert_eq!(c.get("t\na", 1).as_deref(), Some(&[1u64, 2, 3][..]));+ assert!(c.get("t\na", 2).is_none());+ assert!(c.get("t\nb", 1).is_none());+ let s = c.stats();+ assert_eq!((s.hits, s.misses, s.entries), (1, 2, 1));+ }++ #[test]+ fn eviction_drops_the_least_recently_used_entry() {+ let one = entry_bytes(10, "t\nk0");+ let c = ProbeCache::new(one * 8);+ for i in 0..8 {+ c.insert(format!("t\nk{i}"), 0, (0..10).collect());+ }+ assert!(c.get("t\nk0", 0).is_some());+ c.insert("t\nk8".into(), 0, (0..10).collect());+ assert!(c.get("t\nk1", 0).is_none());+ assert!(c.get("t\nk0", 0).is_some());+ assert!(c.get("t\nk8", 0).is_some());+ assert_eq!(c.stats().bytes, one * 8);+ assert_eq!(c.stats().evictions, 1);++ c.set_max_bytes(one * 2);+ assert_eq!(c.stats().entries, 2);+ assert!(c.get("t\nk0", 0).is_some());+ assert!(c.get("t\nk8", 0).is_some());+ }++ #[test]+ fn an_entry_above_an_eighth_of_the_cap_is_not_stored() {+ let c = ProbeCache::new(8 * entry_bytes(100, "t\na"));+ c.insert("t\na".into(), 0, (0..101).collect());+ assert_eq!(c.stats().entries, 0);+ c.insert("t\na".into(), 0, (0..100).collect());+ assert_eq!(c.stats().entries, 1);+ }++ #[test]+ fn a_zero_cap_stores_and_collects_nothing() {+ let c = Arc::new(ProbeCache::new(0));+ c.insert("t\na".into(), 0, vec![1]);+ assert!(c.get("t\na", 0).is_none());+ assert!(c.collector().is_none());+ }++ #[test]+ fn collectors_give_up_when_the_in_flight_pool_is_full_and_release_on_drop() {+ // Pool = max/4 = 16 steps; one entry = max/8 = 8 steps.+ let c = Arc::new(ProbeCache::new(64 * RESERVE_STEP_BYTES));+ let rows_per_entry = c.max_entry_bytes() / 8;+ let mut a = c.collector().unwrap();+ let mut b = c.collector().unwrap();+ let mut late = c.collector().unwrap();+ assert!(a.reserve(rows_per_entry).is_some());+ assert!(b.reserve(rows_per_entry).is_some());+ assert!(late.reserve(1).is_none());+ assert!(late.finish().is_none());+ drop(a);+ let mut late = c.collector().unwrap();+ assert!(late.reserve(1).is_some());+ // Past the entry cap the collector gives up and keeps nothing.+ assert!(late.reserve(rows_per_entry + 1).is_none());+ let mut kept = c.collector().unwrap();+ kept.reserve(1).unwrap().push(7);+ let kept = kept.finish().unwrap();+ drop(b);+ drop(late);+ assert!(c.collecting.load(Ordering::Acquire) > 0);+ kept.store("t\na".into(), 0);+ assert_eq!(c.collecting.load(Ordering::Acquire), 0);+ assert_eq!(c.get("t\na", 0).as_deref(), Some(&[7u64][..]));+ }++ #[test]+ fn a_collector_is_charged_for_its_whole_allocation() {+ let c = Arc::new(ProbeCache::new(1 << 24));+ let mut k = c.collector().unwrap();+ for _ in 0..3 {+ k.reserve(8192)+ .unwrap()+ .extend(std::iter::repeat_n(1, 8192));+ assert!(k.hashes.capacity() * 8 <= k.reserved);+ }+ assert_eq!(c.collecting.load(Ordering::Acquire), k.reserved);+ }++ #[test]+ fn remove_all_frees_the_bytes() {+ let c = ProbeCache::new(1 << 20);+ c.insert("t\na".into(), 0, vec![1, 2]);+ c.insert("t\nb".into(), 0, vec![1]);+ c.remove_all(["t\na", "t\nmissing"]);+ assert_eq!(c.stats().entries, 1);+ assert_eq!(c.stats().bytes, entry_bytes(1, "t\nb"));+ }++ #[test]+ fn sorted_intersect_matches_a_naive_check() {+ let cases: Vec<(Vec<u64>, Vec<u64>)> = vec![+ (vec![], vec![1, 2]),+ (vec![5], vec![1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12]),+ (vec![7], vec![1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12]),+ (+ (0..50).map(|v| v * 2).collect(),+ (0..50).map(|v| v * 2 + 1).collect(),+ ),+ ((0..50).map(|v| v * 2).collect(), vec![1, 3, 98]),+ (vec![u64::MAX], vec![0, u64::MAX]),+ ];+ for (a, b) in cases {+ let naive = a.iter().any(|v| b.contains(v));+ assert_eq!(sorted_intersect(&a, &b), naive, "{a:?} {b:?}");+ assert_eq!(sorted_intersect(&b, &a), naive, "{b:?} {a:?}");+ }+ }+}diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rsindex 1cb20e6ddb63..226b7c7159bc 100644--- a/rust/deltalite/core/src/upsert.rs+++ b/rust/deltalite/core/src/upsert.rs@@ -60,6 +60,9 @@ use crate::limits::{SourceFootprint,};use crate::pkset::PkSet;+use crate::probecache::{+ file_key, key_signature, sorted_intersect, CollectedKeys, KeyCollector, ProbeCache,+};use crate::schema::{cast_to_schema, unknown_columns};use crate::smallfile::{fetch_small_file, reads_whole, SmallFileStore};use crate::writer::StreamingWriter;@@ -264,6 +267,11 @@ pub struct UpsertOptions {pub max_source_bytes: Option<usize>,/// Process-wide concurrency budgets shared with every other concurrent upsert.pub limits: Arc<ProcessLimits>,+ /// Cache of the PK hashes of data files (see [`crate::probecache`]). With+ /// `prune_strategy=probe`, a cached file is decided without a read, a probed file that+ /// holds no match is cached, and the files this upsert writes are cached once its+ /// commit lands. `None` probes every candidate file from storage.+ pub probe_cache: Option<Arc<ProbeCache>>,}impl Default for UpsertOptions {@@ -285,6 +293,7 @@ impl Default for UpsertOptions {max_row_group_bytes: None,max_source_bytes: None,limits: ProcessLimits::global().clone(),+ probe_cache: Some(ProbeCache::global().clone()),}}}@@ -304,6 +313,8 @@ pub struct UpsertStats {pub files_carried_over: usize,/// Files whose PK column(s) were actually read by the probe (prune_strategy=probe).pub files_probed: usize,+ /// Candidate files the probe decided from the probe cache, with no read.+ pub files_probe_cached: usize,/// Existing rows dropped in favour of an incoming row.pub rows_updated: usize,/// Incoming rows with no existing match.@@ -318,6 +329,9 @@ pub struct UpsertStats {pub plan_ms: u64,/// Wall-clock ms spent rewriting the touched partitions (reading + probing targets, writing files).pub rewrite_ms: u64,+ /// Wall-clock ms the partition workers spent probing, summed over partitions (part of+ /// `rewrite_ms`, which runs partitions in parallel).+ pub probe_ms: u64,/// Wall-clock ms spent committing the new file set to the Delta log.pub commit_ms: u64,/// Non-nullable columns relaxed to nullable before this upsert because the table's@@ -511,11 +525,17 @@ struct PartitionOutcome {files_added: usize,files_removed: usize,files_probed: usize,+ files_probe_cached: usize,+ probe_ms: u64,rows_updated: usize,rows_copied: usize,rows_inserted: usize,null_pk_rows: usize,carried: usize,+ /// Cache keys of the files this partition removes.+ removed_keys: Vec<String>,+ /// Key hashes of the files this partition wrote, stored once the commit lands.+ written_keys: Vec<(String, CollectedKeys)>,}/// A unit of byte budget: one permit from the per-call budget and one from the@@ -736,6 +756,8 @@ pub async fn upsert_cached_with_state(counter!("deltalite_files_carried_over_total").increment(stats.files_carried_over as u64);counter!("deltalite_files_probed_total").increment(stats.files_probed as u64);+ counter!("deltalite_files_probe_cached_total")+ .increment(stats.files_probe_cached as u64);counter!("deltalite_rows_updated_total").increment(stats.rows_updated as u64);counter!("deltalite_rows_inserted_total").increment(stats.rows_inserted as u64);counter!("deltalite_rows_copied_total").increment(stats.rows_copied as u64);@@ -1093,6 +1115,8 @@ async fn upsert_inner(..Default::default()};let mut actions: Vec<Action> = Vec::new();+ let mut removed_keys: Vec<String> = Vec::new();+ let mut written_keys: Vec<(String, CollectedKeys)> = Vec::new();for h in handles {let outcome = h@@ -1101,12 +1125,16 @@ async fn upsert_inner(stats.files_added += outcome.files_added;stats.files_removed += outcome.files_removed;stats.files_probed += outcome.files_probed;+ stats.files_probe_cached += outcome.files_probe_cached;+ stats.probe_ms += outcome.probe_ms;stats.files_carried_over += outcome.carried;stats.rows_updated += outcome.rows_updated;stats.rows_copied += outcome.rows_copied;stats.rows_inserted += outcome.rows_inserted;stats.null_pk_rows += outcome.null_pk_rows;actions.extend(outcome.actions);+ removed_keys.extend(outcome.removed_keys);+ written_keys.extend(outcome.written_keys);}let rewrite_ms = rewrite_started.elapsed().as_millis() as u64;@@ -1159,6 +1187,16 @@ async fn upsert_inner(.await?;let commit_ms = commit_started.elapsed().as_millis() as u64;+ // Only a landed commit makes the written files live and the removed files dead. Before+ // it, a written file may be an orphan that no snapshot ever names.+ if let Some(cache) = &opts.probe_cache {+ cache.remove_all(removed_keys.iter().map(String::as_str));+ let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;+ for (key, keys) in written_keys {+ keys.store(key, signature);+ }+ }+if (finalized.version() + 1) % checkpoint_interval == 0 {let maintenance_started = Instant::now();best_effort_log_maintenance(table, finalized.version(), cleanup_enabled).await;@@ -1931,21 +1969,13 @@ async fn probe_file(partition_value: &str,opts: &UpsertOptions,budgets: &Budgets,+ mut collector: Option<&mut KeyCollector>,) -> Result<(bool, Arc<ParquetMetaData>)> {let (builder, held_fetch) = open_builder(store, f, budgets).await?;let metadata = builder.metadata().clone();let file_schema = builder.schema().clone();- let mut pk_types: Vec<DataType> = Vec::with_capacity(opts.primary_keys.len());- for pk in &opts.primary_keys {- pk_types.push(- table_schema- .field_with_name(pk)- .map_err(|e| Error::Generic(format!("PK column lookup: {e}")))?- .data_type()- .clone(),- );- }+ let pk_types = pk_types(table_schema, &opts.primary_keys)?;let mut sources: Vec<PkSource> = Vec::with_capacity(opts.primary_keys.len());let mut projection: Vec<usize> = Vec::new();@@ -1973,7 +2003,8 @@ async fn probe_file(partition_value,&pk_types,)?;- return Ok((pkset.contains_any_columns(&cols, 1)?, metadata));+ let hashes = collector.as_mut().and_then(|c| c.reserve(1));+ return Ok((pkset.probe_columns(&cols, 1, hashes)?, metadata));}// Fetch budget before any data is fetched; held until the stream is gone.@@ -2019,7 +2050,8 @@ async fn probe_file(partition_value,&pk_types,)?;- let hit = pkset.contains_any_columns(&cols, batch.num_rows())?;+ let hashes = collector.as_mut().and_then(|c| c.reserve(batch.num_rows()));+ let hit = pkset.probe_columns(&cols, batch.num_rows(), hashes)?;drop(permit);if hit {drop(stream);@@ -2032,10 +2064,32 @@ async fn probe_file(Ok((false, metadata))}-/// Probe `files` with bounded concurrency, splitting them into (files that contain at-/// least one match, count of files proven match-free). Order is preserved. Up to one-/// reader wave of kept files carries the footer its probe parsed; retaining every hit-/// footer would make memory grow with the partition's file count before rewrites start.+/// What the probe cache contributes to one partition's probe.+struct ProbeCacheView<'a> {+ cache: Arc<ProbeCache>,+ table_url: &'a str,+ signature: u64,+ /// Sorted hashes of the partition's source keys.+ source_hashes: Vec<u64>,+}++/// Result of [`probe_files`].+struct Probed {+ /// Files that contain at least one match, in input order.+ keep: Vec<TargetFile>,+ /// Files proven match-free.+ skipped: usize,+ /// Files decided from the probe cache, with no read.+ cached: usize,+}++/// Probe `files` with bounded concurrency. Order is preserved. Up to one reader wave of+/// kept files carries the footer its probe parsed; retaining every hit footer would make+/// memory grow with the partition's file count before rewrites start.+///+/// With a cache, a cached file is decided from its key hashes, and a probed file that+/// holds no match is cached: the probe read all of its keys to prove that. A file that+/// holds a match is not cached, because this upsert removes it.#[allow(clippy::too_many_arguments)]async fn probe_files(store: &Arc<dyn ObjectStore>,@@ -2046,10 +2100,19 @@ async fn probe_files(partition_value: &str,opts: &UpsertOptions,budgets: &Budgets,-) -> Result<(Vec<TargetFile>, usize)> {+ cache: Option<&ProbeCacheView<'_>>,+) -> Result<Probed> {let results = futures::stream::iter(files.into_iter().map(|f| {let store = store.clone();async move {+ let key = cache.map(|c| file_key(c.table_url, &f.path));+ if let (Some(c), Some(key)) = (cache, &key) {+ if let Some(hashes) = c.cache.get(key, c.signature) {+ let hit = sorted_intersect(&c.source_hashes, &hashes);+ return Ok::<_, Error>((f, hit, None, true));+ }+ }+ let mut collector = cache.and_then(|c| c.cache.collector());let (hit, metadata) = probe_file(&store,&f,@@ -2059,28 +2122,148 @@ async fn probe_files(partition_value,opts,budgets,+ collector.as_mut(),).await?;- Ok::<_, Error>((f, hit, metadata))+ if let (false, Some(c), Some(key), Some(collector)) = (hit, cache, key, collector) {+ collector.store(key, c.signature);+ }+ Ok((f, hit, Some(metadata), false))}})).buffered(opts.probe_concurrency.max(1));futures::pin_mut!(results);let retained_footer_limit = opts.max_parallel_files.max(1);- let mut keep = Vec::new();- let mut skipped = 0usize;- while let Some((mut f, hit, metadata)) = results.try_next().await? {+ let mut out = Probed {+ keep: Vec::new(),+ skipped: 0,+ cached: 0,+ };+ while let Some((mut f, hit, metadata, cached)) = results.try_next().await? {+ out.cached += usize::from(cached);if hit {- if keep.len() < retained_footer_limit {- f.metadata = Some(metadata);+ if out.keep.len() < retained_footer_limit {+ f.metadata = metadata;}- keep.push(f);+ out.keep.push(f);} else {- skipped += 1;+ out.skipped += 1;+ }+ }+ Ok(out)+}++/// The table schema's types of the PK columns, in PK order.+fn pk_types(table_schema: &SchemaRef, primary_keys: &[String]) -> Result<Vec<DataType>> {+ primary_keys+ .iter()+ .map(|pk| {+ Ok(table_schema+ .field_with_name(pk)+ .map_err(|e| Error::Generic(format!("PK column lookup: {e}")))?+ .data_type()+ .clone())+ })+ .collect()+}++/// The probe-cache signature of this table's PK encoding.+fn pk_signature(+ table_schema: &SchemaRef,+ primary_keys: &[String],+ partition_col: &Option<String>,+) -> Result<u64> {+ Ok(key_signature(+ primary_keys,+ &pk_types(table_schema, primary_keys)?,+ partition_col.as_deref(),+ ))+}++/// Collects the key hashes of each file a partition writes, so that the next upsert does+/// not probe a file this one wrote. Every batch the writer receives is in the table+/// schema with the partition column, so its PK columns encode exactly as a later probe+/// of the written file encodes them.+struct OutputKeys {+ cache: Arc<ProbeCache>,+ pk_idx: Vec<usize>,+ pk_types: Vec<DataType>,+ current: Option<KeyCollector>,+ /// Decoded path of each written file with its key hashes.+ done: Vec<(String, CollectedKeys)>,+}++impl OutputKeys {+ fn new(+ cache: Arc<ProbeCache>,+ table_schema: &SchemaRef,+ primary_keys: &[String],+ ) -> Result<Self> {+ let mut pk_idx = Vec::with_capacity(primary_keys.len());+ for pk in primary_keys {+ pk_idx.push(table_schema.index_of(pk).map_err(|_| {+ Error::SchemaMismatch(format!(+ "primary key column '{pk}' not found in table schema"+ ))+ })?);+ }+ let current = cache.collector();+ Ok(Self {+ pk_types: pk_types(table_schema, primary_keys)?,+ cache,+ pk_idx,+ current,+ done: Vec::new(),+ })+ }++ /// Record the keys of `batch`, which goes into the open output file.+ fn note_write(&mut self, pkset: &PkSet, batch: &RecordBatch) -> Result<()> {+ let Some(hashes) = self+ .current+ .as_mut()+ .and_then(|c| c.reserve(batch.num_rows()))+ else {+ return Ok(());+ };+ let cast_opts = CastOptions {+ safe: false,+ ..Default::default()+ };+ let mut cols: Vec<Arc<dyn Array>> = Vec::with_capacity(self.pk_idx.len());+ for (&ci, ty) in self.pk_idx.iter().zip(&self.pk_types) {+ let col = batch.column(ci);+ cols.push(if col.data_type() == ty {+ col.clone()+ } else {+ cast_with_options(col, ty, &cast_opts)?+ });+ }+ pkset.hash_columns(&cols, batch.num_rows(), hashes)+ }++ /// Attach the keys recorded so far to the file a flush closed, if it closed one.+ fn note_flush(&mut self, new_adds: &[Add]) {+ if new_adds.is_empty() {+ return;+ }+ let finished = std::mem::replace(&mut self.current, self.cache.collector());+ // One partition per writer gives one file per flush; anything else is not cached.+ if let ([add], Some(keys)) = (new_adds, finished.and_then(KeyCollector::finish)) {+ let path = percent_encoding::percent_decode_str(&add.path)+ .decode_utf8_lossy()+ .into_owned();+ self.done.push((path, keys));}}- Ok((keep, skipped))++ fn into_written(self, table_url: &str) -> Vec<(String, CollectedKeys)> {+ self.done+ .into_iter()+ .map(|(path, keys)| (file_key(table_url, &path), keys))+ .collect()+ }}/// Per-file result from a reader task.@@ -2239,11 +2422,18 @@ async fn rewrite_partition(let writer = StreamingWriter::for_table(table)?;let store: Arc<dyn ObjectStore> = table.object_store();+ let table_url = table.table_url().as_str().to_string();+ let cache = match opts.prune_strategy {+ PruneStrategy::Probe => opts.probe_cache.clone().filter(|c| c.enabled()),+ _ => None,+ };// Content-based file selection: keep only files that actually contain a matched// row, exactly as MERGE's join does. Files proven match-free are carried over --// neither read again, rewritten, nor tombstoned.let mut files_probed = 0usize;+ let mut files_probe_cached = 0usize;+ let probe_started = Instant::now();if opts.prune_strategy == PruneStrategy::Probe && !rewrite.is_empty() {if pkset.is_empty() {// Every source row has a NULL PK component: nothing can match, so every@@ -2251,8 +2441,17 @@ async fn rewrite_partition(carried += rewrite.len();rewrite = Vec::new();} else {- files_probed = rewrite.len();- let (keep, skipped) = probe_files(+ let candidates = rewrite.len();+ let probe_cache = match &cache {+ Some(cache) => Some(ProbeCacheView {+ cache: cache.clone(),+ table_url: &table_url,+ signature: pk_signature(&table_schema, &opts.primary_keys, &partition_col)?,+ source_hashes: pkset.sorted_key_hashes(),+ }),+ None => None,+ };+ let probed = probe_files(&store,rewrite,&pkset,@@ -2261,22 +2460,45 @@ async fn rewrite_partition(&value,opts,&budgets,+ probe_cache.as_ref(),).await?;- rewrite = keep;- carried += skipped;+ rewrite = probed.keep;+ carried += probed.skipped;+ files_probe_cached = probed.cached;+ files_probed = candidates - probed.cached;}}+ let probe_ms = probe_started.elapsed().as_millis() as u64;+ let removed_keys: Vec<String> = match &cache {+ Some(_) => rewrite+ .iter()+ .map(|f| file_key(&table_url, &f.path))+ .collect(),+ None => Vec::new(),+ };+ let mut writer_keys = match &cache {+ Some(cache) => Some(OutputKeys::new(+ cache.clone(),+ &table_schema,+ &opts.primary_keys,+ )?),+ None => None,+ };// Single writer task per partition: file count and output sizing behave exactly as// in the sequential implementation. The channel is unbounded; memory is bounded by// the byte-budget permits attached to every message, not by message count.let (tx, mut rx) = mpsc::unbounded_channel::<(RecordBatch, BudgetPermit)>();let max_row_group_bytes = opts.max_row_group_bytes;+ let writer_pkset = pkset.clone();let writer_task = tokio::spawn(async move {let mut writer = writer;let mut adds = Vec::new();while let Some((batch, permit)) = rx.recv().await {+ if let Some(keys) = writer_keys.as_mut() {+ keys.note_write(&writer_pkset, &batch)?;+ }writer.write(batch)?;// The batch now lives (encoded) in the writer; release its budget.drop(permit);@@ -2284,6 +2506,7 @@ async fn rewrite_partition(// Bound resident memory: the writer holds the open file until flushed, so// flush on a size threshold instead of once at the end. Each flush yields// Add actions and starts a new file.+ let before = adds.len();roll_writer(&mut writer,&mut adds,@@ -2291,8 +2514,11 @@ async fn rewrite_partition(max_row_group_bytes,).await?;+ if let Some(keys) = writer_keys.as_mut() {+ keys.note_flush(&adds[before..]);+ }}- Ok::<_, Error>((writer, adds))+ Ok::<_, Error>((writer, adds, writer_keys))});// Reader tasks: up to max_parallel_files files stream concurrently (per call AND@@ -2344,14 +2570,18 @@ async fn rewrite_partition(// Writer errors take precedence: when the writer dies, readers fail secondarily on// send, and reporting those would mask the root cause.- let (mut writer, mut adds) = writer_task+ let (mut writer, mut adds, mut output_keys) = writer_task.await.map_err(|e| Error::Generic(format!("writer task panicked: {e}")))??;if let Some(e) = reader_err {return Err(e);}if opts.isolate_source_rows {+ let before = adds.len();adds.extend(writer.flush().await?);+ if let Some(keys) = output_keys.as_mut() {+ keys.note_flush(&adds[before..]);+ }}// Source rows are written LAST: an updated row therefore carries the source values,@@ -2376,8 +2606,13 @@ async fn rewrite_partition(let mut offset = 0usize;while offset < selected.num_rows() {let n = (selected.num_rows() - offset).min(slice_rows);- writer.write(selected.slice(offset, n))?;+ let slice = selected.slice(offset, n);+ if let Some(keys) = output_keys.as_mut() {+ keys.note_write(&pkset, &slice)?;+ }+ writer.write(slice)?;offset += n;+ let before = adds.len();roll_writer(&mut writer,&mut adds,@@ -2385,9 +2620,20 @@ async fn rewrite_partition(max_row_group_bytes,).await?;+ if let Some(keys) = output_keys.as_mut() {+ keys.note_flush(&adds[before..]);+ }}}+ let before = adds.len();adds.extend(writer.flush().await?);+ let written_keys = match output_keys {+ Some(mut keys) => {+ keys.note_flush(&adds[before..]);+ keys.into_written(&table_url)+ }+ None => Vec::new(),+ };// Every source row is written, but a source row that replaced an existing one is an// update, not an insert -- the batch is deduped, so each match consumes exactly one@@ -2404,11 +2650,15 @@ async fn rewrite_partition(files_added,files_removed,files_probed,+ files_probe_cached,+ probe_ms,rows_updated,rows_copied,rows_inserted,null_pk_rows,carried,+ removed_keys,+ written_keys,})}diff --git a/rust/deltalite/core/tests/handle_and_open.rs b/rust/deltalite/core/tests/handle_and_open.rsindex 337a8ce38215..8e07c9d34b0e 100644--- a/rust/deltalite/core/tests/handle_and_open.rs+++ b/rust/deltalite/core/tests/handle_and_open.rs@@ -18,6 +18,7 @@ use deltalake::{DeltaResult, Path, TableProperty};use deltalite_core::handle::TableHandle;use deltalite_core::table::{open_table, open_table_multipart, wrap_multipart, MultipartConfig};use deltalite_core::upsert::{upsert_cached_with_state, RelaxCache, UpsertOptions};+use deltalite_core::ProbeCache;use futures::stream::BoxStream;use object_store::local::LocalFileSystem;use object_store::{@@ -925,6 +926,8 @@ async fn hit_file_footer_reuse_is_bounded_by_reader_concurrency() {s.clone(),UpsertOptions {max_parallel_files: 2,+ // The seeding upserts cached these files; the test needs their probe.+ probe_cache: None,..opts()},MultipartConfig::default(),@@ -943,6 +946,63 @@ async fn hit_file_footer_reuse_is_bounded_by_reader_concurrency() {assert_eq!(counts.iter().filter(|&&count| count == 2).count(), 3);}+/// A file whose keys the cache holds is decided with no data read: first a file the+/// probe read once and found without a match, then a file an earlier upsert wrote.+#[tokio::test]+async fn a_cached_file_is_decided_without_a_data_read() {+ register_counting_scheme();+ let dir = tempfile::tempdir().expect("tempdir");+ let path = dir.path().join("t");+ let uri = format!("dltest://{}", path.to_string_lossy());+ let root = table_root(&path);+ let s = create_disjoint_files_table(&uri, &[&["a0", "a9"], &["m0", "m9"]]).await;+ let mut handle = TableHandle::open(uri.clone(), HashMap::new())+ .await+ .expect("open");+ let data_reads = |ops: &[(Op, String)]| {+ ops.iter()+ .filter(|(op, p)| op.is_read() && p.ends_with(".parquet") && !p.contains("_delta_log"))+ .count()+ };+ let cache = Arc::new(ProbeCache::new(1 << 20));++ // A fresh cache knows nothing yet: the probe reads the file and caches it.+ let probe = |key: &'static str| {+ let opts = UpsertOptions {+ probe_cache: Some(cache.clone()),+ ..opts()+ };+ (vec![batch(&s, &[key], vec![Some(1)])], opts)+ };+ for (key, probed, cached, reads) in [("a5", 1, 0, 1), ("a6", 0, 1, 0)] {+ let (b, o) = probe(key);+ let start = ops_len();+ let stats = handle+ .upsert(b, s.clone(), o, MultipartConfig::default())+ .await+ .expect("upsert");+ let ops = ops_since(start, &root);+ assert_eq!(stats.files_probed, probed, "{key}");+ assert_eq!(stats.files_probe_cached, cached, "{key}");+ assert_eq!(stats.files_removed, 0, "{key}");+ assert_eq!(data_reads(&ops), reads, "{key}: {ops:?}");+ }++ // The upserts above wrote files for `a5` and `a6`; their keys came from the writer.+ let (b, o) = probe("a5");+ let start = ops_len();+ let stats = handle+ .upsert(b, s.clone(), o, MultipartConfig::default())+ .await+ .expect("upsert");+ assert_eq!(stats.files_probed, 0);+ assert_eq!(stats.files_removed, 1);+ assert_eq!(stats.rows_updated, 1);+ let ops = ops_since(start, &root);+ // The one data read is the rewrite of the hit file, not a probe.+ assert_eq!(data_reads(&ops), 1, "{ops:?}");+}+/// After the commit the handle adopts the state delta-rs derived for it instead of/// listing the log and reading `_last_checkpoint` again, and that state is the log's/// truth.@@ -1470,7 +1530,10 @@ async fn a_probed_small_file_costs_one_request() {.upsert(vec![batch(&s, &["a5"], vec![Some(1)])],s.clone(),- opts(),+ UpsertOptions {+ probe_cache: None,+ ..opts()+ },MultipartConfig::default(),).awaitdiff --git a/rust/deltalite/core/tests/probe_cache.rs b/rust/deltalite/core/tests/probe_cache.rsnew file mode 100644index 000000000000..28af5b3a4816--- /dev/null+++ b/rust/deltalite/core/tests/probe_cache.rs@@ -0,0 +1,367 @@+//! The probe cache must not change what an upsert does. Each case runs the same upserts+//! against two copies of one table, one with a cache and one without, and requires the+//! same rows and the same file decisions after every upsert.++use std::collections::HashMap;+use std::sync::Arc;++use arrow_array::{Array, Int64Array, RecordBatch, StringArray};+use arrow_schema::{DataType, Field, Schema, SchemaRef};+use deltalake::kernel::{DataType as KernelType, StructField};+use deltalake::operations::create::CreateBuilder;+use deltalake::writer::{DeltaWriter, RecordBatchWriter};+use deltalake::Path;+use deltalite_core::compact::CompactOptions;+use deltalite_core::handle::TableHandle;+use deltalite_core::limits::ProcessLimits;+use deltalite_core::table::MultipartConfig;+use deltalite_core::upsert::{UpsertOptions, UpsertStats};+use deltalite_core::ProbeCache;+use object_store::ObjectStoreExt;+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;++const PARTITIONS: u64 = 5;+const SEED_KEYS: u64 = 2000;+const SEED_FLUSHES: u64 = 8;+const ROUNDS: u64 = 12;++#[derive(Clone, Copy, Debug)]+enum Keys {+ Int,+ Str,+ Composite,+ /// A PK that includes the partition column, which is not stored in the data files.+ WithPartition,+}++impl Keys {+ fn primary_keys(self) -> Vec<String> {+ let names: &[&str] = match self {+ Keys::Int => &["id_i"],+ Keys::Str => &["id_s"],+ Keys::Composite => &["id_s", "id_i"],+ Keys::WithPartition => &["id_i", "p"],+ };+ names.iter().map(|s| s.to_string()).collect()+ }+}++fn schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("id_i", DataType::Int64, true),+ Field::new("id_s", DataType::Utf8, true),+ Field::new("v", DataType::Int64, true),+ Field::new("p", DataType::Utf8, true),+ ]))+}++/// Rows for keys `ks` (key `k` lives in partition `k % PARTITIONS`), plus `null_rows`+/// rows whose PK columns are NULL, which never match and always insert.+fn rows(ks: &[u64], v: i64, null_rows: usize) -> RecordBatch {+ let mut id_i: Vec<Option<i64>> = ks.iter().map(|k| Some(*k as i64)).collect();+ let mut id_s: Vec<Option<String>> = ks.iter().map(|k| Some(format!("k{k:06}"))).collect();+ let mut p: Vec<String> = ks.iter().map(|k| (k % PARTITIONS).to_string()).collect();+ for i in 0..null_rows {+ id_i.push(None);+ id_s.push(None);+ p.push((i as u64 % PARTITIONS).to_string());+ }+ let n = id_i.len();+ RecordBatch::try_new(+ schema(),+ vec![+ Arc::new(Int64Array::from(id_i)),+ Arc::new(StringArray::from(id_s)),+ Arc::new(Int64Array::from(vec![v; n])),+ Arc::new(StringArray::from(p)),+ ],+ )+ .unwrap()+}++/// A partitioned table holding `SEED_KEYS` keys (offset by `offset`) in+/// `SEED_FLUSHES` files per partition.+async fn seed(dir: &std::path::Path, offset: u64) -> TableHandle {+ let mut table = CreateBuilder::new()+ .with_location(dir.to_str().unwrap())+ .with_columns(vec![+ StructField::new("id_i", KernelType::LONG, true),+ StructField::new("id_s", KernelType::STRING, true),+ StructField::new("v", KernelType::LONG, true),+ StructField::new("p", KernelType::STRING, true),+ ])+ .with_partition_columns(vec!["p".to_string()])+ .await+ .unwrap();+ let per_flush = SEED_KEYS / SEED_FLUSHES;+ for f in 0..SEED_FLUSHES {+ let ks: Vec<u64> = (offset + f * per_flush..offset + (f + 1) * per_flush).collect();+ let mut w = RecordBatchWriter::for_table(&table).unwrap();+ w.write(rows(&ks, 0, 0)).await.unwrap();+ w.flush_and_commit(&mut table).await.unwrap();+ }+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())+ .await+ .unwrap()+}++/// Every live row as (id_i, id_s, v, p), sorted.+async fn read_rows(handle: &TableHandle) -> Vec<(Option<i64>, Option<String>, i64, String)> {+ let store = handle.table().object_store();+ let mut out = Vec::new();+ for f in handle.files().unwrap() {+ let p = f.partition_values["p"].clone().unwrap();+ let bytes = store+ .get(&Path::parse(&f.path).unwrap())+ .await+ .unwrap()+ .bytes()+ .await+ .unwrap();+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)+ .unwrap()+ .build()+ .unwrap()+ {+ let batch = batch.unwrap();+ let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone();+ let (id_i, id_s, v) = (col("id_i"), col("id_s"), col("v"));+ let id_i = id_i.as_any().downcast_ref::<Int64Array>().unwrap();+ let id_s = id_s.as_any().downcast_ref::<StringArray>().unwrap();+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();+ for i in 0..batch.num_rows() {+ out.push((+ (!id_i.is_null(i)).then(|| id_i.value(i)),+ (!id_s.is_null(i)).then(|| id_s.value(i).to_string()),+ v.value(i),+ p.clone(),+ ));+ }+ }+ }+ out.sort();+ out+}++/// Deterministic pseudo-random sequence, so both copies get the same batches.+struct Lcg(u64);++impl Lcg {+ fn next(&mut self, below: u64) -> u64 {+ self.0 = self+ .0+ .wrapping_mul(6364136223846793005)+ .wrapping_add(1442695040888963407);+ (self.0 >> 33) % below+ }+}++/// One table under test: a copy with the cache and a copy without it.+struct Twin {+ cached: TableHandle,+ plain: TableHandle,+ offset: u64,+ /// Keys the last upsert wrote, so the next one hits files that the cache got from+ /// the writer rather than from a probe.+ last_keys: Vec<u64>,+ next_new_key: u64,+ _dirs: (tempfile::TempDir, tempfile::TempDir),+}++impl Twin {+ async fn new(offset: u64) -> Self {+ let (a, b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());+ Self {+ cached: seed(a.path(), offset).await,+ plain: seed(b.path(), offset).await,+ offset,+ last_keys: Vec::new(),+ next_new_key: offset + SEED_KEYS,+ _dirs: (a, b),+ }+ }++ fn batch_keys(&mut self, rng: &mut Lcg) -> Vec<u64> {+ let mut ks: Vec<u64> = Vec::new();+ ks.extend(self.last_keys.iter().take(20));+ ks.extend((0..40).map(|_| self.offset + rng.next(SEED_KEYS)));+ ks.extend(self.next_new_key..self.next_new_key + 15);+ self.next_new_key += 15;+ ks.sort_unstable();+ ks.dedup();+ self.last_keys = ks.clone();+ ks+ }+}++fn opts(keys: Keys, round: u64, cache: Option<Arc<ProbeCache>>) -> UpsertOptions {+ UpsertOptions {+ primary_keys: keys.primary_keys(),+ partition_key: Some("p".to_string()),+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),+ // Small files, so one upsert writes several files per partition.+ target_file_size: Some(2 * 1024),+ isolate_source_rows: round % 3 == 1,+ max_row_group_bytes: (round % 4 == 2).then_some(512),+ // One reader per partition makes the output layout deterministic, so the two+ // copies can be compared file decision by file decision.+ max_parallel_files: 1,+ probe_cache: cache,+ ..Default::default()+ }+}++/// The file decisions of an upsert, which the cache must not change.+fn decisions(s: &UpsertStats) -> [usize; 7] {+ [+ s.files_removed,+ s.files_added,+ s.files_carried_over,+ s.rows_updated,+ s.rows_inserted,+ s.rows_copied,+ s.null_pk_rows,+ ]+}++#[derive(Default)]+struct Totals {+ probed_cached: usize,+ probed_plain: usize,+ decided_from_cache: usize,+}++async fn upsert_round(+ twin: &mut Twin,+ keys: Keys,+ round: u64,+ rng: &mut Lcg,+ cache: &Arc<ProbeCache>,+ totals: &mut Totals,+) {+ let ks = twin.batch_keys(rng);+ let batch = rows(&ks, round as i64 + 1, 2);+ let with = twin+ .cached+ .upsert(+ vec![batch.clone()],+ schema(),+ opts(keys, round, Some(cache.clone())),+ MultipartConfig::default(),+ )+ .await+ .unwrap();+ let without = twin+ .plain+ .upsert(+ vec![batch],+ schema(),+ opts(keys, round, None),+ MultipartConfig::default(),+ )+ .await+ .unwrap();+ assert_eq!(+ decisions(&with),+ decisions(&without),+ "{keys:?} round {round}: {with:?} vs {without:?}"+ );+ assert_eq!(+ read_rows(&twin.cached).await,+ read_rows(&twin.plain).await,+ "{keys:?} round {round}"+ );+ assert_eq!(without.files_probe_cached, 0);+ totals.probed_cached += with.files_probed;+ totals.probed_plain += without.files_probed;+ totals.decided_from_cache += with.files_probe_cached;+}++async fn compact_both(twin: &mut Twin) {+ let copts = || CompactOptions {+ target_file_size: Some(64 * 1024),+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),+ ..Default::default()+ };+ let a = twin+ .cached+ .compact(copts(), MultipartConfig::default())+ .await+ .unwrap();+ let b = twin+ .plain+ .compact(copts(), MultipartConfig::default())+ .await+ .unwrap();+ assert!(a.files_removed > 0);+ assert_eq!(a.files_removed, b.files_removed);+ assert_eq!(read_rows(&twin.cached).await, read_rows(&twin.plain).await);+}++/// Upserts into `tables` tables in turn, all sharing one cache of `cache_bytes`, with+/// one compaction halfway when `compact` is set.+async fn run(+ keys: Keys,+ tables: u64,+ cache_bytes: usize,+ compact: bool,+) -> (Totals, Arc<ProbeCache>) {+ let cache = Arc::new(ProbeCache::new(cache_bytes));+ let mut twins = Vec::new();+ for t in 0..tables {+ twins.push(Twin::new(t * 1_000_000).await);+ }+ let mut rng = Lcg(42);+ let mut totals = Totals::default();+ for round in 0..ROUNDS {+ for twin in twins.iter_mut() {+ upsert_round(twin, keys, round, &mut rng, &cache, &mut totals).await;+ if compact && round == ROUNDS / 2 {+ compact_both(twin).await;+ }+ }+ }+ (totals, cache)+}++#[tokio::test]+async fn cached_upserts_match_uncached_upserts_for_every_key_type() {+ for keys in [Keys::Int, Keys::Str, Keys::Composite, Keys::WithPartition] {+ let (totals, cache) = run(keys, 1, 64 << 20, true).await;+ // The cache must actually take part: most later probes come from it.+ assert!(totals.decided_from_cache > 0, "{keys:?}");+ assert!(+ totals.probed_cached * 2 < totals.probed_plain,+ "{keys:?}: probed {} with the cache, {} without",+ totals.probed_cached,+ totals.probed_plain+ );+ assert_eq!(cache.stats().evictions, 0, "{keys:?}");+ }+}++#[tokio::test]+async fn two_tables_can_share_one_cache() {+ let (totals, cache) = run(Keys::Composite, 2, 64 << 20, true).await;+ assert!(totals.decided_from_cache > 0);+ assert!(cache.stats().entries > 0);+}++#[tokio::test]+async fn a_cache_smaller_than_the_tables_evicts_and_stays_correct() {+ // Without compaction every file is small enough for an entry, but the files of the+ // three tables together are not.+ let (totals, cache) = run(Keys::Str, 3, 32 * 1024, false).await;+ let stats = cache.stats();+ assert!(stats.evictions > 0);+ assert!(stats.bytes <= 32 * 1024);+ assert!(totals.decided_from_cache > 0);+}++#[tokio::test]+async fn a_disabled_cache_probes_every_candidate() {+ let (totals, cache) = run(Keys::Int, 1, 0, true).await;+ assert_eq!(totals.decided_from_cache, 0);+ assert_eq!(totals.probed_cached, totals.probed_plain);+ assert_eq!(cache.stats().entries, 0);+}diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.tomlindex 54ba588631d4..06636d3b831d 100644--- a/rust/deltalite/python/Cargo.toml+++ b/rust/deltalite/python/Cargo.toml@@ -1,7 +1,7 @@[package]name = "deltalite-python"# Mirrors pyproject.toml in this directory (always identical); bump both together.-version = "0.1.13"+version = "0.1.14"edition = "2021"# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.mdindex 034b3da9fcbb..13d9c14c8dbb 100644--- a/rust/deltalite/python/README.md+++ b/rust/deltalite/python/README.md@@ -124,14 +124,17 @@ log.### `UpsertStats`Returned by `upsert`. Counts: `version`, `partitions_touched`, `files_added`,-`files_removed`, `files_carried_over`, `files_probed`, `rows_updated`,-`rows_inserted`, `rows_copied`, `source_rows`, `null_pk_rows`. Per-phase+`files_removed`, `files_carried_over`, `files_probed` (candidate files the probe+read), `files_probe_cached` (candidate files the probe decided from the probe+cache), `rows_updated`, `rows_inserted`, `rows_copied`, `source_rows`,+`null_pk_rows`. Per-phasewall-clock timings (milliseconds): `ingest_ms` (importing the pyarrow source),`initial_open_ms` (the full snapshot load in `DeltaLiteTable.open`, reported onthe first upsert through a handle and `0` afterwards, so a sum over a handle'supserts counts it once), `open_ms` (snapshot refreshes around the upsert),`relax_ms` (nullability-relax check), `plan_ms` (listing + pruning files),-`rewrite_ms` (reading + rewriting the touched partitions), `commit_ms`+`rewrite_ms` (reading + rewriting the touched partitions), `probe_ms` (probing,+summed over partitions, part of `rewrite_ms`), `commit_ms`(committing to the Delta log), `maintenance_ms` (checkpoint/log cleanup,non-zero only on checkpoint-boundary commits). `columns_relaxed` countsnon-nullable columns flipped to nullable before the write.@@ -203,6 +206,7 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:| `multipart_threshold` / `multipart_part_size` | `64 MiB` / `16 MiB` | Multipart upload thresholds (`0` threshold disables). || `commit_max_retries` | `15` | Conditional-put commit retry budget. || `commit_metadata` | `None` | Extra key/values recorded in the Delta commit. |+| `use_probe_cache` | `True` | Use the process-wide probe cache (see below). `False` reads every candidate file from storage. |**Process-global** — environment variables, enforced *on top of* the per-callknobs so that many concurrent upsert threads in one process cannot multiply the@@ -215,6 +219,23 @@ budgets:`DELTALITE_MAX_SOURCE_BYTES`, `DELTALITE_MULTIPART_THRESHOLD_BYTES`,`DELTALITE_MULTIPART_PART_SIZE_BYTES`.+**Probe cache.** The probe reads a footer and the PK column of every candidate+file, so on object storage its cost grows with the number of files in the+touched partitions. A Delta data file never changes, so deltalite keeps the+64-bit hashes of each file's PK tuples in one process-wide LRU cache, keyed by+table URL and file path. A probe that reads a file and finds no match caches it,+and an upsert caches the files it writes once its commit lands. A later probe+decides a cached file with no read. Equal keys always give equal hashes, so a+cached file that holds a key is never skipped. A hash collision only adds a file+to the rewrite, which copies its rows unchanged. A file the commit removes leaves+the cache. `DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` turns the cache off) caps+the entries, and one entry can use at most an eighth of the cap, so a file with+more keys is probed every time. Hashes collected for files still being read or+written use at most a quarter of the cap more, so the cache uses at most+1.25 times the cap. `deltalite.set_probe_cache_max_bytes(n)` changes the cap at+run time, `deltalite.clear_probe_cache()` empties the cache, and+`deltalite.probe_cache_stats()` returns its size and hit counters.+**Checkpoint prefetch.** Opening or reloading a table reads its latestcheckpoint Parquet file whole in one GET and serves the Parquet reader's manysmall range reads from memory, instead of one round trip per footer, metadata@@ -273,7 +294,7 @@ exists.deltalite emits via the Rust [`metrics`](https://docs.rs/metrics) facade (staticlabels only): `deltalite_upserts_total` (`outcome`, `prune_strategy`,`error_kind`), `deltalite_upsert_duration_seconds`,-`deltalite_files_{added,removed,carried_over,probed}_total`,+`deltalite_files_{added,removed,carried_over,probed,probe_cached}_total`,`deltalite_rows_{updated,inserted,copied}_total`,`deltalite_checkpoint_prefetch_total` (`outcome`),`deltalite_refresh_probe_total` (`outcome`: `current`, `new_commits`,diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.tomlindex f1102f1387af..b7fcc623a468 100644--- a/rust/deltalite/python/pyproject.toml+++ b/rust/deltalite/python/pyproject.toml@@ -10,7 +10,7 @@ build-backend = "maturin"[project]name = "deltalite"# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.-version = "0.1.13"+version = "0.1.14"description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"readme = "README.md"requires-python = ">=3.12"diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rsindex b7a440eedd6d..07ac05aca06f 100644--- a/rust/deltalite/python/src/lib.rs+++ b/rust/deltalite/python/src/lib.rs@@ -15,6 +15,7 @@ use deltalake::writer::RecordBatchWriter;use deltalite_core::errors::Error;use deltalite_core::handle::TableHandle;use deltalite_core::limits::ProcessLimits;+use deltalite_core::probecache::ProbeCache;use deltalite_core::schema::import_column;use deltalite_core::table::{open_table, table_exists, MultipartConfig};use deltalite_core::upsert::{PruneStrategy, UpsertOptions};@@ -152,6 +153,8 @@ pub struct UpsertStats {#[pyo3(get)]pub files_probed: usize,#[pyo3(get)]+ pub files_probe_cached: usize,+ #[pyo3(get)]pub rows_updated: usize,#[pyo3(get)]pub rows_inserted: usize,@@ -166,6 +169,8 @@ pub struct UpsertStats {#[pyo3(get)]pub rewrite_ms: u64,#[pyo3(get)]+ pub probe_ms: u64,+ #[pyo3(get)]pub commit_ms: u64,#[pyo3(get)]pub columns_relaxed: usize,@@ -186,14 +191,15 @@ impl UpsertStats {fn __repr__(&self) -> String {format!("UpsertStats(version={}, partitions_touched={}, files_added={}, files_removed={}, \- files_carried_over={}, files_probed={}, rows_updated={}, rows_inserted={}, \- rows_copied={})",+ files_carried_over={}, files_probed={}, files_probe_cached={}, rows_updated={}, \+ rows_inserted={}, rows_copied={})",self.version,self.partitions_touched,self.files_added,self.files_removed,self.files_carried_over,self.files_probed,+ self.files_probe_cached,self.rows_updated,self.rows_inserted,self.rows_copied@@ -210,6 +216,7 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {files_added: s.files_added,files_carried_over: s.files_carried_over,files_probed: s.files_probed,+ files_probe_cached: s.files_probe_cached,rows_updated: s.rows_updated,rows_inserted: s.rows_inserted,rows_copied: s.rows_copied,@@ -217,6 +224,7 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {null_pk_rows: s.null_pk_rows,plan_ms: s.plan_ms,rewrite_ms: s.rewrite_ms,+ probe_ms: s.probe_ms,commit_ms: s.commit_ms,columns_relaxed: s.columns_relaxed,open_ms: s.open_ms,@@ -391,6 +399,9 @@ impl DeltaLiteTable {/// `max_row_group_bytes` closes an output row group once its encoded bytes reach the/// value, so a later reader fetches less per row group. `None` (the default) closes/// row groups only at the encoder's row cap.+ ///+ /// `use_probe_cache=False` makes the probe read every candidate file from storage+ /// instead of using the process-wide probe cache (see `set_probe_cache_max_bytes`).#[pyo3(signature = (data,primary_keys,@@ -412,6 +423,7 @@ impl DeltaLiteTable {max_source_bytes = None,multipart_threshold = None,multipart_part_size = None,+ use_probe_cache = true,))]#[allow(clippy::too_many_arguments)]fn upsert(@@ -436,6 +448,7 @@ impl DeltaLiteTable {max_source_bytes: Option<usize>,multipart_threshold: Option<usize>,multipart_part_size: Option<usize>,+ use_probe_cache: bool,) -> PyResult<UpsertStats> {// Import while holding the GIL (it reads a Python object), then release it for// all the I/O.@@ -472,6 +485,7 @@ impl DeltaLiteTable {// Shared with every other concurrent upsert in this process, so the// process-level memory bound holds regardless of thread count.limits: ProcessLimits::global().clone(),+ probe_cache: use_probe_cache.then(|| ProbeCache::global().clone()),};let multipart = MultipartConfig::resolve(multipart_threshold, multipart_part_size);@@ -659,9 +673,41 @@ impl DeltaLiteTable {}}+/// Set the byte cap of the process-wide probe cache, evicting entries down to it. `0`+/// turns the cache off. The default comes from `DELTALITE_PROBE_CACHE_BYTES` (128 MiB+/// when unset).+#[pyfunction]+fn set_probe_cache_max_bytes(max_bytes: usize) {+ ProbeCache::global().set_max_bytes(max_bytes);+}++/// Drop every entry of the process-wide probe cache.+#[pyfunction]+fn clear_probe_cache() {+ ProbeCache::global().clear();+}++/// Size and counters of the process-wide probe cache.+#[pyfunction]+fn probe_cache_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {+ let s = ProbeCache::global().stats();+ let d = PyDict::new(py);+ d.set_item("entries", s.entries)?;+ d.set_item("bytes", s.bytes)?;+ d.set_item("max_bytes", s.max_bytes)?;+ d.set_item("hits", s.hits)?;+ d.set_item("misses", s.misses)?;+ d.set_item("inserts", s.inserts)?;+ d.set_item("evictions", s.evictions)?;+ Ok(d.into_any().unbind())+}+#[pymodule]fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_class::<DeltaLiteTable>()?;+ m.add_function(wrap_pyfunction!(set_probe_cache_max_bytes, m)?)?;+ m.add_function(wrap_pyfunction!(clear_probe_cache, m)?)?;+ m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;m.add_class::<UpsertStats>()?;m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;m.add(diff --git a/rust/deltalite/python/tests/test_probe.py b/rust/deltalite/python/tests/test_probe.pyindex af7bfa6abe5c..4decfab72ac9 100644--- a/rust/deltalite/python/tests/test_probe.py+++ b/rust/deltalite/python/tests/test_probe.py@@ -464,3 +464,45 @@ def test_invalid_prune_strategy_is_rejected(tmp_path):with pytest.raises(deltalite.DeltaLiteError, match="unknown prune_strategy"):upsert_path(uri, rows(["a"], 2), ["id"], None, prune_strategy="bogus")+++# --------------------------------------------------------------------------------------+# The probe cache: later upserts decide files from cached keys, never differently+# --------------------------------------------------------------------------------------++[email protected]("use_probe_cache", [True, False])[email protected]("partitioned", [False, True])+def test_repeated_upserts_with_the_probe_cache_match_merge(tmp_path, use_probe_cache, partitioned):+ """Each batch re-touches rows the previous batch wrote, plus old rows and new rows, so+ later probes decide files from keys that both the probe and the writer cached."""+ import deltalite++ parts = ["p1", "p2"] if partitioned else ["p1"]+ pkey = PARTITION_KEY if partitioned else None+ base = tmp_path / "base"+ per_file = build_multifile_table(str(base), partitioned=partitioned, n_files=4, partitions=parts)+ old = [(i, p) for p in parts for f in per_file[p] for i in f]++ uri_m, uri_u = str(tmp_path / "m"), str(tmp_path / "u")+ for uri in (uri_m, uri_u):+ shutil.copytree(base, uri)++ rng = random.Random(99)+ previous: list[tuple[str, str]] = []+ cached = 0+ for step in range(6):+ picked = previous[:15] + rng.sample(old, 10) + [(i, rng.choice(parts)) for i in rand_ids(rng, 5)]+ picked = list(dict(picked).items())+ batch = rows([i for i, _ in picked], step + 2, part=[p for _, p in picked])+ merge_path(uri_m, batch, ["id"], pkey)+ stats = upsert_path(uri_u, batch, ["id"], pkey, use_probe_cache=use_probe_cache)+ assert_parity(uri_m, uri_u, f"cache_{use_probe_cache}_step_{step}")+ cached += stats.files_probe_cached+ previous = picked++ if use_probe_cache:+ assert cached > 0+ assert deltalite.probe_cache_stats()["entries"] > 0+ else:+ assert cached == 0