Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(data-warehouse): cache deltalite probe keys per data file (#121060)

Tom Owers45bfdc1bfb27
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 536a2822885a..3d30f7b574b0 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -3973,6 +3973,7 @@ dependencies = [
"metrics",
"object_store",
"parquet",
+ "percent-encoding",
"serde_json",
"sqlparser",
"tempfile",
@@ -3985,7 +3986,7 @@ dependencies = [
[[package]]
name = "deltalite-python"
-version = "0.1.13"
+version = "0.1.14"
dependencies = [
"arrow",
"arrow-array",
diff --git a/rust/deltalite/README.md b/rust/deltalite/README.md
index 8d12d5aaa0fb..56d46eadc35a 100644
--- a/rust/deltalite/README.md
+++ b/rust/deltalite/README.md
@@ -117,7 +117,9 @@ Per-call (arguments to `DeltaLiteTable.upsert`): `max_parallel_partitions` (2),
survivors; on: they get their own files),
`max_row_group_bytes` (`None`: row groups close only at the encoder's row cap),
`max_source_bytes` (2 GiB guard), `multipart_threshold` (64 MiB) /
-`multipart_part_size` (16 MiB), `commit_max_retries` (15).
+`multipart_part_size` (16 MiB), `commit_max_retries` (15), `use_probe_cache`
+(on: the probe decides a file whose PK hashes the process-wide probe cache holds
+without reading it; see `python/README.md`, "Probe cache").
Process-global (environment, enforced on top of the per-call knobs so that
~15 concurrent upsert threads in one Temporal worker cannot multiply the
@@ -125,7 +127,8 @@ budgets): `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS` (8),
`DELTALITE_PROCESS_MAX_PARALLEL_FILES` (16),
`DELTALITE_PROCESS_MAX_BUFFERED_BYTES` (256 MiB),
`DELTALITE_PROCESS_MAX_FETCH_BYTES` (256 MiB), `DELTALITE_MAX_SOURCE_BYTES`,
-`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`.
+`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`,
+`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache).
Kill switches for the request savings on the log and on small files (set to `0`
to restore the earlier requests): `DELTALITE_PROBE_REFRESH`,
@@ -142,7 +145,7 @@ suggests knob values from concurrency + pod memory, e.g.
Metrics (`metrics` crate, static labels only): `deltalite_upserts_total`
(`outcome`, `prune_strategy`, `error_kind`), `deltalite_upsert_duration_seconds`,
-`deltalite_files_{added,removed,carried_over,probed}_total`,
+`deltalite_files_{added,removed,carried_over,probed,probe_cached}_total`,
`deltalite_rows_{updated,inserted,copied}_total`.
## Pre-rollout conditions (from the spike verdict, still open)
diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.toml
index eb61951a67fe..268408c77f37 100644
--- a/rust/deltalite/core/Cargo.toml
+++ b/rust/deltalite/core/Cargo.toml
@@ -53,6 +53,8 @@ arrow-ord = { version = "58" }
delta_kernel = { package = "buoyant_kernel", version = "=0.22.2", default-features = false }
chrono = { workspace = true }
indexmap = "2"
+# Already linked by `deltalake`; decodes Add paths into the form the snapshot reports.
+percent-encoding = "2"
sqlparser = "0.61"
async-trait = { workspace = true }
diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rs
index 15f93945c8ae..898caba68fda 100644
--- a/rust/deltalite/core/src/lib.rs
+++ b/rust/deltalite/core/src/lib.rs
@@ -36,6 +36,7 @@ pub mod limits;
mod logprobe;
pub mod pkset;
pub mod prefetch;
+pub mod probecache;
pub mod schema;
mod smallfile;
mod stats;
@@ -49,6 +50,7 @@ pub use errors::{Error, Result};
pub use handle::{LiveFile, TableHandle};
pub use limits::ProcessLimits;
pub use prefetch::CheckpointCache;
+pub use probecache::{ProbeCache, ProbeCacheStats};
pub use table::{
open_table, open_table_multipart, open_table_prefetched, wrap_multipart, MultipartConfig,
};
diff --git a/rust/deltalite/core/src/pkset.rs b/rust/deltalite/core/src/pkset.rs
index 84750ee3647b..f0243c4be7ce 100644
--- a/rust/deltalite/core/src/pkset.rs
+++ b/rust/deltalite/core/src/pkset.rs
@@ -20,6 +20,7 @@ use arrow_row::{RowConverter, SortField};
use arrow_schema::SchemaRef;
use crate::errors::{Error, Result};
+use crate::probecache::hash_key;
/// A hash set of primary-key tuples with SQL NULL semantics (NULL never matches).
pub struct PkSet {
@@ -124,6 +125,69 @@ impl PkSet {
Ok((0..num_rows).any(|i| mask[i] && self.keys.contains(rows.row(i).as_ref())))
}
+ /// [`Self::contains_any_columns`] that also appends the hash of every matchable row
+ /// to `hashes` (see [`crate::probecache`]). It reads every row only when `hashes` is
+ /// given; without it, it stops at the first match.
+ pub fn probe_columns(
+ &self,
+ cols: &[Arc<dyn Array>],
+ num_rows: usize,
+ hashes: Option<&mut Vec<u64>>,
+ ) -> Result<bool> {
+ let Some(hashes) = hashes else {
+ return self.contains_any_columns(cols, num_rows);
+ };
+ if num_rows == 0 {
+ return Ok(false);
+ }
+ let mask = matchable_mask(cols, num_rows);
+ let rows = self
+ .converter
+ .convert_columns(cols)
+ .map_err(|e| Error::Generic(format!("row encode: {e}")))?;
+ let mut hit = false;
+ for (i, matchable) in mask.iter().enumerate() {
+ if !matchable {
+ continue;
+ }
+ let key = rows.row(i);
+ hashes.push(hash_key(key.as_ref()));
+ hit = hit || self.keys.contains(key.as_ref());
+ }
+ Ok(hit)
+ }
+
+ /// Append the hash of every matchable row of the given PK columns to `hashes`.
+ pub fn hash_columns(
+ &self,
+ cols: &[Arc<dyn Array>],
+ num_rows: usize,
+ hashes: &mut Vec<u64>,
+ ) -> Result<()> {
+ if num_rows == 0 {
+ return Ok(());
+ }
+ let mask = matchable_mask(cols, num_rows);
+ let rows = self
+ .converter
+ .convert_columns(cols)
+ .map_err(|e| Error::Generic(format!("row encode: {e}")))?;
+ hashes.extend(
+ (0..num_rows)
+ .filter(|i| mask[*i])
+ .map(|i| hash_key(rows.row(i).as_ref())),
+ );
+ Ok(())
+ }
+
+ /// The sorted, deduplicated hashes of the keys in the set.
+ pub fn sorted_key_hashes(&self) -> Vec<u64> {
+ let mut out: Vec<u64> = self.keys.iter().map(|k| hash_key(k)).collect();
+ out.sort_unstable();
+ out.dedup();
+ out
+ }
+
/// For each row of `batch`, whether it is replaced by a source row.
pub fn contains_rows(&self, batch: &RecordBatch) -> Result<Vec<bool>> {
let cols = self.pk_columns(batch);
diff --git a/rust/deltalite/core/src/probecache.rs b/rust/deltalite/core/src/probecache.rs
new file mode 100644
index 000000000000..1eec08786ecd
--- /dev/null
+++ b/rust/deltalite/core/src/probecache.rs
@@ -0,0 +1,585 @@
+//! Process-wide cache of the primary-key hashes of data files, for the upsert probe.
+//!
+//! The probe decides, for each candidate file of a touched partition, whether the file
+//! holds any key of the batch. Without a cache that costs a footer read and a PK-column
+//! read per file on every upsert, so on object storage the probe grows with the number of
+//! files in the table, not with the batch. A Delta data file never changes after it is
+//! written, so the keys a file holds can be kept for as long as the process lives.
+//!
+//! Each entry is the sorted, deduplicated 64-bit hashes of the file's matchable PK tuples
+//! (no NULL component), encoded exactly as [`crate::pkset::PkSet`] encodes them. A file is
+//! a candidate when its hashes and the batch's hashes have a common value. A matching key
+//! always gives equal hashes, so the cache never skips a file that holds a key. A hash
+//! collision can only add a file to the rewrite set, and a rewrite of a file without
+//! matches copies every row unchanged, so the table content stays correct.
+//!
+//! Entries are keyed by the table URL plus the URL-decoded file path, and carry a
+//! signature of the PK columns and their types. A PK type change after schema evolution
+//! changes the encoding, so an entry with another signature counts as a miss. An upsert
+//! evicts the entries of the files it removes after its commit lands; files that another
+//! writer removes are never asked for again and age out of the LRU order.
+
+use std::collections::hash_map::DefaultHasher;
+use std::collections::{BTreeMap, HashMap};
+use std::hash::{Hash, Hasher};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, OnceLock};
+
+use arrow_schema::DataType;
+
+/// Environment variable for the byte cap of the process-wide cache; `0` turns it off.
+pub const PROBE_CACHE_BYTES_ENV: &str = "DELTALITE_PROBE_CACHE_BYTES";
+/// Default byte cap of the process-wide cache.
+pub const DEFAULT_PROBE_CACHE_BYTES: usize = 128 * 1024 * 1024;
+
+/// Fixed bookkeeping cost charged per entry on top of its key and hashes.
+const ENTRY_OVERHEAD_BYTES: usize = 128;
+/// Reservations for in-flight key collection grow in steps of this size, so a collector
+/// touches the shared counter once per step instead of once per batch.
+const RESERVE_STEP_BYTES: usize = 64 * 1024;
+
+/// Counters and sizes of a [`ProbeCache`].
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ProbeCacheStats {
+ pub entries: usize,
+ pub bytes: usize,
+ pub max_bytes: usize,
+ /// Lookups that found an entry with the right signature.
+ pub hits: u64,
+ /// Lookups that found no usable entry.
+ pub misses: u64,
+ /// Entries stored, by a probe or by an upsert that wrote the file.
+ pub inserts: u64,
+ /// Entries dropped to stay under the byte cap.
+ pub evictions: u64,
+}
+
+struct Entry {
+ signature: u64,
+ hashes: Arc<[u64]>,
+ tick: u64,
+ bytes: usize,
+}
+
+#[derive(Default)]
+struct Inner {
+ map: HashMap<String, Entry>,
+ /// Least recently used first.
+ order: BTreeMap<u64, String>,
+ tick: u64,
+ bytes: usize,
+}
+
+impl Inner {
+ fn remove(&mut self, key: &str) -> bool {
+ match self.map.remove(key) {
+ Some(e) => {
+ self.order.remove(&e.tick);
+ self.bytes -= e.bytes;
+ true
+ }
+ None => false,
+ }
+ }
+
+ fn evict_to(&mut self, max_bytes: usize) -> u64 {
+ let mut evicted = 0;
+ while self.bytes > max_bytes {
+ let Some((_, key)) = self.order.pop_first() else {
+ break;
+ };
+ if let Some(e) = self.map.remove(&key) {
+ self.bytes -= e.bytes;
+ evicted += 1;
+ }
+ }
+ evicted
+ }
+}
+
+/// A byte-capped LRU map from data file to the PK hashes the file holds.
+pub struct ProbeCache {
+ inner: Mutex<Inner>,
+ max_bytes: AtomicUsize,
+ /// Bytes reserved by collectors that are still reading or writing a file. Kept apart
+ /// from `inner.bytes` and capped at a quarter of `max_bytes`, so the memory of the
+ /// cache feature stays under 1.25 x `max_bytes` whatever the concurrency.
+ collecting: AtomicUsize,
+ hits: AtomicU64,
+ misses: AtomicU64,
+ inserts: AtomicU64,
+ evictions: AtomicU64,
+}
+
+impl std::fmt::Debug for ProbeCache {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("ProbeCache")
+ .field("stats", &self.stats())
+ .finish()
+ }
+}
+
+impl ProbeCache {
+ /// A cache that holds at most `max_bytes`; `0` stores nothing.
+ pub fn new(max_bytes: usize) -> Self {
+ Self {
+ inner: Mutex::new(Inner::default()),
+ max_bytes: AtomicUsize::new(max_bytes),
+ collecting: AtomicUsize::new(0),
+ hits: AtomicU64::new(0),
+ misses: AtomicU64::new(0),
+ inserts: AtomicU64::new(0),
+ evictions: AtomicU64::new(0),
+ }
+ }
+
+ /// The process-wide cache, sized from [`PROBE_CACHE_BYTES_ENV`] on first use.
+ pub fn global() -> &'static Arc<ProbeCache> {
+ static GLOBAL: OnceLock<Arc<ProbeCache>> = OnceLock::new();
+ GLOBAL.get_or_init(|| Arc::new(ProbeCache::new(max_bytes_from_env())))
+ }
+
+ pub fn max_bytes(&self) -> usize {
+ self.max_bytes.load(Ordering::Relaxed)
+ }
+
+ /// Change the byte cap, evicting least recently used entries down to it.
+ pub fn set_max_bytes(&self, max_bytes: usize) {
+ self.max_bytes.store(max_bytes, Ordering::Relaxed);
+ let evicted = self.lock().evict_to(max_bytes);
+ self.evictions.fetch_add(evicted, Ordering::Relaxed);
+ }
+
+ pub fn enabled(&self) -> bool {
+ self.max_bytes() > 0
+ }
+
+ /// Largest entry the cache accepts. A file with more keys than this is probed on every
+ /// upsert, so that one huge file cannot push every other entry out.
+ pub fn max_entry_bytes(&self) -> usize {
+ self.max_bytes() / 8
+ }
+
+ pub fn clear(&self) {
+ let mut inner = self.lock();
+ inner.map.clear();
+ inner.order.clear();
+ inner.bytes = 0;
+ }
+
+ pub fn stats(&self) -> ProbeCacheStats {
+ let inner = self.lock();
+ ProbeCacheStats {
+ entries: inner.map.len(),
+ bytes: inner.bytes,
+ max_bytes: self.max_bytes(),
+ hits: self.hits.load(Ordering::Relaxed),
+ misses: self.misses.load(Ordering::Relaxed),
+ inserts: self.inserts.load(Ordering::Relaxed),
+ evictions: self.evictions.load(Ordering::Relaxed),
+ }
+ }
+
+ fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
+ // A panic while the lock is held leaves the map consistent (every mutation is a
+ // few map operations), so a poisoned lock is still safe to use.
+ self.inner.lock().unwrap_or_else(|e| e.into_inner())
+ }
+
+ /// The sorted key hashes of `key`, when an entry with `signature` exists.
+ pub(crate) fn get(&self, key: &str, signature: u64) -> Option<Arc<[u64]>> {
+ if !self.enabled() {
+ return None;
+ }
+ let mut inner = self.lock();
+ inner.tick += 1;
+ let tick = inner.tick;
+ let found = match inner.map.get_mut(key) {
+ Some(e) if e.signature == signature => {
+ let old = e.tick;
+ e.tick = tick;
+ Some((old, e.hashes.clone()))
+ }
+ _ => None,
+ };
+ match found {
+ Some((old, hashes)) => {
+ inner.order.remove(&old);
+ inner.order.insert(tick, key.to_string());
+ drop(inner);
+ self.hits.fetch_add(1, Ordering::Relaxed);
+ Some(hashes)
+ }
+ None => {
+ drop(inner);
+ self.misses.fetch_add(1, Ordering::Relaxed);
+ None
+ }
+ }
+ }
+
+ /// Store the key hashes of a file. `hashes` need not be sorted or unique.
+ pub(crate) fn insert(&self, key: String, signature: u64, mut hashes: Vec<u64>) {
+ if !self.enabled() {
+ return;
+ }
+ hashes.sort_unstable();
+ hashes.dedup();
+ // The key is held twice: once in `map` and once in `order`.
+ let bytes = hashes.len() * 8 + 2 * key.len() + ENTRY_OVERHEAD_BYTES;
+ let mut inner = self.lock();
+ // Read the cap under the lock: `set_max_bytes` evicts under it too, so a cap lowered
+ // concurrently is never missed by this insertion.
+ let max_bytes = self.max_bytes();
+ if max_bytes == 0 || bytes > max_bytes / 8 {
+ return;
+ }
+ inner.remove(&key);
+ inner.tick += 1;
+ let tick = inner.tick;
+ inner.order.insert(tick, key.clone());
+ inner.map.insert(
+ key,
+ Entry {
+ signature,
+ hashes: hashes.into(),
+ tick,
+ bytes,
+ },
+ );
+ inner.bytes += bytes;
+ let evicted = inner.evict_to(max_bytes);
+ drop(inner);
+ self.inserts.fetch_add(1, Ordering::Relaxed);
+ self.evictions.fetch_add(evicted, Ordering::Relaxed);
+ }
+
+ /// Drop the entries of files that a commit removed from the table.
+ pub(crate) fn remove_all<'a>(&self, keys: impl IntoIterator<Item = &'a str>) {
+ let mut inner = self.lock();
+ for k in keys {
+ inner.remove(k);
+ }
+ }
+
+ /// Reserve `bytes` more for in-flight collection. Never waits: when the pool is full
+ /// the collector gives up and its file is simply not cached.
+ fn try_reserve(&self, bytes: usize) -> bool {
+ let limit = self.max_bytes() / 4;
+ self.collecting
+ .fetch_update(Ordering::AcqRel, Ordering::Acquire, |cur| {
+ let next = cur.checked_add(bytes)?;
+ (next <= limit).then_some(next)
+ })
+ .is_ok()
+ }
+
+ fn release(&self, bytes: usize) {
+ self.collecting.fetch_sub(bytes, Ordering::AcqRel);
+ }
+
+ /// A collector for the key hashes of one file; `None` when the cache is off.
+ pub(crate) fn collector(self: &Arc<Self>) -> Option<KeyCollector> {
+ self.enabled().then(|| KeyCollector {
+ cache: self.clone(),
+ hashes: Vec::new(),
+ reserved: 0,
+ abandoned: false,
+ })
+ }
+}
+
+/// Collects the key hashes of one file while it is read or written. It holds a share of
+/// the cache's in-flight pool for the hashes it keeps, and gives up (keeping nothing)
+/// once the file would make an entry larger than the cache accepts, or the pool is full.
+pub(crate) struct KeyCollector {
+ cache: Arc<ProbeCache>,
+ hashes: Vec<u64>,
+ reserved: usize,
+ abandoned: bool,
+}
+
+impl KeyCollector {
+ /// The buffer to append the hashes of `additional` more rows to, or `None` when the
+ /// collector gave up.
+ pub(crate) fn reserve(&mut self, additional: usize) -> Option<&mut Vec<u64>> {
+ if self.abandoned {
+ return None;
+ }
+ let need_len = self.hashes.len() + additional;
+ let max_entry = self.cache.max_entry_bytes();
+ if need_len * 8 > max_entry {
+ self.abandon();
+ return None;
+ }
+ if need_len > self.hashes.capacity() {
+ // Grow the buffer here, geometrically and with an exact capacity, so that the
+ // pool is charged for the allocation itself and not just the rows in it.
+ let target_len = need_len
+ .max(self.hashes.capacity() * 2)
+ .max(RESERVE_STEP_BYTES.min(max_entry / 8) / 8)
+ .min(max_entry / 8);
+ let target_bytes = target_len * 8;
+ if target_bytes > self.reserved {
+ let step = target_bytes - self.reserved;
+ if !self.cache.try_reserve(step) {
+ self.abandon();
+ return None;
+ }
+ self.reserved += step;
+ }
+ self.hashes.reserve_exact(target_len - self.hashes.len());
+ }
+ Some(&mut self.hashes)
+ }
+
+ fn abandon(&mut self) {
+ self.abandoned = true;
+ self.hashes = Vec::new();
+ self.cache.release(self.reserved);
+ self.reserved = 0;
+ }
+
+ /// What was collected, still holding its share of the pool; `None` if it gave up.
+ pub(crate) fn finish(mut self) -> Option<CollectedKeys> {
+ if self.abandoned {
+ return None;
+ }
+ let reserved = std::mem::take(&mut self.reserved);
+ Some(CollectedKeys {
+ cache: self.cache.clone(),
+ hashes: std::mem::take(&mut self.hashes),
+ reserved,
+ })
+ }
+
+ /// Store what was collected as the entry for `key`. Does nothing if it gave up.
+ pub(crate) fn store(self, key: String, signature: u64) {
+ if let Some(keys) = self.finish() {
+ keys.store(key, signature);
+ }
+ }
+}
+
+impl Drop for KeyCollector {
+ fn drop(&mut self) {
+ self.cache.release(self.reserved);
+ }
+}
+
+/// The finished key hashes of one file, waiting to be stored (for a written file, until
+/// its commit lands).
+pub(crate) struct CollectedKeys {
+ cache: Arc<ProbeCache>,
+ hashes: Vec<u64>,
+ reserved: usize,
+}
+
+impl CollectedKeys {
+ pub(crate) fn store(mut self, key: String, signature: u64) {
+ let hashes = std::mem::take(&mut self.hashes);
+ self.cache.insert(key, signature, hashes);
+ }
+}
+
+impl Drop for CollectedKeys {
+ fn drop(&mut self) {
+ self.cache.release(self.reserved);
+ }
+}
+
+fn max_bytes_from_env() -> usize {
+ match std::env::var(PROBE_CACHE_BYTES_ENV) {
+ Ok(v) => match v.trim().parse::<usize>() {
+ Ok(n) => n,
+ Err(_) => {
+ tracing::warn!(var = PROBE_CACHE_BYTES_ENV, value = %v, "invalid value, using default");
+ DEFAULT_PROBE_CACHE_BYTES
+ }
+ },
+ Err(_) => DEFAULT_PROBE_CACHE_BYTES,
+ }
+}
+
+/// Cache key of a data file: the table URL and the URL-decoded path relative to it.
+pub(crate) fn file_key(table_url: &str, decoded_path: &str) -> String {
+ let mut key = String::with_capacity(table_url.len() + 1 + decoded_path.len());
+ key.push_str(table_url);
+ key.push('\n');
+ key.push_str(decoded_path);
+ key
+}
+
+/// Signature of the key encoding: the PK columns, their table types, and the partition
+/// column (a PK on the partition column takes its value from the file's path).
+pub(crate) fn key_signature(
+ primary_keys: &[String],
+ pk_types: &[DataType],
+ partition_col: Option<&str>,
+) -> u64 {
+ let mut h = DefaultHasher::new();
+ primary_keys.hash(&mut h);
+ for t in pk_types {
+ format!("{t:?}").hash(&mut h);
+ }
+ partition_col.hash(&mut h);
+ h.finish()
+}
+
+/// Hash of one row-encoded PK tuple.
+pub(crate) fn hash_key(encoded: &[u8]) -> u64 {
+ let mut h = DefaultHasher::new();
+ h.write(encoded);
+ h.finish()
+}
+
+/// Whether two sorted slices share a value.
+pub(crate) fn sorted_intersect(a: &[u64], b: &[u64]) -> bool {
+ let (small, large) = if a.len() <= b.len() { (a, b) } else { (b, a) };
+ if small.is_empty() {
+ return false;
+ }
+ // Binary search wins when one side is much smaller; a merge walk otherwise.
+ let log_large = usize::BITS - large.len().leading_zeros();
+ if small.len().saturating_mul(log_large as usize) < small.len() + large.len() {
+ return small.iter().any(|v| large.binary_search(v).is_ok());
+ }
+ let (mut i, mut j) = (0, 0);
+ while i < small.len() && j < large.len() {
+ match small[i].cmp(&large[j]) {
+ std::cmp::Ordering::Less => i += 1,
+ std::cmp::Ordering::Greater => j += 1,
+ std::cmp::Ordering::Equal => return true,
+ }
+ }
+ false
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn entry_bytes(n_hashes: usize, key: &str) -> usize {
+ n_hashes * 8 + 2 * key.len() + ENTRY_OVERHEAD_BYTES
+ }
+
+ #[test]
+ fn get_needs_the_same_signature() {
+ let c = ProbeCache::new(1 << 20);
+ c.insert("t\na".into(), 1, vec![3, 1, 2, 3]);
+ assert_eq!(c.get("t\na", 1).as_deref(), Some(&[1u64, 2, 3][..]));
+ assert!(c.get("t\na", 2).is_none());
+ assert!(c.get("t\nb", 1).is_none());
+ let s = c.stats();
+ assert_eq!((s.hits, s.misses, s.entries), (1, 2, 1));
+ }
+
+ #[test]
+ fn eviction_drops_the_least_recently_used_entry() {
+ let one = entry_bytes(10, "t\nk0");
+ let c = ProbeCache::new(one * 8);
+ for i in 0..8 {
+ c.insert(format!("t\nk{i}"), 0, (0..10).collect());
+ }
+ assert!(c.get("t\nk0", 0).is_some());
+ c.insert("t\nk8".into(), 0, (0..10).collect());
+ assert!(c.get("t\nk1", 0).is_none());
+ assert!(c.get("t\nk0", 0).is_some());
+ assert!(c.get("t\nk8", 0).is_some());
+ assert_eq!(c.stats().bytes, one * 8);
+ assert_eq!(c.stats().evictions, 1);
+
+ c.set_max_bytes(one * 2);
+ assert_eq!(c.stats().entries, 2);
+ assert!(c.get("t\nk0", 0).is_some());
+ assert!(c.get("t\nk8", 0).is_some());
+ }
+
+ #[test]
+ fn an_entry_above_an_eighth_of_the_cap_is_not_stored() {
+ let c = ProbeCache::new(8 * entry_bytes(100, "t\na"));
+ c.insert("t\na".into(), 0, (0..101).collect());
+ assert_eq!(c.stats().entries, 0);
+ c.insert("t\na".into(), 0, (0..100).collect());
+ assert_eq!(c.stats().entries, 1);
+ }
+
+ #[test]
+ fn a_zero_cap_stores_and_collects_nothing() {
+ let c = Arc::new(ProbeCache::new(0));
+ c.insert("t\na".into(), 0, vec![1]);
+ assert!(c.get("t\na", 0).is_none());
+ assert!(c.collector().is_none());
+ }
+
+ #[test]
+ fn collectors_give_up_when_the_in_flight_pool_is_full_and_release_on_drop() {
+ // Pool = max/4 = 16 steps; one entry = max/8 = 8 steps.
+ let c = Arc::new(ProbeCache::new(64 * RESERVE_STEP_BYTES));
+ let rows_per_entry = c.max_entry_bytes() / 8;
+ let mut a = c.collector().unwrap();
+ let mut b = c.collector().unwrap();
+ let mut late = c.collector().unwrap();
+ assert!(a.reserve(rows_per_entry).is_some());
+ assert!(b.reserve(rows_per_entry).is_some());
+ assert!(late.reserve(1).is_none());
+ assert!(late.finish().is_none());
+ drop(a);
+ let mut late = c.collector().unwrap();
+ assert!(late.reserve(1).is_some());
+ // Past the entry cap the collector gives up and keeps nothing.
+ assert!(late.reserve(rows_per_entry + 1).is_none());
+ let mut kept = c.collector().unwrap();
+ kept.reserve(1).unwrap().push(7);
+ let kept = kept.finish().unwrap();
+ drop(b);
+ drop(late);
+ assert!(c.collecting.load(Ordering::Acquire) > 0);
+ kept.store("t\na".into(), 0);
+ assert_eq!(c.collecting.load(Ordering::Acquire), 0);
+ assert_eq!(c.get("t\na", 0).as_deref(), Some(&[7u64][..]));
+ }
+
+ #[test]
+ fn a_collector_is_charged_for_its_whole_allocation() {
+ let c = Arc::new(ProbeCache::new(1 << 24));
+ let mut k = c.collector().unwrap();
+ for _ in 0..3 {
+ k.reserve(8192)
+ .unwrap()
+ .extend(std::iter::repeat_n(1, 8192));
+ assert!(k.hashes.capacity() * 8 <= k.reserved);
+ }
+ assert_eq!(c.collecting.load(Ordering::Acquire), k.reserved);
+ }
+
+ #[test]
+ fn remove_all_frees_the_bytes() {
+ let c = ProbeCache::new(1 << 20);
+ c.insert("t\na".into(), 0, vec![1, 2]);
+ c.insert("t\nb".into(), 0, vec![1]);
+ c.remove_all(["t\na", "t\nmissing"]);
+ assert_eq!(c.stats().entries, 1);
+ assert_eq!(c.stats().bytes, entry_bytes(1, "t\nb"));
+ }
+
+ #[test]
+ fn sorted_intersect_matches_a_naive_check() {
+ let cases: Vec<(Vec<u64>, Vec<u64>)> = vec![
+ (vec![], vec![1, 2]),
+ (vec![5], vec![1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12]),
+ (vec![7], vec![1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12]),
+ (
+ (0..50).map(|v| v * 2).collect(),
+ (0..50).map(|v| v * 2 + 1).collect(),
+ ),
+ ((0..50).map(|v| v * 2).collect(), vec![1, 3, 98]),
+ (vec![u64::MAX], vec![0, u64::MAX]),
+ ];
+ for (a, b) in cases {
+ let naive = a.iter().any(|v| b.contains(v));
+ assert_eq!(sorted_intersect(&a, &b), naive, "{a:?} {b:?}");
+ assert_eq!(sorted_intersect(&b, &a), naive, "{b:?} {a:?}");
+ }
+ }
+}
diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rs
index 1cb20e6ddb63..226b7c7159bc 100644
--- a/rust/deltalite/core/src/upsert.rs
+++ b/rust/deltalite/core/src/upsert.rs
@@ -60,6 +60,9 @@ use crate::limits::{
SourceFootprint,
};
use crate::pkset::PkSet;
+use crate::probecache::{
+ file_key, key_signature, sorted_intersect, CollectedKeys, KeyCollector, ProbeCache,
+};
use crate::schema::{cast_to_schema, unknown_columns};
use crate::smallfile::{fetch_small_file, reads_whole, SmallFileStore};
use crate::writer::StreamingWriter;
@@ -264,6 +267,11 @@ pub struct UpsertOptions {
pub max_source_bytes: Option<usize>,
/// Process-wide concurrency budgets shared with every other concurrent upsert.
pub limits: Arc<ProcessLimits>,
+ /// Cache of the PK hashes of data files (see [`crate::probecache`]). With
+ /// `prune_strategy=probe`, a cached file is decided without a read, a probed file that
+ /// holds no match is cached, and the files this upsert writes are cached once its
+ /// commit lands. `None` probes every candidate file from storage.
+ pub probe_cache: Option<Arc<ProbeCache>>,
}
impl Default for UpsertOptions {
@@ -285,6 +293,7 @@ impl Default for UpsertOptions {
max_row_group_bytes: None,
max_source_bytes: None,
limits: ProcessLimits::global().clone(),
+ probe_cache: Some(ProbeCache::global().clone()),
}
}
}
@@ -304,6 +313,8 @@ pub struct UpsertStats {
pub files_carried_over: usize,
/// Files whose PK column(s) were actually read by the probe (prune_strategy=probe).
pub files_probed: usize,
+ /// Candidate files the probe decided from the probe cache, with no read.
+ pub files_probe_cached: usize,
/// Existing rows dropped in favour of an incoming row.
pub rows_updated: usize,
/// Incoming rows with no existing match.
@@ -318,6 +329,9 @@ pub struct UpsertStats {
pub plan_ms: u64,
/// Wall-clock ms spent rewriting the touched partitions (reading + probing targets, writing files).
pub rewrite_ms: u64,
+ /// Wall-clock ms the partition workers spent probing, summed over partitions (part of
+ /// `rewrite_ms`, which runs partitions in parallel).
+ pub probe_ms: u64,
/// Wall-clock ms spent committing the new file set to the Delta log.
pub commit_ms: u64,
/// Non-nullable columns relaxed to nullable before this upsert because the table's
@@ -511,11 +525,17 @@ struct PartitionOutcome {
files_added: usize,
files_removed: usize,
files_probed: usize,
+ files_probe_cached: usize,
+ probe_ms: u64,
rows_updated: usize,
rows_copied: usize,
rows_inserted: usize,
null_pk_rows: usize,
carried: usize,
+ /// Cache keys of the files this partition removes.
+ removed_keys: Vec<String>,
+ /// Key hashes of the files this partition wrote, stored once the commit lands.
+ written_keys: Vec<(String, CollectedKeys)>,
}
/// A unit of byte budget: one permit from the per-call budget and one from the
@@ -736,6 +756,8 @@ pub async fn upsert_cached_with_state(
counter!("deltalite_files_carried_over_total")
.increment(stats.files_carried_over as u64);
counter!("deltalite_files_probed_total").increment(stats.files_probed as u64);
+ counter!("deltalite_files_probe_cached_total")
+ .increment(stats.files_probe_cached as u64);
counter!("deltalite_rows_updated_total").increment(stats.rows_updated as u64);
counter!("deltalite_rows_inserted_total").increment(stats.rows_inserted as u64);
counter!("deltalite_rows_copied_total").increment(stats.rows_copied as u64);
@@ -1093,6 +1115,8 @@ async fn upsert_inner(
..Default::default()
};
let mut actions: Vec<Action> = Vec::new();
+ let mut removed_keys: Vec<String> = Vec::new();
+ let mut written_keys: Vec<(String, CollectedKeys)> = Vec::new();
for h in handles {
let outcome = h
@@ -1101,12 +1125,16 @@ async fn upsert_inner(
stats.files_added += outcome.files_added;
stats.files_removed += outcome.files_removed;
stats.files_probed += outcome.files_probed;
+ stats.files_probe_cached += outcome.files_probe_cached;
+ stats.probe_ms += outcome.probe_ms;
stats.files_carried_over += outcome.carried;
stats.rows_updated += outcome.rows_updated;
stats.rows_copied += outcome.rows_copied;
stats.rows_inserted += outcome.rows_inserted;
stats.null_pk_rows += outcome.null_pk_rows;
actions.extend(outcome.actions);
+ removed_keys.extend(outcome.removed_keys);
+ written_keys.extend(outcome.written_keys);
}
let rewrite_ms = rewrite_started.elapsed().as_millis() as u64;
@@ -1159,6 +1187,16 @@ async fn upsert_inner(
.await?;
let commit_ms = commit_started.elapsed().as_millis() as u64;
+ // Only a landed commit makes the written files live and the removed files dead. Before
+ // it, a written file may be an orphan that no snapshot ever names.
+ if let Some(cache) = &opts.probe_cache {
+ cache.remove_all(removed_keys.iter().map(String::as_str));
+ let signature = pk_signature(&table_schema, &opts.primary_keys, &partition_col)?;
+ for (key, keys) in written_keys {
+ keys.store(key, signature);
+ }
+ }
+
if (finalized.version() + 1) % checkpoint_interval == 0 {
let maintenance_started = Instant::now();
best_effort_log_maintenance(table, finalized.version(), cleanup_enabled).await;
@@ -1931,21 +1969,13 @@ async fn probe_file(
partition_value: &str,
opts: &UpsertOptions,
budgets: &Budgets,
+ mut collector: Option<&mut KeyCollector>,
) -> Result<(bool, Arc<ParquetMetaData>)> {
let (builder, held_fetch) = open_builder(store, f, budgets).await?;
let metadata = builder.metadata().clone();
let file_schema = builder.schema().clone();
- let mut pk_types: Vec<DataType> = Vec::with_capacity(opts.primary_keys.len());
- for pk in &opts.primary_keys {
- pk_types.push(
- table_schema
- .field_with_name(pk)
- .map_err(|e| Error::Generic(format!("PK column lookup: {e}")))?
- .data_type()
- .clone(),
- );
- }
+ let pk_types = pk_types(table_schema, &opts.primary_keys)?;
let mut sources: Vec<PkSource> = Vec::with_capacity(opts.primary_keys.len());
let mut projection: Vec<usize> = Vec::new();
@@ -1973,7 +2003,8 @@ async fn probe_file(
partition_value,
&pk_types,
)?;
- return Ok((pkset.contains_any_columns(&cols, 1)?, metadata));
+ let hashes = collector.as_mut().and_then(|c| c.reserve(1));
+ return Ok((pkset.probe_columns(&cols, 1, hashes)?, metadata));
}
// Fetch budget before any data is fetched; held until the stream is gone.
@@ -2019,7 +2050,8 @@ async fn probe_file(
partition_value,
&pk_types,
)?;
- let hit = pkset.contains_any_columns(&cols, batch.num_rows())?;
+ let hashes = collector.as_mut().and_then(|c| c.reserve(batch.num_rows()));
+ let hit = pkset.probe_columns(&cols, batch.num_rows(), hashes)?;
drop(permit);
if hit {
drop(stream);
@@ -2032,10 +2064,32 @@ async fn probe_file(
Ok((false, metadata))
}
-/// Probe `files` with bounded concurrency, splitting them into (files that contain at
-/// least one match, count of files proven match-free). Order is preserved. Up to one
-/// reader wave of kept files carries the footer its probe parsed; retaining every hit
-/// footer would make memory grow with the partition's file count before rewrites start.
+/// What the probe cache contributes to one partition's probe.
+struct ProbeCacheView<'a> {
+ cache: Arc<ProbeCache>,
+ table_url: &'a str,
+ signature: u64,
+ /// Sorted hashes of the partition's source keys.
+ source_hashes: Vec<u64>,
+}
+
+/// Result of [`probe_files`].
+struct Probed {
+ /// Files that contain at least one match, in input order.
+ keep: Vec<TargetFile>,
+ /// Files proven match-free.
+ skipped: usize,
+ /// Files decided from the probe cache, with no read.
+ cached: usize,
+}
+
+/// Probe `files` with bounded concurrency. Order is preserved. Up to one reader wave of
+/// kept files carries the footer its probe parsed; retaining every hit footer would make
+/// memory grow with the partition's file count before rewrites start.
+///
+/// With a cache, a cached file is decided from its key hashes, and a probed file that
+/// holds no match is cached: the probe read all of its keys to prove that. A file that
+/// holds a match is not cached, because this upsert removes it.
#[allow(clippy::too_many_arguments)]
async fn probe_files(
store: &Arc<dyn ObjectStore>,
@@ -2046,10 +2100,19 @@ async fn probe_files(
partition_value: &str,
opts: &UpsertOptions,
budgets: &Budgets,
-) -> Result<(Vec<TargetFile>, usize)> {
+ cache: Option<&ProbeCacheView<'_>>,
+) -> Result<Probed> {
let results = futures::stream::iter(files.into_iter().map(|f| {
let store = store.clone();
async move {
+ let key = cache.map(|c| file_key(c.table_url, &f.path));
+ if let (Some(c), Some(key)) = (cache, &key) {
+ if let Some(hashes) = c.cache.get(key, c.signature) {
+ let hit = sorted_intersect(&c.source_hashes, &hashes);
+ return Ok::<_, Error>((f, hit, None, true));
+ }
+ }
+ let mut collector = cache.and_then(|c| c.cache.collector());
let (hit, metadata) = probe_file(
&store,
&f,
@@ -2059,28 +2122,148 @@ async fn probe_files(
partition_value,
opts,
budgets,
+ collector.as_mut(),
)
.await?;
- Ok::<_, Error>((f, hit, metadata))
+ if let (false, Some(c), Some(key), Some(collector)) = (hit, cache, key, collector) {
+ collector.store(key, c.signature);
+ }
+ Ok((f, hit, Some(metadata), false))
}
}))
.buffered(opts.probe_concurrency.max(1));
futures::pin_mut!(results);
let retained_footer_limit = opts.max_parallel_files.max(1);
- let mut keep = Vec::new();
- let mut skipped = 0usize;
- while let Some((mut f, hit, metadata)) = results.try_next().await? {
+ let mut out = Probed {
+ keep: Vec::new(),
+ skipped: 0,
+ cached: 0,
+ };
+ while let Some((mut f, hit, metadata, cached)) = results.try_next().await? {
+ out.cached += usize::from(cached);
if hit {
- if keep.len() < retained_footer_limit {
- f.metadata = Some(metadata);
+ if out.keep.len() < retained_footer_limit {
+ f.metadata = metadata;
}
- keep.push(f);
+ out.keep.push(f);
} else {
- skipped += 1;
+ out.skipped += 1;
+ }
+ }
+ Ok(out)
+}
+
+/// The table schema's types of the PK columns, in PK order.
+fn pk_types(table_schema: &SchemaRef, primary_keys: &[String]) -> Result<Vec<DataType>> {
+ primary_keys
+ .iter()
+ .map(|pk| {
+ Ok(table_schema
+ .field_with_name(pk)
+ .map_err(|e| Error::Generic(format!("PK column lookup: {e}")))?
+ .data_type()
+ .clone())
+ })
+ .collect()
+}
+
+/// The probe-cache signature of this table's PK encoding.
+fn pk_signature(
+ table_schema: &SchemaRef,
+ primary_keys: &[String],
+ partition_col: &Option<String>,
+) -> Result<u64> {
+ Ok(key_signature(
+ primary_keys,
+ &pk_types(table_schema, primary_keys)?,
+ partition_col.as_deref(),
+ ))
+}
+
+/// Collects the key hashes of each file a partition writes, so that the next upsert does
+/// not probe a file this one wrote. Every batch the writer receives is in the table
+/// schema with the partition column, so its PK columns encode exactly as a later probe
+/// of the written file encodes them.
+struct OutputKeys {
+ cache: Arc<ProbeCache>,
+ pk_idx: Vec<usize>,
+ pk_types: Vec<DataType>,
+ current: Option<KeyCollector>,
+ /// Decoded path of each written file with its key hashes.
+ done: Vec<(String, CollectedKeys)>,
+}
+
+impl OutputKeys {
+ fn new(
+ cache: Arc<ProbeCache>,
+ table_schema: &SchemaRef,
+ primary_keys: &[String],
+ ) -> Result<Self> {
+ let mut pk_idx = Vec::with_capacity(primary_keys.len());
+ for pk in primary_keys {
+ pk_idx.push(table_schema.index_of(pk).map_err(|_| {
+ Error::SchemaMismatch(format!(
+ "primary key column '{pk}' not found in table schema"
+ ))
+ })?);
+ }
+ let current = cache.collector();
+ Ok(Self {
+ pk_types: pk_types(table_schema, primary_keys)?,
+ cache,
+ pk_idx,
+ current,
+ done: Vec::new(),
+ })
+ }
+
+ /// Record the keys of `batch`, which goes into the open output file.
+ fn note_write(&mut self, pkset: &PkSet, batch: &RecordBatch) -> Result<()> {
+ let Some(hashes) = self
+ .current
+ .as_mut()
+ .and_then(|c| c.reserve(batch.num_rows()))
+ else {
+ return Ok(());
+ };
+ let cast_opts = CastOptions {
+ safe: false,
+ ..Default::default()
+ };
+ let mut cols: Vec<Arc<dyn Array>> = Vec::with_capacity(self.pk_idx.len());
+ for (&ci, ty) in self.pk_idx.iter().zip(&self.pk_types) {
+ let col = batch.column(ci);
+ cols.push(if col.data_type() == ty {
+ col.clone()
+ } else {
+ cast_with_options(col, ty, &cast_opts)?
+ });
+ }
+ pkset.hash_columns(&cols, batch.num_rows(), hashes)
+ }
+
+ /// Attach the keys recorded so far to the file a flush closed, if it closed one.
+ fn note_flush(&mut self, new_adds: &[Add]) {
+ if new_adds.is_empty() {
+ return;
+ }
+ let finished = std::mem::replace(&mut self.current, self.cache.collector());
+ // One partition per writer gives one file per flush; anything else is not cached.
+ if let ([add], Some(keys)) = (new_adds, finished.and_then(KeyCollector::finish)) {
+ let path = percent_encoding::percent_decode_str(&add.path)
+ .decode_utf8_lossy()
+ .into_owned();
+ self.done.push((path, keys));
}
}
- Ok((keep, skipped))
+
+ fn into_written(self, table_url: &str) -> Vec<(String, CollectedKeys)> {
+ self.done
+ .into_iter()
+ .map(|(path, keys)| (file_key(table_url, &path), keys))
+ .collect()
+ }
}
/// Per-file result from a reader task.
@@ -2239,11 +2422,18 @@ async fn rewrite_partition(
let writer = StreamingWriter::for_table(table)?;
let store: Arc<dyn ObjectStore> = table.object_store();
+ let table_url = table.table_url().as_str().to_string();
+ let cache = match opts.prune_strategy {
+ PruneStrategy::Probe => opts.probe_cache.clone().filter(|c| c.enabled()),
+ _ => None,
+ };
// Content-based file selection: keep only files that actually contain a matched
// row, exactly as MERGE's join does. Files proven match-free are carried over --
// neither read again, rewritten, nor tombstoned.
let mut files_probed = 0usize;
+ let mut files_probe_cached = 0usize;
+ let probe_started = Instant::now();
if opts.prune_strategy == PruneStrategy::Probe && !rewrite.is_empty() {
if pkset.is_empty() {
// Every source row has a NULL PK component: nothing can match, so every
@@ -2251,8 +2441,17 @@ async fn rewrite_partition(
carried += rewrite.len();
rewrite = Vec::new();
} else {
- files_probed = rewrite.len();
- let (keep, skipped) = probe_files(
+ let candidates = rewrite.len();
+ let probe_cache = match &cache {
+ Some(cache) => Some(ProbeCacheView {
+ cache: cache.clone(),
+ table_url: &table_url,
+ signature: pk_signature(&table_schema, &opts.primary_keys, &partition_col)?,
+ source_hashes: pkset.sorted_key_hashes(),
+ }),
+ None => None,
+ };
+ let probed = probe_files(
&store,
rewrite,
&pkset,
@@ -2261,22 +2460,45 @@ async fn rewrite_partition(
&value,
opts,
&budgets,
+ probe_cache.as_ref(),
)
.await?;
- rewrite = keep;
- carried += skipped;
+ rewrite = probed.keep;
+ carried += probed.skipped;
+ files_probe_cached = probed.cached;
+ files_probed = candidates - probed.cached;
}
}
+ let probe_ms = probe_started.elapsed().as_millis() as u64;
+ let removed_keys: Vec<String> = match &cache {
+ Some(_) => rewrite
+ .iter()
+ .map(|f| file_key(&table_url, &f.path))
+ .collect(),
+ None => Vec::new(),
+ };
+ let mut writer_keys = match &cache {
+ Some(cache) => Some(OutputKeys::new(
+ cache.clone(),
+ &table_schema,
+ &opts.primary_keys,
+ )?),
+ None => None,
+ };
// Single writer task per partition: file count and output sizing behave exactly as
// in the sequential implementation. The channel is unbounded; memory is bounded by
// the byte-budget permits attached to every message, not by message count.
let (tx, mut rx) = mpsc::unbounded_channel::<(RecordBatch, BudgetPermit)>();
let max_row_group_bytes = opts.max_row_group_bytes;
+ let writer_pkset = pkset.clone();
let writer_task = tokio::spawn(async move {
let mut writer = writer;
let mut adds = Vec::new();
while let Some((batch, permit)) = rx.recv().await {
+ if let Some(keys) = writer_keys.as_mut() {
+ keys.note_write(&writer_pkset, &batch)?;
+ }
writer.write(batch)?;
// The batch now lives (encoded) in the writer; release its budget.
drop(permit);
@@ -2284,6 +2506,7 @@ async fn rewrite_partition(
// Bound resident memory: the writer holds the open file until flushed, so
// flush on a size threshold instead of once at the end. Each flush yields
// Add actions and starts a new file.
+ let before = adds.len();
roll_writer(
&mut writer,
&mut adds,
@@ -2291,8 +2514,11 @@ async fn rewrite_partition(
max_row_group_bytes,
)
.await?;
+ if let Some(keys) = writer_keys.as_mut() {
+ keys.note_flush(&adds[before..]);
+ }
}
- Ok::<_, Error>((writer, adds))
+ Ok::<_, Error>((writer, adds, writer_keys))
});
// Reader tasks: up to max_parallel_files files stream concurrently (per call AND
@@ -2344,14 +2570,18 @@ async fn rewrite_partition(
// Writer errors take precedence: when the writer dies, readers fail secondarily on
// send, and reporting those would mask the root cause.
- let (mut writer, mut adds) = writer_task
+ let (mut writer, mut adds, mut output_keys) = writer_task
.await
.map_err(|e| Error::Generic(format!("writer task panicked: {e}")))??;
if let Some(e) = reader_err {
return Err(e);
}
if opts.isolate_source_rows {
+ let before = adds.len();
adds.extend(writer.flush().await?);
+ if let Some(keys) = output_keys.as_mut() {
+ keys.note_flush(&adds[before..]);
+ }
}
// Source rows are written LAST: an updated row therefore carries the source values,
@@ -2376,8 +2606,13 @@ async fn rewrite_partition(
let mut offset = 0usize;
while offset < selected.num_rows() {
let n = (selected.num_rows() - offset).min(slice_rows);
- writer.write(selected.slice(offset, n))?;
+ let slice = selected.slice(offset, n);
+ if let Some(keys) = output_keys.as_mut() {
+ keys.note_write(&pkset, &slice)?;
+ }
+ writer.write(slice)?;
offset += n;
+ let before = adds.len();
roll_writer(
&mut writer,
&mut adds,
@@ -2385,9 +2620,20 @@ async fn rewrite_partition(
max_row_group_bytes,
)
.await?;
+ if let Some(keys) = output_keys.as_mut() {
+ keys.note_flush(&adds[before..]);
+ }
}
}
+ let before = adds.len();
adds.extend(writer.flush().await?);
+ let written_keys = match output_keys {
+ Some(mut keys) => {
+ keys.note_flush(&adds[before..]);
+ keys.into_written(&table_url)
+ }
+ None => Vec::new(),
+ };
// Every source row is written, but a source row that replaced an existing one is an
// update, not an insert -- the batch is deduped, so each match consumes exactly one
@@ -2404,11 +2650,15 @@ async fn rewrite_partition(
files_added,
files_removed,
files_probed,
+ files_probe_cached,
+ probe_ms,
rows_updated,
rows_copied,
rows_inserted,
null_pk_rows,
carried,
+ removed_keys,
+ written_keys,
})
}
diff --git a/rust/deltalite/core/tests/handle_and_open.rs b/rust/deltalite/core/tests/handle_and_open.rs
index 337a8ce38215..8e07c9d34b0e 100644
--- a/rust/deltalite/core/tests/handle_and_open.rs
+++ b/rust/deltalite/core/tests/handle_and_open.rs
@@ -18,6 +18,7 @@ use deltalake::{DeltaResult, Path, TableProperty};
use deltalite_core::handle::TableHandle;
use deltalite_core::table::{open_table, open_table_multipart, wrap_multipart, MultipartConfig};
use deltalite_core::upsert::{upsert_cached_with_state, RelaxCache, UpsertOptions};
+use deltalite_core::ProbeCache;
use futures::stream::BoxStream;
use object_store::local::LocalFileSystem;
use object_store::{
@@ -925,6 +926,8 @@ async fn hit_file_footer_reuse_is_bounded_by_reader_concurrency() {
s.clone(),
UpsertOptions {
max_parallel_files: 2,
+ // The seeding upserts cached these files; the test needs their probe.
+ probe_cache: None,
..opts()
},
MultipartConfig::default(),
@@ -943,6 +946,63 @@ async fn hit_file_footer_reuse_is_bounded_by_reader_concurrency() {
assert_eq!(counts.iter().filter(|&&count| count == 2).count(), 3);
}
+/// A file whose keys the cache holds is decided with no data read: first a file the
+/// probe read once and found without a match, then a file an earlier upsert wrote.
+#[tokio::test]
+async fn a_cached_file_is_decided_without_a_data_read() {
+ register_counting_scheme();
+ let dir = tempfile::tempdir().expect("tempdir");
+ let path = dir.path().join("t");
+ let uri = format!("dltest://{}", path.to_string_lossy());
+ let root = table_root(&path);
+ let s = create_disjoint_files_table(&uri, &[&["a0", "a9"], &["m0", "m9"]]).await;
+ let mut handle = TableHandle::open(uri.clone(), HashMap::new())
+ .await
+ .expect("open");
+ let data_reads = |ops: &[(Op, String)]| {
+ ops.iter()
+ .filter(|(op, p)| op.is_read() && p.ends_with(".parquet") && !p.contains("_delta_log"))
+ .count()
+ };
+ let cache = Arc::new(ProbeCache::new(1 << 20));
+
+ // A fresh cache knows nothing yet: the probe reads the file and caches it.
+ let probe = |key: &'static str| {
+ let opts = UpsertOptions {
+ probe_cache: Some(cache.clone()),
+ ..opts()
+ };
+ (vec![batch(&s, &[key], vec![Some(1)])], opts)
+ };
+ for (key, probed, cached, reads) in [("a5", 1, 0, 1), ("a6", 0, 1, 0)] {
+ let (b, o) = probe(key);
+ let start = ops_len();
+ let stats = handle
+ .upsert(b, s.clone(), o, MultipartConfig::default())
+ .await
+ .expect("upsert");
+ let ops = ops_since(start, &root);
+ assert_eq!(stats.files_probed, probed, "{key}");
+ assert_eq!(stats.files_probe_cached, cached, "{key}");
+ assert_eq!(stats.files_removed, 0, "{key}");
+ assert_eq!(data_reads(&ops), reads, "{key}: {ops:?}");
+ }
+
+ // The upserts above wrote files for `a5` and `a6`; their keys came from the writer.
+ let (b, o) = probe("a5");
+ let start = ops_len();
+ let stats = handle
+ .upsert(b, s.clone(), o, MultipartConfig::default())
+ .await
+ .expect("upsert");
+ assert_eq!(stats.files_probed, 0);
+ assert_eq!(stats.files_removed, 1);
+ assert_eq!(stats.rows_updated, 1);
+ let ops = ops_since(start, &root);
+ // The one data read is the rewrite of the hit file, not a probe.
+ assert_eq!(data_reads(&ops), 1, "{ops:?}");
+}
+
/// After the commit the handle adopts the state delta-rs derived for it instead of
/// listing the log and reading `_last_checkpoint` again, and that state is the log's
/// truth.
@@ -1470,7 +1530,10 @@ async fn a_probed_small_file_costs_one_request() {
.upsert(
vec![batch(&s, &["a5"], vec![Some(1)])],
s.clone(),
- opts(),
+ UpsertOptions {
+ probe_cache: None,
+ ..opts()
+ },
MultipartConfig::default(),
)
.await
diff --git a/rust/deltalite/core/tests/probe_cache.rs b/rust/deltalite/core/tests/probe_cache.rs
new file mode 100644
index 000000000000..28af5b3a4816
--- /dev/null
+++ b/rust/deltalite/core/tests/probe_cache.rs
@@ -0,0 +1,367 @@
+//! The probe cache must not change what an upsert does. Each case runs the same upserts
+//! against two copies of one table, one with a cache and one without, and requires the
+//! same rows and the same file decisions after every upsert.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_array::{Array, Int64Array, RecordBatch, StringArray};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use deltalake::kernel::{DataType as KernelType, StructField};
+use deltalake::operations::create::CreateBuilder;
+use deltalake::writer::{DeltaWriter, RecordBatchWriter};
+use deltalake::Path;
+use deltalite_core::compact::CompactOptions;
+use deltalite_core::handle::TableHandle;
+use deltalite_core::limits::ProcessLimits;
+use deltalite_core::table::MultipartConfig;
+use deltalite_core::upsert::{UpsertOptions, UpsertStats};
+use deltalite_core::ProbeCache;
+use object_store::ObjectStoreExt;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+
+const PARTITIONS: u64 = 5;
+const SEED_KEYS: u64 = 2000;
+const SEED_FLUSHES: u64 = 8;
+const ROUNDS: u64 = 12;
+
+#[derive(Clone, Copy, Debug)]
+enum Keys {
+ Int,
+ Str,
+ Composite,
+ /// A PK that includes the partition column, which is not stored in the data files.
+ WithPartition,
+}
+
+impl Keys {
+ fn primary_keys(self) -> Vec<String> {
+ let names: &[&str] = match self {
+ Keys::Int => &["id_i"],
+ Keys::Str => &["id_s"],
+ Keys::Composite => &["id_s", "id_i"],
+ Keys::WithPartition => &["id_i", "p"],
+ };
+ names.iter().map(|s| s.to_string()).collect()
+ }
+}
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("id_i", DataType::Int64, true),
+ Field::new("id_s", DataType::Utf8, true),
+ Field::new("v", DataType::Int64, true),
+ Field::new("p", DataType::Utf8, true),
+ ]))
+}
+
+/// Rows for keys `ks` (key `k` lives in partition `k % PARTITIONS`), plus `null_rows`
+/// rows whose PK columns are NULL, which never match and always insert.
+fn rows(ks: &[u64], v: i64, null_rows: usize) -> RecordBatch {
+ let mut id_i: Vec<Option<i64>> = ks.iter().map(|k| Some(*k as i64)).collect();
+ let mut id_s: Vec<Option<String>> = ks.iter().map(|k| Some(format!("k{k:06}"))).collect();
+ let mut p: Vec<String> = ks.iter().map(|k| (k % PARTITIONS).to_string()).collect();
+ for i in 0..null_rows {
+ id_i.push(None);
+ id_s.push(None);
+ p.push((i as u64 % PARTITIONS).to_string());
+ }
+ let n = id_i.len();
+ RecordBatch::try_new(
+ schema(),
+ vec![
+ Arc::new(Int64Array::from(id_i)),
+ Arc::new(StringArray::from(id_s)),
+ Arc::new(Int64Array::from(vec![v; n])),
+ Arc::new(StringArray::from(p)),
+ ],
+ )
+ .unwrap()
+}
+
+/// A partitioned table holding `SEED_KEYS` keys (offset by `offset`) in
+/// `SEED_FLUSHES` files per partition.
+async fn seed(dir: &std::path::Path, offset: u64) -> TableHandle {
+ let mut table = CreateBuilder::new()
+ .with_location(dir.to_str().unwrap())
+ .with_columns(vec![
+ StructField::new("id_i", KernelType::LONG, true),
+ StructField::new("id_s", KernelType::STRING, true),
+ StructField::new("v", KernelType::LONG, true),
+ StructField::new("p", KernelType::STRING, true),
+ ])
+ .with_partition_columns(vec!["p".to_string()])
+ .await
+ .unwrap();
+ let per_flush = SEED_KEYS / SEED_FLUSHES;
+ for f in 0..SEED_FLUSHES {
+ let ks: Vec<u64> = (offset + f * per_flush..offset + (f + 1) * per_flush).collect();
+ let mut w = RecordBatchWriter::for_table(&table).unwrap();
+ w.write(rows(&ks, 0, 0)).await.unwrap();
+ w.flush_and_commit(&mut table).await.unwrap();
+ }
+ TableHandle::open(dir.to_str().unwrap().into(), HashMap::new())
+ .await
+ .unwrap()
+}
+
+/// Every live row as (id_i, id_s, v, p), sorted.
+async fn read_rows(handle: &TableHandle) -> Vec<(Option<i64>, Option<String>, i64, String)> {
+ let store = handle.table().object_store();
+ let mut out = Vec::new();
+ for f in handle.files().unwrap() {
+ let p = f.partition_values["p"].clone().unwrap();
+ let bytes = store
+ .get(&Path::parse(&f.path).unwrap())
+ .await
+ .unwrap()
+ .bytes()
+ .await
+ .unwrap();
+ for batch in ParquetRecordBatchReaderBuilder::try_new(bytes)
+ .unwrap()
+ .build()
+ .unwrap()
+ {
+ let batch = batch.unwrap();
+ let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone();
+ let (id_i, id_s, v) = (col("id_i"), col("id_s"), col("v"));
+ let id_i = id_i.as_any().downcast_ref::<Int64Array>().unwrap();
+ let id_s = id_s.as_any().downcast_ref::<StringArray>().unwrap();
+ let v = v.as_any().downcast_ref::<Int64Array>().unwrap();
+ for i in 0..batch.num_rows() {
+ out.push((
+ (!id_i.is_null(i)).then(|| id_i.value(i)),
+ (!id_s.is_null(i)).then(|| id_s.value(i).to_string()),
+ v.value(i),
+ p.clone(),
+ ));
+ }
+ }
+ }
+ out.sort();
+ out
+}
+
+/// Deterministic pseudo-random sequence, so both copies get the same batches.
+struct Lcg(u64);
+
+impl Lcg {
+ fn next(&mut self, below: u64) -> u64 {
+ self.0 = self
+ .0
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (self.0 >> 33) % below
+ }
+}
+
+/// One table under test: a copy with the cache and a copy without it.
+struct Twin {
+ cached: TableHandle,
+ plain: TableHandle,
+ offset: u64,
+ /// Keys the last upsert wrote, so the next one hits files that the cache got from
+ /// the writer rather than from a probe.
+ last_keys: Vec<u64>,
+ next_new_key: u64,
+ _dirs: (tempfile::TempDir, tempfile::TempDir),
+}
+
+impl Twin {
+ async fn new(offset: u64) -> Self {
+ let (a, b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
+ Self {
+ cached: seed(a.path(), offset).await,
+ plain: seed(b.path(), offset).await,
+ offset,
+ last_keys: Vec::new(),
+ next_new_key: offset + SEED_KEYS,
+ _dirs: (a, b),
+ }
+ }
+
+ fn batch_keys(&mut self, rng: &mut Lcg) -> Vec<u64> {
+ let mut ks: Vec<u64> = Vec::new();
+ ks.extend(self.last_keys.iter().take(20));
+ ks.extend((0..40).map(|_| self.offset + rng.next(SEED_KEYS)));
+ ks.extend(self.next_new_key..self.next_new_key + 15);
+ self.next_new_key += 15;
+ ks.sort_unstable();
+ ks.dedup();
+ self.last_keys = ks.clone();
+ ks
+ }
+}
+
+fn opts(keys: Keys, round: u64, cache: Option<Arc<ProbeCache>>) -> UpsertOptions {
+ UpsertOptions {
+ primary_keys: keys.primary_keys(),
+ partition_key: Some("p".to_string()),
+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),
+ // Small files, so one upsert writes several files per partition.
+ target_file_size: Some(2 * 1024),
+ isolate_source_rows: round % 3 == 1,
+ max_row_group_bytes: (round % 4 == 2).then_some(512),
+ // One reader per partition makes the output layout deterministic, so the two
+ // copies can be compared file decision by file decision.
+ max_parallel_files: 1,
+ probe_cache: cache,
+ ..Default::default()
+ }
+}
+
+/// The file decisions of an upsert, which the cache must not change.
+fn decisions(s: &UpsertStats) -> [usize; 7] {
+ [
+ s.files_removed,
+ s.files_added,
+ s.files_carried_over,
+ s.rows_updated,
+ s.rows_inserted,
+ s.rows_copied,
+ s.null_pk_rows,
+ ]
+}
+
+#[derive(Default)]
+struct Totals {
+ probed_cached: usize,
+ probed_plain: usize,
+ decided_from_cache: usize,
+}
+
+async fn upsert_round(
+ twin: &mut Twin,
+ keys: Keys,
+ round: u64,
+ rng: &mut Lcg,
+ cache: &Arc<ProbeCache>,
+ totals: &mut Totals,
+) {
+ let ks = twin.batch_keys(rng);
+ let batch = rows(&ks, round as i64 + 1, 2);
+ let with = twin
+ .cached
+ .upsert(
+ vec![batch.clone()],
+ schema(),
+ opts(keys, round, Some(cache.clone())),
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap();
+ let without = twin
+ .plain
+ .upsert(
+ vec![batch],
+ schema(),
+ opts(keys, round, None),
+ MultipartConfig::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ decisions(&with),
+ decisions(&without),
+ "{keys:?} round {round}: {with:?} vs {without:?}"
+ );
+ assert_eq!(
+ read_rows(&twin.cached).await,
+ read_rows(&twin.plain).await,
+ "{keys:?} round {round}"
+ );
+ assert_eq!(without.files_probe_cached, 0);
+ totals.probed_cached += with.files_probed;
+ totals.probed_plain += without.files_probed;
+ totals.decided_from_cache += with.files_probe_cached;
+}
+
+async fn compact_both(twin: &mut Twin) {
+ let copts = || CompactOptions {
+ target_file_size: Some(64 * 1024),
+ limits: Arc::new(ProcessLimits::with_fetch(8, 16, 256 << 20, 256 << 20)),
+ ..Default::default()
+ };
+ let a = twin
+ .cached
+ .compact(copts(), MultipartConfig::default())
+ .await
+ .unwrap();
+ let b = twin
+ .plain
+ .compact(copts(), MultipartConfig::default())
+ .await
+ .unwrap();
+ assert!(a.files_removed > 0);
+ assert_eq!(a.files_removed, b.files_removed);
+ assert_eq!(read_rows(&twin.cached).await, read_rows(&twin.plain).await);
+}
+
+/// Upserts into `tables` tables in turn, all sharing one cache of `cache_bytes`, with
+/// one compaction halfway when `compact` is set.
+async fn run(
+ keys: Keys,
+ tables: u64,
+ cache_bytes: usize,
+ compact: bool,
+) -> (Totals, Arc<ProbeCache>) {
+ let cache = Arc::new(ProbeCache::new(cache_bytes));
+ let mut twins = Vec::new();
+ for t in 0..tables {
+ twins.push(Twin::new(t * 1_000_000).await);
+ }
+ let mut rng = Lcg(42);
+ let mut totals = Totals::default();
+ for round in 0..ROUNDS {
+ for twin in twins.iter_mut() {
+ upsert_round(twin, keys, round, &mut rng, &cache, &mut totals).await;
+ if compact && round == ROUNDS / 2 {
+ compact_both(twin).await;
+ }
+ }
+ }
+ (totals, cache)
+}
+
+#[tokio::test]
+async fn cached_upserts_match_uncached_upserts_for_every_key_type() {
+ for keys in [Keys::Int, Keys::Str, Keys::Composite, Keys::WithPartition] {
+ let (totals, cache) = run(keys, 1, 64 << 20, true).await;
+ // The cache must actually take part: most later probes come from it.
+ assert!(totals.decided_from_cache > 0, "{keys:?}");
+ assert!(
+ totals.probed_cached * 2 < totals.probed_plain,
+ "{keys:?}: probed {} with the cache, {} without",
+ totals.probed_cached,
+ totals.probed_plain
+ );
+ assert_eq!(cache.stats().evictions, 0, "{keys:?}");
+ }
+}
+
+#[tokio::test]
+async fn two_tables_can_share_one_cache() {
+ let (totals, cache) = run(Keys::Composite, 2, 64 << 20, true).await;
+ assert!(totals.decided_from_cache > 0);
+ assert!(cache.stats().entries > 0);
+}
+
+#[tokio::test]
+async fn a_cache_smaller_than_the_tables_evicts_and_stays_correct() {
+ // Without compaction every file is small enough for an entry, but the files of the
+ // three tables together are not.
+ let (totals, cache) = run(Keys::Str, 3, 32 * 1024, false).await;
+ let stats = cache.stats();
+ assert!(stats.evictions > 0);
+ assert!(stats.bytes <= 32 * 1024);
+ assert!(totals.decided_from_cache > 0);
+}
+
+#[tokio::test]
+async fn a_disabled_cache_probes_every_candidate() {
+ let (totals, cache) = run(Keys::Int, 1, 0, true).await;
+ assert_eq!(totals.decided_from_cache, 0);
+ assert_eq!(totals.probed_cached, totals.probed_plain);
+ assert_eq!(cache.stats().entries, 0);
+}
diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.toml
index 54ba588631d4..06636d3b831d 100644
--- a/rust/deltalite/python/Cargo.toml
+++ b/rust/deltalite/python/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "deltalite-python"
# Mirrors pyproject.toml in this directory (always identical); bump both together.
-version = "0.1.13"
+version = "0.1.14"
edition = "2021"
# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to
# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).
diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md
index 034b3da9fcbb..13d9c14c8dbb 100644
--- a/rust/deltalite/python/README.md
+++ b/rust/deltalite/python/README.md
@@ -124,14 +124,17 @@ log.
### `UpsertStats`
Returned by `upsert`. Counts: `version`, `partitions_touched`, `files_added`,
-`files_removed`, `files_carried_over`, `files_probed`, `rows_updated`,
-`rows_inserted`, `rows_copied`, `source_rows`, `null_pk_rows`. Per-phase
+`files_removed`, `files_carried_over`, `files_probed` (candidate files the probe
+read), `files_probe_cached` (candidate files the probe decided from the probe
+cache), `rows_updated`, `rows_inserted`, `rows_copied`, `source_rows`,
+`null_pk_rows`. Per-phase
wall-clock timings (milliseconds): `ingest_ms` (importing the pyarrow source),
`initial_open_ms` (the full snapshot load in `DeltaLiteTable.open`, reported on
the first upsert through a handle and `0` afterwards, so a sum over a handle's
upserts counts it once), `open_ms` (snapshot refreshes around the upsert),
`relax_ms` (nullability-relax check), `plan_ms` (listing + pruning files),
-`rewrite_ms` (reading + rewriting the touched partitions), `commit_ms`
+`rewrite_ms` (reading + rewriting the touched partitions), `probe_ms` (probing,
+summed over partitions, part of `rewrite_ms`), `commit_ms`
(committing to the Delta log), `maintenance_ms` (checkpoint/log cleanup,
non-zero only on checkpoint-boundary commits). `columns_relaxed` counts
non-nullable columns flipped to nullable before the write.
@@ -203,6 +206,7 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:
| `multipart_threshold` / `multipart_part_size` | `64 MiB` / `16 MiB` | Multipart upload thresholds (`0` threshold disables). |
| `commit_max_retries` | `15` | Conditional-put commit retry budget. |
| `commit_metadata` | `None` | Extra key/values recorded in the Delta commit. |
+| `use_probe_cache` | `True` | Use the process-wide probe cache (see below). `False` reads every candidate file from storage. |
**Process-global** — environment variables, enforced *on top of* the per-call
knobs so that many concurrent upsert threads in one process cannot multiply the
@@ -215,6 +219,23 @@ budgets:
`DELTALITE_MAX_SOURCE_BYTES`, `DELTALITE_MULTIPART_THRESHOLD_BYTES`,
`DELTALITE_MULTIPART_PART_SIZE_BYTES`.
+**Probe cache.** The probe reads a footer and the PK column of every candidate
+file, so on object storage its cost grows with the number of files in the
+touched partitions. A Delta data file never changes, so deltalite keeps the
+64-bit hashes of each file's PK tuples in one process-wide LRU cache, keyed by
+table URL and file path. A probe that reads a file and finds no match caches it,
+and an upsert caches the files it writes once its commit lands. A later probe
+decides a cached file with no read. Equal keys always give equal hashes, so a
+cached file that holds a key is never skipped. A hash collision only adds a file
+to the rewrite, which copies its rows unchanged. A file the commit removes leaves
+the cache. `DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` turns the cache off) caps
+the entries, and one entry can use at most an eighth of the cap, so a file with
+more keys is probed every time. Hashes collected for files still being read or
+written use at most a quarter of the cap more, so the cache uses at most
+1.25 times the cap. `deltalite.set_probe_cache_max_bytes(n)` changes the cap at
+run time, `deltalite.clear_probe_cache()` empties the cache, and
+`deltalite.probe_cache_stats()` returns its size and hit counters.
+
**Checkpoint prefetch.** Opening or reloading a table reads its latest
checkpoint Parquet file whole in one GET and serves the Parquet reader's many
small range reads from memory, instead of one round trip per footer, metadata
@@ -273,7 +294,7 @@ exists.
deltalite emits via the Rust [`metrics`](https://docs.rs/metrics) facade (static
labels only): `deltalite_upserts_total` (`outcome`, `prune_strategy`,
`error_kind`), `deltalite_upsert_duration_seconds`,
-`deltalite_files_{added,removed,carried_over,probed}_total`,
+`deltalite_files_{added,removed,carried_over,probed,probe_cached}_total`,
`deltalite_rows_{updated,inserted,copied}_total`,
`deltalite_checkpoint_prefetch_total` (`outcome`),
`deltalite_refresh_probe_total` (`outcome`: `current`, `new_commits`,
diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.toml
index f1102f1387af..b7fcc623a468 100644
--- a/rust/deltalite/python/pyproject.toml
+++ b/rust/deltalite/python/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "maturin"
[project]
name = "deltalite"
# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.
-version = "0.1.13"
+version = "0.1.14"
description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"
readme = "README.md"
requires-python = ">=3.12"
diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs
index b7a440eedd6d..07ac05aca06f 100644
--- a/rust/deltalite/python/src/lib.rs
+++ b/rust/deltalite/python/src/lib.rs
@@ -15,6 +15,7 @@ use deltalake::writer::RecordBatchWriter;
use deltalite_core::errors::Error;
use deltalite_core::handle::TableHandle;
use deltalite_core::limits::ProcessLimits;
+use deltalite_core::probecache::ProbeCache;
use deltalite_core::schema::import_column;
use deltalite_core::table::{open_table, table_exists, MultipartConfig};
use deltalite_core::upsert::{PruneStrategy, UpsertOptions};
@@ -152,6 +153,8 @@ pub struct UpsertStats {
#[pyo3(get)]
pub files_probed: usize,
#[pyo3(get)]
+ pub files_probe_cached: usize,
+ #[pyo3(get)]
pub rows_updated: usize,
#[pyo3(get)]
pub rows_inserted: usize,
@@ -166,6 +169,8 @@ pub struct UpsertStats {
#[pyo3(get)]
pub rewrite_ms: u64,
#[pyo3(get)]
+ pub probe_ms: u64,
+ #[pyo3(get)]
pub commit_ms: u64,
#[pyo3(get)]
pub columns_relaxed: usize,
@@ -186,14 +191,15 @@ impl UpsertStats {
fn __repr__(&self) -> String {
format!(
"UpsertStats(version={}, partitions_touched={}, files_added={}, files_removed={}, \
- files_carried_over={}, files_probed={}, rows_updated={}, rows_inserted={}, \
- rows_copied={})",
+ files_carried_over={}, files_probed={}, files_probe_cached={}, rows_updated={}, \
+ rows_inserted={}, rows_copied={})",
self.version,
self.partitions_touched,
self.files_added,
self.files_removed,
self.files_carried_over,
self.files_probed,
+ self.files_probe_cached,
self.rows_updated,
self.rows_inserted,
self.rows_copied
@@ -210,6 +216,7 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {
files_added: s.files_added,
files_carried_over: s.files_carried_over,
files_probed: s.files_probed,
+ files_probe_cached: s.files_probe_cached,
rows_updated: s.rows_updated,
rows_inserted: s.rows_inserted,
rows_copied: s.rows_copied,
@@ -217,6 +224,7 @@ impl From<deltalite_core::upsert::UpsertStats> for UpsertStats {
null_pk_rows: s.null_pk_rows,
plan_ms: s.plan_ms,
rewrite_ms: s.rewrite_ms,
+ probe_ms: s.probe_ms,
commit_ms: s.commit_ms,
columns_relaxed: s.columns_relaxed,
open_ms: s.open_ms,
@@ -391,6 +399,9 @@ impl DeltaLiteTable {
/// `max_row_group_bytes` closes an output row group once its encoded bytes reach the
/// value, so a later reader fetches less per row group. `None` (the default) closes
/// row groups only at the encoder's row cap.
+ ///
+ /// `use_probe_cache=False` makes the probe read every candidate file from storage
+ /// instead of using the process-wide probe cache (see `set_probe_cache_max_bytes`).
#[pyo3(signature = (
data,
primary_keys,
@@ -412,6 +423,7 @@ impl DeltaLiteTable {
max_source_bytes = None,
multipart_threshold = None,
multipart_part_size = None,
+ use_probe_cache = true,
))]
#[allow(clippy::too_many_arguments)]
fn upsert(
@@ -436,6 +448,7 @@ impl DeltaLiteTable {
max_source_bytes: Option<usize>,
multipart_threshold: Option<usize>,
multipart_part_size: Option<usize>,
+ use_probe_cache: bool,
) -> PyResult<UpsertStats> {
// Import while holding the GIL (it reads a Python object), then release it for
// all the I/O.
@@ -472,6 +485,7 @@ impl DeltaLiteTable {
// Shared with every other concurrent upsert in this process, so the
// process-level memory bound holds regardless of thread count.
limits: ProcessLimits::global().clone(),
+ probe_cache: use_probe_cache.then(|| ProbeCache::global().clone()),
};
let multipart = MultipartConfig::resolve(multipart_threshold, multipart_part_size);
@@ -659,9 +673,41 @@ impl DeltaLiteTable {
}
}
+/// Set the byte cap of the process-wide probe cache, evicting entries down to it. `0`
+/// turns the cache off. The default comes from `DELTALITE_PROBE_CACHE_BYTES` (128 MiB
+/// when unset).
+#[pyfunction]
+fn set_probe_cache_max_bytes(max_bytes: usize) {
+ ProbeCache::global().set_max_bytes(max_bytes);
+}
+
+/// Drop every entry of the process-wide probe cache.
+#[pyfunction]
+fn clear_probe_cache() {
+ ProbeCache::global().clear();
+}
+
+/// Size and counters of the process-wide probe cache.
+#[pyfunction]
+fn probe_cache_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
+ let s = ProbeCache::global().stats();
+ let d = PyDict::new(py);
+ d.set_item("entries", s.entries)?;
+ d.set_item("bytes", s.bytes)?;
+ d.set_item("max_bytes", s.max_bytes)?;
+ d.set_item("hits", s.hits)?;
+ d.set_item("misses", s.misses)?;
+ d.set_item("inserts", s.inserts)?;
+ d.set_item("evictions", s.evictions)?;
+ Ok(d.into_any().unbind())
+}
+
#[pymodule]
fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<DeltaLiteTable>()?;
+ m.add_function(wrap_pyfunction!(set_probe_cache_max_bytes, m)?)?;
+ m.add_function(wrap_pyfunction!(clear_probe_cache, m)?)?;
+ m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;
m.add_class::<UpsertStats>()?;
m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;
m.add(
diff --git a/rust/deltalite/python/tests/test_probe.py b/rust/deltalite/python/tests/test_probe.py
index af7bfa6abe5c..4decfab72ac9 100644
--- a/rust/deltalite/python/tests/test_probe.py
+++ b/rust/deltalite/python/tests/test_probe.py
@@ -464,3 +464,45 @@ def test_invalid_prune_strategy_is_rejected(tmp_path):
with pytest.raises(deltalite.DeltaLiteError, match="unknown prune_strategy"):
upsert_path(uri, rows(["a"], 2), ["id"], None, prune_strategy="bogus")
+
+
+# --------------------------------------------------------------------------------------
+# The probe cache: later upserts decide files from cached keys, never differently
+# --------------------------------------------------------------------------------------
+
+
[email protected]("use_probe_cache", [True, False])
[email protected]("partitioned", [False, True])
+def test_repeated_upserts_with_the_probe_cache_match_merge(tmp_path, use_probe_cache, partitioned):
+ """Each batch re-touches rows the previous batch wrote, plus old rows and new rows, so
+ later probes decide files from keys that both the probe and the writer cached."""
+ import deltalite
+
+ parts = ["p1", "p2"] if partitioned else ["p1"]
+ pkey = PARTITION_KEY if partitioned else None
+ base = tmp_path / "base"
+ per_file = build_multifile_table(str(base), partitioned=partitioned, n_files=4, partitions=parts)
+ old = [(i, p) for p in parts for f in per_file[p] for i in f]
+
+ uri_m, uri_u = str(tmp_path / "m"), str(tmp_path / "u")
+ for uri in (uri_m, uri_u):
+ shutil.copytree(base, uri)
+
+ rng = random.Random(99)
+ previous: list[tuple[str, str]] = []
+ cached = 0
+ for step in range(6):
+ picked = previous[:15] + rng.sample(old, 10) + [(i, rng.choice(parts)) for i in rand_ids(rng, 5)]
+ picked = list(dict(picked).items())
+ batch = rows([i for i, _ in picked], step + 2, part=[p for _, p in picked])
+ merge_path(uri_m, batch, ["id"], pkey)
+ stats = upsert_path(uri_u, batch, ["id"], pkey, use_probe_cache=use_probe_cache)
+ assert_parity(uri_m, uri_u, f"cache_{use_probe_cache}_step_{step}")
+ cached += stats.files_probe_cached
+ previous = picked
+
+ if use_probe_cache:
+ assert cached > 0
+ assert deltalite.probe_cache_stats()["entries"] > 0
+ else:
+ assert cached == 0