:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): bound deltalite s3 calls and permit waits (#124040)
Tom Owers3d575674f83a
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex bf40604bad21..52c49e18bf24 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -4050,7 +4050,7 @@ dependencies = [[[package]]name = "deltalite-python"-version = "0.1.15"+version = "0.1.16"dependencies = ["arrow","arrow-array",diff --git a/rust/deltalite/README.md b/rust/deltalite/README.mdindex bc56808e6117..280d08d0a747 100644--- a/rust/deltalite/README.md+++ b/rust/deltalite/README.md@@ -133,7 +133,12 @@ budgets): `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS` (8),`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`,`DELTALITE_MULTIPART_CONCURRENCY`, `DELTALITE_RANGE_GET_PART_SIZE_BYTES`,`DELTALITE_RANGE_GET_CONCURRENCY`,-`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache).+`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache),+`DELTALITE_IO_DEADLINE_SECS` (600, `0` off: the bound on one data-file upload or ranged read),+`DELTALITE_PERMIT_WAIT_TIMEOUT_SECS` (1800, `0` off: the bound on one wait for a+process-wide budget). S3 tables also get request timeouts that the storage options+can change. `deltalite.limits_snapshot()` shows the free permits of each budget.+See `python/README.md`, "Timeouts".Kill switches for the request savings on the log and on small files (set to `0`to restore the earlier requests): `DELTALITE_PROBE_REFRESH`,diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.tomlindex 268408c77f37..198b803903e8 100644--- a/rust/deltalite/core/Cargo.toml+++ b/rust/deltalite/core/Cargo.toml@@ -68,6 +68,8 @@ tracing = { workspace = true }uuid = { workspace = true }[dev-dependencies]+# Paused time for the deadline and permit-wait tests.+tokio = { workspace = true, features = ["test-util"] }# For the commit_overhead example (counted:// store registration) and table-fixture tests.tempfile = "3"url = { workspace = true }diff --git a/rust/deltalite/core/src/deadline.rs b/rust/deltalite/core/src/deadline.rsnew file mode 100644index 000000000000..e490a9d8c20e--- /dev/null+++ b/rust/deltalite/core/src/deadline.rs@@ -0,0 +1,108 @@+//! An outer bound on object-store calls that deltalite waits for.+//!+//! `object_store` puts a timeout on each HTTP request and a budget on its retries (see+//! [`crate::table::S3_CLIENT_DEFAULTS`]), so a request that the network leaves open+//! fails by itself. The deadline here is a second line for a wait that those timers do+//! not see: a future that stops being woken, or a spawned task that never finishes.+//! The caller can hold budget permits while it waits, and a wait without an end keeps+//! those permits from every other call in the process.+//!+//! When the deadline passes, the wrapped future is dropped. That cancels its request and+//! releases what it holds, and the caller gets a normal `object_store` error.++use std::future::Future;+use std::sync::OnceLock;+use std::time::Duration;++use crate::limits::env_secs;++/// Default deadline of one deltalite object-store call. It is longer than the most+/// time `object_store` gives one call with its default retry budget (3 minutes, then+/// one more request of at most the request timeout), so it fires only when those timers+/// do not.+pub const DEFAULT_IO_DEADLINE: Duration = Duration::from_secs(10 * 60);+/// Environment override for [`DEFAULT_IO_DEADLINE`], in seconds; `0` removes the bound.+pub const IO_DEADLINE_ENV: &str = "DELTALITE_IO_DEADLINE_SECS";++/// The process-wide deadline, read from the environment on first use.+pub(crate) fn io_deadline() -> Option<Duration> {+ static DEADLINE: OnceLock<Option<Duration>> = OnceLock::new();+ *DEADLINE.get_or_init(|| env_secs(IO_DEADLINE_ENV, DEFAULT_IO_DEADLINE))+}++/// Run `fut` for at most `limit`. After `limit`, `fut` is dropped and the result is an+/// error that names the call with `what`. `None` waits without a bound.+pub(crate) async fn with_deadline<T>(+ limit: Option<Duration>,+ what: impl FnOnce() -> String,+ fut: impl Future<Output = object_store::Result<T>>,+) -> object_store::Result<T> {+ let Some(limit) = limit else {+ return fut.await;+ };+ match tokio::time::timeout(limit, fut).await {+ Ok(result) => result,+ Err(_) => Err(object_store::Error::Generic {+ store: "deltalite",+ source: format!(+ "{} did not finish within {}s; {IO_DEADLINE_ENV} sets this bound",+ what(),+ limit.as_secs()+ )+ .into(),+ }),+ }+}++#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use tokio::sync::Semaphore;++ use super::*;++ #[tokio::test(start_paused = true)]+ async fn a_stuck_call_fails_after_the_deadline_and_releases_its_permit() {+ let budget = Arc::new(Semaphore::new(1));+ let permit = budget.clone().acquire_owned().await.unwrap();+ let stuck = async move {+ let _held = permit;+ std::future::pending::<object_store::Result<()>>().await+ };++ let err = with_deadline(+ Some(Duration::from_secs(60)),+ || "upload of part 3 of a.parquet".into(),+ stuck,+ )+ .await+ .unwrap_err()+ .to_string();++ assert!(err.contains("upload of part 3 of a.parquet"), "{err}");+ assert!(err.contains("60s"), "{err}");+ assert_eq!(budget.available_permits(), 1);+ }++ #[tokio::test(start_paused = true)]+ async fn a_call_that_finishes_in_time_keeps_its_result() {+ let slow = async {+ tokio::time::sleep(Duration::from_secs(59)).await;+ Ok(7)+ };+ let got = with_deadline(Some(Duration::from_secs(60)), || "read".into(), slow).await;+ assert_eq!(got.unwrap(), 7);++ let unbounded = async {+ tokio::time::sleep(Duration::from_secs(3600)).await;+ Ok(8)+ };+ assert_eq!(+ with_deadline(None, || "read".into(), unbounded)+ .await+ .unwrap(),+ 8+ );+ }+}diff --git a/rust/deltalite/core/src/errors.rs b/rust/deltalite/core/src/errors.rsindex 613abc415cf9..3ff900982d05 100644--- a/rust/deltalite/core/src/errors.rs+++ b/rust/deltalite/core/src/errors.rs@@ -39,6 +39,11 @@ pub enum Error {/// linearly with the *source*, and this is the guard on that term.#[error("{0}")]SourceTooLarge(String),++ /// A wait on a process-wide budget passed its bound. A call that holds the budget+ /// does not make progress, so the wait fails instead of blocking every other call.+ #[error("{0}")]+ Timeout(String),}impl From<deltalake::DeltaTableError> for Error {@@ -100,6 +105,7 @@ impl Error {Error::Unsupported(_) => "unsupported",Error::Conflict(_) => "conflict",Error::SourceTooLarge(_) => "source_too_large",+ Error::Timeout(_) => "timeout",}}}diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rsindex 9376dac14baf..ae1c9f405bcd 100644--- a/rust/deltalite/core/src/lib.rs+++ b/rust/deltalite/core/src/lib.rs@@ -30,6 +30,7 @@//! crate (built with maturin, imported as `deltalite`).pub mod compact;+pub mod deadline;pub mod errors;pub mod handle;pub mod limits;@@ -49,7 +50,7 @@ mod writer;pub use compact::{compact, CompactOptions, CompactStats};pub use errors::{Error, Result};pub use handle::{LiveFile, TableHandle};-pub use limits::ProcessLimits;+pub use limits::{Budget, BudgetSnapshot, LimitsSnapshot, ProcessLimits};pub use prefetch::CheckpointCache;pub use probecache::{ProbeCache, ProbeCacheStats};pub use table::{diff --git a/rust/deltalite/core/src/limits.rs b/rust/deltalite/core/src/limits.rsindex 9a699f067810..2307ba6858e4 100644--- a/rust/deltalite/core/src/limits.rs+++ b/rust/deltalite/core/src/limits.rs@@ -26,8 +26,17 @@//! static, so tests can build isolated instances; the binding crate passes//! [`ProcessLimits::global()`], a lazily-initialised process-wide singleton configured//! from environment variables.-+//!+//! Each wait on a process-wide budget has an upper bound+//! (`DELTALITE_PERMIT_WAIT_TIMEOUT_SECS`). The waits are fair, so one call that holds+//! permits and stops making progress blocks every later waiter. The bound changes that+//! hang into an [`Error::Timeout`] that names the budget, and+//! [`ProcessLimits::snapshot`] shows how much of each budget is free.++use std::future::Future;+use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, OnceLock};+use std::time::Duration;use tokio::sync::{OwnedSemaphorePermit, Semaphore};@@ -43,24 +52,146 @@ pub const DEFAULT_PROCESS_MAX_BUFFERED_BYTES: usize = 256 * 1024 * 1024;pub const DEFAULT_PROCESS_MAX_FETCH_BYTES: usize = 256 * 1024 * 1024;/// Default ceiling for the source-size guard (resident source + estimated PK set).pub const DEFAULT_MAX_SOURCE_BYTES: usize = 2 * 1024 * 1024 * 1024;+/// Default upper bound on one wait for a process-wide budget. It is much longer than a+/// normal wait, which ends when the holders finish their I/O, so it fires only when a+/// holder does not make progress.+pub const DEFAULT_PERMIT_WAIT_TIMEOUT: Duration = Duration::from_secs(30 * 60);+/// Environment override for [`DEFAULT_PERMIT_WAIT_TIMEOUT`], in seconds; `0` removes the+/// bound.+pub const PERMIT_WAIT_TIMEOUT_ENV: &str = "DELTALITE_PERMIT_WAIT_TIMEOUT_SECS";++/// One of the four process-wide budgets.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Budget {+ Partitions,+ Files,+ /// Decoded bytes in flight, in KiB.+ Buffer,+ /// Compressed row-group bytes held by readers, in KiB.+ Fetch,+}++impl Budget {+ /// Name of the budget in errors and in [`LimitsSnapshot`] consumers.+ pub fn name(self) -> &'static str {+ match self {+ Budget::Partitions => "partitions",+ Budget::Files => "files",+ Budget::Buffer => "buffer_kb",+ Budget::Fetch => "fetch_kb",+ }+ }+}++/// Free and total permits of one budget, and the acquisitions that wait for it. A+/// waiter that needs more permits than are free takes the free ones and waits for the+/// rest, so `available` can be 0 while a part of the budget belongs to a waiter.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct BudgetSnapshot {+ pub available: usize,+ pub total: usize,+ pub waiters: usize,+}++/// [`BudgetSnapshot`] of each process-wide budget.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct LimitsSnapshot {+ pub partitions: BudgetSnapshot,+ pub files: BudgetSnapshot,+ pub buffer_kb: BudgetSnapshot,+ pub fetch_kb: BudgetSnapshot,+}++impl LimitsSnapshot {+ pub fn get(&self, budget: Budget) -> BudgetSnapshot {+ match budget {+ Budget::Partitions => self.partitions,+ Budget::Files => self.files,+ Budget::Buffer => self.buffer_kb,+ Budget::Fetch => self.fetch_kb,+ }+ }+}++/// One process-wide semaphore and the count of the acquisitions that wait for it.+#[derive(Debug)]+struct Pool {+ semaphore: Arc<Semaphore>,+ total: u32,+ waiters: AtomicUsize,+}++impl Pool {+ fn new(total: u32) -> Self {+ Self {+ semaphore: Arc::new(Semaphore::new(total as usize)),+ total,+ waiters: AtomicUsize::new(0),+ }+ }++ fn snapshot(&self) -> BudgetSnapshot {+ BudgetSnapshot {+ available: self.semaphore.available_permits(),+ total: self.total as usize,+ waiters: self.waiters.load(Ordering::Relaxed),+ }+ }+}++/// Counts one waiter of a [`Pool`] until dropped, also when the wait is cancelled.+struct Waiting<'a>(&'a AtomicUsize);++impl<'a> Waiting<'a> {+ fn new(waiters: &'a AtomicUsize) -> Self {+ waiters.fetch_add(1, Ordering::Relaxed);+ Self(waiters)+ }+}++impl Drop for Waiting<'_> {+ fn drop(&mut self) {+ self.0.fetch_sub(1, Ordering::Relaxed);+ }+}/// Process-wide concurrency budgets shared by every concurrent `upsert` call.#[derive(Debug)]pub struct ProcessLimits {- partitions: Arc<Semaphore>,- files: Arc<Semaphore>,- buffer: Arc<Semaphore>,- /// Capacity of `buffer` in KiB (tokio permits are u32-denominated).- buffer_cap_kb: u32,- fetch: Arc<Semaphore>,- /// Capacity of `fetch` in KiB.- fetch_cap_kb: u32,+ partitions: Pool,+ files: Pool,+ /// Capacity in KiB (tokio permits are u32-denominated).+ buffer: Pool,+ /// Capacity in KiB.+ fetch: Pool,+ /// Upper bound on one wait for any of the budgets; `None` waits without a bound.+ wait_timeout: Option<Duration>,}fn cap_kb(bytes: usize) -> u32 {(bytes / 1024).clamp(1, u32::MAX as usize) as u32}+/// Seconds from the environment variable `name` as a duration: unset or invalid gives+/// `default`, and `0` gives `None` (no bound).+pub(crate) fn env_secs(name: &str, default: Duration) -> Option<Duration> {+ secs_setting(name, std::env::var(name).ok().as_deref(), default)+}++fn secs_setting(name: &str, value: Option<&str>, default: Duration) -> Option<Duration> {+ let Some(v) = value else {+ return Some(default);+ };+ match v.trim().parse::<u64>() {+ Ok(0) => None,+ Ok(n) => Some(Duration::from_secs(n)),+ Err(_) => {+ tracing::warn!(var = name, value = %v, "invalid value, using default");+ Some(default)+ }+ }+}+pub(crate) fn env_usize(name: &str, default: usize) -> usize {match std::env::var(name) {Ok(v) => match v.trim().parse::<usize>() {@@ -111,21 +242,26 @@ impl ProcessLimits {max_buffered_bytes: usize,max_fetch_bytes: usize,) -> Self {- let buffer_cap_kb = cap_kb(max_buffered_bytes);- let fetch_cap_kb = cap_kb(max_fetch_bytes);+ let count = |n: usize| n.clamp(1, u32::MAX as usize) as u32;Self {- partitions: Arc::new(Semaphore::new(max_partitions.max(1))),- files: Arc::new(Semaphore::new(max_files.max(1))),- buffer: Arc::new(Semaphore::new(buffer_cap_kb as usize)),- buffer_cap_kb,- fetch: Arc::new(Semaphore::new(fetch_cap_kb as usize)),- fetch_cap_kb,+ partitions: Pool::new(count(max_partitions)),+ files: Pool::new(count(max_files)),+ buffer: Pool::new(cap_kb(max_buffered_bytes)),+ fetch: Pool::new(cap_kb(max_fetch_bytes)),+ wait_timeout: Some(DEFAULT_PERMIT_WAIT_TIMEOUT),}}+ /// Replace the upper bound on one wait for a budget; `None` removes it.+ pub fn with_wait_timeout(mut self, wait_timeout: Option<Duration>) -> Self {+ self.wait_timeout = wait_timeout;+ self+ }+/// Capacities from `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS`,/// `DELTALITE_PROCESS_MAX_PARALLEL_FILES`, `DELTALITE_PROCESS_MAX_BUFFERED_BYTES` and- /// `DELTALITE_PROCESS_MAX_FETCH_BYTES`, with the module defaults as fallback.+ /// `DELTALITE_PROCESS_MAX_FETCH_BYTES`, and the wait bound from+ /// [`PERMIT_WAIT_TIMEOUT_ENV`], with the module defaults as fallback.pub fn from_env() -> Self {Self::with_fetch(env_usize(@@ -145,6 +281,10 @@ impl ProcessLimits {DEFAULT_PROCESS_MAX_FETCH_BYTES,),)+ .with_wait_timeout(env_secs(+ PERMIT_WAIT_TIMEOUT_ENV,+ DEFAULT_PERMIT_WAIT_TIMEOUT,+ ))}/// The process-wide singleton, initialised from the environment on first use.@@ -153,32 +293,88 @@ impl ProcessLimits {GLOBAL.get_or_init(|| Arc::new(ProcessLimits::from_env()))}+ fn pool(&self, budget: Budget) -> &Pool {+ match budget {+ Budget::Partitions => &self.partitions,+ Budget::Files => &self.files,+ Budget::Buffer => &self.buffer,+ Budget::Fetch => &self.fetch,+ }+ }++ /// Free and total permits of every budget, and the acquisitions that wait for each.+ pub fn snapshot(&self) -> LimitsSnapshot {+ LimitsSnapshot {+ partitions: self.partitions.snapshot(),+ files: self.files.snapshot(),+ buffer_kb: self.buffer.snapshot(),+ fetch_kb: self.fetch.snapshot(),+ }+ }++ /// Take `n` permits of `budget`, or fail with [`Error::Timeout`] after the wait bound.+ /// A cancelled acquisition gives back the permits it had collected.+ async fn acquire(&self, budget: Budget, n: u32) -> Result<OwnedSemaphorePermit> {+ let pool = self.pool(budget);+ let _waiting = Waiting::new(&pool.waiters);+ let wait = pool.semaphore.clone().acquire_many_owned(n);+ let closed = || Error::Generic(format!("process {} semaphore closed", budget.name()));+ let Some(limit) = self.wait_timeout else {+ return wait.await.map_err(|_| closed());+ };+ match tokio::time::timeout(limit, wait).await {+ Ok(permit) => permit.map_err(|_| closed()),+ Err(_) => Err(self.timeout_error(budget, n, limit)),+ }+ }++ fn timeout_error(&self, budget: Budget, wanted: u32, limit: Duration) -> Error {+ let s = self.pool(budget).snapshot();+ Error::Timeout(format!(+ "waited more than {}s for {wanted} permit(s) of the process-wide {} budget: \+ {} of {} free, {} waiter(s). A call that holds this budget does not make \+ progress. {PERMIT_WAIT_TIMEOUT_ENV} sets this bound.",+ limit.as_secs(),+ budget.name(),+ s.available,+ s.total,+ s.waiters,+ ))+ }++ /// Run `wait`, an acquisition that ends in a wait for `budget`, under the same bound+ /// as one process-wide wait. Callers use it to bound a per-call wait that comes+ /// before the process-wide one.+ pub(crate) async fn bounded<T>(+ &self,+ budget: Budget,+ wanted: u32,+ wait: impl Future<Output = Result<T>>,+ ) -> Result<T> {+ let Some(limit) = self.wait_timeout else {+ return wait.await;+ };+ match tokio::time::timeout(limit, wait).await {+ Ok(r) => r,+ Err(_) => Err(self.timeout_error(budget, wanted, limit)),+ }+ }+/// Acquire one process-global partition permit.pub async fn acquire_partition(&self) -> Result<OwnedSemaphorePermit> {- self.partitions- .clone()- .acquire_owned()- .await- .map_err(|_| Error::Generic("process partition semaphore closed".into()))+ self.acquire(Budget::Partitions, 1).await}/// Acquire one process-global file-reader permit.pub async fn acquire_file(&self) -> Result<OwnedSemaphorePermit> {- self.files- .clone()- .acquire_owned()- .await- .map_err(|_| Error::Generic("process file semaphore closed".into()))+ self.acquire(Budget::Files, 1).await}/// Acquire `kb` KiB of the process-global byte budget; requests larger than the/// whole budget are capped so an oversized batch still makes progress.pub async fn acquire_buffer_kb(&self, kb: u32) -> Result<OwnedSemaphorePermit> {- self.buffer- .clone()- .acquire_many_owned(kb.min(self.buffer_cap_kb).max(1))+ self.acquire(Budget::Buffer, kb.min(self.buffer.total).max(1)).await- .map_err(|_| Error::Generic("process byte-budget semaphore closed".into()))}/// Try to reserve `kb` KiB without waiting. Callers that can fall back to a@@ -186,29 +382,27 @@ impl ProcessLimits {/// part of the process budget and need one more allocation to finish.pub fn try_acquire_buffer_kb(&self, kb: u32) -> Option<OwnedSemaphorePermit> {self.buffer+ .semaphore.clone()- .try_acquire_many_owned(kb.min(self.buffer_cap_kb).max(1))+ .try_acquire_many_owned(kb.min(self.buffer.total).max(1)).ok()}/// Capacity of the byte budget in KiB.pub fn buffer_cap_kb(&self) -> u32 {- self.buffer_cap_kb+ self.buffer.total}/// Acquire `kb` KiB of the process-global fetch budget, capped at its capacity so a/// row group larger than the whole budget still proceeds (alone).pub async fn acquire_fetch_kb(&self, kb: u32) -> Result<OwnedSemaphorePermit> {- self.fetch- .clone()- .acquire_many_owned(kb.min(self.fetch_cap_kb).max(1))+ self.acquire(Budget::Fetch, kb.min(self.fetch.total).max(1)).await- .map_err(|_| Error::Generic("process fetch-budget semaphore closed".into()))}/// Capacity of the fetch budget in KiB.pub fn fetch_cap_kb(&self) -> u32 {- self.fetch_cap_kb+ self.fetch.total}}@@ -342,6 +536,100 @@ mod tests {drop(p2);}+ async fn take(limits: &ProcessLimits, budget: Budget, n: u32) -> Result<OwnedSemaphorePermit> {+ match budget {+ Budget::Partitions => limits.acquire_partition().await,+ Budget::Files => limits.acquire_file().await,+ Budget::Buffer => limits.acquire_buffer_kb(n).await,+ Budget::Fetch => limits.acquire_fetch_kb(n).await,+ }+ }++ const ALL_BUDGETS: [Budget; 4] = [+ Budget::Partitions,+ Budget::Files,+ Budget::Buffer,+ Budget::Fetch,+ ];++ #[tokio::test(start_paused = true)]+ async fn a_wait_past_the_bound_fails_names_the_budget_and_leaves_no_waiter() {+ for budget in ALL_BUDGETS {+ let limits = ProcessLimits::with_fetch(1, 1, 64 * 1024, 64 * 1024)+ .with_wait_timeout(Some(Duration::from_secs(30)));+ let total = limits.snapshot().get(budget).total as u32;+ let held = take(&limits, budget, total).await.unwrap();++ let err = take(&limits, budget, 1).await.unwrap_err();+ assert_eq!(err.kind(), "timeout", "{budget:?}");+ let msg = err.to_string();+ assert!(msg.contains(budget.name()), "{msg}");+ assert!(msg.contains(&format!("0 of {total} free")), "{msg}");+ assert!(msg.contains("1 waiter(s)"), "{msg}");+ assert_eq!(limits.snapshot().get(budget).waiters, 0, "{budget:?}");++ drop(held);+ assert!(take(&limits, budget, 1).await.is_ok(), "{budget:?}");+ }+ }++ #[tokio::test(start_paused = true)]+ async fn no_bound_waits_past_the_default_bound() {+ let limits = ProcessLimits::with_fetch(1, 1, 1024, 1024).with_wait_timeout(None);+ let _held = limits.acquire_partition().await.unwrap();+ let wait =+ tokio::time::timeout(DEFAULT_PERMIT_WAIT_TIMEOUT * 2, limits.acquire_partition()).await;+ assert!(wait.is_err(), "the acquisition must still wait");+ }++ #[test]+ fn wait_bound_setting_parses_seconds_and_zero_turns_it_off() {+ let default = Duration::from_secs(1800);+ let cases = [+ (None, Some(default)),+ (Some("0"), None),+ (Some(" 45 "), Some(Duration::from_secs(45))),+ (Some("-1"), Some(default)),+ (Some("ten"), Some(default)),+ ];+ for (value, expected) in cases {+ assert_eq!(secs_setting("X", value, default), expected, "{value:?}");+ }+ }++ #[tokio::test]+ async fn snapshot_reports_free_permits_and_waiters() {+ let limits = Arc::new(ProcessLimits::with_fetch(2, 3, 8 * 1024, 4 * 1024));+ let full = limits.snapshot();+ for (budget, total) in [+ (Budget::Partitions, 2),+ (Budget::Files, 3),+ (Budget::Buffer, 8),+ (Budget::Fetch, 4),+ ] {+ let s = full.get(budget);+ assert_eq!(+ (s.available, s.total, s.waiters),+ (total, total, 0),+ "{budget:?}"+ );+ }++ let held = limits.acquire_buffer_kb(6).await.unwrap();+ let waiter = tokio::spawn({+ let limits = limits.clone();+ async move { limits.acquire_buffer_kb(4).await.map(drop) }+ });+ while limits.snapshot().buffer_kb.waiters == 0 {+ tokio::task::yield_now().await;+ }+ assert_eq!(limits.snapshot().buffer_kb.waiters, 1);++ drop(held);+ waiter.await.unwrap().unwrap();+ assert_eq!(limits.snapshot(), full);+ }+#[tokio::test]async fn zero_capacities_are_clamped_to_one() {let limits = ProcessLimits::with_fetch(0, 0, 0, 0);diff --git a/rust/deltalite/core/src/readahead.rs b/rust/deltalite/core/src/readahead.rsindex 346874cbdd12..6677c206f345 100644--- a/rust/deltalite/core/src/readahead.rs+++ b/rust/deltalite/core/src/readahead.rs@@ -10,7 +10,8 @@//! the one it decodes and the one being read. The caller reserves that pair from the//! fetch budget (see [`max_row_group_pair_fetch_bytes`]). The background read is aborted//! when the store is dropped, which happens with the reader, before the reader returns-//! its fetch permit.+//! its fetch permit. The wait for the background read has the deadline of+//! [`crate::deadline`], because the reader holds its fetch permit while it waits.use std::ops::Range;use std::sync::{Arc, Mutex};@@ -27,6 +28,8 @@ use object_store::{use parquet::file::metadata::ParquetMetaData;use tokio::task::JoinHandle;+use crate::deadline::{io_deadline, with_deadline};+/// Byte span of each row group: from the first byte of its first column chunk to the/// last byte of its last one.fn row_group_spans(meta: &ParquetMetaData) -> Vec<Range<u64>> {@@ -182,13 +185,21 @@ impl ObjectStore for ReadAheadStore {let data = match self.take_pending() {Some(mut pending) if pending.group == group => {counter!("deltalite_row_group_read_ahead_total", "outcome" => "hit").increment(1);- let span =+ let read = async {(&mut pending.read.0).await.map_err(|e| object_store::Error::Generic {store: "deltalite",source: Box::new(e),- })??;+ })?+ };+ // After a timeout, `pending` drops at the end of this arm and aborts the read.+ let span = with_deadline(+ io_deadline(),+ || format!("read ahead of row group {group} of {}", self.path),+ read,+ )+ .await?;let Range { start: base, end } = self.groups[group].clone();if span.len() as u64 != end - base {return Err(object_store::Error::Generic {diff --git a/rust/deltalite/core/src/store.rs b/rust/deltalite/core/src/store.rsindex bf0626331b00..c5c58c38644a 100644--- a/rust/deltalite/core/src/store.rs+++ b/rust/deltalite/core/src/store.rs@@ -45,6 +45,7 @@ use object_store::{};use uuid::Uuid;+use crate::deadline::{io_deadline, with_deadline};use crate::logprobe::{commit_path, probe_log, LogAnchor, LogProbe};use crate::table::MultipartConfig;@@ -81,10 +82,16 @@ const DELETE_CONCURRENCY: usize = 16;/// Neither rewrite adds memory: the parts of an upload are slices of the payload the/// caller already holds, and a split read streams each request into its place in one/// buffer of the size that a single request returns.+///+/// Plain-overwrite uploads and ranged reads of data files run under the deadline of+/// [`crate::deadline`]. Conditional puts (the commits) do not: a commit that fails on+/// the client side after the store accepted it would make the caller write the batch a+/// second time.#[derive(Debug)]pub struct MultipartPutStore {inner: Arc<dyn ObjectStore>,config: MultipartConfig,+ deadline: Option<std::time::Duration>,}impl MultipartPutStore {@@ -100,9 +107,16 @@ impl MultipartPutStore {range_concurrency: config.range_concurrency.max(1),..config},+ deadline: io_deadline(),}}+ #[cfg(test)]+ fn with_io_deadline(mut self, deadline: Option<std::time::Duration>) -> Self {+ self.deadline = deadline;+ self+ }+/// Split `payload` into parts of at most `part_size` bytes, zero-copy (`Bytes`/// slices share the underlying allocation).fn split_parts(&self, payload: &PutPayload) -> Vec<PutPayload> {@@ -135,19 +149,26 @@ impl MultipartPutStore {/// order does not change the assembled object.////// After a part fails no new part starts, and the parts already started run to the- /// end: S3 may still store a part whose request was dropped, after the abort.+ /// end: S3 may still store a part whose request was dropped, after the abort. The+ /// exception is a part that passes its deadline, which is dropped so that a stuck+ /// request cannot keep the upload open.async fn upload_parts(&self,+ location: &Path,upload: &mut Box<dyn MultipartUpload>,parts: Vec<PutPayload>,) -> object_store::Result<()> {- let mut parts = parts.into_iter();+ let mut parts = parts.into_iter().enumerate();let mut in_flight = FuturesUnordered::new();let mut first_error = None;loop {while first_error.is_none() && in_flight.len() < self.config.concurrency {- let Some(part) = parts.next() else { break };- in_flight.push(upload.put_part(part));+ let Some((i, part)) = parts.next() else { break };+ in_flight.push(with_deadline(+ self.deadline,+ move || format!("upload of part {} of {location}", i + 1),+ upload.put_part(part),+ ));}match in_flight.next().await {Some(Err(e)) => {@@ -195,8 +216,13 @@ impl MultipartPutStore {.flat_map(move |(range, buf)| {buf.chunks_mut(piece).enumerate().map(move |(i, dst)| {let start = range.start + (i * piece) as u64;- self.read_into(location, start..start + dst.len() as u64, dst)- .boxed()+ let end = start + dst.len() as u64;+ with_deadline(+ self.deadline,+ move || format!("ranged read of {location} for bytes {start}..{end}"),+ self.read_into(location, start..end, dst),+ )+ .boxed()})}).collect();@@ -207,6 +233,19 @@ impl MultipartPutStore {Ok(bufs.into_iter().map(Bytes::from).collect())}+ async fn abort(+ &self,+ location: &Path,+ upload: &mut Box<dyn MultipartUpload>,+ ) -> object_store::Result<()> {+ with_deadline(+ self.deadline,+ || format!("abort of the multipart upload of {location}"),+ upload.abort(),+ )+ .await+ }+async fn read_into(&self,location: &Path,@@ -314,34 +353,47 @@ impl ObjectStore for MultipartPutStore {// Only rewrite the plain overwrite path (data-file uploads). Conditional puts// (PutMode::Create -- Delta log commits -- or Update) must keep their exact// semantics, so they are forwarded untouched.- if self.config.threshold == 0- || !matches!(opts.mode, PutMode::Overwrite)- || payload.content_length() < self.config.threshold- {+ if !matches!(opts.mode, PutMode::Overwrite) {return self.inner.put_opts(location, payload, opts).await;}+ if self.config.threshold == 0 || payload.content_length() < self.config.threshold {+ return with_deadline(+ self.deadline,+ || format!("upload of {location}"),+ self.inner.put_opts(location, payload, opts),+ )+ .await;+ }let multipart_opts = PutMultipartOptions {tags: opts.tags,attributes: opts.attributes,extensions: opts.extensions,};- let mut upload = self- .inner- .put_multipart_opts(location, multipart_opts)- .await?;+ let mut upload = with_deadline(+ self.deadline,+ || format!("start of the multipart upload of {location}"),+ self.inner.put_multipart_opts(location, multipart_opts),+ )+ .await?;let parts = self.split_parts(&payload);- if let Err(e) = self.upload_parts(&mut upload, parts).await {+ if let Err(e) = self.upload_parts(location, &mut upload, parts).await {// Best-effort cleanup of the incomplete upload; the original error is what// the caller needs to see. The parts in flight finished above, and without// `complete` no object is created.- drop(upload.abort().await);+ drop(self.abort(location, &mut upload).await);return Err(e);}- match upload.complete().await {+ match with_deadline(+ self.deadline,+ || format!("completion of the multipart upload of {location}"),+ upload.complete(),+ )+ .await+ {Ok(result) => Ok(result),Err(e) => {- drop(upload.abort().await);+ drop(self.abort(location, &mut upload).await);Err(e)}}@@ -368,13 +420,23 @@ impl ObjectStore for MultipartPutStore {location: &Path,ranges: &[std::ops::Range<u64>],) -> object_store::Result<Vec<Bytes>> {- if !self.splits_reads_of(location) || ranges.iter().any(|r| r.start >= r.end) {+ if location.parts().any(|p| p.as_ref() == "_delta_log") {return self.inner.get_ranges(location, ranges).await;}+ let unsplit = || {+ with_deadline(+ self.deadline,+ || format!("ranged read of {location}"),+ self.inner.get_ranges(location, ranges),+ )+ };+ if !self.splits_reads_of(location) || ranges.iter().any(|r| r.start >= r.end) {+ return unsplit().await;+ }let merged = merge_ranges(ranges);let split_above = self.config.range_part_size as u64;if merged.iter().all(|m| m.end - m.start <= split_above) {- return self.inner.get_ranges(location, ranges).await;+ return unsplit().await;}let fetched = self.get_split(location, &merged).await?;Ok(ranges@@ -729,6 +791,7 @@ mod tests {probe: Arc<Probe>,next: usize,fail_part: Option<usize>,+ stuck_part: Option<usize>,}#[async_trait]@@ -737,9 +800,13 @@ mod tests {let idx = self.next;self.next += 1;let fail = self.fail_part == Some(idx);+ let stuck = self.stuck_part == Some(idx);let probe = self.probe.clone();let put = self.inner.put_part(data);Box::pin(async move {+ if stuck {+ return std::future::pending().await;+ }probe.enter();yield_times(16usize.saturating_sub(idx)).await;probe.leave();@@ -768,6 +835,7 @@ mod tests {inner: Arc<InMemory>,probe: Arc<Probe>,fail_part: Option<usize>,+ stuck_part: Option<usize>,}impl ProbedStore {@@ -776,6 +844,17 @@ mod tests {inner: Arc::new(InMemory::new()),probe: Arc::default(),fail_part,+ stuck_part: None,+ })+ }++ /// A store whose upload part `stuck_part` (0-based) never completes.+ fn with_stuck_part(stuck_part: usize) -> Arc<Self> {+ Arc::new(Self {+ inner: Arc::new(InMemory::new()),+ probe: Arc::default(),+ fail_part: None,+ stuck_part: Some(stuck_part),})}}@@ -806,6 +885,7 @@ mod tests {probe: self.probe.clone(),next: 0,fail_part: self.fail_part,+ stuck_part: self.stuck_part,}))}async fn get_opts(@@ -927,6 +1007,30 @@ mod tests {}}+ #[tokio::test(start_paused = true)]+ async fn a_stuck_part_fails_the_upload_after_the_deadline() {+ for concurrency in [1, 4] {+ let store = ProbedStore::with_stuck_part(1);+ let wrapped = MultipartPutStore::new(store.clone(), config(1024, 5 * MIB, concurrency))+ .with_io_deadline(Some(std::time::Duration::from_secs(60)));+ let path = Path::from("stuck");+ let err = wrapped+ .put(&path, payload_of(20 * MIB))+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("upload of part 2 of stuck"), "{err}");+ assert!(store.probe.aborted(), "the upload must be aborted");+ assert!(+ matches!(+ store.inner.head(&path).await,+ Err(object_store::Error::NotFound { .. })+ ),+ "no object may exist after a stuck upload (concurrency {concurrency})"+ );+ }+ }+#[tokio::test]async fn conditional_puts_are_never_rewritten() {let inner = Arc::new(InMemory::new());diff --git a/rust/deltalite/core/src/table.rs b/rust/deltalite/core/src/table.rsindex f6371e58c094..65d2ff3cfaf8 100644--- a/rust/deltalite/core/src/table.rs+++ b/rust/deltalite/core/src/table.rs@@ -1,6 +1,6 @@//! Table opening helpers.-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use std::sync::Arc;use deltalake::{DeltaTable, DeltaTableBuilder};@@ -108,8 +108,79 @@ impl MultipartConfig {}}+/// HTTP client settings that deltalite gives an S3 table when the caller sets none.+///+/// - `timeout` bounds one request, from connect until the response body ends. The+/// largest single requests are a whole-object read of a checkpoint (up to the+/// checkpoint prefetch cap, 128 MiB) and a single PUT below the multipart threshold+/// (up to 64 MiB when parts upload one after another). `object_store` uses 30s, which+/// cuts such a request below about 4 MB/s on one connection. 120s keeps those+/// requests above about 1 MB/s, and a timed out request is retried.+/// - `connect_timeout` is the `object_store` value, set here so that a change of that+/// default does not change deltalite.+/// - `pool_idle_timeout` stays below the 20s after which S3 closes an idle connection,+/// the same value delta-rs gives its own S3 client. A connection that a proxy or a+/// NAT drops without a reset makes the next request on it wait for the full request+/// timeout; a short idle time makes that less frequent.+///+/// An explicit storage option wins, under any spelling that `object_store` accepts+/// (`timeout`, `aws_timeout`, any case), and so does the environment variable that+/// `object_store` reads for the key (for example `AWS_TIMEOUT`).+pub const S3_CLIENT_DEFAULTS: [(&str, &str); 3] = [+ ("timeout", "120s"),+ ("connect_timeout", "5s"),+ ("pool_idle_timeout", "15s"),+];++/// The `object_store` client key that the option or environment variable `name` sets,+/// for the keys in [`S3_CLIENT_DEFAULTS`]. This is the parsing of+/// `AmazonS3ConfigKey::from_str` for those keys.+fn client_key(name: &str) -> Option<&'static str> {+ let lower = name.to_ascii_lowercase();+ let key = lower.strip_prefix("aws_").unwrap_or(&lower);+ S3_CLIENT_DEFAULTS+ .iter()+ .map(|(k, _)| *k)+ .find(|k| *k == key)+}++/// `storage_options` with each key of [`S3_CLIENT_DEFAULTS`] that neither the options+/// nor the environment (`env_names`) set. Only S3 tables get the defaults; other stores+/// do not read these keys.+fn with_s3_client_defaults<'a>(+ scheme: &str,+ mut storage_options: HashMap<String, String>,+ env_names: impl IntoIterator<Item = &'a str>,+) -> HashMap<String, String> {+ if !matches!(scheme, "s3" | "s3a") {+ return storage_options;+ }+ let set: HashSet<&'static str> = storage_options+ .keys()+ .filter_map(|k| client_key(k))+ .chain(env_names.into_iter().filter_map(client_key))+ .collect();+ for (key, value) in S3_CLIENT_DEFAULTS {+ if !set.contains(key) {+ storage_options.insert(key.to_string(), value.to_string());+ }+ }+ storage_options+}++fn storage_options_for(+ scheme: &str,+ storage_options: HashMap<String, String>,+) -> HashMap<String, String> {+ let env: Vec<String> = std::env::vars_os()+ .filter_map(|(k, _)| k.into_string().ok())+ .collect();+ with_s3_client_defaults(scheme, storage_options, env.iter().map(String::as_str))+}+/// Open and load a Delta table from `uri` with the given delta-rs `storage_options`-/// (the same keys the Python package accepts; the dict passes through unchanged).+/// (the same keys the Python package accepts; the dict passes through unchanged, plus+/// [`S3_CLIENT_DEFAULTS`] for the keys it does not set).////// The load reads its checkpoint through the prefetch wrapper (see [`crate::prefetch`])/// and releases the prefetched bytes before returning, so the table costs no more memory@@ -129,6 +200,7 @@ pub async fn table_exists(uri: &str, storage_options: HashMap<String, String>) -let Ok(url) = deltalake::ensure_table_uri(uri) else {return Ok(false);};+ let storage_options = storage_options_for(url.scheme(), storage_options);let Ok(builder) = DeltaTableBuilder::from_url(url) else {return Ok(false);};@@ -145,6 +217,7 @@ pub async fn open_table_prefetched() -> Result<(DeltaTable, Arc<CheckpointCache>)> {let url = deltalake::ensure_table_uri(uri).map_err(|e| Error::NotFound(format!("invalid table uri {uri}: {e}")))?;+ let storage_options = storage_options_for(url.scheme(), storage_options);let built = DeltaTableBuilder::from_url(url)?.with_storage_options(storage_options).build()?;@@ -217,6 +290,60 @@ mod tests {SERIAL_MULTIPART_THRESHOLD,};+ #[test]+ fn s3_tables_get_client_defaults_for_keys_the_caller_does_not_set() {+ let defaults: HashMap<String, String> = S3_CLIENT_DEFAULTS+ .iter()+ .map(|(k, v)| (k.to_string(), v.to_string()))+ .collect();+ let opts = |pairs: &[(&str, &str)]| -> HashMap<String, String> {+ pairs+ .iter()+ .map(|(k, v)| (k.to_string(), v.to_string()))+ .collect()+ };+ let with = |overrides: &[(&str, &str)]| {+ let mut m = defaults.clone();+ m.extend(opts(overrides));+ m+ };+ type Options = HashMap<String, String>;+ let cases: Vec<(&str, Options, Vec<&str>, Options)> = vec![+ ("s3", opts(&[]), vec![], defaults.clone()),+ ("s3a", opts(&[]), vec![], defaults.clone()),+ (+ "s3",+ opts(&[("region", "us-east-1")]),+ vec![],+ with(&[("region", "us-east-1")]),+ ),+ // An explicit value under any spelling object_store accepts wins.+ (+ "s3",+ opts(&[("timeout", "300s")]),+ vec![],+ with(&[("timeout", "300s")]),+ ),+ ("s3", opts(&[("AWS_CONNECT_TIMEOUT", "1s")]), vec![], {+ let mut m = with(&[("AWS_CONNECT_TIMEOUT", "1s")]);+ m.remove("connect_timeout");+ m+ }),+ // The environment variable object_store reads for a key also wins.+ ("s3", opts(&[]), vec!["AWS_POOL_IDLE_TIMEOUT", "HOME"], {+ let mut m = defaults.clone();+ m.remove("pool_idle_timeout");+ m+ }),+ ("file", opts(&[]), vec![], opts(&[])),+ ("memory", opts(&[("a", "b")]), vec![], opts(&[("a", "b")])),+ ];+ for (scheme, given, env, expected) in cases {+ let got = with_s3_client_defaults(scheme, given.clone(), env);+ assert_eq!(got, expected, "{scheme} {given:?}");+ }+ }+#[test]fn concurrency_one_restores_the_serial_upload_defaults() {let no_env = |_: &str, default: usize| default;diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rsindex 1c3b5d163296..1f6c7ffd9009 100644--- a/rust/deltalite/core/src/upsert.rs+++ b/rust/deltalite/core/src/upsert.rs@@ -26,6 +26,7 @@//! `target_file_size` per partition worker (see `crate::writer`).use std::collections::{HashMap, HashSet};+use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;use std::time::Instant;@@ -56,7 +57,7 @@ use tracing::{debug, info, instrument, warn};use crate::errors::{Error, Result};use crate::limits::{- check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, ProcessLimits,+ check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, Budget, ProcessLimits,SourceFootprint,};use crate::pkset::PkSet;@@ -610,17 +611,23 @@ impl Budgets {let want = Self::kb(bytes);let local_kb = want.min(self.fetch_local_cap_kb as u64) as u32;let global_kb = want.min(self.limits.fetch_cap_kb() as u64) as u32;- let local = self- .fetch_local- .clone()- .acquire_many_owned(local_kb)+ // The per-call wait shares the process-wide bound, so a stuck holder of either+ // budget ends in an error that names the fetch budget.+ self.limits+ .bounded(Budget::Fetch, global_kb, async {+ let local = self+ .fetch_local+ .clone()+ .acquire_many_owned(local_kb)+ .await+ .map_err(|_| Error::Generic("fetch-budget semaphore closed".into()))?;+ let global = self.limits.acquire_fetch_kb(global_kb).await?;+ Ok(FetchPermit {+ _local: local,+ _global: global,+ })+ }).await- .map_err(|_| Error::Generic("fetch-budget semaphore closed".into()))?;- let global = self.limits.acquire_fetch_kb(global_kb).await?;- Ok(FetchPermit {- _local: local,- _global: global,- })}/// Bytes of compressed row-group data one reader can hold without passing either@@ -636,13 +643,7 @@ impl Budgets {let want = Self::kb(bytes);let local_kb = want.min(self.local_cap_kb as u64) as u32;let global_kb = want.min(self.limits.buffer_cap_kb() as u64) as u32;- let local = self- .local- .clone()- .acquire_many_owned(local_kb)- .await- .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;- let global = self.limits.acquire_buffer_kb(global_kb).await?;+ let (local, global) = self.acquire_byte_pair(local_kb, global_kb).await?;Ok(BudgetPermit {local: Some(local),global: Some(global),@@ -651,6 +652,28 @@ impl Budgets {})}+ /// Local before global, the fixed order used everywhere (see `crate::limits`). The+ /// per-call wait shares the process-wide bound, so a stuck holder of either budget+ /// ends in an error that names the buffer budget.+ async fn acquire_byte_pair(+ &self,+ local_kb: u32,+ global_kb: u32,+ ) -> Result<(OwnedSemaphorePermit, OwnedSemaphorePermit)> {+ self.limits+ .bounded(Budget::Buffer, global_kb, async {+ let local = self+ .local+ .clone()+ .acquire_many_owned(local_kb)+ .await+ .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;+ let global = self.limits.acquire_buffer_kb(global_kb).await?;+ Ok((local, global))+ })+ .await+ }+/// Grow `permit` so it covers `total_bytes` (used after a decode turns out larger/// than the pre-decode estimate). Total held stays capped at each budget's capacity,/// preserving the oversized-batch progress guarantee; a decode smaller than the@@ -678,14 +701,7 @@ impl Budgets {permit.global = None;permit.held_local_kb = 0;permit.held_global_kb = 0;- // Local before global, the fixed order used everywhere (see `crate::limits`).- let local = self- .local- .clone()- .acquire_many_owned(want_local)- .await- .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;- let global = self.limits.acquire_buffer_kb(want_global).await?;+ let (local, global) = self.acquire_byte_pair(want_local, want_global).await?;permit.local = Some(local);permit.global = Some(global);permit.held_local_kb = want_local;@@ -2553,10 +2569,18 @@ async fn rewrite_partition(// Reader tasks: up to max_parallel_files files stream concurrently (per call AND// per process). Each task takes its concurrency permits *inside* the task so// spawning never blocks this function.+ //+ // The wait for the per-call file permit has no bound, because files of a large+ // partition queue there for as long as the rewrite takes. After one reader fails,+ // the readers that get a per-call permit later stop instead of reading. Without+ // that, every queued reader would wait the full process-wide bound one after+ // another when a stuck call holds the process-wide file budget.let file_sem = Arc::new(Semaphore::new(opts.max_parallel_files.max(1)));+ let reader_failed = Arc::new(AtomicBool::new(false));let mut readers = Vec::with_capacity(rewrite.len());for f in rewrite {let sem = file_sem.clone();+ let failed = reader_failed.clone();let store = store.clone();let pkset = pkset.clone();let schema = table_schema.clone();@@ -2571,8 +2595,18 @@ async fn rewrite_partition(.acquire_owned().await.map_err(|_| Error::Generic("file semaphore closed".into()))?;- let _global = opts.limits.acquire_file().await?;- filter_file(store, f, pkset, schema, pcol, value, opts, budgets, tx).await+ if failed.load(Ordering::Relaxed) {+ return Ok(None);+ }+ let outcome = async {+ let _global = opts.limits.acquire_file().await?;+ filter_file(store, f, pkset, schema, pcol, value, opts, budgets, tx).await+ }+ .await;+ if outcome.is_err() {+ failed.store(true, Ordering::Relaxed);+ }+ outcome.map(Some)}));}// Drop the parent's sender so the writer task ends when the readers do.@@ -2587,11 +2621,13 @@ async fn rewrite_partition(.await.map_err(|e| Error::Generic(format!("file worker panicked: {e}")))?{- Ok(o) => {+ Ok(Some(o)) => {rows_updated += o.rows_updated;rows_copied += o.rows_copied;removes.push(o.remove);}+ // Stopped because another reader failed; that reader's error is kept below.+ Ok(None) => {}// Keep the first reader error, but keep draining so the writer terminates.Err(e) => reader_err = reader_err.or(Some(e)),}diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.tomlindex afc175eba7d2..7eaadb241338 100644--- a/rust/deltalite/python/Cargo.toml+++ b/rust/deltalite/python/Cargo.toml@@ -1,7 +1,7 @@[package]name = "deltalite-python"# Mirrors pyproject.toml in this directory (always identical); bump both together.-version = "0.1.15"+version = "0.1.16"edition = "2021"# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.mdindex bfa1a7da51c5..bcca01c9761a 100644--- a/rust/deltalite/python/README.md+++ b/rust/deltalite/python/README.md@@ -184,6 +184,7 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:| `DeltaLiteTableNotFoundError` | No Delta table at the URI. || `DeltaLiteUnsupportedTableError` | Table uses a feature deltalite can't handle (e.g. deletion vectors, column mapping). || `DeltaLiteSourceTooLargeError` | Batch exceeds `max_source_bytes` (see below). |+| `DeltaLiteTimeoutError` | A wait for a process-wide budget passed its bound (see "Timeouts" below). |## Operational knobs@@ -224,6 +225,29 @@ budgets:`DELTALITE_RANGE_GET_PART_SIZE_BYTES`, `DELTALITE_RANGE_GET_CONCURRENCY`.The last five are the defaults of the per-call arguments of the same names.+**Timeouts.** All upserts and compactions of one process share the+process-wide budgets above, and their waits are fair. So one call that holds+permits and stops making progress can make every other call wait. Three bounds+stop that:++| Bound | Default | Override | When it passes |+|---|---|---|---|+| S3 request | `timeout` 120s, `connect_timeout` 5s, `pool_idle_timeout` 15s | the storage option of the same name (any spelling `object_store` accepts, for example `aws_timeout`), or its environment variable (`AWS_TIMEOUT`) | `object_store` retries the request, then the call fails. |+| One object-store call | 10 minutes | `DELTALITE_IO_DEADLINE_SECS` (`0` removes it) | An upload of a data file or a part, or a ranged read of a data file, fails with a `DeltaLiteError` that names the call. Conditional puts (commits) have no deadline. |+| One wait for a process-wide budget | 30 minutes | `DELTALITE_PERMIT_WAIT_TIMEOUT_SECS` (`0` removes it) | The call fails with `DeltaLiteTimeoutError`, which names the budget and gives its free permits and waiters. |++Only S3 tables get the request defaults. `object_store` uses a 30s request+timeout, which cuts a 128 MiB checkpoint read or a 64 MiB single PUT on a+connection slower than about 4 MB/s.++`deltalite.limits_snapshot()` returns the free and total permits of each+process-wide budget and the number of acquisitions that wait for it:+`{"partitions": {"available": 8, "total": 8, "waiters": 0}, "files": ...,+"buffer_kb": ..., "fetch_kb": ...}`. It does no I/O and takes no lock, so a+caller can read it while another call of the process hangs. A waiter that needs+more permits than are free takes the free ones and waits for the rest, so+`available` can be 0 while a part of the budget belongs to a waiter.+**Probe cache.** The probe reads a footer and the PK column of every candidatefile, so on object storage its cost grows with the number of files in thetouched partitions. A Delta data file never changes, so deltalite keeps thediff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.tomlindex 80ade625212c..34861e5d9984 100644--- a/rust/deltalite/python/pyproject.toml+++ b/rust/deltalite/python/pyproject.toml@@ -10,7 +10,7 @@ build-backend = "maturin"[project]name = "deltalite"# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.-version = "0.1.15"+version = "0.1.16"description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"readme = "README.md"requires-python = ">=3.12"diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rsindex 8d5199be87f3..f46792891f55 100644--- a/rust/deltalite/python/src/lib.rs+++ b/rust/deltalite/python/src/lib.rs@@ -14,7 +14,7 @@ use arrow_schema::SchemaRef;use deltalake::writer::RecordBatchWriter;use deltalite_core::errors::Error;use deltalite_core::handle::TableHandle;-use deltalite_core::limits::ProcessLimits;+use deltalite_core::limits::{Budget, ProcessLimits};use deltalite_core::probecache::ProbeCache;use deltalite_core::schema::import_column;use deltalite_core::table::{open_table, table_exists, MultipartConfig};@@ -31,6 +31,7 @@ create_exception!(deltalite, DeltaLiteSchemaMismatchError, DeltaLiteError);create_exception!(deltalite, DeltaLiteTableNotFoundError, DeltaLiteError);create_exception!(deltalite, DeltaLiteUnsupportedTableError, DeltaLiteError);create_exception!(deltalite, DeltaLiteSourceTooLargeError, DeltaLiteError);+create_exception!(deltalite, DeltaLiteTimeoutError, DeltaLiteError);/// Map a core error onto the matching Python exception class.fn to_py_err(e: Error) -> PyErr {@@ -41,6 +42,7 @@ fn to_py_err(e: Error) -> PyErr {Error::Unsupported(_) => DeltaLiteUnsupportedTableError::new_err(msg),Error::Conflict(_) => DeltaLiteCommitConflictError::new_err(msg),Error::SourceTooLarge(_) => DeltaLiteSourceTooLargeError::new_err(msg),+ Error::Timeout(_) => DeltaLiteTimeoutError::new_err(msg),Error::Generic(_) => DeltaLiteError::new_err(msg),}}@@ -737,12 +739,38 @@ fn probe_cache_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {Ok(d.into_any().unbind())}+/// Free and total permits of each process-wide budget that every upsert and compaction+/// of this process shares, and the acquisitions that wait for each:+/// `{"partitions": {"available": .., "total": .., "waiters": ..}, "files": {..},+/// "buffer_kb": {..}, "fetch_kb": {..}}`. The byte budgets count KiB. It takes no+/// lock and does no I/O, so a caller can read it while a call of this process hangs.+#[pyfunction]+fn limits_snapshot(py: Python<'_>) -> PyResult<Py<PyAny>> {+ let s = ProcessLimits::global().snapshot();+ let d = PyDict::new(py);+ for budget in [+ Budget::Partitions,+ Budget::Files,+ Budget::Buffer,+ Budget::Fetch,+ ] {+ let b = s.get(budget);+ let entry = PyDict::new(py);+ entry.set_item("available", b.available)?;+ entry.set_item("total", b.total)?;+ entry.set_item("waiters", b.waiters)?;+ d.set_item(budget.name(), entry)?;+ }+ Ok(d.into_any().unbind())+}+#[pymodule]fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_class::<DeltaLiteTable>()?;m.add_function(wrap_pyfunction!(set_probe_cache_max_bytes, m)?)?;m.add_function(wrap_pyfunction!(clear_probe_cache, m)?)?;m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;+ m.add_function(wrap_pyfunction!(limits_snapshot, m)?)?;m.add_class::<UpsertStats>()?;m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;m.add(@@ -765,5 +793,9 @@ fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {"DeltaLiteSourceTooLargeError",m.py().get_type::<DeltaLiteSourceTooLargeError>(),)?;+ m.add(+ "DeltaLiteTimeoutError",+ m.py().get_type::<DeltaLiteTimeoutError>(),+ )?;Ok(())}diff --git a/rust/deltalite/python/tests/test_guards.py b/rust/deltalite/python/tests/test_guards.pyindex 6472d9591ef3..2d0c89bd43ac 100644--- a/rust/deltalite/python/tests/test_guards.py+++ b/rust/deltalite/python/tests/test_guards.py@@ -55,10 +55,11 @@ def test_source_guard_zero_disables_the_ceiling(tmp_path):assert stats.rows_inserted == 1-def test_source_guard_error_is_a_deltalite_error_subclass():[email protected]("name", ["DeltaLiteSourceTooLargeError", "DeltaLiteTimeoutError"])+def test_guard_errors_are_deltalite_error_subclasses(name):import deltalite- assert issubclass(deltalite.DeltaLiteSourceTooLargeError, deltalite.DeltaLiteError)+ assert issubclass(getattr(deltalite, name), deltalite.DeltaLiteError)def test_multipart_forced_upsert_has_merge_parity(tmp_path):@@ -83,6 +84,15 @@ def test_multipart_forced_upsert_has_merge_parity(tmp_path):uri_m, uri_u = run_scenario_with_multipart(scenario, tmp_path)assert_parity(uri_m, uri_u, "multipart_forced")+ import deltalite++ snapshot = deltalite.limits_snapshot()+ assert set(snapshot) == {"partitions", "files", "buffer_kb", "fetch_kb"}+ for name, budget in snapshot.items():+ assert budget["total"] > 0, name+ assert budget["available"] == budget["total"], f"an upsert kept {name} permits"+ assert budget["waiters"] == 0, name+def run_scenario_with_multipart(scenario, tmp_path):from harness.common import dedupe_keep_last, merge_path