Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): bound deltalite s3 calls and permit waits (#124040)

Tom Owers3d575674f83a
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index bf40604bad21..52c49e18bf24 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -4050,7 +4050,7 @@ dependencies = [
[[package]]
name = "deltalite-python"
-version = "0.1.15"
+version = "0.1.16"
dependencies = [
"arrow",
"arrow-array",
diff --git a/rust/deltalite/README.md b/rust/deltalite/README.md
index bc56808e6117..280d08d0a747 100644
--- a/rust/deltalite/README.md
+++ b/rust/deltalite/README.md
@@ -133,7 +133,12 @@ budgets): `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS` (8),
`DELTALITE_MULTIPART_THRESHOLD_BYTES`, `DELTALITE_MULTIPART_PART_SIZE_BYTES`,
`DELTALITE_MULTIPART_CONCURRENCY`, `DELTALITE_RANGE_GET_PART_SIZE_BYTES`,
`DELTALITE_RANGE_GET_CONCURRENCY`,
-`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache).
+`DELTALITE_PROBE_CACHE_BYTES` (128 MiB, `0` off: the byte cap of the probe cache),
+`DELTALITE_IO_DEADLINE_SECS` (600, `0` off: the bound on one data-file upload or ranged read),
+`DELTALITE_PERMIT_WAIT_TIMEOUT_SECS` (1800, `0` off: the bound on one wait for a
+process-wide budget). S3 tables also get request timeouts that the storage options
+can change. `deltalite.limits_snapshot()` shows the free permits of each budget.
+See `python/README.md`, "Timeouts".
Kill switches for the request savings on the log and on small files (set to `0`
to restore the earlier requests): `DELTALITE_PROBE_REFRESH`,
diff --git a/rust/deltalite/core/Cargo.toml b/rust/deltalite/core/Cargo.toml
index 268408c77f37..198b803903e8 100644
--- a/rust/deltalite/core/Cargo.toml
+++ b/rust/deltalite/core/Cargo.toml
@@ -68,6 +68,8 @@ tracing = { workspace = true }
uuid = { workspace = true }
[dev-dependencies]
+# Paused time for the deadline and permit-wait tests.
+tokio = { workspace = true, features = ["test-util"] }
# For the commit_overhead example (counted:// store registration) and table-fixture tests.
tempfile = "3"
url = { workspace = true }
diff --git a/rust/deltalite/core/src/deadline.rs b/rust/deltalite/core/src/deadline.rs
new file mode 100644
index 000000000000..e490a9d8c20e
--- /dev/null
+++ b/rust/deltalite/core/src/deadline.rs
@@ -0,0 +1,108 @@
+//! An outer bound on object-store calls that deltalite waits for.
+//!
+//! `object_store` puts a timeout on each HTTP request and a budget on its retries (see
+//! [`crate::table::S3_CLIENT_DEFAULTS`]), so a request that the network leaves open
+//! fails by itself. The deadline here is a second line for a wait that those timers do
+//! not see: a future that stops being woken, or a spawned task that never finishes.
+//! The caller can hold budget permits while it waits, and a wait without an end keeps
+//! those permits from every other call in the process.
+//!
+//! When the deadline passes, the wrapped future is dropped. That cancels its request and
+//! releases what it holds, and the caller gets a normal `object_store` error.
+
+use std::future::Future;
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use crate::limits::env_secs;
+
+/// Default deadline of one deltalite object-store call. It is longer than the most
+/// time `object_store` gives one call with its default retry budget (3 minutes, then
+/// one more request of at most the request timeout), so it fires only when those timers
+/// do not.
+pub const DEFAULT_IO_DEADLINE: Duration = Duration::from_secs(10 * 60);
+/// Environment override for [`DEFAULT_IO_DEADLINE`], in seconds; `0` removes the bound.
+pub const IO_DEADLINE_ENV: &str = "DELTALITE_IO_DEADLINE_SECS";
+
+/// The process-wide deadline, read from the environment on first use.
+pub(crate) fn io_deadline() -> Option<Duration> {
+ static DEADLINE: OnceLock<Option<Duration>> = OnceLock::new();
+ *DEADLINE.get_or_init(|| env_secs(IO_DEADLINE_ENV, DEFAULT_IO_DEADLINE))
+}
+
+/// Run `fut` for at most `limit`. After `limit`, `fut` is dropped and the result is an
+/// error that names the call with `what`. `None` waits without a bound.
+pub(crate) async fn with_deadline<T>(
+ limit: Option<Duration>,
+ what: impl FnOnce() -> String,
+ fut: impl Future<Output = object_store::Result<T>>,
+) -> object_store::Result<T> {
+ let Some(limit) = limit else {
+ return fut.await;
+ };
+ match tokio::time::timeout(limit, fut).await {
+ Ok(result) => result,
+ Err(_) => Err(object_store::Error::Generic {
+ store: "deltalite",
+ source: format!(
+ "{} did not finish within {}s; {IO_DEADLINE_ENV} sets this bound",
+ what(),
+ limit.as_secs()
+ )
+ .into(),
+ }),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use tokio::sync::Semaphore;
+
+ use super::*;
+
+ #[tokio::test(start_paused = true)]
+ async fn a_stuck_call_fails_after_the_deadline_and_releases_its_permit() {
+ let budget = Arc::new(Semaphore::new(1));
+ let permit = budget.clone().acquire_owned().await.unwrap();
+ let stuck = async move {
+ let _held = permit;
+ std::future::pending::<object_store::Result<()>>().await
+ };
+
+ let err = with_deadline(
+ Some(Duration::from_secs(60)),
+ || "upload of part 3 of a.parquet".into(),
+ stuck,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(err.contains("upload of part 3 of a.parquet"), "{err}");
+ assert!(err.contains("60s"), "{err}");
+ assert_eq!(budget.available_permits(), 1);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_call_that_finishes_in_time_keeps_its_result() {
+ let slow = async {
+ tokio::time::sleep(Duration::from_secs(59)).await;
+ Ok(7)
+ };
+ let got = with_deadline(Some(Duration::from_secs(60)), || "read".into(), slow).await;
+ assert_eq!(got.unwrap(), 7);
+
+ let unbounded = async {
+ tokio::time::sleep(Duration::from_secs(3600)).await;
+ Ok(8)
+ };
+ assert_eq!(
+ with_deadline(None, || "read".into(), unbounded)
+ .await
+ .unwrap(),
+ 8
+ );
+ }
+}
diff --git a/rust/deltalite/core/src/errors.rs b/rust/deltalite/core/src/errors.rs
index 613abc415cf9..3ff900982d05 100644
--- a/rust/deltalite/core/src/errors.rs
+++ b/rust/deltalite/core/src/errors.rs
@@ -39,6 +39,11 @@ pub enum Error {
/// linearly with the *source*, and this is the guard on that term.
#[error("{0}")]
SourceTooLarge(String),
+
+ /// A wait on a process-wide budget passed its bound. A call that holds the budget
+ /// does not make progress, so the wait fails instead of blocking every other call.
+ #[error("{0}")]
+ Timeout(String),
}
impl From<deltalake::DeltaTableError> for Error {
@@ -100,6 +105,7 @@ impl Error {
Error::Unsupported(_) => "unsupported",
Error::Conflict(_) => "conflict",
Error::SourceTooLarge(_) => "source_too_large",
+ Error::Timeout(_) => "timeout",
}
}
}
diff --git a/rust/deltalite/core/src/lib.rs b/rust/deltalite/core/src/lib.rs
index 9376dac14baf..ae1c9f405bcd 100644
--- a/rust/deltalite/core/src/lib.rs
+++ b/rust/deltalite/core/src/lib.rs
@@ -30,6 +30,7 @@
//! crate (built with maturin, imported as `deltalite`).
pub mod compact;
+pub mod deadline;
pub mod errors;
pub mod handle;
pub mod limits;
@@ -49,7 +50,7 @@ mod writer;
pub use compact::{compact, CompactOptions, CompactStats};
pub use errors::{Error, Result};
pub use handle::{LiveFile, TableHandle};
-pub use limits::ProcessLimits;
+pub use limits::{Budget, BudgetSnapshot, LimitsSnapshot, ProcessLimits};
pub use prefetch::CheckpointCache;
pub use probecache::{ProbeCache, ProbeCacheStats};
pub use table::{
diff --git a/rust/deltalite/core/src/limits.rs b/rust/deltalite/core/src/limits.rs
index 9a699f067810..2307ba6858e4 100644
--- a/rust/deltalite/core/src/limits.rs
+++ b/rust/deltalite/core/src/limits.rs
@@ -26,8 +26,17 @@
//! static, so tests can build isolated instances; the binding crate passes
//! [`ProcessLimits::global()`], a lazily-initialised process-wide singleton configured
//! from environment variables.
-
+//!
+//! Each wait on a process-wide budget has an upper bound
+//! (`DELTALITE_PERMIT_WAIT_TIMEOUT_SECS`). The waits are fair, so one call that holds
+//! permits and stops making progress blocks every later waiter. The bound changes that
+//! hang into an [`Error::Timeout`] that names the budget, and
+//! [`ProcessLimits::snapshot`] shows how much of each budget is free.
+
+use std::future::Future;
+use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
+use std::time::Duration;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
@@ -43,24 +52,146 @@ pub const DEFAULT_PROCESS_MAX_BUFFERED_BYTES: usize = 256 * 1024 * 1024;
pub const DEFAULT_PROCESS_MAX_FETCH_BYTES: usize = 256 * 1024 * 1024;
/// Default ceiling for the source-size guard (resident source + estimated PK set).
pub const DEFAULT_MAX_SOURCE_BYTES: usize = 2 * 1024 * 1024 * 1024;
+/// Default upper bound on one wait for a process-wide budget. It is much longer than a
+/// normal wait, which ends when the holders finish their I/O, so it fires only when a
+/// holder does not make progress.
+pub const DEFAULT_PERMIT_WAIT_TIMEOUT: Duration = Duration::from_secs(30 * 60);
+/// Environment override for [`DEFAULT_PERMIT_WAIT_TIMEOUT`], in seconds; `0` removes the
+/// bound.
+pub const PERMIT_WAIT_TIMEOUT_ENV: &str = "DELTALITE_PERMIT_WAIT_TIMEOUT_SECS";
+
+/// One of the four process-wide budgets.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Budget {
+ Partitions,
+ Files,
+ /// Decoded bytes in flight, in KiB.
+ Buffer,
+ /// Compressed row-group bytes held by readers, in KiB.
+ Fetch,
+}
+
+impl Budget {
+ /// Name of the budget in errors and in [`LimitsSnapshot`] consumers.
+ pub fn name(self) -> &'static str {
+ match self {
+ Budget::Partitions => "partitions",
+ Budget::Files => "files",
+ Budget::Buffer => "buffer_kb",
+ Budget::Fetch => "fetch_kb",
+ }
+ }
+}
+
+/// Free and total permits of one budget, and the acquisitions that wait for it. A
+/// waiter that needs more permits than are free takes the free ones and waits for the
+/// rest, so `available` can be 0 while a part of the budget belongs to a waiter.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct BudgetSnapshot {
+ pub available: usize,
+ pub total: usize,
+ pub waiters: usize,
+}
+
+/// [`BudgetSnapshot`] of each process-wide budget.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct LimitsSnapshot {
+ pub partitions: BudgetSnapshot,
+ pub files: BudgetSnapshot,
+ pub buffer_kb: BudgetSnapshot,
+ pub fetch_kb: BudgetSnapshot,
+}
+
+impl LimitsSnapshot {
+ pub fn get(&self, budget: Budget) -> BudgetSnapshot {
+ match budget {
+ Budget::Partitions => self.partitions,
+ Budget::Files => self.files,
+ Budget::Buffer => self.buffer_kb,
+ Budget::Fetch => self.fetch_kb,
+ }
+ }
+}
+
+/// One process-wide semaphore and the count of the acquisitions that wait for it.
+#[derive(Debug)]
+struct Pool {
+ semaphore: Arc<Semaphore>,
+ total: u32,
+ waiters: AtomicUsize,
+}
+
+impl Pool {
+ fn new(total: u32) -> Self {
+ Self {
+ semaphore: Arc::new(Semaphore::new(total as usize)),
+ total,
+ waiters: AtomicUsize::new(0),
+ }
+ }
+
+ fn snapshot(&self) -> BudgetSnapshot {
+ BudgetSnapshot {
+ available: self.semaphore.available_permits(),
+ total: self.total as usize,
+ waiters: self.waiters.load(Ordering::Relaxed),
+ }
+ }
+}
+
+/// Counts one waiter of a [`Pool`] until dropped, also when the wait is cancelled.
+struct Waiting<'a>(&'a AtomicUsize);
+
+impl<'a> Waiting<'a> {
+ fn new(waiters: &'a AtomicUsize) -> Self {
+ waiters.fetch_add(1, Ordering::Relaxed);
+ Self(waiters)
+ }
+}
+
+impl Drop for Waiting<'_> {
+ fn drop(&mut self) {
+ self.0.fetch_sub(1, Ordering::Relaxed);
+ }
+}
/// Process-wide concurrency budgets shared by every concurrent `upsert` call.
#[derive(Debug)]
pub struct ProcessLimits {
- partitions: Arc<Semaphore>,
- files: Arc<Semaphore>,
- buffer: Arc<Semaphore>,
- /// Capacity of `buffer` in KiB (tokio permits are u32-denominated).
- buffer_cap_kb: u32,
- fetch: Arc<Semaphore>,
- /// Capacity of `fetch` in KiB.
- fetch_cap_kb: u32,
+ partitions: Pool,
+ files: Pool,
+ /// Capacity in KiB (tokio permits are u32-denominated).
+ buffer: Pool,
+ /// Capacity in KiB.
+ fetch: Pool,
+ /// Upper bound on one wait for any of the budgets; `None` waits without a bound.
+ wait_timeout: Option<Duration>,
}
fn cap_kb(bytes: usize) -> u32 {
(bytes / 1024).clamp(1, u32::MAX as usize) as u32
}
+/// Seconds from the environment variable `name` as a duration: unset or invalid gives
+/// `default`, and `0` gives `None` (no bound).
+pub(crate) fn env_secs(name: &str, default: Duration) -> Option<Duration> {
+ secs_setting(name, std::env::var(name).ok().as_deref(), default)
+}
+
+fn secs_setting(name: &str, value: Option<&str>, default: Duration) -> Option<Duration> {
+ let Some(v) = value else {
+ return Some(default);
+ };
+ match v.trim().parse::<u64>() {
+ Ok(0) => None,
+ Ok(n) => Some(Duration::from_secs(n)),
+ Err(_) => {
+ tracing::warn!(var = name, value = %v, "invalid value, using default");
+ Some(default)
+ }
+ }
+}
+
pub(crate) fn env_usize(name: &str, default: usize) -> usize {
match std::env::var(name) {
Ok(v) => match v.trim().parse::<usize>() {
@@ -111,21 +242,26 @@ impl ProcessLimits {
max_buffered_bytes: usize,
max_fetch_bytes: usize,
) -> Self {
- let buffer_cap_kb = cap_kb(max_buffered_bytes);
- let fetch_cap_kb = cap_kb(max_fetch_bytes);
+ let count = |n: usize| n.clamp(1, u32::MAX as usize) as u32;
Self {
- partitions: Arc::new(Semaphore::new(max_partitions.max(1))),
- files: Arc::new(Semaphore::new(max_files.max(1))),
- buffer: Arc::new(Semaphore::new(buffer_cap_kb as usize)),
- buffer_cap_kb,
- fetch: Arc::new(Semaphore::new(fetch_cap_kb as usize)),
- fetch_cap_kb,
+ partitions: Pool::new(count(max_partitions)),
+ files: Pool::new(count(max_files)),
+ buffer: Pool::new(cap_kb(max_buffered_bytes)),
+ fetch: Pool::new(cap_kb(max_fetch_bytes)),
+ wait_timeout: Some(DEFAULT_PERMIT_WAIT_TIMEOUT),
}
}
+ /// Replace the upper bound on one wait for a budget; `None` removes it.
+ pub fn with_wait_timeout(mut self, wait_timeout: Option<Duration>) -> Self {
+ self.wait_timeout = wait_timeout;
+ self
+ }
+
/// Capacities from `DELTALITE_PROCESS_MAX_PARALLEL_PARTITIONS`,
/// `DELTALITE_PROCESS_MAX_PARALLEL_FILES`, `DELTALITE_PROCESS_MAX_BUFFERED_BYTES` and
- /// `DELTALITE_PROCESS_MAX_FETCH_BYTES`, with the module defaults as fallback.
+ /// `DELTALITE_PROCESS_MAX_FETCH_BYTES`, and the wait bound from
+ /// [`PERMIT_WAIT_TIMEOUT_ENV`], with the module defaults as fallback.
pub fn from_env() -> Self {
Self::with_fetch(
env_usize(
@@ -145,6 +281,10 @@ impl ProcessLimits {
DEFAULT_PROCESS_MAX_FETCH_BYTES,
),
)
+ .with_wait_timeout(env_secs(
+ PERMIT_WAIT_TIMEOUT_ENV,
+ DEFAULT_PERMIT_WAIT_TIMEOUT,
+ ))
}
/// The process-wide singleton, initialised from the environment on first use.
@@ -153,32 +293,88 @@ impl ProcessLimits {
GLOBAL.get_or_init(|| Arc::new(ProcessLimits::from_env()))
}
+ fn pool(&self, budget: Budget) -> &Pool {
+ match budget {
+ Budget::Partitions => &self.partitions,
+ Budget::Files => &self.files,
+ Budget::Buffer => &self.buffer,
+ Budget::Fetch => &self.fetch,
+ }
+ }
+
+ /// Free and total permits of every budget, and the acquisitions that wait for each.
+ pub fn snapshot(&self) -> LimitsSnapshot {
+ LimitsSnapshot {
+ partitions: self.partitions.snapshot(),
+ files: self.files.snapshot(),
+ buffer_kb: self.buffer.snapshot(),
+ fetch_kb: self.fetch.snapshot(),
+ }
+ }
+
+ /// Take `n` permits of `budget`, or fail with [`Error::Timeout`] after the wait bound.
+ /// A cancelled acquisition gives back the permits it had collected.
+ async fn acquire(&self, budget: Budget, n: u32) -> Result<OwnedSemaphorePermit> {
+ let pool = self.pool(budget);
+ let _waiting = Waiting::new(&pool.waiters);
+ let wait = pool.semaphore.clone().acquire_many_owned(n);
+ let closed = || Error::Generic(format!("process {} semaphore closed", budget.name()));
+ let Some(limit) = self.wait_timeout else {
+ return wait.await.map_err(|_| closed());
+ };
+ match tokio::time::timeout(limit, wait).await {
+ Ok(permit) => permit.map_err(|_| closed()),
+ Err(_) => Err(self.timeout_error(budget, n, limit)),
+ }
+ }
+
+ fn timeout_error(&self, budget: Budget, wanted: u32, limit: Duration) -> Error {
+ let s = self.pool(budget).snapshot();
+ Error::Timeout(format!(
+ "waited more than {}s for {wanted} permit(s) of the process-wide {} budget: \
+ {} of {} free, {} waiter(s). A call that holds this budget does not make \
+ progress. {PERMIT_WAIT_TIMEOUT_ENV} sets this bound.",
+ limit.as_secs(),
+ budget.name(),
+ s.available,
+ s.total,
+ s.waiters,
+ ))
+ }
+
+ /// Run `wait`, an acquisition that ends in a wait for `budget`, under the same bound
+ /// as one process-wide wait. Callers use it to bound a per-call wait that comes
+ /// before the process-wide one.
+ pub(crate) async fn bounded<T>(
+ &self,
+ budget: Budget,
+ wanted: u32,
+ wait: impl Future<Output = Result<T>>,
+ ) -> Result<T> {
+ let Some(limit) = self.wait_timeout else {
+ return wait.await;
+ };
+ match tokio::time::timeout(limit, wait).await {
+ Ok(r) => r,
+ Err(_) => Err(self.timeout_error(budget, wanted, limit)),
+ }
+ }
+
/// Acquire one process-global partition permit.
pub async fn acquire_partition(&self) -> Result<OwnedSemaphorePermit> {
- self.partitions
- .clone()
- .acquire_owned()
- .await
- .map_err(|_| Error::Generic("process partition semaphore closed".into()))
+ self.acquire(Budget::Partitions, 1).await
}
/// Acquire one process-global file-reader permit.
pub async fn acquire_file(&self) -> Result<OwnedSemaphorePermit> {
- self.files
- .clone()
- .acquire_owned()
- .await
- .map_err(|_| Error::Generic("process file semaphore closed".into()))
+ self.acquire(Budget::Files, 1).await
}
/// Acquire `kb` KiB of the process-global byte budget; requests larger than the
/// whole budget are capped so an oversized batch still makes progress.
pub async fn acquire_buffer_kb(&self, kb: u32) -> Result<OwnedSemaphorePermit> {
- self.buffer
- .clone()
- .acquire_many_owned(kb.min(self.buffer_cap_kb).max(1))
+ self.acquire(Budget::Buffer, kb.min(self.buffer.total).max(1))
.await
- .map_err(|_| Error::Generic("process byte-budget semaphore closed".into()))
}
/// Try to reserve `kb` KiB without waiting. Callers that can fall back to a
@@ -186,29 +382,27 @@ impl ProcessLimits {
/// part of the process budget and need one more allocation to finish.
pub fn try_acquire_buffer_kb(&self, kb: u32) -> Option<OwnedSemaphorePermit> {
self.buffer
+ .semaphore
.clone()
- .try_acquire_many_owned(kb.min(self.buffer_cap_kb).max(1))
+ .try_acquire_many_owned(kb.min(self.buffer.total).max(1))
.ok()
}
/// Capacity of the byte budget in KiB.
pub fn buffer_cap_kb(&self) -> u32 {
- self.buffer_cap_kb
+ self.buffer.total
}
/// Acquire `kb` KiB of the process-global fetch budget, capped at its capacity so a
/// row group larger than the whole budget still proceeds (alone).
pub async fn acquire_fetch_kb(&self, kb: u32) -> Result<OwnedSemaphorePermit> {
- self.fetch
- .clone()
- .acquire_many_owned(kb.min(self.fetch_cap_kb).max(1))
+ self.acquire(Budget::Fetch, kb.min(self.fetch.total).max(1))
.await
- .map_err(|_| Error::Generic("process fetch-budget semaphore closed".into()))
}
/// Capacity of the fetch budget in KiB.
pub fn fetch_cap_kb(&self) -> u32 {
- self.fetch_cap_kb
+ self.fetch.total
}
}
@@ -342,6 +536,100 @@ mod tests {
drop(p2);
}
+ async fn take(limits: &ProcessLimits, budget: Budget, n: u32) -> Result<OwnedSemaphorePermit> {
+ match budget {
+ Budget::Partitions => limits.acquire_partition().await,
+ Budget::Files => limits.acquire_file().await,
+ Budget::Buffer => limits.acquire_buffer_kb(n).await,
+ Budget::Fetch => limits.acquire_fetch_kb(n).await,
+ }
+ }
+
+ const ALL_BUDGETS: [Budget; 4] = [
+ Budget::Partitions,
+ Budget::Files,
+ Budget::Buffer,
+ Budget::Fetch,
+ ];
+
+ #[tokio::test(start_paused = true)]
+ async fn a_wait_past_the_bound_fails_names_the_budget_and_leaves_no_waiter() {
+ for budget in ALL_BUDGETS {
+ let limits = ProcessLimits::with_fetch(1, 1, 64 * 1024, 64 * 1024)
+ .with_wait_timeout(Some(Duration::from_secs(30)));
+ let total = limits.snapshot().get(budget).total as u32;
+ let held = take(&limits, budget, total).await.unwrap();
+
+ let err = take(&limits, budget, 1).await.unwrap_err();
+ assert_eq!(err.kind(), "timeout", "{budget:?}");
+ let msg = err.to_string();
+ assert!(msg.contains(budget.name()), "{msg}");
+ assert!(msg.contains(&format!("0 of {total} free")), "{msg}");
+ assert!(msg.contains("1 waiter(s)"), "{msg}");
+ assert_eq!(limits.snapshot().get(budget).waiters, 0, "{budget:?}");
+
+ drop(held);
+ assert!(take(&limits, budget, 1).await.is_ok(), "{budget:?}");
+ }
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn no_bound_waits_past_the_default_bound() {
+ let limits = ProcessLimits::with_fetch(1, 1, 1024, 1024).with_wait_timeout(None);
+ let _held = limits.acquire_partition().await.unwrap();
+ let wait =
+ tokio::time::timeout(DEFAULT_PERMIT_WAIT_TIMEOUT * 2, limits.acquire_partition()).await;
+ assert!(wait.is_err(), "the acquisition must still wait");
+ }
+
+ #[test]
+ fn wait_bound_setting_parses_seconds_and_zero_turns_it_off() {
+ let default = Duration::from_secs(1800);
+ let cases = [
+ (None, Some(default)),
+ (Some("0"), None),
+ (Some(" 45 "), Some(Duration::from_secs(45))),
+ (Some("-1"), Some(default)),
+ (Some("ten"), Some(default)),
+ ];
+ for (value, expected) in cases {
+ assert_eq!(secs_setting("X", value, default), expected, "{value:?}");
+ }
+ }
+
+ #[tokio::test]
+ async fn snapshot_reports_free_permits_and_waiters() {
+ let limits = Arc::new(ProcessLimits::with_fetch(2, 3, 8 * 1024, 4 * 1024));
+ let full = limits.snapshot();
+ for (budget, total) in [
+ (Budget::Partitions, 2),
+ (Budget::Files, 3),
+ (Budget::Buffer, 8),
+ (Budget::Fetch, 4),
+ ] {
+ let s = full.get(budget);
+ assert_eq!(
+ (s.available, s.total, s.waiters),
+ (total, total, 0),
+ "{budget:?}"
+ );
+ }
+
+ let held = limits.acquire_buffer_kb(6).await.unwrap();
+ let waiter = tokio::spawn({
+ let limits = limits.clone();
+ async move { limits.acquire_buffer_kb(4).await.map(drop) }
+ });
+ while limits.snapshot().buffer_kb.waiters == 0 {
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(limits.snapshot().buffer_kb.waiters, 1);
+
+ drop(held);
+ waiter.await.unwrap().unwrap();
+ assert_eq!(limits.snapshot(), full);
+ }
+
#[tokio::test]
async fn zero_capacities_are_clamped_to_one() {
let limits = ProcessLimits::with_fetch(0, 0, 0, 0);
diff --git a/rust/deltalite/core/src/readahead.rs b/rust/deltalite/core/src/readahead.rs
index 346874cbdd12..6677c206f345 100644
--- a/rust/deltalite/core/src/readahead.rs
+++ b/rust/deltalite/core/src/readahead.rs
@@ -10,7 +10,8 @@
//! the one it decodes and the one being read. The caller reserves that pair from the
//! fetch budget (see [`max_row_group_pair_fetch_bytes`]). The background read is aborted
//! when the store is dropped, which happens with the reader, before the reader returns
-//! its fetch permit.
+//! its fetch permit. The wait for the background read has the deadline of
+//! [`crate::deadline`], because the reader holds its fetch permit while it waits.
use std::ops::Range;
use std::sync::{Arc, Mutex};
@@ -27,6 +28,8 @@ use object_store::{
use parquet::file::metadata::ParquetMetaData;
use tokio::task::JoinHandle;
+use crate::deadline::{io_deadline, with_deadline};
+
/// Byte span of each row group: from the first byte of its first column chunk to the
/// last byte of its last one.
fn row_group_spans(meta: &ParquetMetaData) -> Vec<Range<u64>> {
@@ -182,13 +185,21 @@ impl ObjectStore for ReadAheadStore {
let data = match self.take_pending() {
Some(mut pending) if pending.group == group => {
counter!("deltalite_row_group_read_ahead_total", "outcome" => "hit").increment(1);
- let span =
+ let read = async {
(&mut pending.read.0)
.await
.map_err(|e| object_store::Error::Generic {
store: "deltalite",
source: Box::new(e),
- })??;
+ })?
+ };
+ // After a timeout, `pending` drops at the end of this arm and aborts the read.
+ let span = with_deadline(
+ io_deadline(),
+ || format!("read ahead of row group {group} of {}", self.path),
+ read,
+ )
+ .await?;
let Range { start: base, end } = self.groups[group].clone();
if span.len() as u64 != end - base {
return Err(object_store::Error::Generic {
diff --git a/rust/deltalite/core/src/store.rs b/rust/deltalite/core/src/store.rs
index bf0626331b00..c5c58c38644a 100644
--- a/rust/deltalite/core/src/store.rs
+++ b/rust/deltalite/core/src/store.rs
@@ -45,6 +45,7 @@ use object_store::{
};
use uuid::Uuid;
+use crate::deadline::{io_deadline, with_deadline};
use crate::logprobe::{commit_path, probe_log, LogAnchor, LogProbe};
use crate::table::MultipartConfig;
@@ -81,10 +82,16 @@ const DELETE_CONCURRENCY: usize = 16;
/// Neither rewrite adds memory: the parts of an upload are slices of the payload the
/// caller already holds, and a split read streams each request into its place in one
/// buffer of the size that a single request returns.
+///
+/// Plain-overwrite uploads and ranged reads of data files run under the deadline of
+/// [`crate::deadline`]. Conditional puts (the commits) do not: a commit that fails on
+/// the client side after the store accepted it would make the caller write the batch a
+/// second time.
#[derive(Debug)]
pub struct MultipartPutStore {
inner: Arc<dyn ObjectStore>,
config: MultipartConfig,
+ deadline: Option<std::time::Duration>,
}
impl MultipartPutStore {
@@ -100,9 +107,16 @@ impl MultipartPutStore {
range_concurrency: config.range_concurrency.max(1),
..config
},
+ deadline: io_deadline(),
}
}
+ #[cfg(test)]
+ fn with_io_deadline(mut self, deadline: Option<std::time::Duration>) -> Self {
+ self.deadline = deadline;
+ self
+ }
+
/// Split `payload` into parts of at most `part_size` bytes, zero-copy (`Bytes`
/// slices share the underlying allocation).
fn split_parts(&self, payload: &PutPayload) -> Vec<PutPayload> {
@@ -135,19 +149,26 @@ impl MultipartPutStore {
/// order does not change the assembled object.
///
/// After a part fails no new part starts, and the parts already started run to the
- /// end: S3 may still store a part whose request was dropped, after the abort.
+ /// end: S3 may still store a part whose request was dropped, after the abort. The
+ /// exception is a part that passes its deadline, which is dropped so that a stuck
+ /// request cannot keep the upload open.
async fn upload_parts(
&self,
+ location: &Path,
upload: &mut Box<dyn MultipartUpload>,
parts: Vec<PutPayload>,
) -> object_store::Result<()> {
- let mut parts = parts.into_iter();
+ let mut parts = parts.into_iter().enumerate();
let mut in_flight = FuturesUnordered::new();
let mut first_error = None;
loop {
while first_error.is_none() && in_flight.len() < self.config.concurrency {
- let Some(part) = parts.next() else { break };
- in_flight.push(upload.put_part(part));
+ let Some((i, part)) = parts.next() else { break };
+ in_flight.push(with_deadline(
+ self.deadline,
+ move || format!("upload of part {} of {location}", i + 1),
+ upload.put_part(part),
+ ));
}
match in_flight.next().await {
Some(Err(e)) => {
@@ -195,8 +216,13 @@ impl MultipartPutStore {
.flat_map(move |(range, buf)| {
buf.chunks_mut(piece).enumerate().map(move |(i, dst)| {
let start = range.start + (i * piece) as u64;
- self.read_into(location, start..start + dst.len() as u64, dst)
- .boxed()
+ let end = start + dst.len() as u64;
+ with_deadline(
+ self.deadline,
+ move || format!("ranged read of {location} for bytes {start}..{end}"),
+ self.read_into(location, start..end, dst),
+ )
+ .boxed()
})
})
.collect();
@@ -207,6 +233,19 @@ impl MultipartPutStore {
Ok(bufs.into_iter().map(Bytes::from).collect())
}
+ async fn abort(
+ &self,
+ location: &Path,
+ upload: &mut Box<dyn MultipartUpload>,
+ ) -> object_store::Result<()> {
+ with_deadline(
+ self.deadline,
+ || format!("abort of the multipart upload of {location}"),
+ upload.abort(),
+ )
+ .await
+ }
+
async fn read_into(
&self,
location: &Path,
@@ -314,34 +353,47 @@ impl ObjectStore for MultipartPutStore {
// Only rewrite the plain overwrite path (data-file uploads). Conditional puts
// (PutMode::Create -- Delta log commits -- or Update) must keep their exact
// semantics, so they are forwarded untouched.
- if self.config.threshold == 0
- || !matches!(opts.mode, PutMode::Overwrite)
- || payload.content_length() < self.config.threshold
- {
+ if !matches!(opts.mode, PutMode::Overwrite) {
return self.inner.put_opts(location, payload, opts).await;
}
+ if self.config.threshold == 0 || payload.content_length() < self.config.threshold {
+ return with_deadline(
+ self.deadline,
+ || format!("upload of {location}"),
+ self.inner.put_opts(location, payload, opts),
+ )
+ .await;
+ }
let multipart_opts = PutMultipartOptions {
tags: opts.tags,
attributes: opts.attributes,
extensions: opts.extensions,
};
- let mut upload = self
- .inner
- .put_multipart_opts(location, multipart_opts)
- .await?;
+ let mut upload = with_deadline(
+ self.deadline,
+ || format!("start of the multipart upload of {location}"),
+ self.inner.put_multipart_opts(location, multipart_opts),
+ )
+ .await?;
let parts = self.split_parts(&payload);
- if let Err(e) = self.upload_parts(&mut upload, parts).await {
+ if let Err(e) = self.upload_parts(location, &mut upload, parts).await {
// Best-effort cleanup of the incomplete upload; the original error is what
// the caller needs to see. The parts in flight finished above, and without
// `complete` no object is created.
- drop(upload.abort().await);
+ drop(self.abort(location, &mut upload).await);
return Err(e);
}
- match upload.complete().await {
+ match with_deadline(
+ self.deadline,
+ || format!("completion of the multipart upload of {location}"),
+ upload.complete(),
+ )
+ .await
+ {
Ok(result) => Ok(result),
Err(e) => {
- drop(upload.abort().await);
+ drop(self.abort(location, &mut upload).await);
Err(e)
}
}
@@ -368,13 +420,23 @@ impl ObjectStore for MultipartPutStore {
location: &Path,
ranges: &[std::ops::Range<u64>],
) -> object_store::Result<Vec<Bytes>> {
- if !self.splits_reads_of(location) || ranges.iter().any(|r| r.start >= r.end) {
+ if location.parts().any(|p| p.as_ref() == "_delta_log") {
return self.inner.get_ranges(location, ranges).await;
}
+ let unsplit = || {
+ with_deadline(
+ self.deadline,
+ || format!("ranged read of {location}"),
+ self.inner.get_ranges(location, ranges),
+ )
+ };
+ if !self.splits_reads_of(location) || ranges.iter().any(|r| r.start >= r.end) {
+ return unsplit().await;
+ }
let merged = merge_ranges(ranges);
let split_above = self.config.range_part_size as u64;
if merged.iter().all(|m| m.end - m.start <= split_above) {
- return self.inner.get_ranges(location, ranges).await;
+ return unsplit().await;
}
let fetched = self.get_split(location, &merged).await?;
Ok(ranges
@@ -729,6 +791,7 @@ mod tests {
probe: Arc<Probe>,
next: usize,
fail_part: Option<usize>,
+ stuck_part: Option<usize>,
}
#[async_trait]
@@ -737,9 +800,13 @@ mod tests {
let idx = self.next;
self.next += 1;
let fail = self.fail_part == Some(idx);
+ let stuck = self.stuck_part == Some(idx);
let probe = self.probe.clone();
let put = self.inner.put_part(data);
Box::pin(async move {
+ if stuck {
+ return std::future::pending().await;
+ }
probe.enter();
yield_times(16usize.saturating_sub(idx)).await;
probe.leave();
@@ -768,6 +835,7 @@ mod tests {
inner: Arc<InMemory>,
probe: Arc<Probe>,
fail_part: Option<usize>,
+ stuck_part: Option<usize>,
}
impl ProbedStore {
@@ -776,6 +844,17 @@ mod tests {
inner: Arc::new(InMemory::new()),
probe: Arc::default(),
fail_part,
+ stuck_part: None,
+ })
+ }
+
+ /// A store whose upload part `stuck_part` (0-based) never completes.
+ fn with_stuck_part(stuck_part: usize) -> Arc<Self> {
+ Arc::new(Self {
+ inner: Arc::new(InMemory::new()),
+ probe: Arc::default(),
+ fail_part: None,
+ stuck_part: Some(stuck_part),
})
}
}
@@ -806,6 +885,7 @@ mod tests {
probe: self.probe.clone(),
next: 0,
fail_part: self.fail_part,
+ stuck_part: self.stuck_part,
}))
}
async fn get_opts(
@@ -927,6 +1007,30 @@ mod tests {
}
}
+ #[tokio::test(start_paused = true)]
+ async fn a_stuck_part_fails_the_upload_after_the_deadline() {
+ for concurrency in [1, 4] {
+ let store = ProbedStore::with_stuck_part(1);
+ let wrapped = MultipartPutStore::new(store.clone(), config(1024, 5 * MIB, concurrency))
+ .with_io_deadline(Some(std::time::Duration::from_secs(60)));
+ let path = Path::from("stuck");
+ let err = wrapped
+ .put(&path, payload_of(20 * MIB))
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("upload of part 2 of stuck"), "{err}");
+ assert!(store.probe.aborted(), "the upload must be aborted");
+ assert!(
+ matches!(
+ store.inner.head(&path).await,
+ Err(object_store::Error::NotFound { .. })
+ ),
+ "no object may exist after a stuck upload (concurrency {concurrency})"
+ );
+ }
+ }
+
#[tokio::test]
async fn conditional_puts_are_never_rewritten() {
let inner = Arc::new(InMemory::new());
diff --git a/rust/deltalite/core/src/table.rs b/rust/deltalite/core/src/table.rs
index f6371e58c094..65d2ff3cfaf8 100644
--- a/rust/deltalite/core/src/table.rs
+++ b/rust/deltalite/core/src/table.rs
@@ -1,6 +1,6 @@
//! Table opening helpers.
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use deltalake::{DeltaTable, DeltaTableBuilder};
@@ -108,8 +108,79 @@ impl MultipartConfig {
}
}
+/// HTTP client settings that deltalite gives an S3 table when the caller sets none.
+///
+/// - `timeout` bounds one request, from connect until the response body ends. The
+/// largest single requests are a whole-object read of a checkpoint (up to the
+/// checkpoint prefetch cap, 128 MiB) and a single PUT below the multipart threshold
+/// (up to 64 MiB when parts upload one after another). `object_store` uses 30s, which
+/// cuts such a request below about 4 MB/s on one connection. 120s keeps those
+/// requests above about 1 MB/s, and a timed out request is retried.
+/// - `connect_timeout` is the `object_store` value, set here so that a change of that
+/// default does not change deltalite.
+/// - `pool_idle_timeout` stays below the 20s after which S3 closes an idle connection,
+/// the same value delta-rs gives its own S3 client. A connection that a proxy or a
+/// NAT drops without a reset makes the next request on it wait for the full request
+/// timeout; a short idle time makes that less frequent.
+///
+/// An explicit storage option wins, under any spelling that `object_store` accepts
+/// (`timeout`, `aws_timeout`, any case), and so does the environment variable that
+/// `object_store` reads for the key (for example `AWS_TIMEOUT`).
+pub const S3_CLIENT_DEFAULTS: [(&str, &str); 3] = [
+ ("timeout", "120s"),
+ ("connect_timeout", "5s"),
+ ("pool_idle_timeout", "15s"),
+];
+
+/// The `object_store` client key that the option or environment variable `name` sets,
+/// for the keys in [`S3_CLIENT_DEFAULTS`]. This is the parsing of
+/// `AmazonS3ConfigKey::from_str` for those keys.
+fn client_key(name: &str) -> Option<&'static str> {
+ let lower = name.to_ascii_lowercase();
+ let key = lower.strip_prefix("aws_").unwrap_or(&lower);
+ S3_CLIENT_DEFAULTS
+ .iter()
+ .map(|(k, _)| *k)
+ .find(|k| *k == key)
+}
+
+/// `storage_options` with each key of [`S3_CLIENT_DEFAULTS`] that neither the options
+/// nor the environment (`env_names`) set. Only S3 tables get the defaults; other stores
+/// do not read these keys.
+fn with_s3_client_defaults<'a>(
+ scheme: &str,
+ mut storage_options: HashMap<String, String>,
+ env_names: impl IntoIterator<Item = &'a str>,
+) -> HashMap<String, String> {
+ if !matches!(scheme, "s3" | "s3a") {
+ return storage_options;
+ }
+ let set: HashSet<&'static str> = storage_options
+ .keys()
+ .filter_map(|k| client_key(k))
+ .chain(env_names.into_iter().filter_map(client_key))
+ .collect();
+ for (key, value) in S3_CLIENT_DEFAULTS {
+ if !set.contains(key) {
+ storage_options.insert(key.to_string(), value.to_string());
+ }
+ }
+ storage_options
+}
+
+fn storage_options_for(
+ scheme: &str,
+ storage_options: HashMap<String, String>,
+) -> HashMap<String, String> {
+ let env: Vec<String> = std::env::vars_os()
+ .filter_map(|(k, _)| k.into_string().ok())
+ .collect();
+ with_s3_client_defaults(scheme, storage_options, env.iter().map(String::as_str))
+}
+
/// Open and load a Delta table from `uri` with the given delta-rs `storage_options`
-/// (the same keys the Python package accepts; the dict passes through unchanged).
+/// (the same keys the Python package accepts; the dict passes through unchanged, plus
+/// [`S3_CLIENT_DEFAULTS`] for the keys it does not set).
///
/// The load reads its checkpoint through the prefetch wrapper (see [`crate::prefetch`])
/// and releases the prefetched bytes before returning, so the table costs no more memory
@@ -129,6 +200,7 @@ pub async fn table_exists(uri: &str, storage_options: HashMap<String, String>) -
let Ok(url) = deltalake::ensure_table_uri(uri) else {
return Ok(false);
};
+ let storage_options = storage_options_for(url.scheme(), storage_options);
let Ok(builder) = DeltaTableBuilder::from_url(url) else {
return Ok(false);
};
@@ -145,6 +217,7 @@ pub async fn open_table_prefetched(
) -> Result<(DeltaTable, Arc<CheckpointCache>)> {
let url = deltalake::ensure_table_uri(uri)
.map_err(|e| Error::NotFound(format!("invalid table uri {uri}: {e}")))?;
+ let storage_options = storage_options_for(url.scheme(), storage_options);
let built = DeltaTableBuilder::from_url(url)?
.with_storage_options(storage_options)
.build()?;
@@ -217,6 +290,60 @@ mod tests {
SERIAL_MULTIPART_THRESHOLD,
};
+ #[test]
+ fn s3_tables_get_client_defaults_for_keys_the_caller_does_not_set() {
+ let defaults: HashMap<String, String> = S3_CLIENT_DEFAULTS
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect();
+ let opts = |pairs: &[(&str, &str)]| -> HashMap<String, String> {
+ pairs
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect()
+ };
+ let with = |overrides: &[(&str, &str)]| {
+ let mut m = defaults.clone();
+ m.extend(opts(overrides));
+ m
+ };
+ type Options = HashMap<String, String>;
+ let cases: Vec<(&str, Options, Vec<&str>, Options)> = vec![
+ ("s3", opts(&[]), vec![], defaults.clone()),
+ ("s3a", opts(&[]), vec![], defaults.clone()),
+ (
+ "s3",
+ opts(&[("region", "us-east-1")]),
+ vec![],
+ with(&[("region", "us-east-1")]),
+ ),
+ // An explicit value under any spelling object_store accepts wins.
+ (
+ "s3",
+ opts(&[("timeout", "300s")]),
+ vec![],
+ with(&[("timeout", "300s")]),
+ ),
+ ("s3", opts(&[("AWS_CONNECT_TIMEOUT", "1s")]), vec![], {
+ let mut m = with(&[("AWS_CONNECT_TIMEOUT", "1s")]);
+ m.remove("connect_timeout");
+ m
+ }),
+ // The environment variable object_store reads for a key also wins.
+ ("s3", opts(&[]), vec!["AWS_POOL_IDLE_TIMEOUT", "HOME"], {
+ let mut m = defaults.clone();
+ m.remove("pool_idle_timeout");
+ m
+ }),
+ ("file", opts(&[]), vec![], opts(&[])),
+ ("memory", opts(&[("a", "b")]), vec![], opts(&[("a", "b")])),
+ ];
+ for (scheme, given, env, expected) in cases {
+ let got = with_s3_client_defaults(scheme, given.clone(), env);
+ assert_eq!(got, expected, "{scheme} {given:?}");
+ }
+ }
+
#[test]
fn concurrency_one_restores_the_serial_upload_defaults() {
let no_env = |_: &str, default: usize| default;
diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rs
index 1c3b5d163296..1f6c7ffd9009 100644
--- a/rust/deltalite/core/src/upsert.rs
+++ b/rust/deltalite/core/src/upsert.rs
@@ -26,6 +26,7 @@
//! `target_file_size` per partition worker (see `crate::writer`).
use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Instant;
@@ -56,7 +57,7 @@ use tracing::{debug, info, instrument, warn};
use crate::errors::{Error, Result};
use crate::limits::{
- check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, ProcessLimits,
+ check_source_size, estimate_pk_set_bytes, resolve_max_source_bytes, Budget, ProcessLimits,
SourceFootprint,
};
use crate::pkset::PkSet;
@@ -610,17 +611,23 @@ impl Budgets {
let want = Self::kb(bytes);
let local_kb = want.min(self.fetch_local_cap_kb as u64) as u32;
let global_kb = want.min(self.limits.fetch_cap_kb() as u64) as u32;
- let local = self
- .fetch_local
- .clone()
- .acquire_many_owned(local_kb)
+ // The per-call wait shares the process-wide bound, so a stuck holder of either
+ // budget ends in an error that names the fetch budget.
+ self.limits
+ .bounded(Budget::Fetch, global_kb, async {
+ let local = self
+ .fetch_local
+ .clone()
+ .acquire_many_owned(local_kb)
+ .await
+ .map_err(|_| Error::Generic("fetch-budget semaphore closed".into()))?;
+ let global = self.limits.acquire_fetch_kb(global_kb).await?;
+ Ok(FetchPermit {
+ _local: local,
+ _global: global,
+ })
+ })
.await
- .map_err(|_| Error::Generic("fetch-budget semaphore closed".into()))?;
- let global = self.limits.acquire_fetch_kb(global_kb).await?;
- Ok(FetchPermit {
- _local: local,
- _global: global,
- })
}
/// Bytes of compressed row-group data one reader can hold without passing either
@@ -636,13 +643,7 @@ impl Budgets {
let want = Self::kb(bytes);
let local_kb = want.min(self.local_cap_kb as u64) as u32;
let global_kb = want.min(self.limits.buffer_cap_kb() as u64) as u32;
- let local = self
- .local
- .clone()
- .acquire_many_owned(local_kb)
- .await
- .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;
- let global = self.limits.acquire_buffer_kb(global_kb).await?;
+ let (local, global) = self.acquire_byte_pair(local_kb, global_kb).await?;
Ok(BudgetPermit {
local: Some(local),
global: Some(global),
@@ -651,6 +652,28 @@ impl Budgets {
})
}
+ /// Local before global, the fixed order used everywhere (see `crate::limits`). The
+ /// per-call wait shares the process-wide bound, so a stuck holder of either budget
+ /// ends in an error that names the buffer budget.
+ async fn acquire_byte_pair(
+ &self,
+ local_kb: u32,
+ global_kb: u32,
+ ) -> Result<(OwnedSemaphorePermit, OwnedSemaphorePermit)> {
+ self.limits
+ .bounded(Budget::Buffer, global_kb, async {
+ let local = self
+ .local
+ .clone()
+ .acquire_many_owned(local_kb)
+ .await
+ .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;
+ let global = self.limits.acquire_buffer_kb(global_kb).await?;
+ Ok((local, global))
+ })
+ .await
+ }
+
/// Grow `permit` so it covers `total_bytes` (used after a decode turns out larger
/// than the pre-decode estimate). Total held stays capped at each budget's capacity,
/// preserving the oversized-batch progress guarantee; a decode smaller than the
@@ -678,14 +701,7 @@ impl Budgets {
permit.global = None;
permit.held_local_kb = 0;
permit.held_global_kb = 0;
- // Local before global, the fixed order used everywhere (see `crate::limits`).
- let local = self
- .local
- .clone()
- .acquire_many_owned(want_local)
- .await
- .map_err(|_| Error::Generic("byte-budget semaphore closed".into()))?;
- let global = self.limits.acquire_buffer_kb(want_global).await?;
+ let (local, global) = self.acquire_byte_pair(want_local, want_global).await?;
permit.local = Some(local);
permit.global = Some(global);
permit.held_local_kb = want_local;
@@ -2553,10 +2569,18 @@ async fn rewrite_partition(
// Reader tasks: up to max_parallel_files files stream concurrently (per call AND
// per process). Each task takes its concurrency permits *inside* the task so
// spawning never blocks this function.
+ //
+ // The wait for the per-call file permit has no bound, because files of a large
+ // partition queue there for as long as the rewrite takes. After one reader fails,
+ // the readers that get a per-call permit later stop instead of reading. Without
+ // that, every queued reader would wait the full process-wide bound one after
+ // another when a stuck call holds the process-wide file budget.
let file_sem = Arc::new(Semaphore::new(opts.max_parallel_files.max(1)));
+ let reader_failed = Arc::new(AtomicBool::new(false));
let mut readers = Vec::with_capacity(rewrite.len());
for f in rewrite {
let sem = file_sem.clone();
+ let failed = reader_failed.clone();
let store = store.clone();
let pkset = pkset.clone();
let schema = table_schema.clone();
@@ -2571,8 +2595,18 @@ async fn rewrite_partition(
.acquire_owned()
.await
.map_err(|_| Error::Generic("file semaphore closed".into()))?;
- let _global = opts.limits.acquire_file().await?;
- filter_file(store, f, pkset, schema, pcol, value, opts, budgets, tx).await
+ if failed.load(Ordering::Relaxed) {
+ return Ok(None);
+ }
+ let outcome = async {
+ let _global = opts.limits.acquire_file().await?;
+ filter_file(store, f, pkset, schema, pcol, value, opts, budgets, tx).await
+ }
+ .await;
+ if outcome.is_err() {
+ failed.store(true, Ordering::Relaxed);
+ }
+ outcome.map(Some)
}));
}
// Drop the parent's sender so the writer task ends when the readers do.
@@ -2587,11 +2621,13 @@ async fn rewrite_partition(
.await
.map_err(|e| Error::Generic(format!("file worker panicked: {e}")))?
{
- Ok(o) => {
+ Ok(Some(o)) => {
rows_updated += o.rows_updated;
rows_copied += o.rows_copied;
removes.push(o.remove);
}
+ // Stopped because another reader failed; that reader's error is kept below.
+ Ok(None) => {}
// Keep the first reader error, but keep draining so the writer terminates.
Err(e) => reader_err = reader_err.or(Some(e)),
}
diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.toml
index afc175eba7d2..7eaadb241338 100644
--- a/rust/deltalite/python/Cargo.toml
+++ b/rust/deltalite/python/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "deltalite-python"
# Mirrors pyproject.toml in this directory (always identical); bump both together.
-version = "0.1.15"
+version = "0.1.16"
edition = "2021"
# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to
# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).
diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md
index bfa1a7da51c5..bcca01c9761a 100644
--- a/rust/deltalite/python/README.md
+++ b/rust/deltalite/python/README.md
@@ -184,6 +184,7 @@ All inherit from `DeltaLiteError`, so you can catch the base or branch on kind:
| `DeltaLiteTableNotFoundError` | No Delta table at the URI. |
| `DeltaLiteUnsupportedTableError` | Table uses a feature deltalite can't handle (e.g. deletion vectors, column mapping). |
| `DeltaLiteSourceTooLargeError` | Batch exceeds `max_source_bytes` (see below). |
+| `DeltaLiteTimeoutError` | A wait for a process-wide budget passed its bound (see "Timeouts" below). |
## Operational knobs
@@ -224,6 +225,29 @@ budgets:
`DELTALITE_RANGE_GET_PART_SIZE_BYTES`, `DELTALITE_RANGE_GET_CONCURRENCY`.
The last five are the defaults of the per-call arguments of the same names.
+**Timeouts.** All upserts and compactions of one process share the
+process-wide budgets above, and their waits are fair. So one call that holds
+permits and stops making progress can make every other call wait. Three bounds
+stop that:
+
+| Bound | Default | Override | When it passes |
+|---|---|---|---|
+| S3 request | `timeout` 120s, `connect_timeout` 5s, `pool_idle_timeout` 15s | the storage option of the same name (any spelling `object_store` accepts, for example `aws_timeout`), or its environment variable (`AWS_TIMEOUT`) | `object_store` retries the request, then the call fails. |
+| One object-store call | 10 minutes | `DELTALITE_IO_DEADLINE_SECS` (`0` removes it) | An upload of a data file or a part, or a ranged read of a data file, fails with a `DeltaLiteError` that names the call. Conditional puts (commits) have no deadline. |
+| One wait for a process-wide budget | 30 minutes | `DELTALITE_PERMIT_WAIT_TIMEOUT_SECS` (`0` removes it) | The call fails with `DeltaLiteTimeoutError`, which names the budget and gives its free permits and waiters. |
+
+Only S3 tables get the request defaults. `object_store` uses a 30s request
+timeout, which cuts a 128 MiB checkpoint read or a 64 MiB single PUT on a
+connection slower than about 4 MB/s.
+
+`deltalite.limits_snapshot()` returns the free and total permits of each
+process-wide budget and the number of acquisitions that wait for it:
+`{"partitions": {"available": 8, "total": 8, "waiters": 0}, "files": ...,
+"buffer_kb": ..., "fetch_kb": ...}`. It does no I/O and takes no lock, so a
+caller can read it while another call of the process hangs. A waiter that needs
+more permits than are free takes the free ones and waits for the rest, so
+`available` can be 0 while a part of the budget belongs to a waiter.
+
**Probe cache.** The probe reads a footer and the PK column of every candidate
file, so on object storage its cost grows with the number of files in the
touched partitions. A Delta data file never changes, so deltalite keeps the
diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.toml
index 80ade625212c..34861e5d9984 100644
--- a/rust/deltalite/python/pyproject.toml
+++ b/rust/deltalite/python/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "maturin"
[project]
name = "deltalite"
# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.
-version = "0.1.15"
+version = "0.1.16"
description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"
readme = "README.md"
requires-python = ">=3.12"
diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs
index 8d5199be87f3..f46792891f55 100644
--- a/rust/deltalite/python/src/lib.rs
+++ b/rust/deltalite/python/src/lib.rs
@@ -14,7 +14,7 @@ use arrow_schema::SchemaRef;
use deltalake::writer::RecordBatchWriter;
use deltalite_core::errors::Error;
use deltalite_core::handle::TableHandle;
-use deltalite_core::limits::ProcessLimits;
+use deltalite_core::limits::{Budget, ProcessLimits};
use deltalite_core::probecache::ProbeCache;
use deltalite_core::schema::import_column;
use deltalite_core::table::{open_table, table_exists, MultipartConfig};
@@ -31,6 +31,7 @@ create_exception!(deltalite, DeltaLiteSchemaMismatchError, DeltaLiteError);
create_exception!(deltalite, DeltaLiteTableNotFoundError, DeltaLiteError);
create_exception!(deltalite, DeltaLiteUnsupportedTableError, DeltaLiteError);
create_exception!(deltalite, DeltaLiteSourceTooLargeError, DeltaLiteError);
+create_exception!(deltalite, DeltaLiteTimeoutError, DeltaLiteError);
/// Map a core error onto the matching Python exception class.
fn to_py_err(e: Error) -> PyErr {
@@ -41,6 +42,7 @@ fn to_py_err(e: Error) -> PyErr {
Error::Unsupported(_) => DeltaLiteUnsupportedTableError::new_err(msg),
Error::Conflict(_) => DeltaLiteCommitConflictError::new_err(msg),
Error::SourceTooLarge(_) => DeltaLiteSourceTooLargeError::new_err(msg),
+ Error::Timeout(_) => DeltaLiteTimeoutError::new_err(msg),
Error::Generic(_) => DeltaLiteError::new_err(msg),
}
}
@@ -737,12 +739,38 @@ fn probe_cache_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
Ok(d.into_any().unbind())
}
+/// Free and total permits of each process-wide budget that every upsert and compaction
+/// of this process shares, and the acquisitions that wait for each:
+/// `{"partitions": {"available": .., "total": .., "waiters": ..}, "files": {..},
+/// "buffer_kb": {..}, "fetch_kb": {..}}`. The byte budgets count KiB. It takes no
+/// lock and does no I/O, so a caller can read it while a call of this process hangs.
+#[pyfunction]
+fn limits_snapshot(py: Python<'_>) -> PyResult<Py<PyAny>> {
+ let s = ProcessLimits::global().snapshot();
+ let d = PyDict::new(py);
+ for budget in [
+ Budget::Partitions,
+ Budget::Files,
+ Budget::Buffer,
+ Budget::Fetch,
+ ] {
+ let b = s.get(budget);
+ let entry = PyDict::new(py);
+ entry.set_item("available", b.available)?;
+ entry.set_item("total", b.total)?;
+ entry.set_item("waiters", b.waiters)?;
+ d.set_item(budget.name(), entry)?;
+ }
+ Ok(d.into_any().unbind())
+}
+
#[pymodule]
fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<DeltaLiteTable>()?;
m.add_function(wrap_pyfunction!(set_probe_cache_max_bytes, m)?)?;
m.add_function(wrap_pyfunction!(clear_probe_cache, m)?)?;
m.add_function(wrap_pyfunction!(probe_cache_stats, m)?)?;
+ m.add_function(wrap_pyfunction!(limits_snapshot, m)?)?;
m.add_class::<UpsertStats>()?;
m.add("DeltaLiteError", m.py().get_type::<DeltaLiteError>())?;
m.add(
@@ -765,5 +793,9 @@ fn deltalite(m: &Bound<'_, PyModule>) -> PyResult<()> {
"DeltaLiteSourceTooLargeError",
m.py().get_type::<DeltaLiteSourceTooLargeError>(),
)?;
+ m.add(
+ "DeltaLiteTimeoutError",
+ m.py().get_type::<DeltaLiteTimeoutError>(),
+ )?;
Ok(())
}
diff --git a/rust/deltalite/python/tests/test_guards.py b/rust/deltalite/python/tests/test_guards.py
index 6472d9591ef3..2d0c89bd43ac 100644
--- a/rust/deltalite/python/tests/test_guards.py
+++ b/rust/deltalite/python/tests/test_guards.py
@@ -55,10 +55,11 @@ def test_source_guard_zero_disables_the_ceiling(tmp_path):
assert stats.rows_inserted == 1
-def test_source_guard_error_is_a_deltalite_error_subclass():
[email protected]("name", ["DeltaLiteSourceTooLargeError", "DeltaLiteTimeoutError"])
+def test_guard_errors_are_deltalite_error_subclasses(name):
import deltalite
- assert issubclass(deltalite.DeltaLiteSourceTooLargeError, deltalite.DeltaLiteError)
+ assert issubclass(getattr(deltalite, name), deltalite.DeltaLiteError)
def test_multipart_forced_upsert_has_merge_parity(tmp_path):
@@ -83,6 +84,15 @@ def test_multipart_forced_upsert_has_merge_parity(tmp_path):
uri_m, uri_u = run_scenario_with_multipart(scenario, tmp_path)
assert_parity(uri_m, uri_u, "multipart_forced")
+ import deltalite
+
+ snapshot = deltalite.limits_snapshot()
+ assert set(snapshot) == {"partitions", "files", "buffer_kb", "fetch_kb"}
+ for name, budget in snapshot.items():
+ assert budget["total"] > 0, name
+ assert budget["available"] == budget["total"], f"an upsert kept {name} permits"
+ assert budget["waiters"] == 0, name
+
def run_scenario_with_multipart(scenario, tmp_path):
from harness.common import dedupe_keep_last, merge_path