Apache DataFusion SQL Query Engine
feat: log drift between MemoryPool reservations and allocated bytes in SLT (#25651)
Andy Grove1d9be2e10794
diff --git a/datafusion/execution/src/memory_pool/drift.rs b/datafusion/execution/src/memory_pool/drift.rsnew file mode 100644index 0000000000..ded3f7343f--- /dev/null+++ b/datafusion/execution/src/memory_pool/drift.rs@@ -0,0 +1,392 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Logs the drift between what [`MemoryPool`]s have reserved and what the+//! process has actually allocated.+//!+//! [`MemoryPool`] accounting is voluntary: only allocations that an operator+//! explicitly reserves are counted. Anything else (in-flight batches, kernel+//! scratch space, untracked buffers) is invisible to the pool, so a process+//! can run out of memory while its pool reports plenty of headroom. See+//! <https://github.com/apache/datafusion/issues/25650>.+//!+//! This module only observes. Nothing here changes how memory is granted or+//! limited; it logs the gap so that operators which under-report can be found.+//!+//! DataFusion is a library and does not choose the global allocator, so the+//! allocated byte count is supplied by the caller, e.g. from a counting+//! [`GlobalAlloc`](std::alloc::GlobalAlloc) wrapper or allocator statistics.+//!+//! [`MemoryPool`]: super::MemoryPool++use std::{+ fmt::{Debug, Display, Formatter},+ sync::{+ Arc,+ atomic::{AtomicIsize, AtomicUsize, Ordering},+ },+};++use datafusion_common::human_readable_size;+use parking_lot::Mutex;++/// Returns the number of bytes currently allocated by the process.+///+/// This is called on every reservation change (`grow`, `try_grow` and+/// `shrink`) of every pool that reports to the tracker, so it must be cheap,+/// e.g. a single atomic load. Do not read allocator statistics that need a+/// refresh on each call (such as jemalloc's `epoch`).+pub type AllocatedBytesFn = Arc<dyn Fn() -> usize + Send + Sync>;++/// Default rise in drift, in bytes, needed before another line is logged.+pub const DEFAULT_DRIFT_LOG_THRESHOLD: usize = 64 * 1024 * 1024;++/// Compares allocated bytes against the total reserved by every pool that+/// reports to it (see [`PeakRecordingPool::with_drift_tracker`]).+///+/// [`PeakRecordingPool::with_drift_tracker`]: super::PeakRecordingPool::with_drift_tracker+///+/// A single tracker can be shared by many pools, e.g. one pool per+/// `SessionContext` in a process running several at once. The reserved total+/// is then summed across all of them, which is what has to be compared with a+/// process-wide allocated byte count.+///+/// Drift is `allocated - reserved`. A line is logged at `info` level each time+/// drift rises by at least the log threshold, naming the pool and consumer+/// whose reservation change triggered the check.+///+/// # Example+///+/// ```+/// # use std::sync::Arc;+/// # use datafusion_execution::memory_pool::{+/// # MemoryConsumer, MemoryDriftTracker, MemoryPool, PeakRecordingPool, UnboundedMemoryPool,+/// # };+/// // A real caller would read a counting allocator or allocator stats here.+/// let tracker = Arc::new(MemoryDriftTracker::new(Arc::new(|| 10_000)));+/// let pool: Arc<dyn MemoryPool> = Arc::new(+/// PeakRecordingPool::new(Arc::new(UnboundedMemoryPool::default()))+/// .with_drift_tracker(Arc::clone(&tracker), "example"),+/// );+///+/// let reservation = MemoryConsumer::new("op").register(&pool);+/// reservation.grow(4_000);+///+/// assert_eq!(tracker.reserved(), 4_000);+/// assert_eq!(tracker.peak_drift().unwrap().drift, 6_000);+/// ```+pub struct MemoryDriftTracker {+ allocated: AllocatedBytesFn,+ log_threshold: usize,+ /// Total reserved across every pool reporting to this tracker.+ reserved: AtomicUsize,+ /// Positive drift at the time of the last logged line.+ last_logged: AtomicIsize,+ /// Largest drift seen.+ peak_drift: AtomicIsize,+ /// Where the largest drift was seen.+ peak: Mutex<Option<DriftSample>>,+}++/// One observation of drift, recorded by [`MemoryDriftTracker`].+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct DriftSample {+ /// Label of the pool whose reservation change took this sample, or the source passed to [`MemoryDriftTracker::sample`]. When+ /// several pools share a tracker, the untracked memory can come from any+ /// of them.+ pub pool: String,+ /// Consumer whose reservation change took this sample (empty for+ /// [`MemoryDriftTracker::sample`]). This shows when drift was sampled,+ /// not what caused it.+ pub consumer: String,+ /// Bytes reserved across all pools reporting to the tracker.+ pub reserved: usize,+ /// Bytes allocated, as reported by the tracker's [`AllocatedBytesFn`].+ pub allocated: usize,+ /// `allocated - reserved`.+ pub drift: isize,+}++impl Display for DriftSample {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ let drift = if self.drift < 0 {+ format!("-{}", human_readable_size(self.drift.unsigned_abs()))+ } else {+ human_readable_size(self.drift as usize)+ };+ write!(+ f,+ "drift={drift} allocated={} reserved={} pool={} consumer={}",+ human_readable_size(self.allocated),+ human_readable_size(self.reserved),+ self.pool,+ self.consumer,+ )+ }+}++impl MemoryDriftTracker {+ /// Create a tracker that reads allocated bytes from `allocated`, logging+ /// every [`DEFAULT_DRIFT_LOG_THRESHOLD`] of drift.+ pub fn new(allocated: AllocatedBytesFn) -> Self {+ Self {+ allocated,+ log_threshold: DEFAULT_DRIFT_LOG_THRESHOLD,+ reserved: AtomicUsize::new(0),+ last_logged: AtomicIsize::new(0),+ peak_drift: AtomicIsize::new(isize::MIN),+ peak: Mutex::new(None),+ }+ }++ /// Log a line each time drift rises by `log_threshold` bytes.+ pub fn with_log_threshold(mut self, log_threshold: usize) -> Self {+ self.log_threshold = log_threshold;+ self+ }++ /// Bytes currently reserved across all pools reporting to this tracker.+ pub fn reserved(&self) -> usize {+ self.reserved.load(Ordering::Relaxed)+ }++ /// The largest drift seen so far, if any reservation has been made.+ pub fn peak_drift(&self) -> Option<DriftSample> {+ self.peak.lock().clone()+ }++ /// Compare the current allocated bytes with the reserved total now,+ /// without a reservation change.+ ///+ /// Drift is otherwise only sampled when a reservation changes, so memory+ /// allocated by code that reserves little can go unseen until some other+ /// reservation changes. Calling this periodically, e.g. from a timer,+ /// closes that gap. `source` is recorded as the pool label.+ pub fn sample(&self, source: &str) {+ self.observe(source, "", self.reserved());+ }++ pub(super) fn grew(&self, pool: &str, consumer: &str, additional: usize) {+ let reserved =+ self.reserved.fetch_add(additional, Ordering::Relaxed) + additional;+ self.observe(pool, consumer, reserved);+ }++ pub(super) fn shrank(&self, pool: &str, consumer: &str, shrink: usize) {+ let reserved = self.reserved.fetch_sub(shrink, Ordering::Relaxed) - shrink;+ self.observe(pool, consumer, reserved);+ }++ /// Returns `true` if a line was logged.+ fn observe(&self, pool: &str, consumer: &str, reserved: usize) -> bool {+ let allocated = (self.allocated)();+ let drift = allocated as isize - reserved as isize;++ let sample = || DriftSample {+ pool: pool.to_string(),+ consumer: consumer.to_string(),+ reserved,+ allocated,+ drift,+ };++ // Lock-free check first so the lock is only taken for a new peak.+ if self.peak_drift.fetch_max(drift, Ordering::Relaxed) < drift {+ let mut peak = self.peak.lock();+ if peak.as_ref().is_none_or(|p| drift > p.drift) {+ *peak = Some(sample());+ }+ }++ // Only rising positive drift is logged: that is untracked memory, which+ // is what leads to OOM kills. Negative drift (e.g. an operator+ // reserving ahead of allocating) counts as zero. Falling drift quietly+ // lowers the baseline so the next rise is seen.+ let untracked = drift.max(0);+ let last = self.last_logged.load(Ordering::Relaxed);+ let rose = untracked >= last.saturating_add_unsigned(self.log_threshold);+ if !rose && untracked >= last {+ return false;+ }+ let updated = self+ .last_logged+ .compare_exchange(last, untracked, Ordering::Relaxed, Ordering::Relaxed)+ .is_ok();+ let logged = updated && rose;+ if logged {+ log::info!("memory drift: {}", sample());+ }+ logged+ }+}++impl Debug for MemoryDriftTracker {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("MemoryDriftTracker")+ .field("log_threshold", &self.log_threshold)+ .field("reserved", &self.reserved())+ .field("peak", &self.peak_drift())+ .finish()+ }+}++#[cfg(test)]+mod tests {+ use crate::memory_pool::{+ MemoryConsumer, MemoryPool, PeakRecordingPool, UnboundedMemoryPool,+ };++ use super::*;++ /// A tracker whose allocated byte count is set by the test.+ fn tracker(log_threshold: usize) -> (Arc<AtomicUsize>, Arc<MemoryDriftTracker>) {+ let allocated = Arc::new(AtomicUsize::new(0));+ let source = Arc::clone(&allocated);+ let tracker =+ MemoryDriftTracker::new(Arc::new(move || source.load(Ordering::Relaxed)))+ .with_log_threshold(log_threshold);+ (allocated, Arc::new(tracker))+ }++ fn pool(tracker: &Arc<MemoryDriftTracker>, label: &str) -> Arc<dyn MemoryPool> {+ Arc::new(+ PeakRecordingPool::new(Arc::new(UnboundedMemoryPool::default()))+ .with_drift_tracker(Arc::clone(tracker), label),+ )+ }++ #[test]+ fn records_peak_drift_and_where_it_happened() {+ let (allocated, tracker) = tracker(DEFAULT_DRIFT_LOG_THRESHOLD);+ let pool = pool(&tracker, "q1");++ let a = MemoryConsumer::new("a").register(&pool);+ let b = MemoryConsumer::new("b").register(&pool);++ allocated.store(1000, Ordering::Relaxed);+ a.grow(800);+ allocated.store(5000, Ordering::Relaxed);+ b.grow(200);+ allocated.store(1500, Ordering::Relaxed);+ a.shrink(800);++ assert_eq!(tracker.reserved(), 200);+ assert_eq!(+ tracker.peak_drift(),+ Some(DriftSample {+ pool: "q1".to_string(),+ consumer: "b".to_string(),+ reserved: 1000,+ allocated: 5000,+ drift: 4000,+ })+ );+ }++ #[test]+ fn sums_reservations_across_pools_sharing_a_tracker() {+ let (allocated, tracker) = tracker(DEFAULT_DRIFT_LOG_THRESHOLD);+ let one = pool(&tracker, "one");+ let two = pool(&tracker, "two");++ allocated.store(1000, Ordering::Relaxed);+ let a = MemoryConsumer::new("a").register(&one);+ let b = MemoryConsumer::new("b").register(&two);+ a.grow(300);+ b.grow(400);++ // Drift is measured against the sum, so the second pool's growth+ // shrinks it rather than starting from zero.+ assert_eq!(tracker.reserved(), 700);+ let peak = tracker.peak_drift().unwrap();+ assert_eq!(peak.pool, "one");+ assert_eq!(peak.drift, 700);++ drop(a);+ drop(b);+ assert_eq!(tracker.reserved(), 0);+ }++ #[test]+ fn drift_can_be_negative() {+ let (_allocated, tracker) = tracker(DEFAULT_DRIFT_LOG_THRESHOLD);+ let pool = pool(&tracker, "over-reserved");++ let reservation = MemoryConsumer::new("a").register(&pool);+ reservation.grow(2048);++ let peak = tracker.peak_drift().unwrap();+ assert_eq!(peak.drift, -2048);+ assert_eq!(+ peak.to_string(),+ "drift=-2.0 KB allocated=0.0 B reserved=2.0 KB pool=over-reserved consumer=a"+ );+ }++ #[test]+ fn logs_each_rise_by_the_threshold_and_rearms_after_a_fall() {+ let (allocated, tracker) = tracker(1000);+ let observe = |bytes: usize, reserved: usize| {+ allocated.store(bytes, Ordering::Relaxed);+ tracker.observe("p", "c", reserved)+ };++ // Negative drift counts as zero, so the baseline stays at zero.+ assert!(!observe(0, 5000));+ // Below the threshold: nothing logged.+ assert!(!observe(999, 0));+ // Reaching the threshold logs and moves the baseline to 1000.+ assert!(observe(1000, 0));+ assert!(!observe(1999, 0));+ assert!(observe(2000, 0));+ // Falling drift is not logged but lowers the baseline to 500...+ assert!(!observe(500, 0));+ // ...so a rise of the threshold from there logs again.+ assert!(!observe(1499, 0));+ assert!(observe(1500, 0));+ }++ #[test]+ fn a_threshold_above_isize_max_never_logs() {+ let (allocated, tracker) = tracker(usize::MAX);++ allocated.store(1 << 40, Ordering::Relaxed);+ assert!(!tracker.observe("p", "c", 0));+ }++ #[test]+ fn sample_observes_without_a_reservation_change() {+ let (allocated, tracker) = tracker(DEFAULT_DRIFT_LOG_THRESHOLD);+ let pool = pool(&tracker, "q1");+ let reservation = MemoryConsumer::new("a").register(&pool);+ reservation.grow(100);++ allocated.store(5000, Ordering::Relaxed);+ tracker.sample("timer");++ assert_eq!(+ tracker.peak_drift(),+ Some(DriftSample {+ pool: "timer".to_string(),+ consumer: String::new(),+ reserved: 100,+ allocated: 5000,+ drift: 4900,+ })+ );+ }+}diff --git a/datafusion/execution/src/memory_pool/mod.rs b/datafusion/execution/src/memory_pool/mod.rsindex 5df64ba47b..09cef6ddd4 100644--- a/datafusion/execution/src/memory_pool/mod.rs+++ b/datafusion/execution/src/memory_pool/mod.rs@@ -24,6 +24,7 @@ use std::fmt::Display;use std::hash::{Hash, Hasher};use std::{cmp::Ordering, sync::Arc, sync::atomic};+mod drift;mod merge_memory_pool;mod peak_recording;mod pool;@@ -38,6 +39,7 @@ pub mod proxy {pub use datafusion_common::{human_readable_count, human_readable_duration, human_readable_size, units,};+pub use drift::*;pub use merge_memory_pool::{MergeMemoryPool, WorkspaceLoan};pub use peak_recording::*;pub use pool::*;diff --git a/datafusion/execution/src/memory_pool/peak_recording.rs b/datafusion/execution/src/memory_pool/peak_recording.rsindex b407cc0eaf..3c916b5ac7 100644--- a/datafusion/execution/src/memory_pool/peak_recording.rs+++ b/datafusion/execution/src/memory_pool/peak_recording.rs@@ -33,6 +33,10 @@//! This is measurement only: nothing here enforces a relationship between the//! two numbers.//!+//! The same wrapper can also report every reservation change to a+//! [`MemoryDriftTracker`] (see [`PeakRecordingPool::with_drift_tracker`]),+//! which compares reservations with the bytes actually allocated.+//!//! What lands in the peak is whatever the pool accounts for, so this follows//! the accounting rather than fixing it in place. Arrow-side reservations made//! through `ArrowMemoryPool` are included, because that adapter grows a@@ -47,7 +51,9 @@ use std::{},};-use super::{MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation};+use super::{+ MemoryConsumer, MemoryDriftTracker, MemoryLimit, MemoryPool, MemoryReservation,+};use datafusion_common::Result;/// Wraps a [`MemoryPool`], recording the high-water mark of@@ -92,6 +98,8 @@ pub struct PeakRecordingPool {peak: AtomicUsize,/// High-water mark since this pool was created. Never reset.max: AtomicUsize,+ /// Tracker every reservation change is reported to, and this pool's label.+ drift: Option<(Arc<MemoryDriftTracker>, String)>,}impl PeakRecordingPool {@@ -105,9 +113,22 @@ impl PeakRecordingPool {reserved: AtomicUsize::new(0),peak: AtomicUsize::new(0),max: AtomicUsize::new(0),+ drift: None,}}+ /// Also report every reservation change to `tracker`. `label` identifies+ /// this pool in drift log lines, which matters when several pools share+ /// one tracker.+ pub fn with_drift_tracker(+ mut self,+ tracker: Arc<MemoryDriftTracker>,+ label: impl Into<String>,+ ) -> Self {+ self.drift = Some((tracker, label.into()));+ self+ }+/// The recorder installed as `pool`, if there is one.////// Returns `None` whenever a benchmark runs without a memory limit, since@@ -152,11 +173,14 @@ impl PeakRecordingPool {/// The total stays exact because the trait grants exactly what is asked/// for — `grow` is infallible and `try_grow` either grants `additional` or/// returns an error, leaving the reservation untouched.- fn record(&self, additional: usize) {+ fn record(&self, reservation: &MemoryReservation, additional: usize) {let reserved =self.reserved.fetch_add(additional, Ordering::Relaxed) + additional;self.peak.fetch_max(reserved, Ordering::Relaxed);self.max.fetch_max(reserved, Ordering::Relaxed);+ if let Some((tracker, label)) = &self.drift {+ tracker.grew(label, reservation.consumer().name(), additional);+ }}}@@ -166,6 +190,7 @@ impl Debug for PeakRecordingPool {.field("inner", &self.inner).field("peak", &self.peak_reserved()).field("max", &self.max_reserved())+ .field("drift_label", &self.drift.as_ref().map(|(_, label)| label)).finish()}}@@ -193,17 +218,20 @@ impl MemoryPool for PeakRecordingPool {fn grow(&self, reservation: &MemoryReservation, additional: usize) {self.inner.grow(reservation, additional);- self.record(additional);+ self.record(reservation, additional);}fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {self.inner.shrink(reservation, shrink);self.reserved.fetch_sub(shrink, Ordering::Relaxed);+ if let Some((tracker, label)) = &self.drift {+ tracker.shrank(label, reservation.consumer().name(), shrink);+ }}fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {self.inner.try_grow(reservation, additional)?;- self.record(additional);+ self.record(reservation, additional);Ok(())}diff --git a/datafusion/sqllogictest/README.md b/datafusion/sqllogictest/README.mdindex 13ef797dfc..ad6c552f75 100644--- a/datafusion/sqllogictest/README.md+++ b/datafusion/sqllogictest/README.md@@ -101,6 +101,45 @@ SLT_TIMING_SUMMARY=1 cargo test --test sqllogictestsSLT_TIMING_DEBUG_SLOW_FILES=1 cargo test --test sqllogictests```+### Memory drift++The runner compares the bytes reserved in `MemoryPool`s with the bytes actually+allocated, to find operators whose memory is not tracked by the pool (see+[#25650](https://github.com/apache/datafusion/issues/25650)). It is enabled by+default and prints the largest drift seen at the end of the run. It only logs+and never fails a test.++Test files run concurrently by default, so the comparison is process-wide:+allocated bytes across the whole process against reservations summed across all+files. In this mode, the file in the output can be wrong. To attribute drift to+one file, run one file at a time:++```shell+cargo test --test sqllogictests -- --test-threads 1+```++In both modes, the consumer in the output is the reservation change that took+the sample, not necessarily the code that allocated the untracked memory. Most+drift at this scale is memory that `MemoryPool` does not track by design (for+example, in-flight batches and Parquet read buffers), so a large drift is a lead+to investigate, not necessarily a bug.++```shell+# Log each 64 MB rise in drift, with the file and consumer that took the sample+RUST_LOG=datafusion_execution::memory_pool=info cargo test --test sqllogictests+```++```shell+# Log each 4 MB rise instead (also settable via SLT_MEMORY_DRIFT_LOG_THRESHOLD)+RUST_LOG=datafusion_execution::memory_pool=info cargo test --test sqllogictests -- \+ --test-threads 1 --memory-drift-log-threshold 4194304+```++```shell+# Disable (allocations are still counted, but pools are not wrapped)+cargo test --test sqllogictests -- --memory-drift false+```+## Cookbook: Adding Tests1. Add queriesdiff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rsindex 71573c5e53..c64864cb77 100644--- a/datafusion/sqllogictest/bin/sqllogictests.rs+++ b/datafusion/sqllogictest/bin/sqllogictests.rs@@ -19,9 +19,14 @@ use clap::{ColorChoice, Parser};use datafusion::common::instant::Instant;use datafusion::common::utils::get_available_parallelism;use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};+use datafusion::execution::memory_pool::DEFAULT_DRIFT_LOG_THRESHOLD;#[cfg(feature = "substrait")]use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;use datafusion_sqllogictest::TestFile;+use datafusion_sqllogictest::{+ CountingAllocator, enable_memory_drift_logging, flush_thread_allocations,+ memory_drift_tracker,+};use datafusion_sqllogictest::{CurrentlyExecutingSqlTracker, DFColumnType, DataFusion, Filter, TestContext,df_value_validator, read_dir_recursive, run_each_configuration, setup_scratch_dir,@@ -57,6 +62,9 @@ use std::time::Duration;#[cfg(feature = "postgres")]mod postgres_container;+#[global_allocator]+static ALLOC: CountingAllocator = CountingAllocator;+const TEST_DIRECTORY: &str = "test_files/";const DATAFUSION_TESTING_TEST_DIRECTORY: &str = "../../datafusion-testing/data/";const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_";@@ -88,6 +96,7 @@ fn config_change_result(pub fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all()+ .on_thread_stop(flush_thread_allocations).build()?.block_on(run_tests())}@@ -139,6 +148,10 @@ async fn run_tests() -> Result<()> {options.warn_on_ignored();+ if options.memory_drift {+ enable_memory_drift_logging(options.memory_drift_log_threshold);+ }+// Print parallelism info for debugging CI performanceeprintln!("Running with {} test threads (available parallelism: {})",@@ -386,6 +399,16 @@ async fn run_tests() -> Result<()> {HumanDuration(start.elapsed())))?;+ if let Some(peak) = memory_drift_tracker().and_then(|t| t.peak_drift()) {+ eprintln!("Peak memory drift: {peak}");+ if options.test_threads > 1 {+ eprintln!(+ "Test files ran concurrently, so the file and consumer above can be wrong. \+ Run with --test-threads 1 to attribute drift to one file."+ );+ }+ }+#[cfg(feature = "postgres")]terminate_postgres_container().await?;@@ -1106,6 +1129,24 @@ struct Options {)]timing_summary: bool,+ #[clap(+ long,+ env = "SLT_MEMORY_DRIFT",+ default_value_t = true,+ action = clap::ArgAction::Set,+ help = "Log drift between MemoryPool reservations and allocated bytes (RUST_LOG=datafusion_execution::memory_pool=info to log each rise of --memory-drift-log-threshold bytes)"+ )]+ memory_drift: bool,++ #[clap(+ long,+ env = "SLT_MEMORY_DRIFT_LOG_THRESHOLD",+ value_name = "BYTES",+ default_value_t = DEFAULT_DRIFT_LOG_THRESHOLD,+ help = "Rise in memory drift, in bytes, needed before another line is logged"+ )]+ memory_drift_log_threshold: usize,+#[clap(long,value_name = "MODE",diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rsindex 470efa6f4f..67d4c26a54 100644--- a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs@@ -24,6 +24,7 @@ use super::{DFSqlLogicTestError, error::Result, normalize};use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;use crate::engines::output::{DFColumnType, DFOutput};use crate::is_spark_path;+use crate::memory_drift::rewrap_replaced_pool;use arrow::record_batch::RecordBatch;use async_trait::async_trait;use datafusion::physical_plan::common::collect;@@ -156,6 +157,7 @@ impl sqllogictest::AsyncDB for DataFusion {let start = Instant::now();let result = run_query(&self.ctx, is_spark_path(&self.relative_path), sql).await;let duration = start.elapsed();+ rewrap_replaced_pool(&self.ctx, &self.relative_path.display().to_string());self.currently_executing_sql_tracker.remove_sql(tracked_sql);diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rsindex 90ad6646dc..02580eec91 100644--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs@@ -21,6 +21,7 @@ use std::{path::PathBuf, time::Duration};use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;use crate::engines::datafusion_engine::Result;use crate::engines::output::{DFColumnType, DFOutput};+use crate::memory_drift::rewrap_replaced_pool;use crate::{DFSqlLogicTestError, convert_batches, convert_schema_to_types};use arrow::record_batch::RecordBatch;use async_trait::async_trait;@@ -102,6 +103,7 @@ impl sqllogictest::AsyncDB for DataFusionSubstraitRoundTrip {let start = Instant::now();let result = run_query_substrait_round_trip(&self.ctx, sql).await;let duration = start.elapsed();+ rewrap_replaced_pool(&self.ctx, &self.relative_path.display().to_string());self.currently_executing_sql_tracker.remove_sql(tracked_sql);diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rsindex 926b1fb1e8..a397763933 100644--- a/datafusion/sqllogictest/src/lib.rs+++ b/datafusion/sqllogictest/src/lib.rs@@ -44,11 +44,16 @@ pub use engines::convert_schema_to_types;pub use engines::Postgres;mod filters;+mod memory_drift;mod test_context;mod util;pub use config_matrix::{TestConfiguration, run_each_configuration, test_configurations};pub use filters::*;+pub use memory_drift::{+ CountingAllocator, enable_memory_drift_logging, flush_thread_allocations,+ memory_drift_tracker,+};pub use test_context::TestContext;pub use test_file::TestFile;pub use util::*;diff --git a/datafusion/sqllogictest/src/memory_drift.rs b/datafusion/sqllogictest/src/memory_drift.rsnew file mode 100644index 0000000000..2aff6499bf--- /dev/null+++ b/datafusion/sqllogictest/src/memory_drift.rs@@ -0,0 +1,181 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Logs drift between `MemoryPool` reservations and actual allocations while+//! running sqllogictests. See <https://github.com/apache/datafusion/issues/25650>.+//!+//! Test files run concurrently in one process, so allocations cannot be split+//! per file. Instead every file's pool reports to one process-wide+//! [`MemoryDriftTracker`], which compares the sum of all reservations with the+//! bytes counted by [`CountingAllocator`].+//!+//! Files that `SET datafusion.runtime.memory_limit` replace their pool; the+//! runner wraps the replacement too (see [`rewrap_replaced_pool`]).+//!+//! This only logs. It never fails a test.++use std::{+ alloc::{GlobalAlloc, Layout, System},+ cell::Cell,+ sync::{+ Arc, OnceLock,+ atomic::{AtomicIsize, Ordering},+ },+};++use datafusion::execution::SessionStateBuilder;+use datafusion::execution::memory_pool::{+ MemoryDriftTracker, MemoryPool, PeakRecordingPool,+};+use datafusion::execution::runtime_env::RuntimeEnvBuilder;+use datafusion::prelude::SessionContext;++static ALLOCATED: AtomicIsize = AtomicIsize::new(0);+static TRACKER: OnceLock<Arc<MemoryDriftTracker>> = OnceLock::new();++/// A [`GlobalAlloc`] that counts the bytes currently allocated through it,+/// delegating the allocation itself to [`System`].+///+/// Counts requested sizes, so allocator overhead and memory retained by the+/// allocator are not included. Counting starts with the process, so memory+/// freed later was always counted when it was allocated.+pub struct CountingAllocator;++/// Per-thread count is flushed to [`ALLOCATED`] once it moves this far, so+/// threads do not contend on one atomic for every allocation. The global count+/// is therefore off by up to `FLUSH_BYTES` per live thread. Tokio threads+/// flush the rest when they stop (see [`flush_thread_allocations`]); the+/// unflushed count of any other thread that exits is lost.+const FLUSH_BYTES: isize = 256 * 1024;++thread_local! {+ // `const` with no destructor, so accessing it never allocates.+ static UNFLUSHED: Cell<isize> = const { Cell::new(0) };+}++fn count(delta: isize) {+ // `try_with` because this can run while the thread is being torn down.+ let _ = UNFLUSHED.try_with(|unflushed| {+ let pending = unflushed.get() + delta;+ if pending.abs() >= FLUSH_BYTES {+ ALLOCATED.fetch_add(pending, Ordering::Relaxed);+ unflushed.set(0);+ } else {+ unflushed.set(pending);+ }+ });+}++/// Flush this thread's unflushed count into the global count.+///+/// Registered as the Tokio runtime's `on_thread_stop` hook, which runs just+/// before a worker or blocking thread exits, outside the allocator.+pub fn flush_thread_allocations() {+ let _ = UNFLUSHED.try_with(|unflushed| {+ ALLOCATED.fetch_add(unflushed.replace(0), Ordering::Relaxed);+ });+}++// SAFETY: every call is forwarded unchanged to `System`, which upholds the+// `GlobalAlloc` contract. Counting has no effect on the returned memory.+unsafe impl GlobalAlloc for CountingAllocator {+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {+ let ptr = unsafe { System.alloc(layout) };+ if !ptr.is_null() {+ count(layout.size() as isize);+ }+ ptr+ }++ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {+ let ptr = unsafe { System.alloc_zeroed(layout) };+ if !ptr.is_null() {+ count(layout.size() as isize);+ }+ ptr+ }++ unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {+ unsafe { System.dealloc(ptr, layout) };+ count(-(layout.size() as isize));+ }++ unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {+ let new_ptr = unsafe { System.realloc(ptr, layout, new_size) };+ if !new_ptr.is_null() {+ count(new_size as isize - layout.size() as isize);+ }+ new_ptr+ }+}++/// Bytes currently allocated through [`CountingAllocator`], clamped at zero+/// because counts lost with exiting threads can bias it either way.+fn allocated_bytes() -> usize {+ ALLOCATED.load(Ordering::Relaxed).max(0) as usize+}++/// Wrap the memory pool of every test file created from now on, logging each+/// rise in drift of `log_threshold` bytes. Only meaningful if+/// [`CountingAllocator`] is the global allocator.+pub fn enable_memory_drift_logging(log_threshold: usize) {+ TRACKER.get_or_init(|| {+ Arc::new(+ MemoryDriftTracker::new(Arc::new(allocated_bytes))+ .with_log_threshold(log_threshold),+ )+ });+}++/// The process-wide tracker, if [`enable_memory_drift_logging`] was called.+pub fn memory_drift_tracker() -> Option<&'static Arc<MemoryDriftTracker>> {+ TRACKER.get()+}++/// Wrap `pool` so its reservations are reported to the process-wide tracker,+/// or return it unchanged if drift logging is not enabled.+pub fn wrap_pool(pool: Arc<dyn MemoryPool>, label: &str) -> Arc<dyn MemoryPool> {+ match memory_drift_tracker() {+ Some(tracker) => Arc::new(+ PeakRecordingPool::new(pool).with_drift_tracker(Arc::clone(tracker), label),+ ),+ None => pool,+ }+}++/// Wrap the memory pool of `ctx` again if a statement replaced it, e.g.+/// `SET datafusion.runtime.memory_limit`, so its reservations keep being+/// counted. Does nothing if drift logging is not enabled.+pub(crate) fn rewrap_replaced_pool(ctx: &SessionContext, label: &str) {+ if memory_drift_tracker().is_none() {+ return;+ }+ let state = ctx.state_ref();+ let mut state = state.write();+ let runtime = state.runtime_env();+ if runtime.memory_pool.is::<PeakRecordingPool>() {+ return;+ }+ let pool = wrap_pool(Arc::clone(&runtime.memory_pool), label);+ let runtime = RuntimeEnvBuilder::from_runtime_env(runtime)+ .with_memory_pool(pool)+ .build_arc()+ .expect("rebuilding an existing runtime succeeds");+ *state = SessionStateBuilder::from(state.clone())+ .with_runtime_env(runtime)+ .build();+}diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 97d9d66867..bc2859aaa9 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -61,7 +61,8 @@ use range_partitioning::{use async_trait::async_trait;use datafusion::common::cast::as_float64_array;use datafusion::execution::SessionStateBuilder;-use datafusion::execution::runtime_env::RuntimeEnv;+use datafusion::execution::memory_pool::UnboundedMemoryPool;+use datafusion::execution::runtime_env::RuntimeEnvBuilder;use datafusion::physical_plan::operator_statistics::StatisticsRegistry;use log::info;use sqlparser::ast;@@ -110,7 +111,14 @@ impl TestContext {let config = SessionConfig::new()// hardcode target partitions so plans are deterministic.with_target_partitions(4);- let runtime = Arc::new(RuntimeEnv::default());+ let pool = crate::memory_drift::wrap_pool(+ Arc::new(UnboundedMemoryPool::default()),+ &relative_path.display().to_string(),+ );+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(pool)+ .build_arc()+ .expect("default runtime builds");let mut state_builder = SessionStateBuilder::new().with_config(config)