Skip to content

Apache DataFusion SQL Query Engine

Add ObjectStore-backed TempFileFactor / spill example (#23170)

Andrew Lamb551c592ca6b6
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 5f66412e7d..e59f7eb948 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -60,7 +60,7 @@ futures = { workspace = true }
insta = { workspace = true }
log = { workspace = true }
mimalloc = { version = "0.1", default-features = false }
-object_store = { workspace = true, features = ["aws", "http"] }
+object_store = { workspace = true, features = ["aws", "fs", "http"] }
prost = { workspace = true }
rand = { workspace = true }
serde = { version = "1", features = ["derive"] }
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index 4746ac9114..86cfffe1a8 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -93,6 +93,7 @@ cargo run --example dataframe -- dataframe
| catalog | [`data_io/catalog.rs`](examples/data_io/catalog.rs) | Register tables into a custom catalog |
| in_memory_object_store | [`data_io/in_memory_object_store.rs`](examples/data_io/in_memory_object_store.rs) | Read CSV from an in-memory object store (pattern applies to JSON/Parquet) |
| json_shredding | [`data_io/json_shredding.rs`](examples/data_io/json_shredding.rs) | Implement filter rewriting for JSON shredding |
+| object_store_spill | [`data_io/object_store_spill.rs`](examples/data_io/object_store_spill.rs) | Use ObjectStore-backed spill files |
| parquet_adv_idx | [`data_io/parquet_advanced_index.rs`](examples/data_io/parquet_advanced_index.rs) | Create a secondary index across multiple parquet files |
| parquet_emb_idx | [`data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs) | Store a custom index inside Parquet files |
| parquet_enc | [`data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs) | Read & write encrypted Parquet files |
diff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rs
index 0b1c435b93..041308463c 100644
--- a/datafusion-examples/examples/data_io/main.rs
+++ b/datafusion-examples/examples/data_io/main.rs
@@ -21,7 +21,7 @@
//!
//! ## Usage
//! ```bash
-//! cargo run --example data_io -- [all|catalog|in_memory_object_store|json_shredding|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog]
+//! cargo run --example data_io -- [all|catalog|in_memory_object_store|json_shredding|object_store_spill|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog]
//! ```
//!
//! Each subcommand runs a corresponding example:
@@ -36,6 +36,9 @@
//! - `json_shredding`
//! (file: json_shredding.rs, desc: Implement filter rewriting for JSON shredding)
//!
+//! - `object_store_spill`
+//! (file: object_store_spill.rs, desc: Use ObjectStore-backed spill files)
+//!
//! - `parquet_adv_idx`
//! (file: parquet_advanced_index.rs, desc: Create a secondary index across multiple parquet files)
//!
@@ -66,6 +69,7 @@
mod catalog;
mod in_memory_object_store;
mod json_shredding;
+mod object_store_spill;
mod parquet_advanced_index;
mod parquet_embedded_index;
mod parquet_encrypted;
@@ -87,6 +91,7 @@ enum ExampleKind {
Catalog,
InMemoryObjectStore,
JsonShredding,
+ ObjectStoreSpill,
ParquetAdvIdx,
ParquetEmbIdx,
ParquetEnc,
@@ -118,6 +123,9 @@ impl ExampleKind {
in_memory_object_store::in_memory_object_store().await?
}
ExampleKind::JsonShredding => json_shredding::json_shredding().await?,
+ ExampleKind::ObjectStoreSpill => {
+ object_store_spill::object_store_spill().await?
+ }
ExampleKind::ParquetAdvIdx => {
parquet_advanced_index::parquet_advanced_index().await?
}
diff --git a/datafusion-examples/examples/data_io/object_store_spill.rs b/datafusion-examples/examples/data_io/object_store_spill.rs
new file mode 100644
index 0000000000..d7d5392f66
--- /dev/null
+++ b/datafusion-examples/examples/data_io/object_store_spill.rs
@@ -0,0 +1,273 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! See `main.rs` for how to run it.
+//!
+//! [`object_store_spill`] demonstrates how to use the [`TempFileFactory`] API to configure
+//! DataFusion to spill intermediate results to remote storage when it exceeds
+//! the configured memory limits.
+//!
+//! See [`datafusion::execution::memory_pool`] for more information on how
+//! DataFusion decides when operators should spill, and [`SpillFile`] for the
+//! spill file abstraction this example implements.
+use std::future::Future;
+use std::io::Write;
+use std::path::Path as StdPath;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use bytes::Bytes;
+use datafusion::common::Result;
+use datafusion::execution::disk_manager::DiskManagerBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_common::exec_err;
+use futures::{Stream, StreamExt, TryStreamExt, stream};
+use object_store::local::LocalFileSystem;
+use object_store::path::Path;
+use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
+use tempfile::tempdir;
+
+/// Demonstrates configuring DataFusion with spill files backed by an ObjectStore.
+pub async fn object_store_spill() -> Result<()> {
+ // A real system would use S3, GCS, Azure, or some other ObjectStore for
+ // remote spills. This example uses a local-file-backed ObjectStore for
+ // simplicity.
+ let tmp_dir = tempdir()?;
+ let store: Arc<dyn ObjectStore> =
+ Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?);
+
+ // Create the custom TempFileFactory that creates spill files in the ObjectStore.
+ let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(store));
+ let disk_manager_builder =
+ DiskManagerBuilder::default().with_temp_file_factory(temp_file_factory.clone());
+ let runtime = RuntimeEnvBuilder::new()
+ .with_disk_manager_builder(disk_manager_builder) // use the factory
+ // and set a small memory limit so the example spills
+ .with_memory_limit(1024 * 1024, 1.0)
+ .build_arc()?;
+
+ // Configure a SessionContext for running queries; use a single partition
+ // and no sort spill reservation to make the example deterministic and keep
+ // the spill behavior easy to observe.
+ let config = SessionConfig::new()
+ .with_sort_spill_reservation_bytes(0)
+ .with_sort_in_place_threshold_bytes(0)
+ .with_target_partitions(1);
+ let ctx = SessionContext::new_with_config_rt(config, Arc::clone(&runtime));
+
+ // Run an SQL query that sorts a "large" amount of data. Given the
+ // SessionContext's low memory limit, the sort will spill.
+ let row_count = 10_000_000;
+ let mut stream = ctx
+ .sql(&format!(
+ "SELECT * FROM generate_series(1, {row_count}) AS t(v) ORDER BY v DESC"
+ ))
+ .await?
+ .execute_stream()
+ .await?;
+
+ // Drive the query to completion, and verify output
+ let mut output_rows = 0;
+ while let Some(batch) = stream.next().await {
+ output_rows += batch?.num_rows();
+ }
+
+ assert_eq!(output_rows, row_count as usize);
+ assert!(
+ temp_file_factory.created_files() > 0,
+ "expected the custom TempFileFactory to be used for spilling"
+ );
+
+ Ok(())
+}
+
+/// Creates spill files backed by an [`ObjectStore`].
+///
+/// DataFusion calls this factory whenever an operator needs a new temporary
+/// file for spilling. A remote deployment would use the same pattern with an
+/// S3, GCS, Azure, or other remote ObjectStore implementation.
+struct ObjectStoreTempFileFactory {
+ /// ObjectStore used for spill file reads and writes.
+ store: Arc<dyn ObjectStore>,
+ /// Monotonic counter used to create unique object paths.
+ counter: AtomicU64,
+ /// Counts how many spill files DataFusion requested from this factory.
+ created_files: AtomicU64,
+}
+
+impl ObjectStoreTempFileFactory {
+ /// Create a new spill file factory that stores spill data in `store`.
+ fn new(store: Arc<dyn ObjectStore>) -> Self {
+ Self {
+ store,
+ counter: AtomicU64::new(0),
+ created_files: AtomicU64::new(0),
+ }
+ }
+
+ /// Return the number of spill files created through this factory.
+ fn created_files(&self) -> u64 {
+ self.created_files.load(Ordering::Relaxed)
+ }
+}
+
+impl TempFileFactory for ObjectStoreTempFileFactory {
+ /// Create one logical spill file backed by an ObjectStore path.
+ fn create_temp_file(&self, description: &str) -> Result<Arc<dyn SpillFile>> {
+ let id = self.counter.fetch_add(1, Ordering::Relaxed);
+ self.created_files.fetch_add(1, Ordering::Relaxed);
+
+ // Convert a query-provided spill description into an ObjectStore-safe path component.
+ //
+ // For example, `"Sort Spill: partition 0"` becomes `"Sort_Spill__partition_0"`.
+ let cleaned_description: String = description
+ .chars()
+ .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
+ .collect();
+ let location = Path::from(format!("spill/{cleaned_description}-{id}.bin"));
+
+ // Return a SpillFile implementation that reads and writes this ObjectStore path.
+ Ok(Arc::new(ObjectStoreSpillFile {
+ store: Arc::clone(&self.store),
+ location,
+ size: Arc::new(AtomicU64::new(0)),
+ }))
+ }
+}
+
+/// Logical spill file stored at an ObjectStore path.
+///
+/// DataFusion writes spill data by calling [`SpillFile::open_writer`] and reads
+/// it back by calling [`SpillFile::read_stream`].
+struct ObjectStoreSpillFile {
+ /// ObjectStore containing the spill object.
+ store: Arc<dyn ObjectStore>,
+ /// ObjectStore path for this spill object.
+ location: Path,
+ /// Last committed object size, updated when the writer finishes.
+ size: Arc<AtomicU64>,
+}
+
+impl SpillFile for ObjectStoreSpillFile {
+ /// Return no local filesystem path because the spill file is accessed through ObjectStore.
+ fn path(&self) -> Option<&StdPath> {
+ None // Remote ObjectStores do not have a local OS path.
+ }
+
+ /// Return the size of the uploaded object
+ fn size(&self) -> Option<u64> {
+ // Return the last committed size, which this example tracks after upload.
+ Some(self.size.load(Ordering::Relaxed))
+ }
+
+ /// Read the spill file contents as a byte stream.
+ fn read_stream(&self) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes>> + Send>>> {
+ let store = Arc::clone(&self.store);
+ let location = self.location.clone();
+
+ // Use `stream::once` to defer the ObjectStore read until DataFusion
+ // polls the returned stream.
+ let result_stream =
+ async move { store.get(&location).await.map(|r| r.into_stream()) };
+ let stream = stream::once(result_stream)
+ .try_flatten()
+ .map_err(Into::into);
+
+ Ok(Box::pin(stream))
+ }
+
+ /// Open a synchronous writer for this spill file.
+ fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {
+ // Create a writer that buffers bytes and uploads them on finish.
+ Ok(Box::new(ObjectStoreSpillWriter {
+ store: Arc::clone(&self.store),
+ location: self.location.clone(),
+ size: Arc::clone(&self.size),
+ buffer: Vec::new(),
+ }))
+ }
+}
+
+/// Adapts DataFusion's [`SpillWriter`] API to ObjectStore.
+///
+/// This simple example buffers bytes in memory and uploads them in
+/// [`SpillWriter::finish`]. A production remote implementation should consider
+/// multipart or streaming uploads.
+struct ObjectStoreSpillWriter {
+ /// ObjectStore to read/write bytes to.
+ store: Arc<dyn ObjectStore>,
+ /// ObjectStore path to upload to.
+ location: Path,
+ /// Shared size field on the corresponding [`ObjectStoreSpillFile`].
+ size: Arc<AtomicU64>,
+ /// Buffered spill bytes waiting to be uploaded.
+ ///
+ /// This simple example buffers the spill and uploads it on finish.
+ /// Production remote stores should consider multipart or streaming uploads.
+ buffer: Vec<u8>,
+}
+
+impl Write for ObjectStoreSpillWriter {
+ /// Append bytes to the in-memory buffer.
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ // Buffer bytes written through the synchronous Write API.
+ self.buffer.extend_from_slice(buf);
+ Ok(buf.len())
+ }
+
+ /// No-op because data is committed in [`SpillWriter::finish`].
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+}
+
+impl SpillWriter for ObjectStoreSpillWriter {
+ /// Upload buffered bytes to ObjectStore and mark the spill file complete.
+ fn finish(&mut self) -> Result<()> {
+ // Move the buffered bytes into the upload future.
+ let store = Arc::clone(&self.store);
+ let location = self.location.clone();
+ let data = std::mem::take(&mut self.buffer);
+ let size = data.len() as u64;
+
+ // This simple example buffers the spill and uploads it on finish.
+ // Production remote stores should consider multipart or streaming uploads.
+ block_on_object_store(async move {
+ store
+ .put(&location, PutPayload::from_bytes(data.into()))
+ .await?;
+ Ok(())
+ })?;
+
+ self.size.store(size, Ordering::Relaxed);
+ Ok(())
+ }
+}
+
+/// Run an async ObjectStore operation.
+///
+/// Adding a native async API is tracked in <https://github.com/apache/datafusion/issues/23247>
+fn block_on_object_store<T>(future: impl Future<Output = Result<T>>) -> Result<T> {
+ if let Ok(handle) = tokio::runtime::Handle::try_current() {
+ tokio::task::block_in_place(|| handle.block_on(future))
+ } else {
+ exec_err!("No current Tokio runtime available")
+ }
+}
diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs
index 8534c4f4ab..313379f012 100644
--- a/datafusion/execution/src/disk_manager.rs
+++ b/datafusion/execution/src/disk_manager.rs
@@ -74,6 +74,26 @@ impl DiskManagerBuilder {
self
}
+ /// Configure a custom factory for creating temporary spill files.
+ ///
+ /// This sets the disk manager mode to [`DiskManagerMode::Custom`], so
+ /// operators that spill during query execution create files through the
+ /// provided [`TempFileFactory`] instead of using local temporary files.
+ pub fn set_temp_file_factory(&mut self, temp_file_factory: Arc<dyn TempFileFactory>) {
+ self.mode = DiskManagerMode::Custom(temp_file_factory);
+ }
+
+ /// Configure a custom factory for creating temporary spill files.
+ ///
+ /// See details on [`Self::set_temp_file_factory`].
+ pub fn with_temp_file_factory(
+ mut self,
+ temp_file_factory: Arc<dyn TempFileFactory>,
+ ) -> Self {
+ self.set_temp_file_factory(temp_file_factory);
+ self
+ }
+
pub fn set_max_temp_directory_size(&mut self, value: u64) {
self.max_temp_directory_size = value;
}