Apache DataFusion SQL Query Engine
fix: frame IPC messages when reading spill files so batches pin only their own bytes (#24592)
Jay Zhancf083e0815e0
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex addcf78d2d..e3c4b2cf30 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -28,6 +28,7 @@ pub use datafusion_common::utils::memory::get_record_batch_memory_size;#[doc(hidden)]pub use spill_manager::SpillManager;+use std::collections::VecDeque;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};@@ -46,7 +47,7 @@ use arrow::ipc::{};use arrow::record_batch::RecordBatch;use arrow_data::ArrayDataBuilder;-use arrow_ipc::CompressionType;+use arrow_ipc::{CompressionType, root_as_message};use datafusion_common::Result;use datafusion_common::config::SpillCompression;@@ -72,9 +73,15 @@ struct SpillReaderStream {/// see `physical_plan/sort/multi_level_merge.rs`.max_record_batch_memory: Option<usize>,- /// Holds leftover bytes from a chunk when a batch is yielded early+ /// Holds leftover bytes from a chunk not yet consumed by `framer`current_buffer: Buffer,+ /// Assembles the chunks of `byte_stream` into exactly sized messages+ framer: MessageFramer,++ /// Framed message buffers not yet consumed by `decoder`+ pending: VecDeque<Buffer>,+/// Keeps the file alive until the stream is dropped_spill_file: Arc<dyn SpillFile>,@@ -84,6 +91,240 @@ struct SpillReaderStream {// Small margin allowed to accommodate slight memory accounting variationconst SPILL_BATCH_MEMORY_MARGIN: usize = 4096;+/// Reassembles an IPC stream read in arbitrary chunks into one exactly sized+/// allocation per message, so that the zero-copy [`StreamDecoder`] produces+/// batches whose buffers pin only their own message.+///+/// # Why+///+/// The decoder builds arrays on slices of whatever [`Buffer`] it is given,+/// and a slice keeps its whole backing allocation alive. Fed with the raw+/// chunks of the byte stream (128 KB for a file backed spill) that goes+/// wrong in two ways.+///+/// A message that fits inside a chunk pins the entire chunk. With ~5 KB+/// batches, one chunk holds ~27 of them, and each decoded batch retains, and+/// is accounted for, 128 KB:+///+/// ```text+/// chunk (128 KB allocation)+/// ┌──────┬──────┬──────┬─────┬───────┐+/// │ msg1 │ msg2 │ msg3 │ ... │ msg27 │+/// └──────┴──────┴──────┴─────┴───────┘+/// ▲+/// batch1's buffers slice here, yet keep all 128 KB alive+/// ```+///+/// A message that spans two chunks cannot be sliced, so the decoder gathers+/// it into a `Vec` grown by doubling, and the batch keeps the spare+/// capacity: a 256 KB body typically lands in a 512 KB allocation.+///+/// ```text+/// chunk N chunk N+1+/// ┌──────┬────────────────────┬──────────────┬────────┐+/// │ ... │ msgK (first part) │ msgK (rest) │ msgK+1 │+/// └──────┴────────────────────┴──────────────┴────────┘+/// ```+///+/// Either way a batch uses several times the memory recorded for it at+/// spill time, breaking the `max_record_batch_memory` budgeting that the+/// multi-level merge relies on.+///+/// # How+///+/// Each message is copied out of the chunks into allocations sized from its+/// own headers: a head buffer (the length prefix and flatbuffer metadata,+/// whose `bodyLength` gives the body size) and, when non-empty, a body+/// buffer of exactly that size. The decoder then zero-copies from the body+/// buffer, so a batch pins exactly its own message:+///+/// ```text+/// body for msg1 (5 KB) body for msgK (256 KB)+/// ┌──────┐ ┌────────────────────┐+/// │ msg1 │ ◀── batch1 │ msgK │ ◀── batchK+/// └──────┘ └────────────────────┘+/// ```+///+/// This costs one copy per message, which the decoder already paid for+/// spanning messages, without the doubling reallocation. Measured on the+/// `spill_io` benchmark, the copy is not visible end to end for batches at+/// or above the 128 KB read chunk, where skipping the doubling gather makes+/// framing the faster of the two, and costs a few percent for batches small+/// enough that several share a chunk, which are exactly the batches whose+/// accounting it fixes.+struct MessageFramer {+ state: FramerState,+}++enum FramerState {+ /// Reading the 4 byte continuation marker or metadata length.+ Prefix {+ head: Vec<u8>,+ read: usize,+ continuation: bool,+ },+ /// Reading the flatbuffer metadata into `head`, which already holds the+ /// prefix and is allocated for `head_len` bytes.+ Metadata {+ head: Vec<u8>,+ metadata_start: usize,+ head_len: usize,+ },+ /// Reading the body into `body`, which is allocated for `body_len` bytes.+ Body {+ head: Vec<u8>,+ body: Vec<u8>,+ body_len: usize,+ },+ /// The end-of-stream marker was read.+ Finished,+}++impl MessageFramer {+ fn new() -> Self {+ Self {+ state: FramerState::prefix(),+ }+ }++ /// Consumes bytes from `input` until a message is complete or `input` is+ /// exhausted, returning the buffers of a completed message.+ fn push(&mut self, input: &mut Buffer) -> Result<Option<Vec<Buffer>>> {+ while !input.is_empty() {+ match &mut self.state {+ FramerState::Prefix {+ head,+ read,+ continuation,+ } => {+ let to_read = input.len().min(4 - *read);+ head.extend_from_slice(&input[..to_read]);+ input.advance(to_read);+ *read += to_read;+ if *read < 4 {+ continue;+ }+ let word: [u8; 4] = head[head.len() - 4..].try_into().unwrap();+ if !*continuation && word == CONTINUATION_MARKER {+ *continuation = true;+ *read = 0;+ continue;+ }+ let metadata_len = u32::from_le_bytes(word) as usize;+ let head = std::mem::take(head);+ if metadata_len == 0 {+ self.state = FramerState::Finished;+ return Ok(Some(vec![Buffer::from_vec(head)]));+ }+ let metadata_start = head.len();+ let head_len = metadata_start + metadata_len;+ let mut sized = Vec::with_capacity(head_len);+ sized.extend_from_slice(&head);+ self.state = FramerState::Metadata {+ head: sized,+ metadata_start,+ head_len,+ };+ }+ FramerState::Metadata {+ head,+ metadata_start,+ head_len,+ } => {+ let to_read = input.len().min(*head_len - head.len());+ head.extend_from_slice(&input[..to_read]);+ input.advance(to_read);+ if head.len() < *head_len {+ continue;+ }+ let message =+ root_as_message(&head[*metadata_start..]).map_err(|e| {+ datafusion_common::exec_datafusion_err!(+ "Invalid IPC message in spill file: {e}"+ )+ })?;+ let body_len =+ usize::try_from(message.bodyLength()).map_err(|_| {+ datafusion_common::exec_datafusion_err!(+ "Invalid IPC message body length in spill file: {}",+ message.bodyLength()+ )+ })?;+ let head = std::mem::take(head);+ if body_len == 0 {+ self.state = FramerState::prefix();+ return Ok(Some(vec![Buffer::from_vec(head)]));+ }+ self.state = FramerState::Body {+ head,+ body: Vec::with_capacity(body_len),+ body_len,+ };+ }+ FramerState::Body {+ head,+ body,+ body_len,+ } => {+ let to_read = input.len().min(*body_len - body.len());+ body.extend_from_slice(&input[..to_read]);+ input.advance(to_read);+ if body.len() < *body_len {+ continue;+ }+ let (head, body) = (std::mem::take(head), std::mem::take(body));+ self.state = FramerState::prefix();+ return Ok(Some(vec![+ Buffer::from_vec(head),+ Buffer::from_vec(body),+ ]));+ }+ FramerState::Finished => {+ return datafusion_common::exec_err!(+ "Unexpected bytes after the end of the IPC stream in spill file"+ );+ }+ }+ }+ Ok(None)+ }++ /// Checks that the stream ended on a message boundary.+ ///+ /// This mirrors [`StreamDecoder::finish`]: a stream is complete either+ /// when the end-of-stream marker was read, or when it ends right after a+ /// message. Framing moves the partial bytes of a truncated file out of+ /// the decoder's own scratch space and into the framer, so without this+ /// check a truncated spill file would silently decode as a shorter+ /// stream instead of erroring.+ fn finish(&self) -> Result<()> {+ match &self.state {+ FramerState::Finished+ | FramerState::Prefix {+ read: 0,+ continuation: false,+ ..+ } => Ok(()),+ _ => datafusion_common::exec_err!(+ "Unexpected end of spill file: the IPC stream ends mid-message"+ ),+ }+ }+}++impl FramerState {+ fn prefix() -> Self {+ Self::Prefix {+ head: Vec::with_capacity(8),+ read: 0,+ continuation: false,+ }+ }+}++/// Marks a length prefix in the IPC stream format, see `arrow_ipc`.+const CONTINUATION_MARKER: [u8; 4] = [0xff; 4];+impl SpillReaderStream {fn new(schema: SchemaRef,@@ -101,6 +342,8 @@ impl SpillReaderStream {max_record_batch_memory,is_done: false,current_buffer: Buffer::from(&[]),+ framer: MessageFramer::new(),+ pending: VecDeque::new(),_spill_file: spill_file,schema_validated: false,})@@ -118,8 +361,13 @@ impl Stream for SpillReaderStream {}loop {- if !this.current_buffer.is_empty() {- match this.decoder.decode(&mut this.current_buffer) {+ // Decode the framed messages first+ if let Some(buffer) = this.pending.front_mut() {+ if buffer.is_empty() {+ this.pending.pop_front();+ continue;+ }+ match this.decoder.decode(buffer) {Ok(Some(batch)) => {// One-time schema validation on the first decoded batch.// The IPC stream embeds the writer's schema in its header;@@ -159,17 +407,31 @@ impl Stream for SpillReaderStream {return Poll::Ready(Some(Ok(batch)));}Ok(None) => {- // The chunk didn't form a complete message. Arrow consumed the partial bytes- // into its internal scratch pad, leaving our current_buffer completely empty.- // We do nothing and fall through to fetch more data.+ // A schema or dictionary message, or the buffer was+ // only part of a message. Carry on with the next one.}Err(e) => {this.is_done = true;return Poll::Ready(Some(Err(e.into())));}}+ continue;}+ // Then frame the next message out of the current chunk+ if !this.current_buffer.is_empty() {+ match this.framer.push(&mut this.current_buffer) {+ Ok(Some(buffers)) => this.pending.extend(buffers),+ Ok(None) => {}+ Err(e) => {+ this.is_done = true;+ return Poll::Ready(Some(Err(e)));+ }+ }+ continue;+ }++ // Finally fetch another chunkmatch futures::ready!(this.byte_stream.as_mut().poll_next(cx)) {Some(Ok(chunk)) => {this.current_buffer = Buffer::from(chunk);@@ -181,6 +443,11 @@ impl Stream for SpillReaderStream {None => {this.is_done = true;+ // The framer holds the bytes of an incomplete trailing+ // message, so it is the one that detects truncation.+ if let Err(e) = this.framer.finish() {+ return Poll::Ready(Some(Err(e)));+ }if let Err(e) = this.decoder.finish() {return Poll::Ready(Some(Err(e.into())));}@@ -585,6 +852,361 @@ mod tests {Ok(())}+ /// Reading a spill file back must not inflate the batches' memory+ /// footprint: the decoder is zero-copy, so without framing every small+ /// batch would keep a whole read chunk alive, see [`MessageFramer`].+ ///+ /// Regression test for+ /// <https://github.com/apache/datafusion/issues/17340>. Without framing+ /// these 50 batches read back at 131072 bytes each, against a 22992 byte+ /// maximum recorded at spill time, and the whole 345 KB stream is+ /// accounted for as 5.9 MB. Note that this is not a counting bug that a+ /// shared-buffer aware counter could net out: holding only 5 of the 50+ /// batches still pins every read chunk, so the memory really is retained.+ #[tokio::test]+ async fn test_read_back_does_not_inflate_batch_memory() -> Result<()> {+ use arrow::array::{ListArray, StringViewArray};+ use arrow::buffer::OffsetBuffer;++ let schema = Arc::new(Schema::new(vec![+ Field::new("i", DataType::Int32, false),+ Field::new("s", DataType::Utf8, false),+ Field::new("v", DataType::Utf8View, true),+ Field::new(+ "l",+ DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),+ false,+ ),+ ]));++ // Small batches: many fit in one 128 KB read chunk.+ let batches: Vec<RecordBatch> = (0..50)+ .map(|b| {+ let n = 100;+ let ints = Int32Array::from_iter_values((0..n).map(|i| b * n + i));+ let strs = StringArray::from_iter_values(+ (0..n).map(|i| format!("string value number {i}")),+ );+ let views = StringViewArray::from_iter((0..n).map(|i| {+ (i % 3 != 0).then(|| format!("a longer view value {b}/{i}"))+ }));+ let values = Int32Array::from_iter_values(0..n * 2);+ let offsets =+ OffsetBuffer::from_lengths(std::iter::repeat_n(2, n as usize));+ let list = ListArray::new(+ Arc::new(Field::new("item", DataType::Int32, true)),+ offsets,+ Arc::new(values),+ None,+ );+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(ints),+ Arc::new(strs),+ Arc::new(views),+ Arc::new(list),+ ],+ )+ .unwrap()+ })+ .collect();++ let max_written = batches+ .iter()+ .map(get_record_batch_memory_size)+ .max()+ .unwrap();++ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, Arc::clone(&schema));+ let spill_file = spill_manager+ .spill_record_batch_and_finish(&batches, "Test")?+ .unwrap();++ let stream = spill_manager.read_spill_as_stream(spill_file, None)?;+ let read_back = collect(stream).await?;+ assert_eq!(read_back.len(), batches.len());++ for (written, read) in batches.iter().zip(&read_back) {+ assert_eq!(written, read);+ let size = get_record_batch_memory_size(read);+ assert!(+ size <= max_written + SPILL_BATCH_MEMORY_MARGIN,+ "read-back batch retains {size} bytes, written max was {max_written}"+ );+ }+ Ok(())+ }++ /// The workload from <https://github.com/apache/datafusion/issues/17340>,+ /// reduced to a single spilled run.+ ///+ /// `memory_limit::test_stringview_external_sort` sorts 200 batches of 1000+ /// random 100 byte strings under a 60 MB pool, producing around 10 spilled+ /// runs of about 20 batches each. This spills one such run and reads it+ /// back through the same [`SpillManager`] calls the external sorter and the+ /// multi-level merge use: the spill records the largest batch's memory size,+ /// and the merge reserves its budget from that number, so a batch that comes+ /// back bigger is memory the merge is using without having reserved it.+ ///+ /// Each batch here is around 167 KB, larger than the 128 KB read chunk, so+ /// every IPC message spans chunks. Without framing the decoder gathers such+ /// a message into a `Vec` grown by doubling and the batch keeps the spare+ /// capacity, so these batches read back at about 255 KB each against the+ /// ~167 KB recorded at spill time - half again the budget the merge+ /// reserved for them, which is the discrepancy the issue reports.+ #[tokio::test]+ async fn test_stringview_spill_read_back_memory_accounting() -> Result<()> {+ use arrow::array::StringViewArray;+ use rand::Rng;++ let schema = Arc::new(Schema::new(vec![+ Field::new("strings", DataType::Utf8View, false),+ Field::new("random_numbers", DataType::Int32, false),+ ]));++ // 100 random bytes per string, as in the issue's reproducer: long+ // enough that the views point into a data buffer rather than inlining,+ // and random so nothing dedupes.+ let mut rng = rand::rng();+ let batches: Vec<RecordBatch> = (0..20)+ .map(|_| {+ let strings: Vec<String> = (0..1000)+ .map(|_| {+ (0..100)+ .map(|_| rng.random_range(0..=u8::MAX) as char)+ .collect()+ })+ .collect();+ let numbers: Vec<i32> =+ (0..1000).map(|_| rng.random_range(0..=1000)).collect();+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(StringViewArray::from(strings)) as ArrayRef,+ Arc::new(Int32Array::from(numbers)) as ArrayRef,+ ],+ )+ .unwrap()+ })+ .collect();++ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, Arc::clone(&schema));++ // The same call the external sorter spills a sorted run with: the+ // returned size is what the merge budgets against.+ let (spill_file, max_record_batch_memory) = spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(+ batches+ .iter()+ .map(Ok::<_, datafusion_common::DataFusionError>),+ "Test",+ )?+ .unwrap();++ let stream = spill_manager+ .read_spill_as_stream(spill_file, Some(max_record_batch_memory))?;+ let read_back = collect(stream).await?;+ assert_eq!(read_back.len(), batches.len());++ for (written, read) in batches.iter().zip(&read_back) {+ assert_eq!(written, read);+ let size = get_record_batch_memory_size(read);+ assert!(+ size <= max_record_batch_memory + SPILL_BATCH_MEMORY_MARGIN,+ "read-back batch retains {size} bytes, but the merge only \+ reserved for {max_record_batch_memory} bytes"+ );+ }+ Ok(())+ }++ /// Frames an IPC stream delivered in chunks of `chunk_size` bytes and+ /// decodes it, checking that every batch is intact and that its buffers+ /// are backed by an allocation no larger than its own message body.+ fn frame_and_decode(ipc: &[u8], chunk_size: usize, expected: &[RecordBatch]) {+ let mut framer = MessageFramer::new();+ let mut decoder = StreamDecoder::new();+ let mut decoded = vec![];+ for chunk in ipc.chunks(chunk_size) {+ let mut input = Buffer::from(chunk);+ while !input.is_empty() {+ let Some(buffers) = framer.push(&mut input).unwrap() else {+ continue;+ };+ let body_size = buffers.last().unwrap().len();+ for mut buffer in buffers {+ while !buffer.is_empty() {+ if let Some(batch) = decoder.decode(&mut buffer).unwrap() {+ let retained = get_record_batch_memory_size(&batch);+ assert!(+ retained <= body_size,+ "batch retains {retained} bytes for a {body_size} byte body"+ );+ decoded.push(batch);+ }+ }+ }+ }+ }+ decoder.finish().unwrap();+ assert!(matches!(framer.state, FramerState::Finished));+ assert_eq!(decoded, expected);+ }++ #[test]+ fn test_message_framer_across_chunk_boundaries() {+ let batches: Vec<RecordBatch> = (0..5)+ .map(|b| {+ let n = 10 * (b + 1);+ build_table_i32(+ ("a", &(0..n).collect::<Vec<_>>()),+ ("b", &(n..2 * n).collect::<Vec<_>>()),+ ("c", &(2 * n..3 * n).collect::<Vec<_>>()),+ )+ })+ .collect();+ let schema = batches[0].schema();++ let mut ipc = vec![];+ let mut writer = StreamWriter::try_new(&mut ipc, &schema).unwrap();+ for batch in &batches {+ writer.write(batch).unwrap();+ }+ writer.finish().unwrap();++ for chunk_size in [1, 3, 7, 64, 1000, ipc.len()] {+ frame_and_decode(&ipc, chunk_size, &batches);+ }+ }++ /// Decodes `ipc` the way the reader did before framing, returning the+ /// number of batches or `Err` if the stream is not a complete one.+ fn decode_unframed(ipc: &[u8]) -> std::result::Result<usize, ()> {+ let mut decoder = StreamDecoder::new();+ let mut buffer = Buffer::from(ipc);+ let mut batches = 0;+ while !buffer.is_empty() {+ match decoder.decode(&mut buffer) {+ Ok(Some(_)) => batches += 1,+ Ok(None) => {}+ Err(_) => return Err(()),+ }+ }+ decoder.finish().map_err(|_| ())?;+ Ok(batches)+ }++ /// Same as [`decode_unframed`], but through the [`MessageFramer`].+ fn decode_framed(ipc: &[u8]) -> std::result::Result<usize, ()> {+ let mut framer = MessageFramer::new();+ let mut decoder = StreamDecoder::new();+ let mut input = Buffer::from(ipc);+ let mut batches = 0;+ while !input.is_empty() {+ let Some(buffers) = framer.push(&mut input).map_err(|_| ())? else {+ continue;+ };+ for mut buffer in buffers {+ while !buffer.is_empty() {+ match decoder.decode(&mut buffer) {+ Ok(Some(_)) => batches += 1,+ Ok(None) => {}+ Err(_) => return Err(()),+ }+ }+ }+ }+ framer.finish().map_err(|_| ())?;+ decoder.finish().map_err(|_| ())?;+ Ok(batches)+ }++ /// Framing must not weaken the reader's detection of a corrupt or cut+ /// short spill file. The partial bytes of an incomplete trailing message+ /// end up in the [`MessageFramer`] instead of the decoder's own scratch+ /// space, so the framer has to report them, see [`MessageFramer::finish`].+ ///+ /// Truncated at every possible offset, framed decoding must accept and+ /// reject exactly what unframed decoding does.+ #[test]+ fn test_message_framer_reports_truncation_like_the_decoder() {+ let batches: Vec<RecordBatch> = (0..3)+ .map(|b| {+ build_table_i32(+ ("a", &(0..50).map(|i| b * 50 + i).collect::<Vec<_>>()),+ ("b", &(0..50).collect::<Vec<_>>()),+ ("c", &(0..50).collect::<Vec<_>>()),+ )+ })+ .collect();++ let mut ipc = vec![];+ let mut writer = StreamWriter::try_new(&mut ipc, &batches[0].schema()).unwrap();+ for batch in &batches {+ writer.write(batch).unwrap();+ }+ writer.finish().unwrap();++ let mut complete = 0;+ for cut in 0..=ipc.len() {+ let truncated = &ipc[..cut];+ let framed = decode_framed(truncated);+ assert_eq!(+ framed,+ decode_unframed(truncated),+ "framed and unframed decoding disagree on a stream cut to {cut} bytes"+ );+ complete += usize::from(framed.is_ok());+ }+ // Cutting anywhere but on a message boundary must be an error, so+ // only the empty stream, the three batches and the end-of-stream+ // marker may decode cleanly.+ assert_eq!(complete, 5);+ assert_eq!(decode_framed(&ipc), Ok(batches.len()));+ }++ /// End-to-end: a spill file cut short mid-message must error rather than+ /// silently read back as a shorter stream.+ #[tokio::test]+ async fn test_truncated_spill_file_errors() -> Result<()> {+ let batch = build_table_i32(+ ("a", &(0..100).collect::<Vec<_>>()),+ ("b", &(100..200).collect::<Vec<_>>()),+ ("c", &(200..300).collect::<Vec<_>>()),+ );+ let schema = batch.schema();+ let batches = vec![batch.clone(), batch];++ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, Arc::clone(&schema));+ let spill_file = spill_manager+ .spill_record_batch_and_finish(&batches, "Test")?+ .unwrap();++ // Truncate the file mid-message: drop the 8 byte end-of-stream+ // marker plus part of the last batch's body.+ let path = spill_file.path().unwrap().to_path_buf();+ let len = std::fs::metadata(&path)?.len();+ let f = std::fs::OpenOptions::new().write(true).open(&path)?;+ f.set_len(len - 8 - 13)?;+ drop(f);++ let stream = spill_manager.read_spill_as_stream(spill_file, None)?;+ let result = collect(stream).await;+ assert!(+ result.is_err(),+ "truncated spill file should error, got Ok with {} batches",+ result.as_ref().map(|b| b.len()).unwrap_or(0)+ );+ Ok(())+ }+#[tokio::test]async fn test_batch_spill_and_read_dictionary_arrays() -> Result<()> {// See https://github.com/apache/datafusion/issues/4658