Apache DataFusion SQL Query Engine
test: add tests for spill file sizes to verify View GC (#21750)
Ratul Dawar7acbe03efb85
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 1c900b7579..e8c9e9167b 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -1421,4 +1421,92 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn test_spill_file_size_gc_verification_string_view() -> Result<()> {+ use arrow::array::StringViewArray;+ use std::fs;++ // 1. Setup bloated data (large buffers)+ let num_rows = 1000;+ let string_array: StringViewArray = (0..num_rows)+ .map(|i| Some(format!("this_is_a_long_string_to_ensure_it_is_not_inlined_and_causes_waste_{i}")))+ .collect();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Utf8View,+ false,+ )]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(string_array.clone()) as ArrayRef],+ )?;++ // 2. Slice it heavily (1% of the data)+ let sliced_batch = batch.slice(0, 10);++ // 3. Spill to disk using SpillManager+ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, schema);+ let spill_file = spill_manager+ .spill_record_batch_and_finish(&[sliced_batch], "TestGC")?+ .unwrap();++ // 4. Check file size on disk+ let file_size = fs::metadata(spill_file.path())?.len();++ // The original buffer size is around 70KB.+ // Without GC, the spill file would be > 70KB.+ // With GC, it should be much smaller (only 10 rows of ~70 bytes each + metadata).+ assert!(+ file_size < 10 * 1024,+ "Spill file is too large ({file_size} bytes)! GC might not be working."+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_spill_file_size_gc_verification_binary_view() -> Result<()> {+ use arrow::array::BinaryViewArray;+ use std::fs;++ // 1. Setup bloated data (large buffers)+ let num_rows = 1000;+ let binary_array: BinaryViewArray =+ (0..num_rows).map(|i| Some(vec![i as u8; 100])).collect();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "b",+ DataType::BinaryView,+ false,+ )]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(binary_array.clone()) as ArrayRef],+ )?;++ // 2. Slice it heavily (1% of the data)+ let sliced_batch = batch.slice(0, 10);++ // 3. Spill to disk using SpillManager+ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, schema);+ let spill_file = spill_manager+ .spill_record_batch_and_finish(&[sliced_batch], "TestGCBinary")?+ .unwrap();++ // 4. Check file size on disk+ let file_size = fs::metadata(spill_file.path())?.len();++ // Original buffer is 100KB.+ // With GC, it should be much smaller.+ assert!(+ file_size < 10 * 1024,+ "Spill file is too large ({file_size} bytes)! GC might not be working."+ );++ Ok(())+ }}