Apache DataFusion SQL Query Engine
feat: add DataFrame fill_nan (#22702)
Nagato Yuzuruc83a981b5564
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 3d6b832aa6..be5011cdbf 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -58,13 +58,11 @@ use datafusion_common::{};use datafusion_expr::select_expr::SelectExpr;use datafusion_expr::{- ExplainOption, SortExpr, TableProviderFilterPushDown, UNNAMED_TABLE, case,- dml::InsertOp,- expr::{Alias, ScalarFunction},- is_null, lit,- utils::COUNT_STAR_EXPANSION,+ ExplainOption, ScalarUDF, SortExpr, TableProviderFilterPushDown, UNNAMED_TABLE, case,+ dml::InsertOp, is_null, lit, utils::COUNT_STAR_EXPANSION,};use datafusion_functions::core::coalesce;+use datafusion_functions::math::nanvl;use datafusion_functions_aggregate::expr_fn::{avg, count, max, median, min, stddev, sum,};@@ -2471,6 +2469,64 @@ impl DataFrame {&self,value: ScalarValue,columns: Vec<String>,+ ) -> Result<DataFrame> {+ self.fill_columns(&value, &columns, &coalesce(), |_| true)+ }++ // Helper to find columns from names+ fn find_columns(&self, names: &[impl AsRef<str>]) -> Result<Vec<FieldRef>> {+ let schema = self.logical_plan().schema();+ names+ .iter()+ .map(|name| {+ let name = name.as_ref();+ schema+ .field_with_name(None, name)+ .cloned()+ .map_err(|_| plan_datafusion_err!("Column '{}' not found", name))+ })+ .collect()+ }++ /// Fill NaN values in specified floating-point columns with a given value+ /// If no columns are specified (empty slice), applies to all columns+ /// Only floating-point columns are affected; other columns are left unchanged+ /// Only fills if the value can be cast to the column's type+ ///+ /// # Arguments+ /// * `value` - Value to fill NaNs with+ /// * `columns` - List of column names to fill. If empty, fills all columns.+ ///+ /// # Example+ /// ```+ /// # use datafusion::prelude::*;+ /// # use datafusion::error::Result;+ /// # use datafusion_common::ScalarValue;+ /// # #[tokio::main]+ /// # async fn main() -> Result<()> {+ /// let ctx = SessionContext::new();+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// // Fill NaN in only columns "a" and "c":+ /// let df = df.fill_nan(&ScalarValue::from(0.0), &["a", "c"])?;+ /// // Fill NaN across all columns:+ /// let df = df.fill_nan(&ScalarValue::from(0.0), &[])?;+ /// # Ok(())+ /// # }+ /// ```+ pub fn fill_nan(&self, value: &ScalarValue, columns: &[&str]) -> Result<DataFrame> {+ self.fill_columns(value, columns, &nanvl(), |field| {+ field.data_type().is_floating()+ })+ }++ fn fill_columns(+ &self,+ value: &ScalarValue,+ columns: &[impl AsRef<str>],+ func: &Arc<ScalarUDF>,+ applies: impl Fn(&FieldRef) -> bool,) -> Result<DataFrame> {let cols = if columns.is_empty() {self.logical_plan()@@ -2480,28 +2536,21 @@ impl DataFrame {.map(Arc::clone).collect()} else {- self.find_columns(&columns)?+ self.find_columns(columns)?};- // Create projections for each columnlet projections = self.logical_plan().schema().fields().iter().map(|field| {- if cols.contains(field) {+ if cols.contains(field) && applies(field) {// Try to cast fill value to column type. If the cast fails, fallback to the original column.match value.clone().cast_to(field.data_type()) {- Ok(fill_value) => Expr::Alias(Alias {- expr: Box::new(Expr::ScalarFunction(ScalarFunction {- func: coalesce(),- args: vec![col(field.name()), lit(fill_value)],- })),- relation: None,- name: field.name().to_string(),- metadata: None,- }),+ Ok(fill_value) => func+ .call(vec![col(field.name()), lit(fill_value)])+ .alias(field.name()),Err(_) => col(field.name()),}} else {@@ -2513,20 +2562,6 @@ impl DataFrame {self.clone().select(projections)}- // Helper to find columns from names- fn find_columns(&self, names: &[String]) -> Result<Vec<FieldRef>> {- let schema = self.logical_plan().schema();- names- .iter()- .map(|name| {- schema- .field_with_name(None, name)- .cloned()- .map_err(|_| plan_datafusion_err!("Column '{}' not found", name))- })- .collect()- }-/// Find qualified columns for this dataframe from names////// # Argumentsdiff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex bc1ad4c4c6..3b92b92004 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -6539,6 +6539,173 @@ async fn test_fill_null_all_columns() -> Result<()> {Ok(())}+async fn create_nan_table() -> Result<DataFrame> {+ // create a DataFrame with a NaN value in a float column "a" and a+ // non-float column "b" that must stay untouched by fill_nan.+ // "+-----+---+",+ // "| a | b |",+ // "+-----+---+",+ // "| 1.0 | 1 |",+ // "| NaN | 2 |",+ // "| 3.0 | 3 |",+ // "+-----+---+",+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Float64, true),+ Field::new("b", DataType::Int32, true),+ ]));+ let a_values = Float64Array::from(vec![Some(1.0), Some(f64::NAN), Some(3.0)]);+ let b_values = Int32Array::from(vec![Some(1), Some(2), Some(3)]);+ let batch = RecordBatch::try_new(+ schema.clone(),+ vec![Arc::new(a_values), Arc::new(b_values)],+ )?;++ let ctx = SessionContext::new();+ let table = MemTable::try_new(schema.clone(), vec![vec![batch]])?;+ ctx.register_table("t_nan", Arc::new(table))?;+ let df = ctx.table("t_nan").await?;+ Ok(df)+}++#[tokio::test]+async fn test_fill_nan() -> Result<()> {+ let df = create_nan_table().await?;++ // Fill NaNs in the float column "a" with 0.0.+ let df_filled = df.fill_nan(&ScalarValue::Float64(Some(0.0)), &["a"])?;++ let results = df_filled.collect().await?;+ assert_snapshot!(+ batches_to_sort_string(&results),+ @r"+ +-----+---++ | a | b |+ +-----+---++ | 0.0 | 2 |+ | 1.0 | 1 |+ | 3.0 | 3 |+ +-----+---++ "+ );++ Ok(())+}++#[tokio::test]+async fn test_fill_nan_all_columns() -> Result<()> {+ let df = create_nan_table().await?;++ // Fill NaNs across all columns. Only the float column "a" is affected;+ // the non-float column "b" is left unchanged since NaN only exists for+ // floating-point types.+ let df_filled = df.fill_nan(&ScalarValue::Float64(Some(0.0)), &[])?;++ let results = df_filled.collect().await?;+ assert_snapshot!(+ batches_to_sort_string(&results),+ @r"+ +-----+---++ | a | b |+ +-----+---++ | 0.0 | 2 |+ | 1.0 | 1 |+ | 3.0 | 3 |+ +-----+---++ "+ );+ Ok(())+}++#[tokio::test]+async fn test_fill_nan_non_float_column() -> Result<()> {+ let df = create_nan_table().await?;++ // Explicitly naming a non-float column is a no-op, not an error: NaN does+ // not exist for Int32, so column "b" (and the un-targeted "a") are unchanged.+ let df_filled = df.fill_nan(&ScalarValue::Float64(Some(0.0)), &["b"])?;++ let results = df_filled.collect().await?;+ assert_snapshot!(+ batches_to_sort_string(&results),+ @r"+ +-----+---++ | a | b |+ +-----+---++ | 1.0 | 1 |+ | 3.0 | 3 |+ | NaN | 2 |+ +-----+---++ "+ );++ Ok(())+}++#[tokio::test]+async fn test_fill_nan_unknown_column() -> Result<()> {+ let df = create_nan_table().await?;++ // A column name that is not in the schema is propagated as an error.+ let err = df+ .fill_nan(&ScalarValue::Float64(Some(0.0)), &["does_not_exist"])+ .unwrap_err();++ assert_snapshot!(err.strip_backtrace(), @"Error during planning: Column 'does_not_exist' not found");++ Ok(())+}++#[tokio::test]+async fn test_fill_nan_casts_fill_value() -> Result<()> {+ let df = create_nan_table().await?;++ // Int32(0) is not the column's type (Float64) but can be cast to it, so the+ // NaN is replaced with 0.0. Exercises the cross-type cast path — the other+ // positive tests pass a Float64 value, which skips the actual cast.+ let df_filled = df.fill_nan(&ScalarValue::Int32(Some(0)), &["a"])?;++ let results = df_filled.collect().await?;+ assert_snapshot!(+ batches_to_sort_string(&results),+ @r"+ +-----+---++ | a | b |+ +-----+---++ | 0.0 | 2 |+ | 1.0 | 1 |+ | 3.0 | 3 |+ +-----+---++ "+ );++ Ok(())+}++#[tokio::test]+async fn test_fill_nan_uncastable_value() -> Result<()> {+ let df = create_nan_table().await?;++ // The float column "a" is targeted, but "abc" cannot be cast to Float64, so+ // the fill is skipped and column "a" keeps its original NaN value.+ let df_filled = df.fill_nan(&ScalarValue::Utf8(Some("abc".to_string())), &["a"])?;++ let results = df_filled.collect().await?;+ assert_snapshot!(+ batches_to_sort_string(&results),+ @r"+ +-----+---++ | a | b |+ +-----+---++ | 1.0 | 1 |+ | 3.0 | 3 |+ | NaN | 2 |+ +-----+---++ "+ );++ Ok(())+}+#[tokio::test]async fn test_insert_into_casting_support() -> Result<()> {// Testing case1: