Skip to content

Apache DataFusion SQL Query Engine

fix: Preserve column identifiers in DataFrame fill methods (#25857)

limadog97c04517bea24
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
index bce0ab44f8..1a06487399 100644
--- a/datafusion/core/src/dataframe/mod.rs
+++ b/datafusion/core/src/dataframe/mod.rs
@@ -2804,19 +2804,19 @@ impl DataFrame {
let projections = self
.logical_plan()
.schema()
- .fields()
.iter()
- .map(|field| {
+ .map(|(qualifier, field)| {
+ let column = Expr::Column(Column::from((qualifier, field)));
if cols.contains(field) && applies(field) {
// Try to cast fill value to column type. If the cast fails, fallback to the original column.
match value.clone().cast_to(field.data_type()) {
- Ok(fill_value) => func
- .call(vec![col(field.name()), lit(fill_value)])
- .alias(field.name()),
- Err(_) => col(field.name()),
+ Ok(fill_value) => {
+ func.call(vec![column, lit(fill_value)]).alias(field.name())
+ }
+ Err(_) => column,
}
} else {
- col(field.name())
+ column
}
})
.collect::<Vec<_>>();
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 4bbf6e2bcc..791fd73ff2 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -7236,6 +7236,132 @@ async fn test_fill_nan_uncastable_value() -> Result<()> {
Ok(())
}
+#[rstest]
+#[tokio::test]
+async fn test_fill_special_column_names(
+ #[values(false, true)] fill_nan: bool,
+ #[values("Name", "a.b")] name: &str,
+ #[values(false, true)] uncastable: bool,
+) -> Result<()> {
+ let fill = if fill_nan {
+ DataFrame::fill_nan
+ } else {
+ DataFrame::fill_null
+ };
+ let missing = if fill_nan { Some(f64::NAN) } else { None };
+ let other_name = if name == "Name" { "a.b" } else { "Name" };
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(name, DataType::Float64, true),
+ Field::new(other_name, DataType::Utf8, true),
+ ]));
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Float64Array::from(vec![Some(1.0), missing, Some(3.0)])),
+ Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])),
+ ],
+ )?;
+ let ctx = SessionContext::new();
+ let df = ctx.read_batch(batch.clone())?;
+ let value = if uncastable {
+ ScalarValue::from("abc")
+ } else {
+ ScalarValue::from(0.0)
+ };
+
+ let results = fill(&df, &value, &[name])?.collect().await?;
+ let expected = if uncastable {
+ // An uncastable replacement leaves the selected column unchanged.
+ batch
+ } else {
+ RecordBatch::try_new(
+ batch.schema(),
+ vec![
+ Arc::new(Float64Array::from(vec![1.0, 0.0, 3.0])),
+ Arc::clone(batch.column(1)),
+ ],
+ )?
+ };
+ // Check names and types directly without requiring unchanged nullability.
+ for result in &results {
+ let schema = result.schema();
+ assert_eq!(
+ schema
+ .fields()
+ .iter()
+ .map(|field| (field.name().as_str(), field.data_type()))
+ .collect::<Vec<_>>(),
+ vec![(name, &DataType::Float64), (other_name, &DataType::Utf8)]
+ );
+ }
+ assert_eq!(batches_to_string(&results), batches_to_string(&[expected]));
+ Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn test_fill_preserves_qualified_columns(
+ #[values(false, true)] fill_nan: bool,
+) -> Result<()> {
+ let fill = if fill_nan {
+ DataFrame::fill_nan
+ } else {
+ DataFrame::fill_null
+ };
+ let missing = if fill_nan { Some(f64::NAN) } else { None };
+ let ctx = SessionContext::new();
+ let left = ctx
+ .read_batch(record_batch!(
+ ("id", Int32, [1, 2]),
+ ("value", Float64, [missing, Some(3.0)])
+ )?)?
+ .alias("t1")?;
+ let right = ctx
+ .read_batch(record_batch!(("id", Int32, [1, 2]))?)?
+ .alias("t2")?;
+ let joined = left.join(right, JoinType::Inner, &["t1.id"], &["t2.id"], None)?;
+ let filled = fill(&joined, &ScalarValue::from(0.0), &["value"])?;
+
+ assert_eq!(
+ filled.schema().qualified_field(0).0,
+ Some(&TableReference::bare("t1"))
+ );
+ assert_eq!(
+ filled.schema().qualified_field(2).0,
+ Some(&TableReference::bare("t2"))
+ );
+ let results = filled.collect().await?;
+ for result in &results {
+ let schema = result.schema();
+ assert_eq!(
+ schema
+ .fields()
+ .iter()
+ .map(|field| (field.name().as_str(), field.data_type()))
+ .collect::<Vec<_>>(),
+ vec![
+ ("id", &DataType::Int32),
+ ("value", &DataType::Float64),
+ ("id", &DataType::Int32),
+ ]
+ );
+ }
+ insta::allow_duplicates! {
+ assert_snapshot!(
+ batches_to_sort_string(&results),
+ @r"
+ +----+-------+----+
+ | id | value | id |
+ +----+-------+----+
+ | 1 | 0.0 | 1 |
+ | 2 | 3.0 | 2 |
+ +----+-------+----+
+ "
+ );
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn test_insert_into_casting_support() -> Result<()> {
// Testing case1: