Apache DataFusion SQL Query Engine
fix: `DataFrame::select_columns` and `DataFrame::drop_columns` for qualified duplicated field names (#18236)
Khanh Duong9435513e5d65
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex b164b050da..965181b27c 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -310,12 +310,12 @@ impl DataFrame {pub fn select_columns(self, columns: &[&str]) -> Result<DataFrame> {let fields = columns.iter()- .map(|name| {+ .flat_map(|name| {self.plan.schema()- .qualified_field_with_unqualified_name(name)+ .qualified_fields_with_unqualified_name(name)})- .collect::<Result<Vec<_>>>()?;+ .collect::<Vec<_>>();let expr: Vec<Expr> = fields.into_iter().map(|(qualifier, field)| Expr::Column(Column::from((qualifier, field))))@@ -439,13 +439,12 @@ impl DataFrame {pub fn drop_columns(self, columns: &[&str]) -> Result<DataFrame> {let fields_to_drop = columns.iter()- .map(|name| {+ .flat_map(|name| {self.plan.schema()- .qualified_field_with_unqualified_name(name)+ .qualified_fields_with_unqualified_name(name)})- .filter(|r| r.is_ok())- .collect::<Result<Vec<_>>>()?;+ .collect::<Vec<_>>();let expr: Vec<Expr> = self.plan.schema()diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex c35e3b2eb3..2aac1768ac 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -404,6 +404,55 @@ async fn select_with_periods() -> Result<()> {Ok(())}+#[tokio::test]+async fn select_columns_duplicated_names_from_different_qualifiers() -> Result<()> {+ let t1 = test_table_with_name("t1")+ .await?+ .select_columns(&["c1"])?+ .limit(0, Some(3))?;+ let t2 = test_table_with_name("t2")+ .await?+ .select_columns(&["c1"])?+ .limit(3, Some(3))?;+ let t3 = test_table_with_name("t3")+ .await?+ .select_columns(&["c1"])?+ .limit(6, Some(3))?;++ let join_res = t1+ .join(t2, JoinType::Left, &["t1.c1"], &["t2.c1"], None)?+ .join(t3, JoinType::Left, &["t1.c1"], &["t3.c1"], None)?;+ assert_snapshot!(+ batches_to_sort_string(&join_res.clone().collect().await.unwrap()),+ @r"+ +----+----+----++ | c1 | c1 | c1 |+ +----+----+----++ | b | b | |+ | b | b | |+ | c | | |+ | d | | d |+ +----+----+----++ "+ );++ let select_res = join_res.select_columns(&["c1"])?;+ assert_snapshot!(+ batches_to_sort_string(&select_res.clone().collect().await.unwrap()),+ @r"+ +----+----+----++ | c1 | c1 | c1 |+ +----+----+----++ | b | b | |+ | b | b | |+ | c | | |+ | d | | d |+ +----+----+----++ "+ );+ Ok(())+}+#[tokio::test]async fn drop_columns() -> Result<()> {// build plan using Table API@@ -542,6 +591,54 @@ async fn drop_with_periods() -> Result<()> {Ok(())}+#[tokio::test]+async fn drop_columns_duplicated_names_from_different_qualifiers() -> Result<()> {+ let t1 = test_table_with_name("t1")+ .await?+ .select_columns(&["c1"])?+ .limit(0, Some(3))?;+ let t2 = test_table_with_name("t2")+ .await?+ .select_columns(&["c1"])?+ .limit(3, Some(3))?;+ let t3 = test_table_with_name("t3")+ .await?+ .select_columns(&["c1"])?+ .limit(6, Some(3))?;++ let join_res = t1+ .join(t2, JoinType::LeftMark, &["c1"], &["c1"], None)?+ .join(t3, JoinType::LeftMark, &["c1"], &["c1"], None)?;+ assert_snapshot!(+ batches_to_sort_string(&join_res.clone().collect().await.unwrap()),+ @r"+ +----+-------+-------++ | c1 | mark | mark |+ +----+-------+-------++ | b | true | false |+ | c | false | false |+ | d | false | true |+ +----+-------+-------++ "+ );++ let drop_res = join_res.drop_columns(&["mark"])?;+ assert_snapshot!(+ batches_to_sort_string(&drop_res.clone().collect().await.unwrap()),+ @r"+ +----++ | c1 |+ +----++ | b |+ | c |+ | d |+ +----++ "+ );++ Ok(())+}+#[tokio::test]async fn aggregate() -> Result<()> {// build plan using DataFrame API