Skip to content

Apache DataFusion SQL Query Engine

fix: `DataFrame::select_columns` and `DataFrame::drop_columns` for qualified duplicated field names (#18236)

Khanh Duong9435513e5d65
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
index b164b050da..965181b27c 100644
--- a/datafusion/core/src/dataframe/mod.rs
+++ b/datafusion/core/src/dataframe/mod.rs
@@ -310,12 +310,12 @@ impl DataFrame {
pub fn select_columns(self, columns: &[&str]) -> Result<DataFrame> {
let fields = columns
.iter()
- .map(|name| {
+ .flat_map(|name| {
self.plan
.schema()
- .qualified_field_with_unqualified_name(name)
+ .qualified_fields_with_unqualified_name(name)
})
- .collect::<Result<Vec<_>>>()?;
+ .collect::<Vec<_>>();
let expr: Vec<Expr> = fields
.into_iter()
.map(|(qualifier, field)| Expr::Column(Column::from((qualifier, field))))
@@ -439,13 +439,12 @@ impl DataFrame {
pub fn drop_columns(self, columns: &[&str]) -> Result<DataFrame> {
let fields_to_drop = columns
.iter()
- .map(|name| {
+ .flat_map(|name| {
self.plan
.schema()
- .qualified_field_with_unqualified_name(name)
+ .qualified_fields_with_unqualified_name(name)
})
- .filter(|r| r.is_ok())
- .collect::<Result<Vec<_>>>()?;
+ .collect::<Vec<_>>();
let expr: Vec<Expr> = self
.plan
.schema()
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index c35e3b2eb3..2aac1768ac 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -404,6 +404,55 @@ async fn select_with_periods() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn select_columns_duplicated_names_from_different_qualifiers() -> Result<()> {
+ let t1 = test_table_with_name("t1")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(0, Some(3))?;
+ let t2 = test_table_with_name("t2")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(3, Some(3))?;
+ let t3 = test_table_with_name("t3")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(6, Some(3))?;
+
+ let join_res = t1
+ .join(t2, JoinType::Left, &["t1.c1"], &["t2.c1"], None)?
+ .join(t3, JoinType::Left, &["t1.c1"], &["t3.c1"], None)?;
+ assert_snapshot!(
+ batches_to_sort_string(&join_res.clone().collect().await.unwrap()),
+ @r"
+ +----+----+----+
+ | c1 | c1 | c1 |
+ +----+----+----+
+ | b | b | |
+ | b | b | |
+ | c | | |
+ | d | | d |
+ +----+----+----+
+ "
+ );
+
+ let select_res = join_res.select_columns(&["c1"])?;
+ assert_snapshot!(
+ batches_to_sort_string(&select_res.clone().collect().await.unwrap()),
+ @r"
+ +----+----+----+
+ | c1 | c1 | c1 |
+ +----+----+----+
+ | b | b | |
+ | b | b | |
+ | c | | |
+ | d | | d |
+ +----+----+----+
+ "
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn drop_columns() -> Result<()> {
// build plan using Table API
@@ -542,6 +591,54 @@ async fn drop_with_periods() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn drop_columns_duplicated_names_from_different_qualifiers() -> Result<()> {
+ let t1 = test_table_with_name("t1")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(0, Some(3))?;
+ let t2 = test_table_with_name("t2")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(3, Some(3))?;
+ let t3 = test_table_with_name("t3")
+ .await?
+ .select_columns(&["c1"])?
+ .limit(6, Some(3))?;
+
+ let join_res = t1
+ .join(t2, JoinType::LeftMark, &["c1"], &["c1"], None)?
+ .join(t3, JoinType::LeftMark, &["c1"], &["c1"], None)?;
+ assert_snapshot!(
+ batches_to_sort_string(&join_res.clone().collect().await.unwrap()),
+ @r"
+ +----+-------+-------+
+ | c1 | mark | mark |
+ +----+-------+-------+
+ | b | true | false |
+ | c | false | false |
+ | d | false | true |
+ +----+-------+-------+
+ "
+ );
+
+ let drop_res = join_res.drop_columns(&["mark"])?;
+ assert_snapshot!(
+ batches_to_sort_string(&drop_res.clone().collect().await.unwrap()),
+ @r"
+ +----+
+ | c1 |
+ +----+
+ | b |
+ | c |
+ | d |
+ +----+
+ "
+ );
+
+ Ok(())
+}
+
#[tokio::test]
async fn aggregate() -> Result<()> {
// build plan using DataFrame API