Apache DataFusion SQL Query Engine
fix(sql): reject duplicate unqualified names in CTAS, CREATE VIEW, and SELECT INTO (#22290)
Kumar Ujjawale960c4b88553
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 72d6c4c3a6..5770c20262 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -67,10 +67,10 @@ use datafusion_catalog::{use datafusion_common::config::{ConfigField, ConfigOptions};use datafusion_common::metadata::ScalarAndMetadata;use datafusion_common::{- DFSchema, DataFusionError, ParamValues, SchemaReference, TableReference,+ DFSchema, DataFusionError, ParamValues, SchemaError, SchemaReference, TableReference,config::{ConfigExtension, TableOptions},exec_datafusion_err, exec_err, internal_datafusion_err, not_impl_err,- plan_datafusion_err, plan_err,+ plan_datafusion_err, plan_err, schema_err,tree_node::{TreeNodeRecursion, TreeNodeVisitor},};pub use datafusion_execution::TaskContext;@@ -887,6 +887,7 @@ impl SessionContext {match (if_not_exists, or_replace, table) {(true, false, Ok(_)) => self.return_empty_dataframe(),(false, true, Ok(_)) => {+ Self::ensure_unique_column_names(input.schema())?;self.deregister_table(name.clone())?;let schema = Arc::clone(input.schema().inner());let physical = DataFrame::new(self.state(), input);@@ -906,6 +907,7 @@ impl SessionContext {exec_err!("'IF NOT EXISTS' cannot coexist with 'REPLACE'")}(_, _, Err(_)) => {+ Self::ensure_unique_column_names(input.schema())?;let schema = Arc::clone(input.schema().inner());let physical = DataFrame::new(self.state(), input);@@ -951,14 +953,16 @@ impl SessionContext {match (or_replace, view) {(true, Ok(_)) => {- self.deregister_table(name.clone())?;let input = Self::apply_type_coercion(Arc::unwrap_or_clone(input))?;+ Self::ensure_unique_column_names(input.schema())?;+ self.deregister_table(name.clone())?;let table = Arc::new(ViewTable::new(input, definition));self.register_table(name, table)?;self.return_empty_dataframe()}(_, Err(_)) => {let input = Self::apply_type_coercion(Arc::unwrap_or_clone(input))?;+ Self::ensure_unique_column_names(input.schema())?;let table = Arc::new(ViewTable::new(input, definition));self.register_table(name, table)?;self.return_empty_dataframe()@@ -967,6 +971,21 @@ impl SessionContext {}}+ fn ensure_unique_column_names(schema: &DFSchema) -> Result<()> {+ // DFSchema name checks allow duplicate unqualified names as long as their+ // qualifiers differ. DDL persistence drops qualifiers, so this helper must+ // enforce uniqueness on the final unqualified names instead.+ let mut seen = HashSet::with_capacity(schema.fields().len());+ for field in schema.fields() {+ if !seen.insert(field.name().as_str()) {+ return schema_err!(SchemaError::DuplicateUnqualifiedField {+ name: field.name().to_string(),+ });+ }+ }+ Ok(())+ }+async fn create_catalog_schema(&self, cmd: CreateCatalogSchema) -> Result<DataFrame> {let CreateCatalogSchema {schema_name,diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex d1dc964da8..e55a373ada 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -75,10 +75,11 @@ use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_expr::expr::{GroupingSet, NullTreatment, Sort, WindowFunction};use datafusion_expr::var_provider::{VarProvider, VarType};use datafusion_expr::{- Expr, ExprFunctionExt, ExprSchemable, LogicalPlan, LogicalPlanBuilder,- ScalarFunctionImplementation, SortExpr, TableType, WindowFrame, WindowFrameBound,- WindowFrameUnits, WindowFunctionDefinition, cast, col, create_udf, exists,- in_subquery, lit, out_ref_col, placeholder, scalar_subquery, when, wildcard,+ CreateMemoryTable, CreateView, DdlStatement, Expr, ExprFunctionExt, ExprSchemable,+ LogicalPlan, LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, TableType,+ WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, cast, col,+ create_udf, exists, in_subquery, lit, out_ref_col, placeholder, scalar_subquery,+ when, wildcard,};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::aggregate::AggregateExprBuilder;@@ -6750,6 +6751,169 @@ async fn test_copy_to_preserves_order() -> Result<()> {Ok(())}+fn duplicate_unqualified_name_batch() -> Result<RecordBatch> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Int32, false),+ ]));++ RecordBatch::try_new(+ schema,+ vec![+ Arc::new(StringArray::from(vec![+ "abcDEF",+ "abc123",+ "CBAdef",+ "123AbcDef",+ ])),+ Arc::new(Int32Array::from(vec![1, 10, 10, 100])),+ ],+ )+ .map_err(Into::into)+}++async fn duplicate_unqualified_name_input(+ ctx: &SessionContext,+ name: &str,+) -> Result<LogicalPlan> {+ ctx.register_batch(name, duplicate_unqualified_name_batch()?)?;++ let left = ctx.table(name).await?;+ let right = left.clone().alias("t2")?;+ Ok(left+ .join(right, JoinType::Inner, &["b"], &["b"], None)?+ .into_unoptimized_plan())+}++#[tokio::test]+async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_create_memory_table()+-> Result<()> {+ let ctx = SessionContext::new();+ let input = duplicate_unqualified_name_input(&ctx, "t1").await?;++ let plan = LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(CreateMemoryTable {+ name: TableReference::bare("dup_out"),+ constraints: Constraints::default(),+ input: Arc::new(input),+ if_not_exists: false,+ or_replace: false,+ temporary: false,+ column_defaults: vec![],+ }));++ let err = ctx.execute_logical_plan(plan).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Schema contains duplicate unqualified field name a"+ );+ assert!(ctx.table("dup_out").await.is_err());++ Ok(())+}++#[tokio::test]+async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_table()+-> Result<()> {+ let ctx = SessionContext::new();+ ctx.sql("CREATE TABLE dup_out AS VALUES ('seed', 0)")+ .await?+ .collect()+ .await?;++ let input = duplicate_unqualified_name_input(&ctx, "t1").await?;+ let plan = LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(CreateMemoryTable {+ name: TableReference::bare("dup_out"),+ constraints: Constraints::default(),+ input: Arc::new(input),+ if_not_exists: false,+ or_replace: true,+ temporary: false,+ column_defaults: vec![],+ }));++ let err = ctx.execute_logical_plan(plan).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Schema contains duplicate unqualified field name a"+ );++ let rows = ctx.sql("SELECT * FROM dup_out").await?.collect().await?;+ assert_batches_eq!(+ [+ "+---------+---------+",+ "| column1 | column2 |",+ "+---------+---------+",+ "| seed | 0 |",+ "+---------+---------+",+ ],+ &rows+ );++ Ok(())+}++#[tokio::test]+async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_create_view()+-> Result<()> {+ let ctx = SessionContext::new();+ let input = duplicate_unqualified_name_input(&ctx, "t1").await?;+ let plan = LogicalPlan::Ddl(DdlStatement::CreateView(CreateView {+ name: TableReference::bare("dup_view"),+ input: Arc::new(input),+ or_replace: false,+ definition: None,+ temporary: false,+ }));++ let err = ctx.execute_logical_plan(plan).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Schema contains duplicate unqualified field name a"+ );+ assert!(ctx.table("dup_view").await.is_err());++ Ok(())+}++#[tokio::test]+async fn execute_logical_plan_rejects_duplicate_unqualified_names_in_replace_view()+-> Result<()> {+ let ctx = SessionContext::new();+ ctx.sql("CREATE VIEW dup_view AS SELECT 'seed' AS column1, 0 AS column2")+ .await?+ .collect()+ .await?;++ let input = duplicate_unqualified_name_input(&ctx, "t1").await?;+ let plan = LogicalPlan::Ddl(DdlStatement::CreateView(CreateView {+ name: TableReference::bare("dup_view"),+ input: Arc::new(input),+ or_replace: true,+ definition: None,+ temporary: false,+ }));++ let err = ctx.execute_logical_plan(plan).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Schema contains duplicate unqualified field name a"+ );++ let rows = ctx.sql("SELECT * FROM dup_view").await?.collect().await?;+ assert_batches_eq!(+ [+ "+---------+---------+",+ "| column1 | column2 |",+ "+---------+---------+",+ "| seed | 0 |",+ "+---------+---------+",+ ],+ &rows+ );++ Ok(())+}+#[tokio::test]async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> {let ctx = SessionContext::new();diff --git a/datafusion/sqllogictest/test_files/ddl.slt b/datafusion/sqllogictest/test_files/ddl.sltindex 977d2d03a1..82c30e9aba 100644--- a/datafusion/sqllogictest/test_files/ddl.slt+++ b/datafusion/sqllogictest/test_files/ddl.slt@@ -971,3 +971,23 @@ drop view v;statement count 0drop table t;++# https://github.com/apache/datafusion/issues/22167+# https://github.com/apache/datafusion/issues/13487+# CREATE TABLE AS / CREATE VIEW must reject queries whose result schema+# has duplicate (unqualified) column names, since the resulting object+# stores only unqualified names and would otherwise be unselectable.+statement ok+CREATE TABLE dup_src AS VALUES(1, 2);++statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1+CREATE TABLE dup_ctas AS SELECT * FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2;++statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1+CREATE VIEW dup_view AS SELECT * FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2;++statement error DataFusion error: Schema error: Schema contains duplicate unqualified field name column1+SELECT * INTO dup_into FROM dup_src LEFT JOIN dup_src y ON dup_src.column1 = y.column2;++statement ok+DROP TABLE dup_src;