Apache DataFusion SQL Query Engine
fix: enhance CTE resolution with identifier normalization (#19519)
Siyuan Huang4e1bc79e004b
diff --git a/datafusion/sql/src/resolve.rs b/datafusion/sql/src/resolve.rsindex 148e886161..955dbb8660 100644--- a/datafusion/sql/src/resolve.rs+++ b/datafusion/sql/src/resolve.rs@@ -15,10 +15,12 @@// specific language governing permissions and limitations// under the License.-use crate::TableReference;use std::collections::BTreeSet;use std::ops::ControlFlow;+use datafusion_common::{DataFusionError, Result};++use crate::TableReference;use crate::parser::{CopyToSource, CopyToStatement, Statement as DFStatement};use crate::planner::object_name_to_table_reference;use sqlparser::ast::*;@@ -45,27 +47,40 @@ const INFORMATION_SCHEMA_TABLES: &[&str] = &[PARAMETERS,];+// Collect table/CTE references as `TableReference`s and normalize them during traversal.+// This avoids a second normalization/conversion pass after visiting the AST.struct RelationVisitor {- relations: BTreeSet<ObjectName>,- all_ctes: BTreeSet<ObjectName>,- ctes_in_scope: Vec<ObjectName>,+ relations: BTreeSet<TableReference>,+ all_ctes: BTreeSet<TableReference>,+ ctes_in_scope: Vec<TableReference>,+ enable_ident_normalization: bool,}impl RelationVisitor {/// Record the reference to `relation`, if it's not a CTE reference.- fn insert_relation(&mut self, relation: &ObjectName) {- if !self.relations.contains(relation) && !self.ctes_in_scope.contains(relation) {- self.relations.insert(relation.clone());+ fn insert_relation(&mut self, relation: &ObjectName) -> ControlFlow<DataFusionError> {+ match object_name_to_table_reference(+ relation.clone(),+ self.enable_ident_normalization,+ ) {+ Ok(relation) => {+ if !self.relations.contains(&relation)+ && !self.ctes_in_scope.contains(&relation)+ {+ self.relations.insert(relation);+ }+ ControlFlow::Continue(())+ }+ Err(e) => ControlFlow::Break(e),}}}impl Visitor for RelationVisitor {- type Break = ();+ type Break = DataFusionError;- fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow<()> {- self.insert_relation(relation);- ControlFlow::Continue(())+ fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow<Self::Break> {+ self.insert_relation(relation)}fn pre_visit_query(&mut self, q: &Query) -> ControlFlow<Self::Break> {@@ -78,10 +93,16 @@ impl Visitor for RelationVisitor {if !with.recursive {// This is a bit hackish as the CTE will be visited again as part of visiting `q`,// but thankfully `insert_relation` is idempotent.- let _ = cte.visit(self);+ cte.visit(self)?;+ }+ let cte_name = ObjectName::from(vec![cte.alias.name.clone()]);+ match object_name_to_table_reference(+ cte_name,+ self.enable_ident_normalization,+ ) {+ Ok(cte_ref) => self.ctes_in_scope.push(cte_ref),+ Err(e) => return ControlFlow::Break(e),}- self.ctes_in_scope- .push(ObjectName::from(vec![cte.alias.name.clone()]));}}ControlFlow::Continue(())@@ -97,13 +118,13 @@ impl Visitor for RelationVisitor {ControlFlow::Continue(())}- fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<()> {+ fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<Self::Break> {if let Statement::ShowCreate {obj_type: ShowCreateObject::Table | ShowCreateObject::View,obj_name,} = statement{- self.insert_relation(obj_name)+ self.insert_relation(obj_name)?;}// SHOW statements will later be rewritten into a SELECT from the information_schema@@ -120,35 +141,53 @@ impl Visitor for RelationVisitor {);if requires_information_schema {for s in INFORMATION_SCHEMA_TABLES {- self.relations.insert(ObjectName::from(vec![+ // Information schema references are synthesized here, so convert directly.+ let obj = ObjectName::from(vec![Ident::new(INFORMATION_SCHEMA),Ident::new(*s),- ]));+ ]);+ match object_name_to_table_reference(obj, self.enable_ident_normalization)+ {+ Ok(tbl_ref) => {+ self.relations.insert(tbl_ref);+ }+ Err(e) => return ControlFlow::Break(e),+ }}}ControlFlow::Continue(())}}-fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) {+fn control_flow_to_result(flow: ControlFlow<DataFusionError>) -> Result<()> {+ match flow {+ ControlFlow::Continue(()) => Ok(()),+ ControlFlow::Break(err) => Err(err),+ }+}++fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) -> Result<()> {match statement {DFStatement::Statement(s) => {- let _ = s.as_ref().visit(visitor);+ control_flow_to_result(s.as_ref().visit(visitor))?;}DFStatement::CreateExternalTable(table) => {- visitor.relations.insert(table.name.clone());+ control_flow_to_result(visitor.insert_relation(&table.name))?;}DFStatement::CopyTo(CopyToStatement { source, .. }) => match source {CopyToSource::Relation(table_name) => {- visitor.insert_relation(table_name);+ control_flow_to_result(visitor.insert_relation(table_name))?;}CopyToSource::Query(query) => {- let _ = query.visit(visitor);+ control_flow_to_result(query.visit(visitor))?;}},- DFStatement::Explain(explain) => visit_statement(&explain.statement, visitor),+ DFStatement::Explain(explain) => {+ visit_statement(&explain.statement, visitor)?;+ }DFStatement::Reset(_) => {}}+ Ok(())}/// Collects all tables and views referenced in the SQL statement. CTEs are collected separately.@@ -188,26 +227,20 @@ fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) {pub fn resolve_table_references(statement: &crate::parser::Statement,enable_ident_normalization: bool,-) -> datafusion_common::Result<(Vec<TableReference>, Vec<TableReference>)> {+) -> Result<(Vec<TableReference>, Vec<TableReference>)> {let mut visitor = RelationVisitor {relations: BTreeSet::new(),all_ctes: BTreeSet::new(),ctes_in_scope: vec![],+ enable_ident_normalization,};- visit_statement(statement, &mut visitor);-- let table_refs = visitor- .relations- .into_iter()- .map(|x| object_name_to_table_reference(x, enable_ident_normalization))- .collect::<datafusion_common::Result<_>>()?;- let ctes = visitor- .all_ctes- .into_iter()- .map(|x| object_name_to_table_reference(x, enable_ident_normalization))- .collect::<datafusion_common::Result<_>>()?;- Ok((table_refs, ctes))+ visit_statement(statement, &mut visitor)?;++ Ok((+ visitor.relations.into_iter().collect(),+ visitor.all_ctes.into_iter().collect(),+ ))}#[cfg(test)]@@ -270,4 +303,57 @@ mod tests {assert_eq!(ctes.len(), 1);assert_eq!(ctes[0].to_string(), "nodes");}++ #[test]+ fn resolve_table_references_cte_with_quoted_reference() {+ use crate::parser::DFParser;++ let query = r#"with barbaz as (select 1) select * from "barbaz""#;+ let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();+ let (table_refs, ctes) = resolve_table_references(&statement, true).unwrap();+ assert_eq!(ctes.len(), 1);+ assert_eq!(ctes[0].to_string(), "barbaz");+ // Quoted reference should still resolve to the CTE when normalization is on+ assert_eq!(table_refs.len(), 0);+ }++ #[test]+ fn resolve_table_references_cte_with_quoted_reference_normalization_off() {+ use crate::parser::DFParser;++ let query = r#"with barbaz as (select 1) select * from "barbaz""#;+ let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();+ let (table_refs, ctes) = resolve_table_references(&statement, false).unwrap();+ assert_eq!(ctes.len(), 1);+ assert_eq!(ctes[0].to_string(), "barbaz");+ // Even with normalization off, quoted reference matches same-case CTE name+ assert_eq!(table_refs.len(), 0);+ }++ #[test]+ fn resolve_table_references_cte_with_quoted_reference_uppercase_normalization_on() {+ use crate::parser::DFParser;++ let query = r#"with FOObar as (select 1) select * from "FOObar""#;+ let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();+ let (table_refs, ctes) = resolve_table_references(&statement, true).unwrap();+ // CTE name is normalized to lowercase, quoted reference preserves case, so they differ+ assert_eq!(ctes.len(), 1);+ assert_eq!(ctes[0].to_string(), "foobar");+ assert_eq!(table_refs.len(), 1);+ assert_eq!(table_refs[0].to_string(), "FOObar");+ }++ #[test]+ fn resolve_table_references_cte_with_quoted_reference_uppercase_normalization_off() {+ use crate::parser::DFParser;++ let query = r#"with FOObar as (select 1) select * from "FOObar""#;+ let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap();+ let (table_refs, ctes) = resolve_table_references(&statement, false).unwrap();+ // Without normalization, cases match exactly, so quoted reference resolves to the CTE+ assert_eq!(ctes.len(), 1);+ assert_eq!(ctes[0].to_string(), "FOObar");+ assert_eq!(table_refs.len(), 0);+ }}diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 9ec085b41e..a9aa3baa24 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -30,7 +30,7 @@ use arrow::buffer::ScalarBuffer;use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit, UnionFields};use arrow::record_batch::RecordBatch;use datafusion::catalog::{- CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, Session,+ CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider, Session,};use datafusion::common::{DataFusionError, Result, not_impl_err};use datafusion::functions::math::abs;@@ -96,6 +96,10 @@ impl TestContext {let file_name = relative_path.file_name().unwrap().to_str().unwrap();match file_name {+ "cte_quoted_reference.slt" => {+ info!("Registering strict catalog provider for CTE tests");+ register_strict_orders_catalog(test_ctx.session_ctx());+ }"information_schema_table_types.slt" => {info!("Registering local temporary table");register_temp_table(test_ctx.session_ctx()).await;@@ -171,6 +175,104 @@ impl TestContext {}}+// ==============================================================================+// Strict Catalog / Schema Provider (sqllogictest-only)+// ==============================================================================+//+// The goal of `cte_quoted_reference.slt` is to exercise end-to-end query planning+// while detecting *unexpected* catalog lookups.+//+// Specifically, if DataFusion incorrectly treats a CTE reference (e.g. `"barbaz"`)+// as a real table reference, the planner will attempt to resolve it through the+// schema provider. The types below deliberately `panic!` on any lookup other than+// the one table we expect (`orders`).+//+// This makes the "extra provider lookup" bug observable in an end-to-end test,+// rather than being silently ignored by default providers that return `Ok(None)`+// for unknown tables.++#[derive(Debug)]+struct StrictOrdersCatalog {+ schema: Arc<dyn SchemaProvider>,+}++impl CatalogProvider for StrictOrdersCatalog {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema_names(&self) -> Vec<String> {+ vec!["public".to_string()]+ }++ fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {+ (name == "public").then(|| Arc::clone(&self.schema))+ }+}++#[derive(Debug)]+struct StrictOrdersSchema {+ orders: Arc<dyn TableProvider>,+}++#[async_trait]+impl SchemaProvider for StrictOrdersSchema {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn table_names(&self) -> Vec<String> {+ vec!["orders".to_string()]+ }++ async fn table(+ &self,+ name: &str,+ ) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {+ match name {+ "orders" => Ok(Some(Arc::clone(&self.orders))),+ other => panic!(+ "unexpected table lookup: {other}. This maybe indicates a CTE reference was \+ incorrectly treated as a catalog table reference."+ ),+ }+ }++ fn table_exist(&self, name: &str) -> bool {+ name == "orders"+ }+}++fn register_strict_orders_catalog(ctx: &SessionContext) {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "order_id",+ DataType::Int32,+ false,+ )]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![1, 2]))],+ )+ .expect("record batch should be valid");++ let orders =+ MemTable::try_new(schema, vec![vec![batch]]).expect("memtable should be valid");++ let schema_provider: Arc<dyn SchemaProvider> = Arc::new(StrictOrdersSchema {+ orders: Arc::new(orders),+ });++ // Override the default "datafusion" catalog for this test file so that any+ // unexpected lookup is caught immediately.+ ctx.register_catalog(+ "datafusion",+ Arc::new(StrictOrdersCatalog {+ schema: schema_provider,+ }),+ );+}+#[cfg(feature = "avro")]pub async fn register_avro_tables(ctx: &mut TestContext) {use datafusion::prelude::AvroReadOptions;diff --git a/datafusion/sqllogictest/test_files/cte_quoted_reference.slt b/datafusion/sqllogictest/test_files/cte_quoted_reference.sltnew file mode 100644index 0000000000..6142157e5e--- /dev/null+++ b/datafusion/sqllogictest/test_files/cte_quoted_reference.slt@@ -0,0 +1,70 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+## CTE Reference Resolution+##########++# These tests exercise CTE reference resolution with and without identifier+# normalization. The session is configured with a strict catalog/schema provider+# (see `datafusion/sqllogictest/src/test_context.rs`) that only provides the+# `orders` table and panics on any unexpected table lookup.+#+# This makes it observable if DataFusion incorrectly treats a CTE reference as a+# catalog lookup.+#+# Refs: https://github.com/apache/datafusion/issues/18932+#+# NOTE: This test relies on a strict catalog/schema provider registered in+# `datafusion/sqllogictest/src/test_context.rs` that provides only the `orders`+# table and panics on unexpected lookups.++statement ok+set datafusion.sql_parser.enable_ident_normalization = true;++query I+with barbaz as (select * from orders) select * from "barbaz";+----+1+2++query I+with BarBaz as (select * from orders) select * from "barbaz";+----+1+2++query I+with barbaz as (select * from orders) select * from barbaz;+----+1+2++statement ok+set datafusion.sql_parser.enable_ident_normalization = false;++query I+with barbaz as (select * from orders) select * from "barbaz";+----+1+2++query I+with barbaz as (select * from orders) select * from barbaz;+----+1+2