Apache DataFusion SQL Query Engine
feat(sql): expose CTE scope to relation planners (#24755)
Geoffrey Claude565f933d6fc1
diff --git a/datafusion/core/tests/user_defined/relation_planner.rs b/datafusion/core/tests/user_defined/relation_planner.rsindex 54af53ad85..cc1bbb71b0 100644--- a/datafusion/core/tests/user_defined/relation_planner.rs+++ b/datafusion/core/tests/user_defined/relation_planner.rs@@ -30,7 +30,7 @@ use datafusion_expr::logical_plan::builder::LogicalPlanBuilder;use datafusion_expr::planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,};-use datafusion_sql::sqlparser::ast::TableFactor;+use datafusion_sql::sqlparser::ast::{TableAlias, TableFactor};use insta::assert_snapshot;// ============================================================================@@ -51,29 +51,54 @@ use insta::assert_snapshot;/// example planners below.fn plan_static_values_table(relation: TableFactor,+ context: &dyn RelationPlannerContext,table_name: &str,column_name: &str,values: Vec<ScalarValue>,) -> Result<RelationPlanning> {- match relation {+ let alias = match &relation {TableFactor::Table { name, alias, .. }if name.to_string().eq_ignore_ascii_case(table_name) =>{- let rows = values- .into_iter()- .map(|v| vec![Expr::Literal(v, None)])- .collect::<Vec<_>>();-- let plan = LogicalPlanBuilder::values(rows)?- .project(vec![col("column1").alias(column_name)])?- .build()?;-- Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new(- plan, alias,- ))))+ alias.clone()+ }+ _ => return Ok(RelationPlanning::Original(Box::new(relation))),+ };++ if let TableFactor::Table {+ name, args: None, ..+ } = &relation+ {+ let table_ref = context.object_name_to_table_reference(name.clone())?;++ // RelationPlanner extensions run before DataFusion resolves CTEs. A+ // name-based planner should defer when a CTE with that name is visible,+ // so SQL's normal lexical CTE precedence is preserved.+ if context.get_cte(&table_ref).is_some() {+ return Ok(RelationPlanning::Original(Box::new(relation)));}- other => Ok(RelationPlanning::Original(Box::new(other))),}++ plan_static_values(alias, column_name, values)+}++fn plan_static_values(+ alias: Option<TableAlias>,+ column_name: &str,+ values: Vec<ScalarValue>,+) -> Result<RelationPlanning> {+ let rows = values+ .into_iter()+ .map(|v| vec![Expr::Literal(v, None)])+ .collect::<Vec<_>>();++ let plan = LogicalPlanBuilder::values(rows)?+ .project(vec![col("column1").alias(column_name)])?+ .build()?;++ Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new(+ plan, alias,+ ))))}/// Example planner that provides a virtual `numbers` table with values@@ -85,10 +110,11 @@ impl RelationPlanner for NumbersPlanner {fn plan_relation(&self,relation: TableFactor,- _context: &mut dyn RelationPlannerContext,+ context: &mut dyn RelationPlannerContext,) -> Result<RelationPlanning> {plan_static_values_table(relation,+ context,"numbers","number",vec![@@ -100,6 +126,28 @@ impl RelationPlanner for NumbersPlanner {}}+/// Example planner for a qualified virtual relation. This is kept separate+/// from `NumbersPlanner` so CTE tests can verify that extension lookup matches+/// DataFusion's existing qualified-name resolution.+#[derive(Debug)]+struct QualifiedNumbersPlanner;++impl RelationPlanner for QualifiedNumbersPlanner {+ fn plan_relation(+ &self,+ relation: TableFactor,+ context: &mut dyn RelationPlannerContext,+ ) -> Result<RelationPlanning> {+ plan_static_values_table(+ relation,+ context,+ "foo.bar",+ "number",+ vec![ScalarValue::Int64(Some(7))],+ )+ }+}+/// Example planner that provides a virtual `colors` table with three string/// values: `red`, `green`, `blue`.#[derive(Debug)]@@ -109,10 +157,11 @@ impl RelationPlanner for ColorsPlanner {fn plan_relation(&self,relation: TableFactor,- _context: &mut dyn RelationPlannerContext,+ context: &mut dyn RelationPlannerContext,) -> Result<RelationPlanning> {plan_static_values_table(relation,+ context,"colors","color",vec![@@ -133,10 +182,11 @@ impl RelationPlanner for AlternativeNumbersPlanner {fn plan_relation(&self,relation: TableFactor,- _context: &mut dyn RelationPlannerContext,+ context: &mut dyn RelationPlannerContext,) -> Result<RelationPlanning> {plan_static_values_table(relation,+ context,"numbers","number",vec![ScalarValue::Int64(Some(100)), ScalarValue::Int64(Some(200))],@@ -144,6 +194,28 @@ impl RelationPlanner for AlternativeNumbersPlanner {}}+/// Example planner with a deliberately reserved `numbers` name. Unlike the+/// ordinary name-based planners above, it intentionally does not defer to CTEs.+#[derive(Debug)]+struct ReservedNumbersPlanner;++impl RelationPlanner for ReservedNumbersPlanner {+ fn plan_relation(+ &self,+ relation: TableFactor,+ _context: &mut dyn RelationPlannerContext,+ ) -> Result<RelationPlanning> {+ match relation {+ TableFactor::Table { name, alias, .. }+ if name.to_string().eq_ignore_ascii_case("numbers") =>+ {+ plan_static_values(alias, "number", vec![ScalarValue::Int64(Some(999))])+ }+ other => Ok(RelationPlanning::Original(Box::new(other))),+ }+ }+}+/// Example planner that intercepts nested joins and samples both sides (limit 2)/// before joining, demonstrating recursive planning with `context.plan()`.#[derive(Debug)]@@ -357,6 +429,145 @@ mod tests {");}+ // A name-based relation planner can defer to a visible CTE, preserving the+ // same shadowing behavior as a catalog table with that name.+ #[tokio::test]+ async fn cte_takes_precedence_over_virtual_table() {+ let ctx = ctx_with_numbers();++ let result = execute_sql_to_string(+ &ctx,+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers",+ )+ .await;++ assert_snapshot!(result, @r"+ +--------++ | number |+ +--------++ | 42 |+ +--------++ ");+ }++ // A derived query inherits CTEs from its outer query, so a name-based+ // relation planner still defers to the visible CTE in the nested scope.+ #[tokio::test]+ async fn derived_query_inherits_outer_cte() {+ let ctx = ctx_with_numbers();++ let result = execute_sql_to_string(+ &ctx,+ "WITH numbers AS (SELECT 42 AS number) \+ SELECT * FROM (SELECT * FROM numbers) nested",+ )+ .await;++ assert_snapshot!(result, @r"+ +--------++ | number |+ +--------++ | 42 |+ +--------++ ");+ }++ // Deferring a visible CTE continues through the planner chain. A later+ // planner with a deliberately reserved name can still intercept it.+ #[tokio::test]+ async fn cte_defer_still_allows_reserved_planner() {+ let ctx = SessionContext::new()+ .with_planner(ReservedNumbersPlanner)+ .with_planner(NumbersPlanner);++ let result = execute_sql_to_string(+ &ctx,+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers",+ )+ .await;++ assert_snapshot!(result, @r"+ +--------++ | number |+ +--------++ | 999 |+ +--------++ ");+ }++ // The extension lookup follows DataFusion's existing name matching. Both+ // the extension and the default planner therefore resolve the same CTE for+ // a quoted dotted name and a qualified reference with the same rendering.+ #[tokio::test]+ async fn cte_lookup_matches_default_qualified_name_resolution() {+ let ctx = SessionContext::new().with_planner(QualifiedNumbersPlanner);++ let result = execute_sql_to_string(+ &ctx,+ "WITH \"foo.bar\" AS (SELECT 42 AS number) SELECT * FROM foo.bar",+ )+ .await;++ assert_snapshot!(result, @r"+ +--------++ | number |+ +--------++ | 42 |+ +--------++ ");+ }++ // A CTE shadows an ordinary table reference, not a same-named table+ // function call with arguments.+ #[tokio::test]+ async fn cte_does_not_shadow_same_named_table_function() {+ let ctx = ctx_with_numbers();++ let result = execute_sql_to_string(+ &ctx,+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers(1)",+ )+ .await;++ assert_snapshot!(result, @r"+ +--------++ | number |+ +--------++ | 1 |+ | 2 |+ | 3 |+ +--------++ ");+ }++ // CTE visibility follows the query's lexical scope: the nested CTE wins+ // inside the derived table but does not hide the outer virtual relation.+ #[tokio::test]+ async fn cte_visibility_is_scoped_to_nested_query() {+ let ctx = ctx_with_numbers();++ let result = execute_sql_to_string(+ &ctx,+ "SELECT 'cte' AS source, number \+ FROM (WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers) nested \+ UNION ALL \+ SELECT 'virtual' AS source, number FROM numbers \+ ORDER BY source, number",+ )+ .await;++ assert_snapshot!(result, @r"+ +---------+--------++ | source | number |+ +---------+--------++ | cte | 42 |+ | virtual | 1 |+ | virtual | 2 |+ | virtual | 3 |+ +---------+--------++ ");+ }+// Virtual table supports standard SQL operations (projection, filter, aggregation).#[tokio::test]async fn virtual_table_filters_and_aggregation() {diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rsindex 4009dfb3e9..31aad67d17 100644--- a/datafusion/expr/src/planner.rs+++ b/datafusion/expr/src/planner.rs@@ -430,6 +430,27 @@ pub trait RelationPlannerContext {/// from the first registered relation planner.fn plan(&mut self, relation: TableFactor) -> Result<LogicalPlan>;+ /// Returns the common table expression (CTE) visible for the specified+ /// table reference in the current query scope, if any.+ ///+ /// Relation planners run before DataFusion's built-in relation planning. A+ /// planner that recognizes ordinary tables by name can use this method to+ /// pass a same-named relation to the remaining planners. When every+ /// ordinary name-based planner does so, built-in CTE resolution is+ /// preserved. Use [`Self::object_name_to_table_reference`] to normalize an+ /// [`ObjectName`] before looking it up.+ ///+ /// The lookup follows the same string-key matching as DataFusion's built-in+ /// relation planning, including for qualified table references. Different+ /// [`TableReference`] variants with the same string representation therefore+ /// use the same CTE lookup key.+ ///+ /// The default implementation returns `None` for compatibility with custom+ /// implementations of this context.+ fn get_cte(&self, _name: &TableReference) -> Option<&LogicalPlan> {+ None+ }+/// Converts a SQL expression into a logical expression using the current/// planner context.fn sql_to_expr(&mut self, expr: SQLExpr, schema: &DFSchema) -> Result<Expr>;diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rsindex 8549bd833e..beb6acd422 100644--- a/datafusion/sql/src/relation/mod.rs+++ b/datafusion/sql/src/relation/mod.rs@@ -38,6 +38,15 @@ struct SqlToRelRelationContext<'a, 'b, S: ContextProvider> {planner_context: &'a mut PlannerContext,}+/// Look up a CTE using the same string key as DataFusion's built-in relation+/// planning.+fn get_cte_for_table_reference<'a>(+ planner_context: &'a PlannerContext,+ name: &TableReference,+) -> Option<&'a LogicalPlan> {+ planner_context.get_cte(&name.to_string())+}+// Implement RelationPlannerContextimpl<S: ContextProvider> RelationPlannerContext for SqlToRelRelationContext<'_, '_, S> {fn context_provider(&self) -> &dyn ContextProvider {@@ -48,6 +57,10 @@ impl<S: ContextProvider> RelationPlannerContext for SqlToRelRelationContext<'_,self.planner.create_relation(relation, self.planner_context)}+ fn get_cte(&self, name: &TableReference) -> Option<&LogicalPlan> {+ get_cte_for_table_reference(self.planner_context, name)+ }+fn sql_to_expr(&mut self,expr: sqlparser::ast::Expr,@@ -184,8 +197,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {} else {// Normalize name and aliaslet table_ref = self.object_name_to_table_reference(name)?;- let table_name = table_ref.to_string();- let cte = planner_context.get_cte(&table_name);+ let cte = get_cte_for_table_reference(planner_context, &table_ref);(match (cte,diff --git a/docs/source/library-user-guide/extending-sql.md b/docs/source/library-user-guide/extending-sql.mdindex eea5b3b1ac..120bc8fab2 100644--- a/docs/source/library-user-guide/extending-sql.md+++ b/docs/source/library-user-guide/extending-sql.md@@ -264,15 +264,38 @@ enables you to implement SQL constructs like:When implementing [`RelationPlanner`], you receive a [`RelationPlannerContext`] thatprovides utilities for planning:-| Method | Purpose |-| --------------------------- | ----------------------------------------------- |-| `plan(relation)` | Recursively plan a nested relation |-| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr |-| `context_provider()` | Access session configuration, tables, functions |+| Method | Purpose |+| --------------------------- | ------------------------------------------------ |+| `plan(relation)` | Recursively plan a nested relation |+| `get_cte(name)` | Look up a CTE visible in the current query scope |+| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr |+| `context_provider()` | Access session configuration, tables, functions |See the [RelationPlanner API documentation] for additional methods like`normalize_ident()` and `object_name_to_table_reference()`.+Relation planners are called before DataFusion's built-in CTE and catalog lookup.+If your planner recognizes ordinary relations by name, check `get_cte()` after+determining the relation is an ordinary name that your planner would otherwise+handle. Deliberately reserved names and custom syntax can keep their existing+precedence. Return `RelationPlanning::Original` when a CTE is visible so the+next planner can handle the relation. When every ordinary name-based planner in+the chain performs this check, planning reaches DataFusion's built-in CTE+resolution. Normalize the parsed name first so the lookup follows DataFusion's+existing name matching. Only do this for a table without function arguments: a+CTE named `numbers` shadows `numbers`, but it does not shadow a call such as+`numbers(10)`.++```rust+// Run this after recognizing `relation` as an ordinary name handled here.+if let TableFactor::Table { name, args: None, .. } = &relation {+ let table_ref = ctx.object_name_to_table_reference(name.clone())?;+ if ctx.get_cte(&table_ref).is_some() {+ return Ok(RelationPlanning::Original(Box::new(relation)));+ }+}+```+#### Implementation StrategiesThere are two main approaches when implementing a [`RelationPlanner`]: