Skip to content

Apache DataFusion SQL Query Engine

feat(sql): expose CTE scope to relation planners (#24755)

Geoffrey Claude565f933d6fc1
diff --git a/datafusion/core/tests/user_defined/relation_planner.rs b/datafusion/core/tests/user_defined/relation_planner.rs
index 54af53ad85..cc1bbb71b0 100644
--- a/datafusion/core/tests/user_defined/relation_planner.rs
+++ b/datafusion/core/tests/user_defined/relation_planner.rs
@@ -30,7 +30,7 @@ use datafusion_expr::logical_plan::builder::LogicalPlanBuilder;
use datafusion_expr::planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
};
-use datafusion_sql::sqlparser::ast::TableFactor;
+use datafusion_sql::sqlparser::ast::{TableAlias, TableFactor};
use insta::assert_snapshot;
// ============================================================================
@@ -51,29 +51,54 @@ use insta::assert_snapshot;
/// example planners below.
fn plan_static_values_table(
relation: TableFactor,
+ context: &dyn RelationPlannerContext,
table_name: &str,
column_name: &str,
values: Vec<ScalarValue>,
) -> Result<RelationPlanning> {
- match relation {
+ let alias = match &relation {
TableFactor::Table { name, alias, .. }
if name.to_string().eq_ignore_ascii_case(table_name) =>
{
- let rows = values
- .into_iter()
- .map(|v| vec![Expr::Literal(v, None)])
- .collect::<Vec<_>>();
-
- let plan = LogicalPlanBuilder::values(rows)?
- .project(vec![col("column1").alias(column_name)])?
- .build()?;
-
- Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new(
- plan, alias,
- ))))
+ alias.clone()
+ }
+ _ => return Ok(RelationPlanning::Original(Box::new(relation))),
+ };
+
+ if let TableFactor::Table {
+ name, args: None, ..
+ } = &relation
+ {
+ let table_ref = context.object_name_to_table_reference(name.clone())?;
+
+ // RelationPlanner extensions run before DataFusion resolves CTEs. A
+ // name-based planner should defer when a CTE with that name is visible,
+ // so SQL's normal lexical CTE precedence is preserved.
+ if context.get_cte(&table_ref).is_some() {
+ return Ok(RelationPlanning::Original(Box::new(relation)));
}
- other => Ok(RelationPlanning::Original(Box::new(other))),
}
+
+ plan_static_values(alias, column_name, values)
+}
+
+fn plan_static_values(
+ alias: Option<TableAlias>,
+ column_name: &str,
+ values: Vec<ScalarValue>,
+) -> Result<RelationPlanning> {
+ let rows = values
+ .into_iter()
+ .map(|v| vec![Expr::Literal(v, None)])
+ .collect::<Vec<_>>();
+
+ let plan = LogicalPlanBuilder::values(rows)?
+ .project(vec![col("column1").alias(column_name)])?
+ .build()?;
+
+ Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new(
+ plan, alias,
+ ))))
}
/// Example planner that provides a virtual `numbers` table with values
@@ -85,10 +110,11 @@ impl RelationPlanner for NumbersPlanner {
fn plan_relation(
&self,
relation: TableFactor,
- _context: &mut dyn RelationPlannerContext,
+ context: &mut dyn RelationPlannerContext,
) -> Result<RelationPlanning> {
plan_static_values_table(
relation,
+ context,
"numbers",
"number",
vec![
@@ -100,6 +126,28 @@ impl RelationPlanner for NumbersPlanner {
}
}
+/// Example planner for a qualified virtual relation. This is kept separate
+/// from `NumbersPlanner` so CTE tests can verify that extension lookup matches
+/// DataFusion's existing qualified-name resolution.
+#[derive(Debug)]
+struct QualifiedNumbersPlanner;
+
+impl RelationPlanner for QualifiedNumbersPlanner {
+ fn plan_relation(
+ &self,
+ relation: TableFactor,
+ context: &mut dyn RelationPlannerContext,
+ ) -> Result<RelationPlanning> {
+ plan_static_values_table(
+ relation,
+ context,
+ "foo.bar",
+ "number",
+ vec![ScalarValue::Int64(Some(7))],
+ )
+ }
+}
+
/// Example planner that provides a virtual `colors` table with three string
/// values: `red`, `green`, `blue`.
#[derive(Debug)]
@@ -109,10 +157,11 @@ impl RelationPlanner for ColorsPlanner {
fn plan_relation(
&self,
relation: TableFactor,
- _context: &mut dyn RelationPlannerContext,
+ context: &mut dyn RelationPlannerContext,
) -> Result<RelationPlanning> {
plan_static_values_table(
relation,
+ context,
"colors",
"color",
vec![
@@ -133,10 +182,11 @@ impl RelationPlanner for AlternativeNumbersPlanner {
fn plan_relation(
&self,
relation: TableFactor,
- _context: &mut dyn RelationPlannerContext,
+ context: &mut dyn RelationPlannerContext,
) -> Result<RelationPlanning> {
plan_static_values_table(
relation,
+ context,
"numbers",
"number",
vec![ScalarValue::Int64(Some(100)), ScalarValue::Int64(Some(200))],
@@ -144,6 +194,28 @@ impl RelationPlanner for AlternativeNumbersPlanner {
}
}
+/// Example planner with a deliberately reserved `numbers` name. Unlike the
+/// ordinary name-based planners above, it intentionally does not defer to CTEs.
+#[derive(Debug)]
+struct ReservedNumbersPlanner;
+
+impl RelationPlanner for ReservedNumbersPlanner {
+ fn plan_relation(
+ &self,
+ relation: TableFactor,
+ _context: &mut dyn RelationPlannerContext,
+ ) -> Result<RelationPlanning> {
+ match relation {
+ TableFactor::Table { name, alias, .. }
+ if name.to_string().eq_ignore_ascii_case("numbers") =>
+ {
+ plan_static_values(alias, "number", vec![ScalarValue::Int64(Some(999))])
+ }
+ other => Ok(RelationPlanning::Original(Box::new(other))),
+ }
+ }
+}
+
/// Example planner that intercepts nested joins and samples both sides (limit 2)
/// before joining, demonstrating recursive planning with `context.plan()`.
#[derive(Debug)]
@@ -357,6 +429,145 @@ mod tests {
");
}
+ // A name-based relation planner can defer to a visible CTE, preserving the
+ // same shadowing behavior as a catalog table with that name.
+ #[tokio::test]
+ async fn cte_takes_precedence_over_virtual_table() {
+ let ctx = ctx_with_numbers();
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +--------+
+ | number |
+ +--------+
+ | 42 |
+ +--------+
+ ");
+ }
+
+ // A derived query inherits CTEs from its outer query, so a name-based
+ // relation planner still defers to the visible CTE in the nested scope.
+ #[tokio::test]
+ async fn derived_query_inherits_outer_cte() {
+ let ctx = ctx_with_numbers();
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "WITH numbers AS (SELECT 42 AS number) \
+ SELECT * FROM (SELECT * FROM numbers) nested",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +--------+
+ | number |
+ +--------+
+ | 42 |
+ +--------+
+ ");
+ }
+
+ // Deferring a visible CTE continues through the planner chain. A later
+ // planner with a deliberately reserved name can still intercept it.
+ #[tokio::test]
+ async fn cte_defer_still_allows_reserved_planner() {
+ let ctx = SessionContext::new()
+ .with_planner(ReservedNumbersPlanner)
+ .with_planner(NumbersPlanner);
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +--------+
+ | number |
+ +--------+
+ | 999 |
+ +--------+
+ ");
+ }
+
+ // The extension lookup follows DataFusion's existing name matching. Both
+ // the extension and the default planner therefore resolve the same CTE for
+ // a quoted dotted name and a qualified reference with the same rendering.
+ #[tokio::test]
+ async fn cte_lookup_matches_default_qualified_name_resolution() {
+ let ctx = SessionContext::new().with_planner(QualifiedNumbersPlanner);
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "WITH \"foo.bar\" AS (SELECT 42 AS number) SELECT * FROM foo.bar",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +--------+
+ | number |
+ +--------+
+ | 42 |
+ +--------+
+ ");
+ }
+
+ // A CTE shadows an ordinary table reference, not a same-named table
+ // function call with arguments.
+ #[tokio::test]
+ async fn cte_does_not_shadow_same_named_table_function() {
+ let ctx = ctx_with_numbers();
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers(1)",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +--------+
+ | number |
+ +--------+
+ | 1 |
+ | 2 |
+ | 3 |
+ +--------+
+ ");
+ }
+
+ // CTE visibility follows the query's lexical scope: the nested CTE wins
+ // inside the derived table but does not hide the outer virtual relation.
+ #[tokio::test]
+ async fn cte_visibility_is_scoped_to_nested_query() {
+ let ctx = ctx_with_numbers();
+
+ let result = execute_sql_to_string(
+ &ctx,
+ "SELECT 'cte' AS source, number \
+ FROM (WITH numbers AS (SELECT 42 AS number) SELECT * FROM numbers) nested \
+ UNION ALL \
+ SELECT 'virtual' AS source, number FROM numbers \
+ ORDER BY source, number",
+ )
+ .await;
+
+ assert_snapshot!(result, @r"
+ +---------+--------+
+ | source | number |
+ +---------+--------+
+ | cte | 42 |
+ | virtual | 1 |
+ | virtual | 2 |
+ | virtual | 3 |
+ +---------+--------+
+ ");
+ }
+
// Virtual table supports standard SQL operations (projection, filter, aggregation).
#[tokio::test]
async fn virtual_table_filters_and_aggregation() {
diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rs
index 4009dfb3e9..31aad67d17 100644
--- a/datafusion/expr/src/planner.rs
+++ b/datafusion/expr/src/planner.rs
@@ -430,6 +430,27 @@ pub trait RelationPlannerContext {
/// from the first registered relation planner.
fn plan(&mut self, relation: TableFactor) -> Result<LogicalPlan>;
+ /// Returns the common table expression (CTE) visible for the specified
+ /// table reference in the current query scope, if any.
+ ///
+ /// Relation planners run before DataFusion's built-in relation planning. A
+ /// planner that recognizes ordinary tables by name can use this method to
+ /// pass a same-named relation to the remaining planners. When every
+ /// ordinary name-based planner does so, built-in CTE resolution is
+ /// preserved. Use [`Self::object_name_to_table_reference`] to normalize an
+ /// [`ObjectName`] before looking it up.
+ ///
+ /// The lookup follows the same string-key matching as DataFusion's built-in
+ /// relation planning, including for qualified table references. Different
+ /// [`TableReference`] variants with the same string representation therefore
+ /// use the same CTE lookup key.
+ ///
+ /// The default implementation returns `None` for compatibility with custom
+ /// implementations of this context.
+ fn get_cte(&self, _name: &TableReference) -> Option<&LogicalPlan> {
+ None
+ }
+
/// Converts a SQL expression into a logical expression using the current
/// planner context.
fn sql_to_expr(&mut self, expr: SQLExpr, schema: &DFSchema) -> Result<Expr>;
diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rs
index 8549bd833e..beb6acd422 100644
--- a/datafusion/sql/src/relation/mod.rs
+++ b/datafusion/sql/src/relation/mod.rs
@@ -38,6 +38,15 @@ struct SqlToRelRelationContext<'a, 'b, S: ContextProvider> {
planner_context: &'a mut PlannerContext,
}
+/// Look up a CTE using the same string key as DataFusion's built-in relation
+/// planning.
+fn get_cte_for_table_reference<'a>(
+ planner_context: &'a PlannerContext,
+ name: &TableReference,
+) -> Option<&'a LogicalPlan> {
+ planner_context.get_cte(&name.to_string())
+}
+
// Implement RelationPlannerContext
impl<S: ContextProvider> RelationPlannerContext for SqlToRelRelationContext<'_, '_, S> {
fn context_provider(&self) -> &dyn ContextProvider {
@@ -48,6 +57,10 @@ impl<S: ContextProvider> RelationPlannerContext for SqlToRelRelationContext<'_,
self.planner.create_relation(relation, self.planner_context)
}
+ fn get_cte(&self, name: &TableReference) -> Option<&LogicalPlan> {
+ get_cte_for_table_reference(self.planner_context, name)
+ }
+
fn sql_to_expr(
&mut self,
expr: sqlparser::ast::Expr,
@@ -184,8 +197,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
} else {
// Normalize name and alias
let table_ref = self.object_name_to_table_reference(name)?;
- let table_name = table_ref.to_string();
- let cte = planner_context.get_cte(&table_name);
+ let cte = get_cte_for_table_reference(planner_context, &table_ref);
(
match (
cte,
diff --git a/docs/source/library-user-guide/extending-sql.md b/docs/source/library-user-guide/extending-sql.md
index eea5b3b1ac..120bc8fab2 100644
--- a/docs/source/library-user-guide/extending-sql.md
+++ b/docs/source/library-user-guide/extending-sql.md
@@ -264,15 +264,38 @@ enables you to implement SQL constructs like:
When implementing [`RelationPlanner`], you receive a [`RelationPlannerContext`] that
provides utilities for planning:
-| Method | Purpose |
-| --------------------------- | ----------------------------------------------- |
-| `plan(relation)` | Recursively plan a nested relation |
-| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr |
-| `context_provider()` | Access session configuration, tables, functions |
+| Method | Purpose |
+| --------------------------- | ------------------------------------------------ |
+| `plan(relation)` | Recursively plan a nested relation |
+| `get_cte(name)` | Look up a CTE visible in the current query scope |
+| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr |
+| `context_provider()` | Access session configuration, tables, functions |
See the [RelationPlanner API documentation] for additional methods like
`normalize_ident()` and `object_name_to_table_reference()`.
+Relation planners are called before DataFusion's built-in CTE and catalog lookup.
+If your planner recognizes ordinary relations by name, check `get_cte()` after
+determining the relation is an ordinary name that your planner would otherwise
+handle. Deliberately reserved names and custom syntax can keep their existing
+precedence. Return `RelationPlanning::Original` when a CTE is visible so the
+next planner can handle the relation. When every ordinary name-based planner in
+the chain performs this check, planning reaches DataFusion's built-in CTE
+resolution. Normalize the parsed name first so the lookup follows DataFusion's
+existing name matching. Only do this for a table without function arguments: a
+CTE named `numbers` shadows `numbers`, but it does not shadow a call such as
+`numbers(10)`.
+
+```rust
+// Run this after recognizing `relation` as an ordinary name handled here.
+if let TableFactor::Table { name, args: None, .. } = &relation {
+ let table_ref = ctx.object_name_to_table_reference(name.clone())?;
+ if ctx.get_cte(&table_ref).is_some() {
+ return Ok(RelationPlanning::Original(Box::new(relation)));
+ }
+}
+```
+
#### Implementation Strategies
There are two main approaches when implementing a [`RelationPlanner`]: