Apache DataFusion SQL Query Engine
Improve plan_to_sql handling of empty projections with dialect-specific SELECT list support (#19221)
kosiew6ce237492d9f
diff --git a/datafusion/sql/src/expr/value.rs b/datafusion/sql/src/expr/value.rsindex 7808f8cb5d..bd75ac3630 100644--- a/datafusion/sql/src/expr/value.rs+++ b/datafusion/sql/src/expr/value.rs@@ -191,75 +191,79 @@ impl<S: ContextProvider> SqlToRel<'_, S> {negative: bool,interval: Interval,) -> Result<Expr> {- if interval.leading_precision.is_some() {- return not_impl_err!(- "Unsupported Interval Expression with leading_precision {:?}",- interval.leading_precision- );- }-- if interval.last_field.is_some() {- return not_impl_err!(- "Unsupported Interval Expression with last_field {:?}",- interval.last_field- );- }+ sql_interval_to_expr_impl(negative, interval)+ }+}- if interval.fractional_seconds_precision.is_some() {- return not_impl_err!(- "Unsupported Interval Expression with fractional_seconds_precision {:?}",- interval.fractional_seconds_precision- );- }+fn sql_interval_to_expr_impl(negative: bool, interval: Interval) -> Result<Expr> {+ if interval.leading_precision.is_some() {+ return not_impl_err!(+ "Unsupported Interval Expression with leading_precision {:?}",+ interval.leading_precision+ );+ }- if let SQLExpr::BinaryOp { left, op, right } = *interval.value {- let df_op = match op {- BinaryOperator::Plus => Operator::Plus,- BinaryOperator::Minus => Operator::Minus,- _ => {- return not_impl_err!("Unsupported interval operator: {op:?}");- }- };- let left_expr = self.sql_interval_to_expr(- negative,- Interval {- value: left,- leading_field: interval.leading_field.clone(),- leading_precision: None,- last_field: None,- fractional_seconds_precision: None,- },- )?;- let right_expr = self.sql_interval_to_expr(- false,- Interval {- value: right,- leading_field: interval.leading_field,- leading_precision: None,- last_field: None,- fractional_seconds_precision: None,- },- )?;- return Ok(Expr::BinaryExpr(BinaryExpr::new(- Box::new(left_expr),- df_op,- Box::new(right_expr),- )));- }+ if interval.last_field.is_some() {+ return not_impl_err!(+ "Unsupported Interval Expression with last_field {:?}",+ interval.last_field+ );+ }- let value = interval_literal(*interval.value, negative)?;+ if interval.fractional_seconds_precision.is_some() {+ return not_impl_err!(+ "Unsupported Interval Expression with fractional_seconds_precision {:?}",+ interval.fractional_seconds_precision+ );+ }- // leading_field really means the unit if specified- // For example, "month" in `INTERVAL '5' month`- let value = match interval.leading_field.as_ref() {- Some(leading_field) => format!("{value} {leading_field}"),- None => value,+ if let SQLExpr::BinaryOp { left, op, right } = *interval.value {+ let df_op = match op {+ BinaryOperator::Plus => Operator::Plus,+ BinaryOperator::Minus => Operator::Minus,+ _ => {+ return not_impl_err!("Unsupported interval operator: {op:?}");+ }};-- let config = IntervalParseConfig::new(IntervalUnit::Second);- let val = parse_interval_month_day_nano_config(&value, config)?;- Ok(lit(ScalarValue::IntervalMonthDayNano(Some(val))))+ let left_expr = sql_interval_to_expr_impl(+ negative,+ Interval {+ value: left,+ leading_field: interval.leading_field.clone(),+ leading_precision: None,+ last_field: None,+ fractional_seconds_precision: None,+ },+ )?;+ let right_expr = sql_interval_to_expr_impl(+ false,+ Interval {+ value: right,+ leading_field: interval.leading_field,+ leading_precision: None,+ last_field: None,+ fractional_seconds_precision: None,+ },+ )?;+ return Ok(Expr::BinaryExpr(BinaryExpr::new(+ Box::new(left_expr),+ df_op,+ Box::new(right_expr),+ )));}++ let value = interval_literal(*interval.value, negative)?;++ // leading_field really means the unit if specified+ // For example, "month" in `INTERVAL '5' month`+ let value = match interval.leading_field.as_ref() {+ Some(leading_field) => format!("{value} {leading_field}"),+ None => value,+ };++ let config = IntervalParseConfig::new(IntervalUnit::Second);+ let val = parse_interval_month_day_nano_config(&value, config)?;+ Ok(lit(ScalarValue::IntervalMonthDayNano(Some(val))))}fn interval_literal(interval_value: SQLExpr, negative: bool) -> Result<String> {diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rsindex 586ecea328..ec78a42d65 100644--- a/datafusion/sql/src/unparser/ast.rs+++ b/datafusion/sql/src/unparser/ast.rs@@ -139,7 +139,16 @@ impl Default for QueryBuilder {pub struct SelectBuilder {distinct: Option<ast::Distinct>,top: Option<ast::Top>,- projection: Vec<ast::SelectItem>,+ /// Projection items for the SELECT clause.+ ///+ /// This field uses `Option` to distinguish between three distinct states:+ /// - `None`: No projection has been set (not yet initialized)+ /// - `Some(vec![])`: Empty projection explicitly set (generates `SELECT FROM ...` or `SELECT 1 FROM ...`)+ /// - `Some(vec![SelectItem::Wildcard(...)])`: Wildcard projection (generates `SELECT * FROM ...`)+ /// - `Some(vec![...])`: Non-empty projection with specific columns/expressions+ ///+ /// Use `projection()` to set this field and `already_projected()` to check if it has been set.+ projection: Option<Vec<ast::SelectItem>>,into: Option<ast::SelectInto>,from: Vec<TableWithJoinsBuilder>,lateral_views: Vec<ast::LateralView>,@@ -165,16 +174,37 @@ impl SelectBuilder {self}pub fn projection(&mut self, value: Vec<ast::SelectItem>) -> &mut Self {- self.projection = value;+ self.projection = Some(value);self}pub fn pop_projections(&mut self) -> Vec<ast::SelectItem> {- let ret = self.projection.clone();- self.projection.clear();- ret+ self.projection.take().unwrap_or_default()}+ /// Returns true if a projection has been explicitly set via `projection()`.+ ///+ /// This method is used to determine whether the SELECT clause has already been+ /// defined, which helps avoid creating duplicate projection nodes during query+ /// unparsing. It returns `true` for both empty and non-empty projections.+ ///+ /// # Returns+ ///+ /// - `true` if `projection()` has been called (regardless of whether it was empty or not)+ /// - `false` if no projection has been set yet+ ///+ /// # Example+ ///+ /// ```ignore+ /// let mut builder = SelectBuilder::default();+ /// assert!(!builder.already_projected());+ ///+ /// builder.projection(vec![]);+ /// assert!(builder.already_projected()); // true even for empty projection+ ///+ /// builder.projection(vec![SelectItem::Wildcard(...)]);+ /// assert!(builder.already_projected()); // true for non-empty projection+ /// ```pub fn already_projected(&self) -> bool {- !self.projection.is_empty()+ self.projection.is_some()}pub fn into(&mut self, value: Option<ast::SelectInto>) -> &mut Self {self.into = value;@@ -288,7 +318,7 @@ impl SelectBuilder {distinct: self.distinct.clone(),top_before_distinct: false,top: self.top.clone(),- projection: self.projection.clone(),+ projection: self.projection.clone().unwrap_or_default(),into: self.into.clone(),from: self.from@@ -325,7 +355,7 @@ impl SelectBuilder {Self {distinct: Default::default(),top: Default::default(),- projection: Default::default(),+ projection: None,into: Default::default(),from: Default::default(),lateral_views: Default::default(),diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rsindex 82202a405f..844fa6febc 100644--- a/datafusion/sql/src/unparser/dialect.rs+++ b/datafusion/sql/src/unparser/dialect.rs@@ -218,6 +218,36 @@ pub trait Dialect: Send + Sync {fn timestamp_with_tz_to_string(&self, dt: DateTime<Tz>, _unit: TimeUnit) -> String {dt.to_string()}++ /// Whether the dialect supports an empty select list such as `SELECT FROM table`.+ ///+ /// An empty select list returns rows without any column data, which is useful for:+ /// - Counting rows: `SELECT FROM users WHERE active = true` (combined with `COUNT(*)`)+ /// - Testing row existence without retrieving column data+ /// - Performance optimization when only row counts or existence checks are needed+ ///+ /// # Default+ ///+ /// Returns `false` for maximum compatibility across SQL dialects. When `false`,+ /// the unparser falls back to `SELECT 1 FROM table`.+ ///+ /// # Implementation Note+ ///+ /// Specific dialects should override this method to return `true` if they support+ /// the empty select list syntax (e.g., PostgreSQL).+ ///+ /// # Example SQL Output+ ///+ /// ```sql+ /// -- When supported:+ /// SELECT FROM users WHERE active = true;+ ///+ /// -- Fallback when unsupported:+ /// SELECT 1 FROM users WHERE active = true;+ /// ```+ fn supports_empty_select_list(&self) -> bool {+ false+ }}/// `IntervalStyle` to use for unparsing@@ -289,6 +319,10 @@ impl Dialect for PostgreSqlDialect {true}+ fn supports_empty_select_list(&self) -> bool {+ true+ }+fn identifier_quote_style(&self, _: &str) -> Option<char> {Some('"')}diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 61a06d9c38..d56ecb4f15 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -340,7 +340,7 @@ impl Unparser<'_> {) -> Result<()> {match plan {LogicalPlan::TableScan(scan) => {- if let Some(unparsed_table_scan) = Self::unparse_table_scan_pushdown(+ if let Some(unparsed_table_scan) = self.unparse_table_scan_pushdown(plan,None,select.already_projected(),@@ -822,7 +822,7 @@ impl Unparser<'_> {LogicalPlan::SubqueryAlias(plan_alias) => {let (plan, mut columns) =subquery_alias_inner_query_and_columns(plan_alias);- let unparsed_table_scan = Self::unparse_table_scan_pushdown(+ let unparsed_table_scan = self.unparse_table_scan_pushdown(plan,Some(plan_alias.alias.clone()),select.already_projected(),@@ -1062,6 +1062,7 @@ impl Unparser<'_> {/// Try to unparse a table scan with pushdown operations into a new subquery plan./// If the table scan is without any pushdown operations, return None.fn unparse_table_scan_pushdown(+ &self,plan: &LogicalPlan,alias: Option<TableReference>,already_projected: bool,@@ -1099,10 +1100,7 @@ impl Unparser<'_> {// information included in the TableScan node.if !already_projected && let Some(project_vec) = &table_scan.projection {if project_vec.is_empty() {- builder = builder.project(vec![Expr::Literal(- ScalarValue::Int64(Some(1)),- None,- )])?;+ builder = builder.project(self.empty_projection_fallback())?;} else {let project_columns = project_vec.iter()@@ -1164,7 +1162,7 @@ impl Unparser<'_> {Ok(Some(builder.build()?))}LogicalPlan::SubqueryAlias(subquery_alias) => {- let ret = Self::unparse_table_scan_pushdown(+ let ret = self.unparse_table_scan_pushdown(&subquery_alias.input,Some(subquery_alias.alias.clone()),already_projected,@@ -1180,7 +1178,7 @@ impl Unparser<'_> {// SubqueryAlias could be rewritten to a plan with a projection as the top node by [rewrite::subquery_alias_inner_query_and_columns].// The inner table scan could be a scan with pushdown operations.LogicalPlan::Projection(projection) => {- if let Some(plan) = Self::unparse_table_scan_pushdown(+ if let Some(plan) = self.unparse_table_scan_pushdown(&projection.input,alias.clone(),already_projected,@@ -1413,6 +1411,17 @@ impl Unparser<'_> {fn dml_to_sql(&self, plan: &LogicalPlan) -> Result<ast::Statement> {not_impl_err!("Unsupported plan: {plan:?}")}++ /// Generates appropriate projection expression for empty projection lists.+ /// Returns an empty vec for dialects supporting empty select lists,+ /// or a dummy literal `1` for other dialects.+ fn empty_projection_fallback(&self) -> Vec<Expr> {+ if self.dialect.supports_empty_select_list() {+ Vec::new()+ } else {+ vec![Expr::Literal(ScalarValue::Int64(Some(1)), None)]+ }+ }}impl From<BuilderError> for DataFusionError {diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex 39d2cb241b..27f4e2cd55 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -1234,6 +1234,84 @@ fn test_table_scan_with_empty_projection_in_plan_to_sql_3() {);}+#[test]+fn test_table_scan_with_empty_projection_in_plan_to_sql_postgres() {+ let schema = test_schema();+ let table_name = "table";+ let plan = table_scan_with_empty_projection_and_none_projection_helper(+ table_name,+ schema,+ Some(vec![]),+ );+ let unparser = Unparser::new(&UnparserPostgreSqlDialect {});+ let sql = unparser.plan_to_sql(&plan).unwrap();+ assert_snapshot!(+ sql,+ @r#"SELECT FROM "table""#+ );+}++#[test]+fn test_table_scan_with_empty_projection_in_plan_to_sql_default_dialect() {+ let schema = test_schema();+ let table_name = "table";+ let plan = table_scan_with_empty_projection_and_none_projection_helper(+ table_name,+ schema,+ Some(vec![]),+ );+ let unparser = Unparser::new(&UnparserDefaultDialect {});+ let sql = unparser.plan_to_sql(&plan).unwrap();+ assert_snapshot!(+ sql,+ @r#"SELECT 1 FROM "table""#+ );+}++#[test]+fn test_table_scan_with_empty_projection_and_filter_postgres() {+ let schema = test_schema();+ let table_name = "table";+ let plan = table_scan_with_filter_and_fetch(+ Some(table_name),+ &schema,+ Some(vec![]),+ vec![col("id").gt(lit(10))],+ None,+ )+ .unwrap()+ .build()+ .unwrap();+ let unparser = Unparser::new(&UnparserPostgreSqlDialect {});+ let sql = unparser.plan_to_sql(&plan).unwrap();+ assert_snapshot!(+ sql,+ @r#"SELECT FROM "table" WHERE ("table"."id" > 10)"#+ );+}++#[test]+fn test_table_scan_with_empty_projection_and_filter_default_dialect() {+ let schema = test_schema();+ let table_name = "table";+ let plan = table_scan_with_filter_and_fetch(+ Some(table_name),+ &schema,+ Some(vec![]),+ vec![col("id").gt(lit(10))],+ None,+ )+ .unwrap()+ .build()+ .unwrap();+ let unparser = Unparser::new(&UnparserDefaultDialect {});+ let sql = unparser.plan_to_sql(&plan).unwrap();+ assert_snapshot!(+ sql,+ @r#"SELECT 1 FROM "table" WHERE ("table".id > 10)"#+ );+}+fn table_scan_with_empty_projection_and_none_projection_helper(table_name: &str,table_schema: Schema,