Apache DataFusion SQL Query Engine
fix: prevent unparser stack overflow on deeply nested expressions (#23058)
Adrian Garcia Badaraccof9c1e9ed2e13
diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rsindex 872af63533..d7dad04014 100644--- a/datafusion/sql/src/unparser/dialect.rs+++ b/datafusion/sql/src/unparser/dialect.rs@@ -434,9 +434,11 @@ impl PostgreSqlDialect {};Ok(Some(ast::Expr::AnyOp {- left: Box::new(unparser.expr_to_sql(needle)?),+ // Recurse through the annotated entry point so the stack-growth+ // protection engages on nested arguments; see issue #23056.+ left: Box::new(unparser.expr_to_sql_with_nesting(needle)?),compare_op: BinaryOperator::Eq,- right: Box::new(unparser.expr_to_sql(haystack)?),+ right: Box::new(unparser.expr_to_sql_with_nesting(haystack)?),is_some: false,}))}diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 6096e17140..e2e2601895 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -31,6 +31,7 @@ use std::vec;use super::Unparser;use super::dialect::{DistinctFromStyle, IntervalStyle};+use crate::stack::StackGuard;use arrow::array::{ArrayRef, Date32Array, Date64Array, PrimitiveArray,types::{@@ -94,6 +95,26 @@ const IS: &BinaryOperator = &BinaryOperator::BitwiseAnd;impl Unparser<'_> {pub fn expr_to_sql(&self, expr: &Expr) -> Result<ast::Expr> {+ // Unparsing recurses once per nesting level. The function-argument and+ // dialect scalar-function-override paths cost more per level than the+ // default `recursive` red zone, so without raising the minimum stack+ // size the stack-growing trampoline engages too late and the OS stack+ // overflows on deeply nested expressions (issue #23056). The size+ // mirrors the planner's `StackGuard` usage in `query.rs`.+ let _guard = StackGuard::new(256 * 1024);+ self.expr_to_sql_with_nesting(expr)+ }++ /// Recursive entry point shared by the public [`Self::expr_to_sql`] and the+ /// internal recursion sites (scalar-function arguments, arrays, maps, and+ /// dialect scalar-function overrides).+ ///+ /// This carries the `recursive` annotation so every nesting level becomes a+ /// stack-growth checkpoint. Internal recursion must call this rather than+ /// the public [`Self::expr_to_sql`]: the public entry point is not+ /// annotated and would re-install the [`StackGuard`] on every level.+ #[cfg_attr(feature = "recursive_protection", recursive::recursive)]+ pub(crate) fn expr_to_sql_with_nesting(&self, expr: &Expr) -> Result<ast::Expr> {let mut root_expr = self.expr_to_sql_inner(expr)?;if self.pretty {root_expr = self.remove_unnecessary_nesting(root_expr, LOWEST, LOWEST);@@ -681,7 +702,7 @@ impl Unparser<'_> {fn make_array_to_sql(&self, args: &[Expr]) -> Result<ast::Expr> {let args = args.iter()- .map(|e| self.expr_to_sql(e))+ .map(|e| self.expr_to_sql_with_nesting(e)).collect::<Result<Vec<_>>>()?;Ok(ast::Expr::Array(Array {elem: args,@@ -708,8 +729,8 @@ impl Unparser<'_> {2,"array_element must have exactly 2 arguments");- let array = self.expr_to_sql(&args[0])?;- let index = self.expr_to_sql(&args[1])?;+ let array = self.expr_to_sql_with_nesting(&args[0])?;+ let index = self.expr_to_sql_with_nesting(&args[1])?;Ok(ast::Expr::CompoundFieldAccess {root: Box::new(array),access_chain: vec![ast::AccessExpr::Subscript(Subscript::Index { index })],@@ -732,7 +753,7 @@ impl Unparser<'_> {Ok(ast::DictionaryField {key,- value: Box::new(self.expr_to_sql(&chunk[1])?),+ value: Box::new(self.expr_to_sql_with_nesting(&chunk[1])?),})}).collect::<Result<Vec<_>>>()?;@@ -802,7 +823,8 @@ impl Unparser<'_> {fn map_to_sql(&self, args: &[Expr]) -> Result<ast::Expr> {assert_eq_or_internal_err!(args.len(), 2, "map must have exactly 2 arguments");- let ast::Expr::Array(Array { elem: keys, .. }) = self.expr_to_sql(&args[0])?+ let ast::Expr::Array(Array { elem: keys, .. }) =+ self.expr_to_sql_with_nesting(&args[0])?else {return internal_err!("map expects first argument to be an array, but received: {:?}",@@ -810,7 +832,8 @@ impl Unparser<'_> {);};- let ast::Expr::Array(Array { elem: values, .. }) = self.expr_to_sql(&args[1])?+ let ast::Expr::Array(Array { elem: values, .. }) =+ self.expr_to_sql_with_nesting(&args[1])?else {return internal_err!("map expects second argument to be an array, but received: {:?}",@@ -944,7 +967,7 @@ impl Unparser<'_> {) {Ok(ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard))} else {- self.expr_to_sql(e)+ self.expr_to_sql_with_nesting(e).map(|e| ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)))}})@@ -989,6 +1012,7 @@ impl Unparser<'_> {////// Also note that when fetching the precedence of a nested expression, we ignore other nested/// expressions, so precedence of expr `(a * (b + c))` equals `*` and not `+`.+ #[cfg_attr(feature = "recursive_protection", recursive::recursive)]fn remove_unnecessary_nesting(&self,expr: ast::Expr,@@ -3307,6 +3331,72 @@ mod tests {Ok(())}+ /// Regression test for https://github.com/apache/datafusion/issues/23056+ ///+ /// Deeply-nested expressions whose unparse path routes through scalar+ /// function arguments and dialect scalar-function overrides used to+ /// overflow the OS stack even with `recursive_protection` enabled,+ /// because the per-level stack cost of those paths exceeds the default+ /// `recursive` red zone and the unparser installed no [`StackGuard`].+ ///+ /// This test only asserts the protected behavior, so it is gated on the+ /// `recursive_protection` feature. Without that feature the unparser is+ /// not stack-safe by design and a deep enough expression will overflow.+ #[cfg(feature = "recursive_protection")]+ #[test]+ fn test_deeply_nested_expr_does_not_overflow_stack() {+ // Far deeper than the ~60 levels that overflow without protection, but+ // bounded so the trampoline's heap stacks stay reasonable in debug.+ const DEPTH: usize = 2_000;++ // Run on an explicit, realistically-sized thread stack. The work is+ // performed on a spawned thread so an overflow (in the unfixed code)+ // aborts the process and fails the test deterministically rather than+ // depending on the harness thread's stack size.+ let handle = std::thread::Builder::new()+ .stack_size(2 * 1024 * 1024)+ .spawn(|| {+ // 1. Linear chain through a dialect scalar-function override:+ // array_has(array_has(... array_has(col, 'x') ...), 'x').+ // PostgreSqlDialect unparses array_has via array_has_to_sql_any,+ // which recurses back into the unparser for each argument.+ let mut nested_fn: Expr = col("c");+ for _ in 0..DEPTH {+ nested_fn = array_has(nested_fn, lit("x"));+ }+ let pg = PostgreSqlDialect {};+ Unparser::new(&pg)+ .expr_to_sql(&nested_fn)+ .expect("deeply nested scalar function should unparse");++ // 2. Linear chain of plain binary operators, exercising the+ // inner -> inner recursion on the default dialect.+ let mut nested_binary: Expr = col("c");+ for _ in 0..DEPTH {+ nested_binary = nested_binary + lit(1);+ }+ Unparser::default()+ .expr_to_sql(&nested_binary)+ .expect("deeply nested binary expression should unparse");++ // 3. Same binary chain in pretty mode. Pretty mode runs+ // `remove_unnecessary_nesting` at every level, which recurses+ // alongside the unparse itself; this locks down that second+ // recursion site fixed by this PR.+ Unparser::default()+ .with_pretty(true)+ .expr_to_sql(&nested_binary)+ .expect(+ "deeply nested binary expression should unparse in pretty mode",+ );+ })+ .unwrap();++ // If the unparser overflows, the process aborts and this join is never+ // reached; otherwise the spawned thread returns cleanly.+ handle.join().expect("unparsing thread should not panic");+ }+#[test]fn test_window_func_support_window_frame() -> Result<()> {let default_dialect: Arc<dyn Dialect> =diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rsindex 732e030b33..1cc023d112 100644--- a/datafusion/sql/src/unparser/utils.rs+++ b/datafusion/sql/src/unparser/utils.rs@@ -448,7 +448,7 @@ pub(crate) fn date_part_to_sql() -> Result<Option<ast::Expr>> {match (style, date_part_args.len()) {(DateFieldExtractStyle::Extract, 2) => {- let date_expr = unparser.expr_to_sql(&date_part_args[1])?;+ let date_expr = unparser.expr_to_sql_with_nesting(&date_part_args[1])?;if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] {let field = match field.to_lowercase().as_str() {"year" => ast::DateTimeField::Year,@@ -468,7 +468,7 @@ pub(crate) fn date_part_to_sql(}}(DateFieldExtractStyle::Strftime, 2) => {- let column = unparser.expr_to_sql(&date_part_args[1])?;+ let column = unparser.expr_to_sql_with_nesting(&date_part_args[1])?;if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] {let field = match field.to_lowercase().as_str() {