Apache DataFusion SQL Query Engine
fix(cli): respect `sql_parser.recursion_limit` when parsing and validating input (#25807)
Kassahun Sanayew98360e3d4172
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rsindex 213ae9c5d4..e1c12a315b 100644--- a/datafusion-cli/src/exec.rs+++ b/datafusion-cli/src/exec.rs@@ -23,7 +23,7 @@ use datafusion::logical_expr::{DdlStatement, LogicalPlan};use datafusion::physical_plan::execution_plan::EmissionType;use datafusion::physical_plan::spill::get_record_batch_memory_size;use datafusion::physical_plan::{ExecutionPlanProperties, execute_stream};-use datafusion::sql::parser::{DFParser, Statement};+use datafusion::sql::parser::{DFParserBuilder, Statement};use datafusion::sql::sqlparser;use datafusion::sql::sqlparser::dialect::dialect_from_str;use futures::StreamExt;@@ -120,6 +120,12 @@ pub async fn exec_from_repl(rl.load_history(".history").ok();loop {+ // Parser options can change through any input, including `\i` files,+ // so sync the validator before every read.+ rl.helper_mut()+ .unwrap()+ .set_parser_options(&ctx.task_ctx().session_config().options().sql_parser);+match rl.readline("> ") {Ok(line) if line.starts_with('\\') => {rl.add_history_entry(line.trim_end())?;@@ -167,13 +173,8 @@ pub async fn exec_from_repl(},_ = signal::ctrl_c() => {println!("^C");- continue},}- // dialect might have changed- rl.helper_mut().unwrap().set_dialect(- &ctx.task_ctx().session_config().options().sql_parser.dialect,- );}}Err(ReadlineError::Interrupted) => {@@ -209,7 +210,11 @@ pub(super) async fn exec_and_print()})?;- let statements = DFParser::parse_sql_with_dialect(&sql, dialect.as_ref())?;+ let statements = DFParserBuilder::new(sql.as_str())+ .with_dialect(dialect.as_ref())+ .with_recursion_limit(options.sql_parser.recursion_limit.get())+ .build()?+ .parse_statements()?;for statement in statements {StatementExecutor::new(statement).execute(ctx, print_options)@@ -506,6 +511,7 @@ mod tests {use super::*;use datafusion::common::plan_err;+ use datafusion::sql::parser::DFParser;use datafusion::prelude::SessionContext;use datafusion_common::assert_contains;diff --git a/datafusion-cli/src/helper.rs b/datafusion-cli/src/helper.rsindex 4ba4db377c..b3fbbb0e03 100644--- a/datafusion-cli/src/helper.rs+++ b/datafusion-cli/src/helper.rs@@ -10,9 +10,9 @@ use std::cell::Cell;use crate::highlighter::{Color, NoSyntaxHighlighter, SyntaxHighlighter};-use datafusion::sql::parser::{DFParser, Statement};+use datafusion::sql::parser::{DFParser, DFParserBuilder, Statement};use datafusion::sql::sqlparser::dialect::dialect_from_str;-use datafusion_common::config::Dialect;+use datafusion_common::config::{Dialect, SqlParserOptions};use rustyline::completion::{Completer, FilenameCompleter, Pair};use rustyline::error::ReadlineError;@@ -27,6 +27,7 @@ const DEFAULT_HINT_SUGGESTION: &str = " \\? for help, \\q to quit";pub struct CliHelper {completer: FilenameCompleter,dialect: Dialect,+ recursion_limit: usize,highlighter: Box<dyn Highlighter>,/// Tracks whether to show the default hint. Set to `false` once the user/// types anything, so the hint doesn't reappear after deleting back to@@ -44,6 +45,7 @@ impl CliHelper {Self {completer: FilenameCompleter::new(),dialect: *dialect,+ recursion_limit: SqlParserOptions::default().recursion_limit.get(),highlighter,show_hint: Cell::new(true),}@@ -55,6 +57,13 @@ impl CliHelper {}}+ /// Sets the parser options (dialect and recursion limit) used when+ /// validating input, see `datafusion.sql_parser.*`.+ pub fn set_parser_options(&mut self, options: &SqlParserOptions) {+ self.set_dialect(&options.dialect);+ self.recursion_limit = options.recursion_limit.get();+ }+/// Re-enable the default hint for the next prompt.pub fn reset_hint(&self) {self.show_hint.set(true);@@ -70,7 +79,12 @@ impl CliHelper {};let lines = split_from_semicolon(sql);for line in lines {- match DFParser::parse_sql_with_dialect(&line, dialect.as_ref()) {+ let statements = DFParserBuilder::new(line.as_str())+ .with_dialect(dialect.as_ref())+ .with_recursion_limit(self.recursion_limit)+ .build()+ .and_then(|mut parser| parser.parse_statements());+ match statements {Ok(statements) if statements.is_empty() => {return Ok(ValidationResult::Invalid(Some(" 🤔 You entered an empty statement".to_string(),@@ -201,6 +215,8 @@ mod tests {use super::*;+ use datafusion_common::config::ConfigNonZeroUsize;+fn readline_direct(mut reader: impl BufRead,validator: &CliHelper,@@ -305,6 +321,36 @@ mod tests {Ok(())}+ #[test]+ fn sql_recursion_limit() -> Result<()> {+ let mut validator = CliHelper::default();+ let nested = |depth: usize| {+ format!("select {}1{};", "abs(".repeat(depth), ")".repeat(depth))+ };+ let with_limit = |limit: usize| SqlParserOptions {+ recursion_limit: ConfigNonZeroUsize::try_new(limit).unwrap(),+ ..Default::default()+ };++ // exceeds the default recursion limit+ let result = readline_direct(Cursor::new(nested(60)), &validator)?;+ assert!(+ matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))+ );++ validator.set_parser_options(&with_limit(100));+ let result = readline_direct(Cursor::new(nested(60)), &validator)?;+ assert!(matches!(result, ValidationResult::Valid(None)));++ validator.set_parser_options(&with_limit(5));+ let result = readline_direct(Cursor::new(nested(10)), &validator)?;+ assert!(+ matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))+ );++ Ok(())+ }+#[test]fn test_split_from_semicolon() {let sql = "SELECT 1; SELECT 2;";diff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]index df3a10b6bb..c139c19c2f 100644--- a/datafusion-cli/tests/snapshots/[email protected]+++ b/datafusion-cli/tests/snapshots/[email protected]@@ -15,5 +15,11 @@ exit_code: 0+----------+| 1 |+----------+++------++| deep |++------++| 1 |++------+----- stderr -----+SQL error: RecursionLimitExceeded (current limit: 5)diff --git a/datafusion-cli/tests/sql/select.sql b/datafusion-cli/tests/sql/select.sqlindex 9e13a3eff4..8ecbb3cc42 100644--- a/datafusion-cli/tests/sql/select.sql+++ b/datafusion-cli/tests/sql/select.sql@@ -1 +1,6 @@-select 1;\ No newline at end of file+select 1;+-- datafusion.sql_parser.recursion_limit is respected when parsing (#24913)+set datafusion.sql_parser.recursion_limit = 100;+select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) as deep;+set datafusion.sql_parser.recursion_limit = 5;+select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))) as shallow;