Skip to content

Apache DataFusion SQL Query Engine

fix(cli): respect `sql_parser.recursion_limit` when parsing and validating input (#25807)

Kassahun Sanayew98360e3d4172
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
index 213ae9c5d4..e1c12a315b 100644
--- a/datafusion-cli/src/exec.rs
+++ b/datafusion-cli/src/exec.rs
@@ -23,7 +23,7 @@ use datafusion::logical_expr::{DdlStatement, LogicalPlan};
use datafusion::physical_plan::execution_plan::EmissionType;
use datafusion::physical_plan::spill::get_record_batch_memory_size;
use datafusion::physical_plan::{ExecutionPlanProperties, execute_stream};
-use datafusion::sql::parser::{DFParser, Statement};
+use datafusion::sql::parser::{DFParserBuilder, Statement};
use datafusion::sql::sqlparser;
use datafusion::sql::sqlparser::dialect::dialect_from_str;
use futures::StreamExt;
@@ -120,6 +120,12 @@ pub async fn exec_from_repl(
rl.load_history(".history").ok();
loop {
+ // Parser options can change through any input, including `\i` files,
+ // so sync the validator before every read.
+ rl.helper_mut()
+ .unwrap()
+ .set_parser_options(&ctx.task_ctx().session_config().options().sql_parser);
+
match rl.readline("> ") {
Ok(line) if line.starts_with('\\') => {
rl.add_history_entry(line.trim_end())?;
@@ -167,13 +173,8 @@ pub async fn exec_from_repl(
},
_ = signal::ctrl_c() => {
println!("^C");
- continue
},
}
- // dialect might have changed
- rl.helper_mut().unwrap().set_dialect(
- &ctx.task_ctx().session_config().options().sql_parser.dialect,
- );
}
}
Err(ReadlineError::Interrupted) => {
@@ -209,7 +210,11 @@ pub(super) async fn exec_and_print(
)
})?;
- let statements = DFParser::parse_sql_with_dialect(&sql, dialect.as_ref())?;
+ let statements = DFParserBuilder::new(sql.as_str())
+ .with_dialect(dialect.as_ref())
+ .with_recursion_limit(options.sql_parser.recursion_limit.get())
+ .build()?
+ .parse_statements()?;
for statement in statements {
StatementExecutor::new(statement)
.execute(ctx, print_options)
@@ -506,6 +511,7 @@ mod tests {
use super::*;
use datafusion::common::plan_err;
+ use datafusion::sql::parser::DFParser;
use datafusion::prelude::SessionContext;
use datafusion_common::assert_contains;
diff --git a/datafusion-cli/src/helper.rs b/datafusion-cli/src/helper.rs
index 4ba4db377c..b3fbbb0e03 100644
--- a/datafusion-cli/src/helper.rs
+++ b/datafusion-cli/src/helper.rs
@@ -10,9 +10,9 @@ use std::cell::Cell;
use crate::highlighter::{Color, NoSyntaxHighlighter, SyntaxHighlighter};
-use datafusion::sql::parser::{DFParser, Statement};
+use datafusion::sql::parser::{DFParser, DFParserBuilder, Statement};
use datafusion::sql::sqlparser::dialect::dialect_from_str;
-use datafusion_common::config::Dialect;
+use datafusion_common::config::{Dialect, SqlParserOptions};
use rustyline::completion::{Completer, FilenameCompleter, Pair};
use rustyline::error::ReadlineError;
@@ -27,6 +27,7 @@ const DEFAULT_HINT_SUGGESTION: &str = " \\? for help, \\q to quit";
pub struct CliHelper {
completer: FilenameCompleter,
dialect: Dialect,
+ recursion_limit: usize,
highlighter: Box<dyn Highlighter>,
/// Tracks whether to show the default hint. Set to `false` once the user
/// types anything, so the hint doesn't reappear after deleting back to
@@ -44,6 +45,7 @@ impl CliHelper {
Self {
completer: FilenameCompleter::new(),
dialect: *dialect,
+ recursion_limit: SqlParserOptions::default().recursion_limit.get(),
highlighter,
show_hint: Cell::new(true),
}
@@ -55,6 +57,13 @@ impl CliHelper {
}
}
+ /// Sets the parser options (dialect and recursion limit) used when
+ /// validating input, see `datafusion.sql_parser.*`.
+ pub fn set_parser_options(&mut self, options: &SqlParserOptions) {
+ self.set_dialect(&options.dialect);
+ self.recursion_limit = options.recursion_limit.get();
+ }
+
/// Re-enable the default hint for the next prompt.
pub fn reset_hint(&self) {
self.show_hint.set(true);
@@ -70,7 +79,12 @@ impl CliHelper {
};
let lines = split_from_semicolon(sql);
for line in lines {
- match DFParser::parse_sql_with_dialect(&line, dialect.as_ref()) {
+ let statements = DFParserBuilder::new(line.as_str())
+ .with_dialect(dialect.as_ref())
+ .with_recursion_limit(self.recursion_limit)
+ .build()
+ .and_then(|mut parser| parser.parse_statements());
+ match statements {
Ok(statements) if statements.is_empty() => {
return Ok(ValidationResult::Invalid(Some(
" 🤔 You entered an empty statement".to_string(),
@@ -201,6 +215,8 @@ mod tests {
use super::*;
+ use datafusion_common::config::ConfigNonZeroUsize;
+
fn readline_direct(
mut reader: impl BufRead,
validator: &CliHelper,
@@ -305,6 +321,36 @@ mod tests {
Ok(())
}
+ #[test]
+ fn sql_recursion_limit() -> Result<()> {
+ let mut validator = CliHelper::default();
+ let nested = |depth: usize| {
+ format!("select {}1{};", "abs(".repeat(depth), ")".repeat(depth))
+ };
+ let with_limit = |limit: usize| SqlParserOptions {
+ recursion_limit: ConfigNonZeroUsize::try_new(limit).unwrap(),
+ ..Default::default()
+ };
+
+ // exceeds the default recursion limit
+ let result = readline_direct(Cursor::new(nested(60)), &validator)?;
+ assert!(
+ matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))
+ );
+
+ validator.set_parser_options(&with_limit(100));
+ let result = readline_direct(Cursor::new(nested(60)), &validator)?;
+ assert!(matches!(result, ValidationResult::Valid(None)));
+
+ validator.set_parser_options(&with_limit(5));
+ let result = readline_direct(Cursor::new(nested(10)), &validator)?;
+ assert!(
+ matches!(result, ValidationResult::Invalid(Some(e)) if e.contains("RecursionLimitExceeded"))
+ );
+
+ Ok(())
+ }
+
#[test]
fn test_split_from_semicolon() {
let sql = "SELECT 1; SELECT 2;";
diff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]
index df3a10b6bb..c139c19c2f 100644
--- a/datafusion-cli/tests/snapshots/[email protected]
+++ b/datafusion-cli/tests/snapshots/[email protected]
@@ -15,5 +15,11 @@ exit_code: 0
+----------+
| 1 |
+----------+
++------+
+| deep |
++------+
+| 1 |
++------+
----- stderr -----
+SQL error: RecursionLimitExceeded (current limit: 5)
diff --git a/datafusion-cli/tests/sql/select.sql b/datafusion-cli/tests/sql/select.sql
index 9e13a3eff4..8ecbb3cc42 100644
--- a/datafusion-cli/tests/sql/select.sql
+++ b/datafusion-cli/tests/sql/select.sql
@@ -1 +1,6 @@
-select 1;
\ No newline at end of file
+select 1;
+-- datafusion.sql_parser.recursion_limit is respected when parsing (#24913)
+set datafusion.sql_parser.recursion_limit = 100;
+select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) as deep;
+set datafusion.sql_parser.recursion_limit = 5;
+select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(1)))))))))) as shallow;