Apache DataFusion SQL Query Engine
chore: Update sqlparser to 0.63.0 (#25278)
Andrew Lamb1dea7142016e
diff --git a/Cargo.lock b/Cargo.lockindex f0677c38d8..061df543c5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2961,7 +2961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"dependencies = ["libc",- "windows-sys 0.61.2",+ "windows-sys 0.52.0",][[package]]@@ -5050,7 +5050,7 @@ dependencies = ["once_cell","socket2","tracing",- "windows-sys 0.60.2",+ "windows-sys 0.52.0",][[package]]@@ -5431,7 +5431,7 @@ dependencies = ["errno","libc","linux-raw-sys",- "windows-sys 0.61.2",+ "windows-sys 0.52.0",][[package]]@@ -5939,9 +5939,9 @@ dependencies = [[[package]]name = "sqlparser"-version = "0.62.0"+version = "0.63.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"+checksum = "3679862809bd1f92e563cf6fd820e28f39135303668001c10331d5027183e8b4"dependencies = ["log","recursive",@@ -5950,13 +5950,13 @@ dependencies = [[[package]]name = "sqlparser_derive"-version = "0.5.0"+version = "0.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"+checksum = "2905ae8e8148d8f7f5748ea95d1e949fa4fc1d7f5161547b758ad118b2e4b0aa"dependencies = ["proc-macro2","quote",- "syn 2.0.119",+ "syn 3.0.5",][[package]]@@ -6189,7 +6189,7 @@ dependencies = ["getrandom 0.4.2","once_cell","rustix",- "windows-sys 0.61.2",+ "windows-sys 0.52.0",][[package]]@@ -7134,7 +7134,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.61.2",+ "windows-sys 0.52.0",][[package]]@@ -7250,16 +7250,7 @@ version = "0.52.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"dependencies = [- "windows-targets 0.52.6",-]--[[package]]-name = "windows-sys"-version = "0.60.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"-dependencies = [- "windows-targets 0.53.5",+ "windows-targets",][[package]]@@ -7277,31 +7268,14 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"dependencies = [- "windows_aarch64_gnullvm 0.52.6",- "windows_aarch64_msvc 0.52.6",- "windows_i686_gnu 0.52.6",- "windows_i686_gnullvm 0.52.6",- "windows_i686_msvc 0.52.6",- "windows_x86_64_gnu 0.52.6",- "windows_x86_64_gnullvm 0.52.6",- "windows_x86_64_msvc 0.52.6",-]--[[package]]-name = "windows-targets"-version = "0.53.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"-dependencies = [- "windows-link",- "windows_aarch64_gnullvm 0.53.1",- "windows_aarch64_msvc 0.53.1",- "windows_i686_gnu 0.53.1",- "windows_i686_gnullvm 0.53.1",- "windows_i686_msvc 0.53.1",- "windows_x86_64_gnu 0.53.1",- "windows_x86_64_gnullvm 0.53.1",- "windows_x86_64_msvc 0.53.1",+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc",+ "windows_i686_gnu",+ "windows_i686_gnullvm",+ "windows_i686_msvc",+ "windows_x86_64_gnu",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc",][[package]]@@ -7319,96 +7293,48 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"-[[package]]-name = "windows_aarch64_gnullvm"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"-[[package]]name = "windows_aarch64_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"-[[package]]-name = "windows_aarch64_msvc"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"-[[package]]name = "windows_i686_gnu"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"-[[package]]-name = "windows_i686_gnu"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"-[[package]]name = "windows_i686_gnullvm"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"-[[package]]-name = "windows_i686_gnullvm"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"-[[package]]name = "windows_i686_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"-[[package]]-name = "windows_i686_msvc"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"-[[package]]name = "windows_x86_64_gnu"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"-[[package]]-name = "windows_x86_64_gnu"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"-[[package]]name = "windows_x86_64_gnullvm"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"-[[package]]-name = "windows_x86_64_gnullvm"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"-[[package]]name = "windows_x86_64_msvc"version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"-[[package]]-name = "windows_x86_64_msvc"-version = "0.53.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"-[[package]]name = "winnow"version = "1.0.2"diff --git a/Cargo.toml b/Cargo.tomlindex c92821c7fe..34ff33de37 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -199,7 +199,7 @@ regex = "1.12"rstest = "0.27.0"serde_json = "1"sha2 = "^0.11.0"-sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor"] }+sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] }stacker = "0.1.24"strum = "0.28.0"strum_macros = "0.28.0"diff --git a/datafusion-testing b/datafusion-testingindex 13bbae3877..81c7440e08 160000--- a/datafusion-testing+++ b/datafusion-testing@@ -1 +1 @@-Subproject commit 13bbae38776c2bfbc1fab1be7e7220222d4284bf+Subproject commit 81c7440e08b9fe81a982e6b0b7e0fe3a6e82fb78diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 360586b0e9..e36fa80f53 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -301,7 +301,7 @@ config_namespace! {pub collect_spans: bool, default = false/// Specifies the recursion depth limit when parsing complex SQL Queries- pub recursion_limit: ConfigNonZeroUsize, default = non_zero_usize_default(50)+ pub recursion_limit: ConfigNonZeroUsize, default = non_zero_usize_default(51)/// Specifies the default null ordering for query results. There are 4 options:/// - `nulls_max`: Nulls appear last in ascending order.diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rsindex a7f7979a67..4bbf3834de 100644--- a/datafusion/sql/src/expr/function.rs+++ b/datafusion/sql/src/expr/function.rs@@ -166,6 +166,11 @@ impl FunctionArgs {"Calling {name}: LIMIT not supported in function arguments: {limit}");}+ FunctionArgumentClause::Where(predicate) => {+ return not_impl_err!(+ "Calling {name}: WHERE not supported in function arguments: {predicate}"+ );+ }FunctionArgumentClause::OnOverflow(overflow) => {return not_impl_err!("Calling {name}: ON OVERFLOW not supported in function arguments: {overflow}"diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex 0fa1fc16bf..ea2ed8eea3 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -450,16 +450,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {planner_context,),- SQLExpr::Cast { array: true, .. } => {- not_impl_err!("`CAST(... AS type ARRAY`) not supported")- }-SQLExpr::Cast {kind: CastKind::Cast | CastKind::DoubleColon,expr,data_type,format,- array: false,} => {self.sql_cast_to_expr(*expr, &data_type, format, schema, planner_context)}@@ -469,7 +464,6 @@ impl<S: ContextProvider> SqlToRel<'_, S> {expr,data_type,format,- array: false,} => {if let Some(format) = format {return not_impl_err!("CAST with format is not supported: {format}");@@ -1068,7 +1062,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {negated: bool,expr: SQLExpr,pattern: SQLExpr,- escape_char: Option<ValueWithSpan>,+ escape_char: Option<Box<SQLExpr>>,schema: &DFSchema,planner_context: &mut PlannerContext,case_insensitive: bool,@@ -1078,13 +1072,14 @@ impl<S: ContextProvider> SqlToRel<'_, S> {return not_impl_err!("ANY in LIKE expression");}let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?;- let escape_char = match escape_char.map(|v| v.value) {- Some(Value::SingleQuotedString(char)) if char.len() == 1 => {- Some(char.chars().next().unwrap())- }- Some(value) => {+ let escape_char = match escape_char.map(|e| *e) {+ Some(SQLExpr::Value(ValueWithSpan {+ value: Value::SingleQuotedString(char),+ ..+ })) if char.len() == 1 => Some(char.chars().next().unwrap()),+ Some(expr) => {return plan_err!(- "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {value}"+ "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {expr}");}None => None,@@ -1103,18 +1098,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {negated: bool,expr: SQLExpr,pattern: SQLExpr,- escape_char: Option<ValueWithSpan>,+ escape_char: Option<Box<SQLExpr>>,schema: &DFSchema,planner_context: &mut PlannerContext,) -> Result<Expr> {let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?;- let escape_char = match escape_char.map(|v| v.value) {- Some(Value::SingleQuotedString(char)) if char.len() == 1 => {- Some(char.chars().next().unwrap())- }- Some(value) => {+ let escape_char = match escape_char.map(|e| *e) {+ Some(SQLExpr::Value(ValueWithSpan {+ value: Value::SingleQuotedString(char),+ ..+ })) if char.len() == 1 => Some(char.chars().next().unwrap()),+ Some(expr) => {return plan_err!(- "Invalid escape character in SIMILAR TO expression. Expected a single character wrapped with single quotes, got {value}"+ "Invalid escape character in SIMILAR TO expression. Expected a single character wrapped with single quotes, got {expr}");}None => None,diff --git a/datafusion/sql/src/expr/order_by.rs b/datafusion/sql/src/expr/order_by.rsindex 0067a1ebd7..fcf14d5dc8 100644--- a/datafusion/sql/src/expr/order_by.rs+++ b/datafusion/sql/src/expr/order_by.rs@@ -22,7 +22,7 @@ use datafusion_common::{use datafusion_expr::expr::Sort;use datafusion_expr::{Expr, SortExpr};use sqlparser::ast::{- Expr as SQLExpr, OrderByExpr, OrderByOptions, Value, ValueWithSpan,+ Expr as SQLExpr, OrderByExpr, OrderByOptions, OrderBySort, Value, ValueWithSpan,};impl<S: ContextProvider> SqlToRel<'_, S> {@@ -75,7 +75,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {for order_by_expr in order_by_exprs {let OrderByExpr {expr,- options: OrderByOptions { asc, nulls_first },+ options: OrderByOptions { sort, nulls_first },with_fill,} = order_by_expr;@@ -83,6 +83,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {return not_impl_err!("ORDER BY WITH FILL is not supported: {with_fill}");}+ let asc = match sort {+ Some(OrderBySort::Asc) => Some(true),+ Some(OrderBySort::Desc) => Some(false),+ Some(OrderBySort::Using(op)) => {+ return not_impl_err!("ORDER BY USING is not supported: {op}");+ }+ None => None,+ };+let expr = match expr {SQLExpr::Value(ValueWithSpan {value: Value::Number(v, _),diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rsindex fcf4708f1b..12c2569740 100644--- a/datafusion/sql/src/parser.rs+++ b/datafusion/sql/src/parser.rs@@ -24,7 +24,7 @@ use datafusion_common::DataFusionError;use datafusion_common::config::{ConfigNonZeroUsize, SqlParserOptions};use datafusion_common::format::{ExplainFormat, ExplainStatementOptions};use datafusion_common::{Diagnostic, Span, sql_err};-use sqlparser::ast::{ExprWithAlias, Ident, OrderByOptions};+use sqlparser::ast::{ExprWithAlias, Ident, OrderByOptions, OrderBySort};use sqlparser::tokenizer::TokenWithSpan;use sqlparser::{ast::{@@ -381,8 +381,12 @@ pub struct DFParser<'a> {supports_explain_with_utility_options: bool,}-/// Same as `sqlparser`-const DEFAULT_RECURSION_LIMIT: usize = 50;+/// Default recursion limit for SQL parsing. Higher than the `sqlparser` default+/// (50) for backwards compatibility with older versions.+///+/// Newer `sqlparser` versions added recursion guards to more parse functions, so+/// the same query consumes more depth than it used to.+const DEFAULT_RECURSION_LIMIT: usize = 51;const DEFAULT_DIALECT: GenericDialect = GenericDialect {};/// Builder for [`DFParser`]@@ -464,7 +468,7 @@ impl<'a, 'b> DFParserBuilder<'a, 'b> {self}- /// Adjust the recursion limit of sql parsing. Defaults to 50+ /// Adjust the recursion limit of sql parsing. Defaults to 51pub fn with_recursion_limit(mut self, recursion_limit: usize) -> Self {self.recursion_limit = recursion_limit;self@@ -998,10 +1002,10 @@ impl<'a> DFParser<'a> {pub fn parse_order_by_expr(&mut self) -> Result<OrderByExpr, DataFusionError> {let expr = self.parser.parse_expr()?;- let asc = if self.parser.parse_keyword(Keyword::ASC) {- Some(true)+ let sort = if self.parser.parse_keyword(Keyword::ASC) {+ Some(OrderBySort::Asc)} else if self.parser.parse_keyword(Keyword::DESC) {- Some(false)+ Some(OrderBySort::Desc)} else {None};@@ -1019,7 +1023,7 @@ impl<'a> DFParser<'a> {Ok(OrderByExpr {expr,- options: OrderByOptions { asc, nulls_first },+ options: OrderByOptions { sort, nulls_first },with_fill: None,})}@@ -1619,7 +1623,16 @@ mod tests {quote_style: None,span: Span::empty(),}),- options: OrderByOptions { asc, nulls_first },+ options: OrderByOptions {+ sort: asc.map(|asc| {+ if asc {+ OrderBySort::Asc+ } else {+ OrderBySort::Desc+ }+ }),+ nulls_first,+ },with_fill: None,}]],..make_create_external_table("foo.csv")@@ -1643,7 +1656,7 @@ mod tests {span: Span::empty(),}),options: OrderByOptions {- asc: Some(true),+ sort: Some(OrderBySort::Asc),nulls_first: None,},with_fill: None,@@ -1655,7 +1668,7 @@ mod tests {span: Span::empty(),}),options: OrderByOptions {- asc: Some(false),+ sort: Some(OrderBySort::Desc),nulls_first: Some(true),},with_fill: None,@@ -1688,7 +1701,7 @@ mod tests {})),},options: OrderByOptions {- asc: Some(true),+ sort: Some(OrderBySort::Asc),nulls_first: None,},with_fill: None,@@ -1731,7 +1744,7 @@ mod tests {})),},options: OrderByOptions {- asc: Some(true),+ sort: Some(OrderBySort::Asc),nulls_first: None,},with_fill: None,@@ -1795,7 +1808,7 @@ mod tests {})),},options: OrderByOptions {- asc: Some(true),+ sort: Some(OrderBySort::Asc),nulls_first: None,},with_fill: None,diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rsindex a3e6d75fdb..e3f51cb075 100644--- a/datafusion/sql/src/planner.rs+++ b/datafusion/sql/src/planner.rs@@ -838,7 +838,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {.collect::<Result<Vec<_>>>()?;Ok(DataType::Struct(Fields::from(fields)))}- SQLDataType::Map(key_type, value_type) => {+ SQLDataType::Map(key_type, value_type, _) => {let key_field = Arc::new(Field::new("key", self.convert_data_type_to_field(key_type)?.data_type().clone(), false));diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rsindex d23d1a008f..f26411cd60 100644--- a/datafusion/sql/src/query.rs+++ b/datafusion/sql/src/query.rs@@ -27,9 +27,9 @@ use datafusion_expr::{CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder,};use sqlparser::ast::{- Expr as SQLExpr, ExprWithAliasAndOrderBy, LimitClause, Offset, OffsetRows, OrderBy,- OrderByExpr, OrderByKind, PipeOperator, Query, SelectInto, SetExpr, SetOperator,- SetQuantifier, TableAlias, Value,+ Expr as SQLExpr, ExprWithAliasAndOrderBy, LimitClause, ObjectName, Offset,+ OffsetRows, OrderBy, OrderByExpr, OrderByKind, PipeOperator, Query, SelectInto,+ SetExpr, SetOperator, SetQuantifier, TableAlias, Value,};impl<S: ContextProvider> SqlToRel<'_, S> {@@ -362,17 +362,30 @@ impl<S: ContextProvider> SqlToRel<'_, S> {select_into: Option<SelectInto>,) -> Result<LogicalPlan> {match select_into {- Some(into) => Ok(LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(- CreateMemoryTable {- name: self.object_name_to_table_reference(into.name)?,- constraints: Constraints::default(),- input: Arc::new(plan),- if_not_exists: false,- or_replace: false,- temporary: false,- column_defaults: vec![],- },- ))),+ Some(into) => {+ let name = match into.targets.as_slice() {+ [SQLExpr::Identifier(ident)] => ObjectName::from(vec![ident.clone()]),+ [SQLExpr::CompoundIdentifier(idents)] => {+ ObjectName::from(idents.clone())+ }+ _ => {+ return not_impl_err!(+ "SELECT INTO only supports a single table name target"+ );+ }+ };+ Ok(LogicalPlan::Ddl(DdlStatement::CreateMemoryTable(+ CreateMemoryTable {+ name: self.object_name_to_table_reference(name)?,+ constraints: Constraints::default(),+ input: Arc::new(plan),+ if_not_exists: false,+ or_replace: false,+ temporary: false,+ column_defaults: vec![],+ },+ )))+ }_ => Ok(plan),}}@@ -393,7 +406,7 @@ fn to_order_by_exprs(OrderByKind::All(options) => Ok((1..=order_by_all_column_count).map(|position| OrderByExpr {expr: SQLExpr::value(Value::Number(position.to_string(), false)),- options,+ options: options.clone(),with_fill: None,}).collect()),diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex c61f09f6f5..38687efeb0 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -118,6 +118,7 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConstindex_type_display: _index_type_display,index_type: _index_type,columns: _column,+ include: _include,index_options: _index_options,nulls_distinct: _nulls_distinct,}) => constraints.push(TableConstraint::Unique(UniqueConstraint {@@ -129,13 +130,14 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConstcolumn: OrderByExpr {expr: SQLExpr::Identifier(column.name.clone()),options: OrderByOptions {- asc: None,+ sort: None,nulls_first: None,},with_fill: None,},operator_class: None,}],+ include: vec![],index_options: vec![],characteristics: *characteristics,nulls_distinct: NullsDistinctOption::None,@@ -146,6 +148,7 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConstindex_name: _index_name,index_type: _index_type,columns: _columns,+ include: _include,index_options: _index_options,}) => {constraints.push(TableConstraint::PrimaryKey(PrimaryKeyConstraint {@@ -156,13 +159,14 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConstcolumn: OrderByExpr {expr: SQLExpr::Identifier(column.name.clone()),options: OrderByOptions {- asc: None,+ sort: None,nulls_first: None,},with_fill: None,},operator_class: None,}],+ include: vec![],index_options: vec![],characteristics: *characteristics,}))@@ -193,10 +197,12 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConstast::ColumnOption::Check(CheckConstraint {name,expr,+ no_inherit: _no_inherit,enforced: _enforced,}) => constraints.push(TableConstraint::Check(CheckConstraint {name: name.clone(),expr: expr.clone(),+ no_inherit: false,enforced: None,})),ast::ColumnOption::Default(_)@@ -362,6 +368,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {distkey,sortkey,backup,+ unlogged,+ with_connection,+ multiset,+ fallback,+ with_data,}) => {if temporary {return not_impl_err!("Temporary tables not supported");@@ -533,6 +544,21 @@ impl<S: ContextProvider> SqlToRel<'_, S> {if backup.is_some() {return not_impl_err!("BACKUP not supported");}+ if unlogged {+ return not_impl_err!("UNLOGGED tables not supported");+ }+ if with_connection.is_some() {+ return not_impl_err!("WITH CONNECTION not supported");+ }+ if multiset.is_some() {+ return not_impl_err!("MULTISET tables not supported");+ }+ if fallback.is_some() {+ return not_impl_err!("FALLBACK not supported");+ }+ if with_data.is_some() {+ return not_impl_err!("WITH [NO] DATA not supported");+ }// Merge inline constraints and existing constraintslet mut all_constraints = constraints;let inline_constraints = calc_inline_constraints_from_columns(&columns);@@ -1765,7 +1791,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {schema,planner_context,)?;- let asc = order_by_expr.options.asc.unwrap_or(true);+ let asc = match &order_by_expr.options.sort {+ Some(ast::OrderBySort::Asc) | None => true,+ Some(ast::OrderBySort::Desc) => false,+ Some(ast::OrderBySort::Using(op)) => {+ return not_impl_err!(+ "ORDER BY USING is not supported: {op}"+ );+ }+ };let nulls_first =order_by_expr.options.nulls_first.unwrap_or_else(|| {self.options.default_null_ordering.nulls_first(asc)@@ -1932,6 +1966,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {index_type_display: _,index_type: _,columns,+ include: _,index_options: _,characteristics: _,nulls_distinct: _,@@ -1953,6 +1988,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {index_name: _,index_type: _,columns,+ include: _,index_options: _,characteristics: _,}) => {@@ -1967,6 +2003,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {TableConstraint::ForeignKey { .. } => {_plan_err!("Foreign key constraints are not currently supported")}+ TableConstraint::Exclude(_) => {+ _plan_err!("Exclude constraints are not currently supported")+ }TableConstraint::Check { .. } => {_plan_err!("Check constraints are not currently supported")}@@ -2723,8 +2762,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {"MERGE UPDATE DELETE WHERE predicates are not supported");}- let assignments = update_expr- .assignments+ let sql_assignments = match update_expr.kind {+ ast::MergeUpdateKind::Set(assignments) => assignments,+ ast::MergeUpdateKind::Wildcard => {+ return not_impl_err!("MERGE UPDATE SET * is not supported");+ }+ };+ let assignments = sql_assignments.into_iter().map(|assign| {let col_name = match &assign.target {@@ -2802,11 +2846,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {ast::MergeInsertKind::Row => {return not_impl_err!("MERGE INSERT ROW is not supported");}+ ast::MergeInsertKind::Wildcard => {+ return not_impl_err!("MERGE INSERT * is not supported");+ }};MergeIntoAction::Insert { columns, values }}ast::MergeAction::Delete { .. } => MergeIntoAction::Delete,+ ast::MergeAction::DoNothing { .. } => {+ return not_impl_err!(+ "MERGE WHEN ... THEN DO NOTHING action is not supported"+ );+ }};Ok(MergeIntoClause {diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rsindex 545da7a95c..bea4e35300 100644--- a/datafusion/sql/src/unparser/dialect.rs+++ b/datafusion/sql/src/unparser/dialect.rs@@ -465,7 +465,6 @@ impl PostgreSqlDialect {kind: ast::CastKind::Cast,expr: Box::new(expr.clone()),data_type: ast::DataType::Numeric(ast::ExactNumberInfo::None),- array: false,format: None,};}diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex d5cbe64cae..0f2cef167c 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -360,8 +360,9 @@ impl Unparser<'_> {negated: *negated,expr: Box::new(self.expr_to_sql_inner(expr)?),pattern: Box::new(self.expr_to_sql_inner(pattern)?),- escape_char: escape_char- .map(|c| SingleQuotedString(c.to_string()).into()),+ escape_char: escape_char.map(|c| {+ Box::new(ast::Expr::Value(SingleQuotedString(c.to_string()).into()))+ }),any: false,}),Expr::Like(Like {@@ -374,8 +375,9 @@ impl Unparser<'_> {let negated = *negated;let expr = Box::new(self.expr_to_sql_inner(expr)?);let pattern = Box::new(self.expr_to_sql_inner(pattern)?);- let escape_char =- escape_char.map(|c| SingleQuotedString(c.to_string()).into());+ let escape_char = escape_char.map(|c| {+ Box::new(ast::Expr::Value(SingleQuotedString(c.to_string()).into()))+ });if *case_insensitive {Ok(ast::Expr::ILike {@@ -562,7 +564,6 @@ impl Unparser<'_> {kind: ast::CastKind::TryCast,expr: Box::new(inner_expr),data_type: self.arrow_dtype_to_ast_dtype(field)?,- array: false,format: None,})}@@ -868,7 +869,11 @@ impl Unparser<'_> {Ok(ast::OrderByExpr {expr: sql_parser_expr,options: OrderByOptions {- asc: Some(*asc),+ sort: Some(if *asc {+ ast::OrderBySort::Asc+ } else {+ ast::OrderBySort::Desc+ }),nulls_first,},with_fill: None,@@ -1249,7 +1254,6 @@ impl Unparser<'_> {kind: ast::CastKind::Cast,expr: Box::new(ast::Expr::value(SingleQuotedString(ts))),data_type: self.dialect.timestamp_cast_dtype(&time_unit, &None),- array: false,format: None,})}@@ -1272,7 +1276,6 @@ impl Unparser<'_> {kind: ast::CastKind::Cast,expr: Box::new(ast::Expr::value(SingleQuotedString(time))),data_type: ast::DataType::Time(None, TimezoneInfo::None),- array: false,format: None,})}@@ -1293,7 +1296,6 @@ impl Unparser<'_> {kind: ast::CastKind::Cast,expr: Box::new(inner_expr),data_type: self.arrow_dtype_to_ast_dtype(field)?,- array: false,format: None,}),},@@ -1301,7 +1303,6 @@ impl Unparser<'_> {kind: ast::CastKind::Cast,expr: Box::new(inner_expr),data_type: self.arrow_dtype_to_ast_dtype(field)?,- array: false,format: None,}),}@@ -1452,7 +1453,6 @@ impl Unparser<'_> {date.to_string(),))),data_type: ast::DataType::Date,- array: false,format: None,})}@@ -1476,7 +1476,6 @@ impl Unparser<'_> {datetime.to_string(),))),data_type: self.ast_type_for_date64_in_cast(),- array: false,format: None,})}diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rsindex 8b025d3945..d7ad61b2bc 100644--- a/datafusion/sql/tests/cases/params.rs+++ b/datafusion/sql/tests/cases/params.rs@@ -171,7 +171,7 @@ fn test_non_prepare_statement_should_infer_types() {#[test]#[should_panic(- expected = "Expected: [NOT] NULL | TRUE | FALSE | DISTINCT | [form] NORMALIZED FROM after IS, found: $1"+ expected = "Expected: [NOT] NULL | TRUE | FALSE | DISTINCT | [NOT] JSON [VALUE | SCALAR | ARRAY | OBJECT] [WITH | WITHOUT UNIQUE [KEYS]] | [form] NORMALIZED FROM after IS, found: $1")]fn test_prepare_statement_to_plan_panic_is_param() {let sql = "PREPARE my_plan(INT) AS SELECT id, age FROM person WHERE age is $1";diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.sltindex 451241756d..5e9fad36e3 100644--- a/datafusion/sqllogictest/test_files/array_agg.slt+++ b/datafusion/sqllogictest/test_files/array_agg.slt@@ -596,6 +596,9 @@ physical_planstatement error This feature is not implemented: Calling array_agg: LIMIT not supported in function arguments: 1SELECT array_agg(c13 LIMIT 1) FROM aggregate_test_100+statement error This feature is not implemented: Calling array_agg: WHERE not supported in function arguments: c2 > 1+SELECT array_agg(c13 WHERE c2 > 1) FROM aggregate_test_100+query error Error during planning: WITHIN GROUP is only supported for ordered-set aggregate functionsSELECT array_agg(a_varchar) WITHIN GROUP (ORDER BY a_varchar)FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar);diff --git a/datafusion/sqllogictest/test_files/ddl.slt b/datafusion/sqllogictest/test_files/ddl.sltindex d47aad0b19..21c195c3d2 100644--- a/datafusion/sqllogictest/test_files/ddl.slt+++ b/datafusion/sqllogictest/test_files/ddl.slt@@ -377,6 +377,10 @@ SELECT * FROM new_tablestatement okDROP TABLE new_table;+# Select into only supports a single table name target+statement error DataFusion error: This feature is not implemented: SELECT INTO only supports a single table name target+SELECT * INTO new_table, other_table FROM test_table+# Create table as without alias names of window aggregatesstatement okCREATE TABLE new_table AS SELECT SUM(c1) OVER(ORDER BY c2), c2, c3 FROM test_table@@ -846,6 +850,18 @@ CREATE EXTERNAL TEMPORARY TABLE tty STORED as ARROW LOCATION '../datasource-arrostatement error DataFusion error: This feature is not implemented: Temporary views not supportedCREATE TEMPORARY VIEW y AS VALUES (1,2,3);+statement error DataFusion error: This feature is not implemented: UNLOGGED tables not supported+CREATE UNLOGGED TABLE foo (x int);++statement error DataFusion error: This feature is not implemented: MULTISET tables not supported+CREATE MULTISET TABLE foo (x int);++statement error DataFusion error: This feature is not implemented: WITH \[NO\] DATA not supported+CREATE TABLE foo AS SELECT 1 WITH NO DATA;++statement error DataFusion error: Error during planning: Exclude constraints are not currently supported+CREATE TABLE foo (x int, EXCLUDE USING gist (x WITH =));+query error DataFusion error: Schema error: No field named a\.EXPLAIN CREATE TABLE t(a int) AS VALUES (a + a);diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex b270eba99d..924883da42 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -359,7 +359,7 @@ datafusion.sql_parser.enable_options_value_normalization falsedatafusion.sql_parser.enable_subquery_sort_elimination truedatafusion.sql_parser.map_string_types_to_utf8view truedatafusion.sql_parser.parse_float_as_decimal false-datafusion.sql_parser.recursion_limit 50+datafusion.sql_parser.recursion_limit 51datafusion.sql_parser.support_varchar_with_length true# show all variables with verbose@@ -520,7 +520,7 @@ datafusion.sql_parser.enable_options_value_normalization false When set to true,datafusion.sql_parser.enable_subquery_sort_elimination true When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query.datafusion.sql_parser.map_string_types_to_utf8view true If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true.datafusion.sql_parser.parse_float_as_decimal false When set to true, SQL parser will parse float as decimal type-datafusion.sql_parser.recursion_limit 50 Specifies the recursion depth limit when parsing complex SQL Queries+datafusion.sql_parser.recursion_limit 51 Specifies the recursion depth limit when parsing complex SQL Queriesdatafusion.sql_parser.support_varchar_with_length true If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits.# show_variable_in_config_optionsdiff --git a/datafusion/sqllogictest/test_files/merge_into.slt b/datafusion/sqllogictest/test_files/merge_into.sltindex 5049bf8825..6b0fe64c8a 100644--- a/datafusion/sqllogictest/test_files/merge_into.slt+++ b/datafusion/sqllogictest/test_files/merge_into.slt@@ -241,6 +241,18 @@ statement error DataFusion error: This feature is not implemented: MERGE INSERTmerge into target using source on target.id = source.idwhen not matched then insert row;+statement error DataFusion error: This feature is not implemented: MERGE UPDATE SET \* is not supported+merge into target using source on target.id = source.id+when matched then update set *;++statement error DataFusion error: This feature is not implemented: MERGE INSERT \* is not supported+merge into target using source on target.id = source.id+when not matched then insert *;++statement error DataFusion error: This feature is not implemented: MERGE WHEN ... THEN DO NOTHING action is not supported+merge into target using source on target.id = source.id+when matched then do nothing;+statement okdrop table target;diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex d080139bc7..d974eb9dd0 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -634,6 +634,16 @@ ORDER BY sum(value) + sum(value);statement error DataFusion error: This feature is not implemented: SORT BYselect * from t SORT BY time;+## ORDER BY USING <operator> (PostgreSQL) is not supported+statement ok+set datafusion.sql_parser.dialect = 'PostgreSQL';++statement error DataFusion error: This feature is not implemented: ORDER BY USING is not supported: <+select * from t ORDER BY time USING <;++statement ok+reset datafusion.sql_parser.dialect;+# distinct on a column not in the select list should not workstatement error DataFusion error: Error during planning: For SELECT DISTINCT, ORDER BY expressions t\.time must appear in select listdiff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex f83c8b3d5c..7e6543cefc 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -416,6 +416,24 @@ let description = ChildFilterDescription::from_child_with_column_mapping()?;```+### Default `datafusion.sql_parser.recursion_limit` raised from 50 to 51++DataFusion now uses `sqlparser` 0.63.0, which adds recursion guards to more+parse functions (for example, data type and `INTERVAL` parsing). As a result,+the same SQL statement consumes slightly more of the parser recursion budget+than it did with previous `sqlparser` versions. In particular, every expression+now needs one additional level of depth.++To avoid rejecting queries that previously parsed successfully, the default+value of the `datafusion.sql_parser.recursion_limit` configuration option, and+of `DFParserBuilder::with_recursion_limit`, has been raised from `50` to `51`.++If you set `datafusion.sql_parser.recursion_limit` explicitly to a value close+to the depth of your queries, you may see `RecursionLimitExceeded` errors after+upgrading and should raise the limit accordingly.++See [PR #25278](https://github.com/apache/datafusion/pull/25278) for details.+### `ensure_distribution` is deprecated in favour of `ensure_distribution_with_stats``datafusion_physical_optimizer::enforce_distribution::ensure_distribution` nowdiff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex 0085d4ac7c..5922c9e196 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -200,7 +200,7 @@ The following configuration settings are available:| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. || datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. || datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. |-| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries |+| datafusion.sql_parser.recursion_limit | 51 | Specifies the recursion depth limit when parsing complex SQL Queries || datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: <https://www.postgresql.org/docs/current/queries-order.html> || datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. || datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] |