Apache DataFusion SQL Query Engine
chore: Remove usage of `paste` crate (#20946)
Bhargava Vadlamani5d37babf3685
diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.ymlindex 3652506c32..b1cabc2d00 100644--- a/.github/workflows/audit.yml+++ b/.github/workflows/audit.yml@@ -48,4 +48,4 @@ jobs:- name: Run audit check# Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:# run: cargo audit --ignore RUSTSEC-2026-0001- run: cargo audit --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2024-0014+ run: cargo audit --ignore RUSTSEC-2024-0014diff --git a/Cargo.lock b/Cargo.lockindex 168d3bd0c1..35660359ce 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1768,7 +1768,6 @@ dependencies = ["object_store","parking_lot","parquet",- "paste","pretty_assertions","rand 0.9.2","rand_distr",@@ -1910,7 +1909,6 @@ dependencies = ["log","object_store","parquet",- "paste","rand 0.9.2","recursive","sqlparser",@@ -2166,7 +2164,6 @@ dependencies = ["indexmap 2.13.0","insta","itertools 0.14.0",- "paste","recursive","serde_json","sqlparser",@@ -2181,7 +2178,6 @@ dependencies = ["indexmap 2.13.0","insta","itertools 0.14.0",- "paste",][[package]]@@ -2270,7 +2266,6 @@ dependencies = ["half","log","num-traits",- "paste","rand 0.9.2",]@@ -2308,7 +2303,6 @@ dependencies = ["itertools 0.14.0","itoa","log",- "paste","rand 0.9.2",]@@ -2323,7 +2317,6 @@ dependencies = ["datafusion-expr","datafusion-physical-plan","parking_lot",- "paste",][[package]]@@ -2340,7 +2333,6 @@ dependencies = ["datafusion-physical-expr","datafusion-physical-expr-common","log",- "paste",][[package]]@@ -2406,7 +2398,6 @@ dependencies = ["insta","itertools 0.14.0","parking_lot",- "paste","petgraph","rand 0.9.2","recursive",@@ -2625,7 +2616,6 @@ dependencies = ["insta","itertools 0.14.0","log",- "paste","recursive","regex","rstest",diff --git a/Cargo.toml b/Cargo.tomlindex a185cd874a..73e8a61d33 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -173,7 +173,6 @@ parquet = { version = "58.0.0", default-features = false, features = ["async","object_store",] }-paste = "1.0.15"pbjson = { version = "0.9.0" }pbjson-types = "0.9"# Should match arrow-flight's version of prost.diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex 4441fe9035..36435580b2 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -85,7 +85,6 @@ libc = "0.2.180"log = { workspace = true }object_store = { workspace = true, optional = true }parquet = { workspace = true, optional = true, default-features = true }-paste = { workspace = true }recursive = { workspace = true, optional = true }sqlparser = { workspace = true, optional = true }tokio = { workspace = true }diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rsindex 4f681896df..b7a30f868a 100644--- a/datafusion/common/src/error.rs+++ b/datafusion/common/src/error.rs@@ -903,76 +903,125 @@ macro_rules! assert_ne_or_internal_err {/// plan_err!("Error {val:?}")////// `NAME_ERR` - macro name for wrapping Err(DataFusionError::*)+/// `PREFIXED_NAME_ERR` - underscore-prefixed alias for NAME_ERR (e.g., _plan_err)+/// (Needed to avoid compiler error when using macro in the same crate: `macros from the current crate cannot be referred to by absolute paths`)/// `NAME_DF_ERR` - macro name for wrapping DataFusionError::*. Needed to keep backtrace opportunity/// in construction where DataFusionError::* used directly, like `map_err`, `ok_or_else`, etc+/// `PREFIXED_NAME_DF_ERR` - underscore-prefixed alias for NAME_DF_ERR (e.g., _plan_datafusion_err).+/// (Needed to avoid compiler error when using macro in the same crate: `macros from the current crate cannot be referred to by absolute paths`)macro_rules! make_error {- ($NAME_ERR:ident, $NAME_DF_ERR: ident, $ERR:ident) => { make_error!(@inner ($), $NAME_ERR, $NAME_DF_ERR, $ERR); };- (@inner ($d:tt), $NAME_ERR:ident, $NAME_DF_ERR:ident, $ERR:ident) => {- ::paste::paste!{- /// Macro wraps `$ERR` to add backtrace feature- #[macro_export]- macro_rules! $NAME_DF_ERR {- ($d($d args:expr),* $d(; diagnostic=$d DIAG:expr)?) => {{- let err =$crate::DataFusionError::$ERR(- ::std::format!(- "{}{}",- ::std::format!($d($d args),*),- $crate::DataFusionError::get_back_trace(),- ).into()- );- $d (- let err = err.with_diagnostic($d DIAG);- )?- err- }- }+ ($NAME_ERR:ident, $PREFIXED_NAME_ERR:ident, $NAME_DF_ERR:ident, $PREFIXED_NAME_DF_ERR:ident, $ERR:ident) => {+ make_error!(@inner ($), $NAME_ERR, $PREFIXED_NAME_ERR, $NAME_DF_ERR, $PREFIXED_NAME_DF_ERR, $ERR);+ };+ (@inner ($d:tt), $NAME_ERR:ident, $PREFIXED_NAME_ERR:ident, $NAME_DF_ERR:ident, $PREFIXED_NAME_DF_ERR:ident, $ERR:ident) => {+ /// Macro wraps `$ERR` to add backtrace feature+ #[macro_export]+ macro_rules! $NAME_DF_ERR {+ ($d($d args:expr),* $d(; diagnostic = $d DIAG:expr)?) => {{+ let err = $crate::DataFusionError::$ERR(+ ::std::format!(+ "{}{}",+ ::std::format!($d($d args),*),+ $crate::DataFusionError::get_back_trace(),+ ).into()+ );+ $d (+ let err = err.with_diagnostic($d DIAG);+ )?+ err+ }}}- /// Macro wraps Err(`$ERR`) to add backtrace feature- #[macro_export]- macro_rules! $NAME_ERR {- ($d($d args:expr),* $d(; diagnostic = $d DIAG:expr)?) => {{- let err = $crate::[<_ $NAME_DF_ERR>]!($d($d args),*);- $d (- let err = err.with_diagnostic($d DIAG);- )?- Err(err)-- }}- }--- #[doc(hidden)]- pub use $NAME_ERR as [<_ $NAME_ERR>];- #[doc(hidden)]- pub use $NAME_DF_ERR as [<_ $NAME_DF_ERR>];+ /// Macro wraps Err(`$ERR`) to add backtrace feature+ #[macro_export]+ macro_rules! $NAME_ERR {+ ($d($d args:expr),* $d(; diagnostic = $d DIAG:expr)?) => {{+ let err = $crate::$PREFIXED_NAME_DF_ERR!($d($d args),*);+ $d (+ let err = err.with_diagnostic($d DIAG);+ )?+ Err(err)+ }}}++ #[doc(hidden)]+ pub use $NAME_ERR as $PREFIXED_NAME_ERR;+ #[doc(hidden)]+ pub use $NAME_DF_ERR as $PREFIXED_NAME_DF_ERR;};}// Exposes a macro to create `DataFusionError::Plan` with optional backtrace-make_error!(plan_err, plan_datafusion_err, Plan);+make_error!(+ plan_err,+ _plan_err,+ plan_datafusion_err,+ _plan_datafusion_err,+ Plan+);// Exposes a macro to create `DataFusionError::Internal` with optional backtrace-make_error!(internal_err, internal_datafusion_err, Internal);+make_error!(+ internal_err,+ _internal_err,+ internal_datafusion_err,+ _internal_datafusion_err,+ Internal+);// Exposes a macro to create `DataFusionError::NotImplemented` with optional backtrace-make_error!(not_impl_err, not_impl_datafusion_err, NotImplemented);+make_error!(+ not_impl_err,+ _not_impl_err,+ not_impl_datafusion_err,+ _not_impl_datafusion_err,+ NotImplemented+);// Exposes a macro to create `DataFusionError::Execution` with optional backtrace-make_error!(exec_err, exec_datafusion_err, Execution);+make_error!(+ exec_err,+ _exec_err,+ exec_datafusion_err,+ _exec_datafusion_err,+ Execution+);// Exposes a macro to create `DataFusionError::Configuration` with optional backtrace-make_error!(config_err, config_datafusion_err, Configuration);+make_error!(+ config_err,+ _config_err,+ config_datafusion_err,+ _config_datafusion_err,+ Configuration+);// Exposes a macro to create `DataFusionError::Substrait` with optional backtrace-make_error!(substrait_err, substrait_datafusion_err, Substrait);+make_error!(+ substrait_err,+ _substrait_err,+ substrait_datafusion_err,+ _substrait_datafusion_err,+ Substrait+);// Exposes a macro to create `DataFusionError::ResourcesExhausted` with optional backtrace-make_error!(resources_err, resources_datafusion_err, ResourcesExhausted);+make_error!(+ resources_err,+ _resources_err,+ resources_datafusion_err,+ _resources_datafusion_err,+ ResourcesExhausted+);// Exposes a macro to create `DataFusionError::Ffi` with optional backtrace-make_error!(ffi_err, ffi_datafusion_err, Ffi);+make_error!(+ ffi_err,+ _ffi_err,+ ffi_datafusion_err,+ _ffi_datafusion_err,+ Ffi+);// Exposes a macro to create `DataFusionError::SQL` with optional backtrace#[macro_export]diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex be507e0691..326b791a2f 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -172,7 +172,6 @@ bytes = { workspace = true }env_logger = { workspace = true }glob = { workspace = true }insta = { workspace = true }-paste = { workspace = true }pretty_assertions = "1.0"rand = { workspace = true, features = ["small_rng"] }rand_distr = "0.5"diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rsindex 6d49e0bcc6..a41803191a 100644--- a/datafusion/core/tests/parquet/page_pruning.rs+++ b/datafusion/core/tests/parquet/page_pruning.rs@@ -366,281 +366,367 @@ async fn prune_date64() {}macro_rules! int_tests {- ($bits:expr) => {- paste::item! {- #[tokio::test]- // null count min max- // page-0 0 -5 -1- // page-1 0 -4 0- // page-2 0 0 4- // page-3 0 5 9- async fn [<prune_int $bits _lt>]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{} < 1", $bits),- Some(0),- Some(5),- 11,- 5,- )- .await;- // result of sql "SELECT * FROM t where i < 1" is same as- // "SELECT * FROM t where -i > -1"- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where -i{} > -1", $bits),- Some(0),- Some(5),- 11,- 5,- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _gt >]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{} > 8", $bits),- Some(0),- Some(15),- 1,- 5,- )- .await;-- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where -i{} < -8", $bits),- Some(0),- Some(15),- 1,- 5,- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq >]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{} = 1", $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }- #[tokio::test]- async fn [<prune_int $bits _scalar_fun_and_eq >]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where abs(i{}) = 1 and i{} = 1", $bits, $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _scalar_fun >]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where abs(i{}) = 1", $bits),- Some(0),- Some(0),- 3,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _complex_expr>]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{}+1 = 1", $bits),- Some(0),- Some(0),- 2,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _complex_expr_subtract >]() {- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where 1-i{} > 1", $bits),- Some(0),- Some(0),- 9,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq_in_list >]() {- // result of sql "SELECT * FROM t where in (1)"- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{} in (1)", $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq_in_list_negated >]() {- // result of sql "SELECT * FROM t where not in (1)" prune nothing- test_prune(- Scenario::Int,- &format!("SELECT * FROM t where i{} not in (1)", $bits),- Some(0),- Some(0),- 19,- 5- )- .await;- }+ ($bits:expr, $fn_lt:ident, $fn_gt:ident, $fn_eq:ident, $fn_scalar_fun_and_eq:ident, $fn_scalar_fun:ident, $fn_complex_expr:ident, $fn_complex_expr_subtract:ident, $fn_eq_in_list:ident, $fn_eq_in_list_negated:ident) => {+ #[tokio::test]+ // null count min max+ // page-0 0 -5 -1+ // page-1 0 -4 0+ // page-2 0 0 4+ // page-3 0 5 9+ async fn $fn_lt() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{} < 1", $bits),+ Some(0),+ Some(5),+ 11,+ 5,+ )+ .await;+ // result of sql "SELECT * FROM t where i < 1" is same as+ // "SELECT * FROM t where -i > -1"+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where -i{} > -1", $bits),+ Some(0),+ Some(5),+ 11,+ 5,+ )+ .await;}- }++ #[tokio::test]+ async fn $fn_gt() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{} > 8", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;++ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where -i{} < -8", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{} = 1", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }+ #[tokio::test]+ async fn $fn_scalar_fun_and_eq() {+ test_prune(+ Scenario::Int,+ &format!(+ "SELECT * FROM t where abs(i{}) = 1 and i{} = 1",+ $bits, $bits+ ),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_scalar_fun() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where abs(i{}) = 1", $bits),+ Some(0),+ Some(0),+ 3,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{}+1 = 1", $bits),+ Some(0),+ Some(0),+ 2,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr_subtract() {+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where 1-i{} > 1", $bits),+ Some(0),+ Some(0),+ 9,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list() {+ // result of sql "SELECT * FROM t where in (1)"+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{} in (1)", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_negated() {+ // result of sql "SELECT * FROM t where not in (1)" prune nothing+ test_prune(+ Scenario::Int,+ &format!("SELECT * FROM t where i{} not in (1)", $bits),+ Some(0),+ Some(0),+ 19,+ 5,+ )+ .await;+ }+ };}-int_tests!(8);-int_tests!(16);-int_tests!(32);-int_tests!(64);+int_tests!(+ 8,+ prune_int8_lt,+ prune_int8_gt,+ prune_int8_eq,+ prune_int8_scalar_fun_and_eq,+ prune_int8_scalar_fun,+ prune_int8_complex_expr,+ prune_int8_complex_expr_subtract,+ prune_int8_eq_in_list,+ prune_int8_eq_in_list_negated+);+int_tests!(+ 16,+ prune_int16_lt,+ prune_int16_gt,+ prune_int16_eq,+ prune_int16_scalar_fun_and_eq,+ prune_int16_scalar_fun,+ prune_int16_complex_expr,+ prune_int16_complex_expr_subtract,+ prune_int16_eq_in_list,+ prune_int16_eq_in_list_negated+);+int_tests!(+ 32,+ prune_int32_lt,+ prune_int32_gt,+ prune_int32_eq,+ prune_int32_scalar_fun_and_eq,+ prune_int32_scalar_fun,+ prune_int32_complex_expr,+ prune_int32_complex_expr_subtract,+ prune_int32_eq_in_list,+ prune_int32_eq_in_list_negated+);+int_tests!(+ 64,+ prune_int64_lt,+ prune_int64_gt,+ prune_int64_eq,+ prune_int64_scalar_fun_and_eq,+ prune_int64_scalar_fun,+ prune_int64_complex_expr,+ prune_int64_complex_expr_subtract,+ prune_int64_eq_in_list,+ prune_int64_eq_in_list_negated+);macro_rules! uint_tests {- ($bits:expr) => {- paste::item! {- #[tokio::test]- // null count min max- // page-0 0 0 4- // page-1 0 1 5- // page-2 0 5 9- // page-3 0 250 254- async fn [<prune_uint $bits _lt>]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{} < 6", $bits),- Some(0),- Some(5),- 11,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _gt >]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{} > 253", $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq >]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{} = 6", $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _scalar_fun_and_eq >]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where power(u{}, 2) = 36 and u{} = 6", $bits, $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _scalar_fun >]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where power(u{}, 2) = 25", $bits),- Some(0),- Some(0),- 2,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _complex_expr>]() {- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{}+1 = 6", $bits),- Some(0),- Some(0),- 2,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq_in_list >]() {- // result of sql "SELECT * FROM t where in (1)"- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{} in (6)", $bits),- Some(0),- Some(15),- 1,- 5- )- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq_in_list_negated >]() {- // result of sql "SELECT * FROM t where not in (6)" prune nothing- test_prune(- Scenario::UInt,- &format!("SELECT * FROM t where u{} not in (6)", $bits),- Some(0),- Some(0),- 19,- 5- )- .await;- }+ ($bits:expr, $fn_lt:ident, $fn_gt:ident, $fn_eq:ident, $fn_scalar_fun_and_eq:ident, $fn_scalar_fun:ident, $fn_complex_expr:ident, $fn_eq_in_list:ident, $fn_eq_in_list_negated:ident) => {+ #[tokio::test]+ // null count min max+ // page-0 0 0 4+ // page-1 0 1 5+ // page-2 0 5 9+ // page-3 0 250 254+ async fn $fn_lt() {+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{} < 6", $bits),+ Some(0),+ Some(5),+ 11,+ 5,+ )+ .await;}- }++ #[tokio::test]+ async fn $fn_gt() {+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{} > 253", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq() {+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{} = 6", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_scalar_fun_and_eq() {+ test_prune(+ Scenario::UInt,+ &format!(+ "SELECT * FROM t where power(u{}, 2) = 36 and u{} = 6",+ $bits, $bits+ ),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_scalar_fun() {+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where power(u{}, 2) = 25", $bits),+ Some(0),+ Some(0),+ 2,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr() {+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{}+1 = 6", $bits),+ Some(0),+ Some(0),+ 2,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list() {+ // result of sql "SELECT * FROM t where in (1)"+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{} in (6)", $bits),+ Some(0),+ Some(15),+ 1,+ 5,+ )+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_negated() {+ // result of sql "SELECT * FROM t where not in (6)" prune nothing+ test_prune(+ Scenario::UInt,+ &format!("SELECT * FROM t where u{} not in (6)", $bits),+ Some(0),+ Some(0),+ 19,+ 5,+ )+ .await;+ }+ };}-uint_tests!(8);-uint_tests!(16);-uint_tests!(32);-uint_tests!(64);+uint_tests!(+ 8,+ prune_uint8_lt,+ prune_uint8_gt,+ prune_uint8_eq,+ prune_uint8_scalar_fun_and_eq,+ prune_uint8_scalar_fun,+ prune_uint8_complex_expr,+ prune_uint8_eq_in_list,+ prune_uint8_eq_in_list_negated+);+uint_tests!(+ 16,+ prune_uint16_lt,+ prune_uint16_gt,+ prune_uint16_eq,+ prune_uint16_scalar_fun_and_eq,+ prune_uint16_scalar_fun,+ prune_uint16_complex_expr,+ prune_uint16_eq_in_list,+ prune_uint16_eq_in_list_negated+);+uint_tests!(+ 32,+ prune_uint32_lt,+ prune_uint32_gt,+ prune_uint32_eq,+ prune_uint32_scalar_fun_and_eq,+ prune_uint32_scalar_fun,+ prune_uint32_complex_expr,+ prune_uint32_eq_in_list,+ prune_uint32_eq_in_list_negated+);+uint_tests!(+ 64,+ prune_uint64_lt,+ prune_uint64_gt,+ prune_uint64_eq,+ prune_uint64_scalar_fun_and_eq,+ prune_uint64_scalar_fun,+ prune_uint64_complex_expr,+ prune_uint64_eq_in_list,+ prune_uint64_eq_in_list_negated+);#[tokio::test]// null count min maxdiff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rsindex 445ae7e97f..3ec3541af9 100644--- a/datafusion/core/tests/parquet/row_group_pruning.rs+++ b/datafusion/core/tests/parquet/row_group_pruning.rs@@ -399,321 +399,365 @@ async fn prune_disabled() {// https://github.com/apache/datafusion/issues/9779 bug so that tests pass// if and only if Bloom filters on Int8 and Int16 columns are still buggy.macro_rules! int_tests {- ($bits:expr) => {- paste::item! {- #[tokio::test]- async fn [<prune_int $bits _lt >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{} < 1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(3))- .with_pruned_by_stats(Some(1))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(3))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(11)- .test_row_group_prune()- .await;-- // result of sql "SELECT * FROM t where i < 1" is same as- // "SELECT * FROM t where -i > -1"- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where -i{} > -1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(3))- .with_pruned_by_stats(Some(1))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(3))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(11)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{} = 1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }- #[tokio::test]- async fn [<prune_int $bits _scalar_fun_and_eq >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where abs(i{}) = 1 and i{} = 1", $bits, $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _scalar_fun >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where abs(i{}) = 1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(3)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _complex_expr >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{}+1 = 1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(2)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _complex_expr_subtract >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where 1-i{} > 1", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(9)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq_in_list >]() {- // result of sql "SELECT * FROM t where in (1)"- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{} in (1)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq_in_list_2 >]() {- // result of sql "SELECT * FROM t where in (1000)", prune all- // test whether statistics works- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{} in (100)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(1))- .with_matched_by_bloom_filter(Some(0))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(0)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_int $bits _eq_in_list_negated >]() {- // result of sql "SELECT * FROM t where not in (1)" prune nothing- RowGroupPruningTest::new()- .with_scenario(Scenario::Int)- .with_query(&format!("SELECT * FROM t where i{} not in (1)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(19)- .test_row_group_prune()- .await;- }+ ($bits:expr, $fn_lt:ident, $fn_eq:ident, $fn_scalar_fun_and_eq:ident, $fn_scalar_fun:ident, $fn_complex_expr:ident, $fn_complex_expr_subtract:ident, $fn_eq_in_list:ident, $fn_eq_in_list_2:ident, $fn_eq_in_list_negated:ident) => {+ #[tokio::test]+ async fn $fn_lt() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{} < 1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(3))+ .with_pruned_by_stats(Some(1))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(3))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(11)+ .test_row_group_prune()+ .await;++ // result of sql "SELECT * FROM t where i < 1" is same as+ // "SELECT * FROM t where -i > -1"+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where -i{} > -1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(3))+ .with_pruned_by_stats(Some(1))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(3))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(11)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{} = 1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }+ #[tokio::test]+ async fn $fn_scalar_fun_and_eq() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!(+ "SELECT * FROM t where abs(i{}) = 1 and i{} = 1",+ $bits, $bits+ ))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_scalar_fun() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where abs(i{}) = 1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(3)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{}+1 = 1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(2)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr_subtract() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where 1-i{} > 1", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(9)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list() {+ // result of sql "SELECT * FROM t where in (1)"+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{} in (1)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_2() {+ // result of sql "SELECT * FROM t where in (1000)", prune all+ // test whether statistics works+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{} in (100)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(0))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(1))+ .with_matched_by_bloom_filter(Some(0))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(0)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_negated() {+ // result of sql "SELECT * FROM t where not in (1)" prune nothing+ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(&format!("SELECT * FROM t where i{} not in (1)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(19)+ .test_row_group_prune()+ .await;}};}// int8/int16 are incorrect: https://github.com/apache/datafusion/issues/9779-int_tests!(32);-int_tests!(64);+int_tests!(+ 32,+ prune_int32_lt,+ prune_int32_eq,+ prune_int32_scalar_fun_and_eq,+ prune_int32_scalar_fun,+ prune_int32_complex_expr,+ prune_int32_complex_expr_subtract,+ prune_int32_eq_in_list,+ prune_int32_eq_in_list_2,+ prune_int32_eq_in_list_negated+);+int_tests!(+ 64,+ prune_int64_lt,+ prune_int64_eq,+ prune_int64_scalar_fun_and_eq,+ prune_int64_scalar_fun,+ prune_int64_complex_expr,+ prune_int64_complex_expr_subtract,+ prune_int64_eq_in_list,+ prune_int64_eq_in_list_2,+ prune_int64_eq_in_list_negated+);// $bits: number of bits of the integer to test (8, 16, 32, 64)// $correct_bloom_filters: if false, replicates the// https://github.com/apache/datafusion/issues/9779 bug so that tests pass// if and only if Bloom filters on UInt8 and UInt16 columns are still buggy.macro_rules! uint_tests {- ($bits:expr) => {- paste::item! {- #[tokio::test]- async fn [<prune_uint $bits _lt >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{} < 6", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(3))- .with_pruned_by_stats(Some(1))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(3))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(11)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{} = 6", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }- #[tokio::test]- async fn [<prune_uint $bits _scalar_fun_and_eq >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where power(u{}, 2) = 36 and u{} = 6", $bits, $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _scalar_fun >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where power(u{}, 2) = 25", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(2)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _complex_expr >]() {- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{}+1 = 6", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(2)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq_in_list >]() {- // result of sql "SELECT * FROM t where in (1)"- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{} in (6)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(1))- .with_pruned_by_stats(Some(3))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(1))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(1)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq_in_list_2 >]() {- // result of sql "SELECT * FROM t where in (1000)", prune all- // test whether statistics works- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{} in (100)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))- .with_pruned_by_stats(Some(4))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(0))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(0)- .test_row_group_prune()- .await;- }-- #[tokio::test]- async fn [<prune_uint $bits _eq_in_list_negated >]() {- // result of sql "SELECT * FROM t where not in (1)" prune nothing- RowGroupPruningTest::new()- .with_scenario(Scenario::UInt)- .with_query(&format!("SELECT * FROM t where u{} not in (6)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(4))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(0))- .with_matched_by_bloom_filter(Some(4))- .with_pruned_by_bloom_filter(Some(0))- .with_expected_rows(19)- .test_row_group_prune()- .await;- }+ ($bits:expr, $fn_lt:ident, $fn_eq:ident, $fn_scalar_fun_and_eq:ident, $fn_scalar_fun:ident, $fn_complex_expr:ident, $fn_eq_in_list:ident, $fn_eq_in_list_2:ident, $fn_eq_in_list_negated:ident) => {+ #[tokio::test]+ async fn $fn_lt() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{} < 6", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(3))+ .with_pruned_by_stats(Some(1))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(3))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(11)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{} = 6", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }+ #[tokio::test]+ async fn $fn_scalar_fun_and_eq() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!(+ "SELECT * FROM t where power(u{}, 2) = 36 and u{} = 6",+ $bits, $bits+ ))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_scalar_fun() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where power(u{}, 2) = 25", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(2)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_complex_expr() {+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{}+1 = 6", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(2)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list() {+ // result of sql "SELECT * FROM t where in (1)"+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{} in (6)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_pruned_by_stats(Some(3))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(1))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(1)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_2() {+ // result of sql "SELECT * FROM t where in (1000)", prune all+ // test whether statistics works+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{} in (100)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(0))+ .with_pruned_by_stats(Some(4))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(0))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(0)+ .test_row_group_prune()+ .await;+ }++ #[tokio::test]+ async fn $fn_eq_in_list_negated() {+ // result of sql "SELECT * FROM t where not in (1)" prune nothing+ RowGroupPruningTest::new()+ .with_scenario(Scenario::UInt)+ .with_query(&format!("SELECT * FROM t where u{} not in (6)", $bits))+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_matched_by_bloom_filter(Some(4))+ .with_pruned_by_bloom_filter(Some(0))+ .with_expected_rows(19)+ .test_row_group_prune()+ .await;}};}// uint8/uint16 are incorrect: https://github.com/apache/datafusion/issues/9779-uint_tests!(32);-uint_tests!(64);+uint_tests!(+ 32,+ prune_uint32_lt,+ prune_uint32_eq,+ prune_uint32_scalar_fun_and_eq,+ prune_uint32_scalar_fun,+ prune_uint32_complex_expr,+ prune_uint32_eq_in_list,+ prune_uint32_eq_in_list_2,+ prune_uint32_eq_in_list_negated+);+uint_tests!(+ 64,+ prune_uint64_lt,+ prune_uint64_eq,+ prune_uint64_scalar_fun_and_eq,+ prune_uint64_scalar_fun,+ prune_uint64_complex_expr,+ prune_uint64_eq_in_list,+ prune_uint64_eq_in_list_2,+ prune_uint64_eq_in_list_negated+);#[tokio::test]async fn prune_int32_eq_large_in_list() {diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.tomlindex d66d8ee858..072c8f14da 100644--- a/datafusion/expr-common/Cargo.toml+++ b/datafusion/expr-common/Cargo.toml@@ -45,7 +45,6 @@ arrow = { workspace = true }datafusion-common = { workspace = true }indexmap = { workspace = true }itertools = { workspace = true }-paste = { workspace = true }[dev-dependencies]insta = { workspace = true }diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex f93ef3b795..0f88723d11 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -37,7 +37,7 @@ use datafusion_common::{};macro_rules! get_extreme_value {- ($extreme:ident, $value:expr) => {+ ($extreme:ident, $DECIMAL128_ARRAY:ident, $DECIMAL256_ARRAY:ident, $value:expr) => {match $value {DataType::UInt8 => ScalarValue::UInt8(Some(u8::$extreme)),DataType::UInt16 => ScalarValue::UInt16(Some(u16::$extreme)),@@ -83,18 +83,12 @@ macro_rules! get_extreme_value {ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::$extreme))}DataType::Decimal128(precision, scale) => ScalarValue::Decimal128(- Some(- paste::paste! {[<$extreme _DECIMAL128_FOR_EACH_PRECISION>]}- [*precision as usize],- ),+ Some($DECIMAL128_ARRAY[*precision as usize]),*precision,*scale,),DataType::Decimal256(precision, scale) => ScalarValue::Decimal256(- Some(- paste::paste! {[<$extreme _DECIMAL256_FOR_EACH_PRECISION>]}- [*precision as usize],- ),+ Some($DECIMAL256_ARRAY[*precision as usize]),*precision,*scale,),@@ -1162,10 +1156,20 @@ fn handle_overflow<const UPPER: bool>(match (UPPER, positive_sign) {(true, true) | (false, false) => ScalarValue::try_from(dt).unwrap(),(true, false) => {- get_extreme_value!(MIN, dt)+ get_extreme_value!(+ MIN,+ MIN_DECIMAL128_FOR_EACH_PRECISION,+ MIN_DECIMAL256_FOR_EACH_PRECISION,+ dt+ )}(false, true) => {- get_extreme_value!(MAX, dt)+ get_extreme_value!(+ MAX,+ MAX_DECIMAL128_FOR_EACH_PRECISION,+ MAX_DECIMAL256_FOR_EACH_PRECISION,+ dt+ )}}}@@ -4218,12 +4222,8 @@ mod tests {}macro_rules! capture_mode_change {- ($TYPE:ty) => {- paste::item! {- capture_mode_change_helper!([<capture_mode_change_ $TYPE>],- [<create_interval_ $TYPE>],- $TYPE);- }+ ($TYPE:ty, $TEST_FN_NAME:ident, $CREATE_FN_NAME:ident) => {+ capture_mode_change_helper!($TEST_FN_NAME, $CREATE_FN_NAME, $TYPE);};}@@ -4251,8 +4251,8 @@ mod tests {};}- capture_mode_change!(f32);- capture_mode_change!(f64);+ capture_mode_change!(f32, capture_mode_change_f32, create_interval_f32);+ capture_mode_change!(f64, capture_mode_change_f64, create_interval_f64);#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"),diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.tomlindex 75aa59595b..6990714585 100644--- a/datafusion/expr/Cargo.toml+++ b/datafusion/expr/Cargo.toml@@ -57,7 +57,6 @@ datafusion-functions-window-common = { workspace = true }datafusion-physical-expr-common = { workspace = true }indexmap = { workspace = true }itertools = { workspace = true }-paste = { workspace = true }recursive = { workspace = true, optional = true }serde_json = { workspace = true }sqlparser = { workspace = true, optional = true }diff --git a/datafusion/expr/src/test/function_stub.rs b/datafusion/expr/src/test/function_stub.rsindex d784169d54..26ac16d90d 100644--- a/datafusion/expr/src/test/function_stub.rs+++ b/datafusion/expr/src/test/function_stub.rs@@ -41,7 +41,6 @@ use crate::{macro_rules! create_func {($UDAF:ty, $AGGREGATE_UDF_FN:ident) => {- paste::paste! {#[doc = concat!("AggregateFunction that returns a [AggregateUDF](crate::AggregateUDF) for [`", stringify!($UDAF), "`]")]pub fn $AGGREGATE_UDF_FN() -> std::sync::Arc<crate::AggregateUDF> {// Singleton instance of [$UDAF], ensures the UDAF is only created once@@ -51,7 +50,6 @@ macro_rules! create_func {});std::sync::Arc::clone(&INSTANCE)}- }}}diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.tomlindex 39337e44bb..1ca494e38e 100644--- a/datafusion/functions-aggregate/Cargo.toml+++ b/datafusion/functions-aggregate/Cargo.toml@@ -54,7 +54,6 @@ datafusion-physical-expr-common = { workspace = true }half = { workspace = true }log = { workspace = true }num-traits = { workspace = true }-paste = { workspace = true }[dev-dependencies]arrow = { workspace = true, features = ["test_utils"] }diff --git a/datafusion/functions-aggregate/src/macros.rs b/datafusion/functions-aggregate/src/macros.rsindex 6c6bf72838..0c919a1e5e 100644--- a/datafusion/functions-aggregate/src/macros.rs+++ b/datafusion/functions-aggregate/src/macros.rs@@ -67,7 +67,6 @@ macro_rules! create_func {create_func!($UDAF, $AGGREGATE_UDF_FN, <$UDAF>::default());};($UDAF:ty, $AGGREGATE_UDF_FN:ident, $CREATE:expr) => {- paste::paste! {#[doc = concat!("AggregateFunction that returns a [`AggregateUDF`](datafusion_expr::AggregateUDF) for [`", stringify!($UDAF), "`]")]pub fn $AGGREGATE_UDF_FN() -> std::sync::Arc<datafusion_expr::AggregateUDF> {// Singleton instance of [$UDAF], ensures the UDAF is only created once@@ -76,7 +75,6 @@ macro_rules! create_func {std::sync::Arc::new(datafusion_expr::AggregateUDF::from($CREATE))});std::sync::Arc::clone(&INSTANCE)- }}}}diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.tomlindex 0fdb69e6e7..5fce3e854e 100644--- a/datafusion/functions-nested/Cargo.toml+++ b/datafusion/functions-nested/Cargo.toml@@ -61,7 +61,6 @@ hashbrown = { workspace = true }itertools = { workspace = true, features = ["use_std"] }itoa = { workspace = true }log = { workspace = true }-paste = { workspace = true }[dev-dependencies]criterion = { workspace = true, features = ["async_tokio"] }diff --git a/datafusion/functions-nested/src/macros.rs b/datafusion/functions-nested/src/macros.rsindex 5380f6b127..5f12113150 100644--- a/datafusion/functions-nested/src/macros.rs+++ b/datafusion/functions-nested/src/macros.rs@@ -50,7 +50,6 @@ macro_rules! make_udf_expr_and_func {make_udf_expr_and_func!($UDF, $EXPR_FN, $($arg)*, $DOC, $SCALAR_UDF_FN, $UDF::new);};($UDF:ident, $EXPR_FN:ident, $($arg:ident)*, $DOC:expr, $SCALAR_UDF_FN:ident, $CTOR:path) => {- paste::paste! {// "fluent expr_fn" style function#[doc = $DOC]pub fn $EXPR_FN($($arg: datafusion_expr::Expr),*) -> datafusion_expr::Expr {@@ -60,13 +59,11 @@ macro_rules! make_udf_expr_and_func {))}create_func!($UDF, $SCALAR_UDF_FN, $CTOR);- }};($UDF:ident, $EXPR_FN:ident, $DOC:expr, $SCALAR_UDF_FN:ident) => {make_udf_expr_and_func!($UDF, $EXPR_FN, $DOC, $SCALAR_UDF_FN, $UDF::new);};($UDF:ident, $EXPR_FN:ident, $DOC:expr, $SCALAR_UDF_FN:ident, $CTOR:path) => {- paste::paste! {// "fluent expr_fn" style function#[doc = $DOC]pub fn $EXPR_FN(arg: Vec<datafusion_expr::Expr>) -> datafusion_expr::Expr {@@ -76,7 +73,6 @@ macro_rules! make_udf_expr_and_func {))}create_func!($UDF, $SCALAR_UDF_FN, $CTOR);- }};}@@ -97,7 +93,6 @@ macro_rules! create_func {create_func!($UDF, $SCALAR_UDF_FN, $UDF::new);};($UDF:ident, $SCALAR_UDF_FN:ident, $CTOR:path) => {- paste::paste! {#[doc = concat!("ScalarFunction that returns a [`ScalarUDF`](datafusion_expr::ScalarUDF) for ")]#[doc = stringify!($UDF)]pub fn $SCALAR_UDF_FN() -> std::sync::Arc<datafusion_expr::ScalarUDF> {@@ -110,6 +105,5 @@ macro_rules! create_func {});std::sync::Arc::clone(&INSTANCE)}- }};}diff --git a/datafusion/functions-table/Cargo.toml b/datafusion/functions-table/Cargo.tomlindex aa401fbd7d..4edb640cb2 100644--- a/datafusion/functions-table/Cargo.toml+++ b/datafusion/functions-table/Cargo.toml@@ -48,7 +48,6 @@ datafusion-common = { workspace = true }datafusion-expr = { workspace = true }datafusion-physical-plan = { workspace = true }parking_lot = { workspace = true }-paste = { workspace = true }[dev-dependencies]arrow = { workspace = true, features = ["test_utils"] }diff --git a/datafusion/functions-table/src/lib.rs b/datafusion/functions-table/src/lib.rsindex cd9ade041a..668e964901 100644--- a/datafusion/functions-table/src/lib.rs+++ b/datafusion/functions-table/src/lib.rs@@ -38,25 +38,27 @@ pub fn all_default_table_functions() -> Vec<Arc<TableFunction>> {/// Creates a singleton instance of a table function/// - `$module`: A struct implementing `TableFunctionImpl` to create the function from/// - `$name`: The name to give to the created function-///-/// This is used to ensure creating the list of `TableFunction` only happens once.+/// - `$func_name`: The name of the function to be called+/// This is used to ensure creating the list of `TableFunction` only happens once.#[macro_export]macro_rules! create_udtf_function {- ($module:path, $name:expr) => {- paste::paste! {- pub fn [<$name:lower>]() -> Arc<TableFunction> {- static INSTANCE: std::sync::LazyLock<Arc<TableFunction>> =- std::sync::LazyLock::new(|| {- std::sync::Arc::new(TableFunction::new(- $name.to_string(),- Arc::new($module {}),- ))- });- std::sync::Arc::clone(&INSTANCE)- }+ ($module:expr, $func_name:ident, $name:expr) => {+ pub fn $func_name() -> Arc<TableFunction> {+ static INSTANCE: std::sync::LazyLock<Arc<TableFunction>> =+ std::sync::LazyLock::new(|| {+ std::sync::Arc::new(TableFunction::new(+ $name.to_string(),+ Arc::new($module),+ ))+ });+ std::sync::Arc::clone(&INSTANCE)}};}-create_udtf_function!(generate_series::GenerateSeriesFunc, "generate_series");-create_udtf_function!(generate_series::RangeFunc, "range");+create_udtf_function!(+ generate_series::GenerateSeriesFunc {},+ generate_series,+ "generate_series"+);+create_udtf_function!(generate_series::RangeFunc {}, range, "range");diff --git a/datafusion/functions-window/Cargo.toml b/datafusion/functions-window/Cargo.tomlindex fae71e180e..9c4342adae 100644--- a/datafusion/functions-window/Cargo.toml+++ b/datafusion/functions-window/Cargo.toml@@ -50,7 +50,6 @@ datafusion-macros = { workspace = true }datafusion-physical-expr = { workspace = true }datafusion-physical-expr-common = { workspace = true }log = { workspace = true }-paste = { workspace = true }[dev-dependencies]arrow = { workspace = true, features = ["test_utils"] }diff --git a/datafusion/functions-window/src/cume_dist.rs b/datafusion/functions-window/src/cume_dist.rsindex dccb9148d2..8e1cb1b1e6 100644--- a/datafusion/functions-window/src/cume_dist.rs+++ b/datafusion/functions-window/src/cume_dist.rs@@ -39,6 +39,7 @@ use std::sync::Arc;define_udwf_and_expr!(CumeDist,cume_dist,+ cume_dist_udwf,"Calculates the cumulative distribution of a value in a group of values.");diff --git a/datafusion/functions-window/src/lead_lag.rs b/datafusion/functions-window/src/lead_lag.rsindex 7569dac9ac..fbb84483e2 100644--- a/datafusion/functions-window/src/lead_lag.rs+++ b/datafusion/functions-window/src/lead_lag.rs@@ -43,6 +43,7 @@ use std::sync::{Arc, LazyLock};get_or_init_udwf!(Lag,lag,+ lag_udwf,"Returns the row value that precedes the current row by a specified \offset within partition. If no such row exists, then returns the \default value.",@@ -51,6 +52,7 @@ get_or_init_udwf!(get_or_init_udwf!(Lead,lead,+ lead_udwf,"Returns the value from a row that follows the current row by a \specified offset within the partition. If no such row exists, then \returns the default value.",diff --git a/datafusion/functions-window/src/macros.rs b/datafusion/functions-window/src/macros.rsindex 890ced90a9..aeb54356f8 100644--- a/datafusion/functions-window/src/macros.rs+++ b/datafusion/functions-window/src/macros.rs@@ -30,8 +30,8 @@////// * `$UDWF`: The struct which defines the [`Signature`](datafusion_expr::Signature)/// of the user-defined window function.-/// * `$OUT_FN_NAME`: The basename to generate a unique function name like-/// `$OUT_FN_NAME_udwf`.+/// * `$OUT_FN_NAME`: The expression function name+/// `UDWF_FN` : The unique function name/// * `$DOC`: Doc comments for UDWF./// * (optional) `$CTOR`: Pass a custom constructor. When omitted it/// automatically resolves to `$UDWF::default()`.@@ -52,6 +52,7 @@/// get_or_init_udwf!(/// SimpleUDWF,/// simple,+/// simple_udwf,/// "Simple user-defined window function doc comment."/// );/// #@@ -94,16 +95,15 @@/// ```#[macro_export]macro_rules! get_or_init_udwf {- ($UDWF:ident, $OUT_FN_NAME:ident, $DOC:expr) => {- get_or_init_udwf!($UDWF, $OUT_FN_NAME, $DOC, $UDWF::default);+ ($UDWF:ident, $OUT_FN_NAME:ident, $UDWF_FN:ident, $DOC:expr) => {+ get_or_init_udwf!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC, $UDWF::default);};- ($UDWF:ident, $OUT_FN_NAME:ident, $DOC:expr, $CTOR:path) => {- paste::paste! {+ ($UDWF:ident, $OUT_FN_NAME:ident, $UDWF_FN:ident, $DOC:expr, $CTOR:path) => {#[doc = concat!(" Returns a [`WindowUDF`](datafusion_expr::WindowUDF) for [`", stringify!($OUT_FN_NAME), "`].")]#[doc = ""]#[doc = concat!(" ", $DOC)]- pub fn [<$OUT_FN_NAME _udwf>]() -> std::sync::Arc<datafusion_expr::WindowUDF> {+ pub fn $UDWF_FN() -> std::sync::Arc<datafusion_expr::WindowUDF> {// Singleton instance of UDWF, ensures it is only created once.static INSTANCE: std::sync::LazyLock<std::sync::Arc<datafusion_expr::WindowUDF>> =std::sync::LazyLock::new(|| {@@ -111,7 +111,6 @@ macro_rules! get_or_init_udwf {});std::sync::Arc::clone(&INSTANCE)}- }};}@@ -149,6 +148,7 @@ macro_rules! get_or_init_udwf {/// # get_or_init_udwf!(/// # RowNumber,/// # row_number,+/// # row_number_udwf,/// # "Returns a unique row number for each row in window partition beginning at 1."/// # );/// /// Creates `row_number()` API which has zero parameters:@@ -163,6 +163,7 @@ macro_rules! get_or_init_udwf {/// create_udwf_expr!(/// RowNumber,/// row_number,+/// row_number_udwf,/// "Returns a unique row number for each row in window partition beginning at 1."/// );/// #@@ -221,7 +222,7 @@ macro_rules! get_or_init_udwf {/// # use datafusion_expr::{col, lit};/// # use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;/// #-/// # get_or_init_udwf!(Lead, lead, "user-defined window function");+/// # get_or_init_udwf!(Lead, lead,lead_udwf, "user-defined window function");/// #/// /// Creates `lead(expr, offset, default)` with 3 parameters:/// ///@@ -240,6 +241,7 @@ macro_rules! get_or_init_udwf {/// Lead,/// lead,/// [expr, offset, default],+/// lead_udwf,/// "Returns a value evaluated at the row that is offset rows after the current row within the partition."/// );/// #@@ -298,21 +300,18 @@ macro_rules! get_or_init_udwf {#[macro_export]macro_rules! create_udwf_expr {// zero arguments- ($UDWF:ident, $OUT_FN_NAME:ident, $DOC:expr) => {- paste::paste! {+ ($UDWF:ident, $OUT_FN_NAME:ident, $UDWF_FN:ident, $DOC:expr) => {#[doc = " Create a [`WindowFunction`](datafusion_expr::Expr::WindowFunction) expression for"]#[doc = concat!(" `", stringify!($UDWF), "` user-defined window function.")]#[doc = ""]#[doc = concat!(" ", $DOC)]pub fn $OUT_FN_NAME() -> datafusion_expr::Expr {- [<$OUT_FN_NAME _udwf>]().call(vec![])+ $UDWF_FN().call(vec![])}- }};// 1 or more arguments- ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+], $DOC:expr) => {- paste::paste! {+ ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+], $UDWF_FN:ident, $DOC:expr) => {#[doc = " Create a [`WindowFunction`](datafusion_expr::Expr::WindowFunction) expression for"]#[doc = concat!(" `", stringify!($UDWF), "` user-defined window function.")]#[doc = ""]@@ -320,10 +319,9 @@ macro_rules! create_udwf_expr {pub fn $OUT_FN_NAME($($PARAM: datafusion_expr::Expr),+) -> datafusion_expr::Expr {- [<$OUT_FN_NAME _udwf>]()+ $UDWF_FN().call(vec![$($PARAM),+])}- }};}@@ -374,6 +372,7 @@ macro_rules! create_udwf_expr {/// define_udwf_and_expr!(/// SimpleUDWF,/// simple,+/// simple_udwf,/// "a simple user-defined window function"/// );/// #@@ -437,6 +436,7 @@ macro_rules! create_udwf_expr {/// define_udwf_and_expr!(/// RowNumber,/// row_number,+/// row_number_udwf,/// "Returns a unique row number for each row in window partition beginning at 1.",/// RowNumber::new // <-- custom constructor/// );@@ -514,6 +514,7 @@ macro_rules! create_udwf_expr {/// Lead,/// lead,/// [expr, offset, default], // <- 3 parameters+/// lead_udwf,/// "user-defined window function"/// );/// #@@ -603,6 +604,7 @@ macro_rules! create_udwf_expr {/// Lead,/// lead,/// [expr, offset, default], // <- 3 parameters+/// lead_udwf,/// "user-defined window function",/// Lead::new // <- Custom constructor/// );@@ -663,29 +665,29 @@ macro_rules! create_udwf_expr {macro_rules! define_udwf_and_expr {// Defines UDWF with default constructor// Defines expression API with zero parameters- ($UDWF:ident, $OUT_FN_NAME:ident, $DOC:expr) => {- get_or_init_udwf!($UDWF, $OUT_FN_NAME, $DOC);- create_udwf_expr!($UDWF, $OUT_FN_NAME, $DOC);+ ($UDWF:ident, $OUT_FN_NAME:ident, $UDWF_FN:ident, $DOC:expr) => {+ get_or_init_udwf!($UDWF, $OUT_FN_NAME,$UDWF_FN, $DOC);+ create_udwf_expr!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC);};// Defines UDWF by passing a custom constructor// Defines expression API with zero parameters- ($UDWF:ident, $OUT_FN_NAME:ident, $DOC:expr, $CTOR:path) => {- get_or_init_udwf!($UDWF, $OUT_FN_NAME, $DOC, $CTOR);- create_udwf_expr!($UDWF, $OUT_FN_NAME, $DOC);+ ($UDWF:ident, $OUT_FN_NAME:ident, $UDWF_FN:ident, $DOC:expr, $CTOR:path) => {+ get_or_init_udwf!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC, $CTOR);+ create_udwf_expr!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC);};// Defines UDWF with default constructor// Defines expression API with multiple parameters- ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+], $DOC:expr) => {- get_or_init_udwf!($UDWF, $OUT_FN_NAME, $DOC);- create_udwf_expr!($UDWF, $OUT_FN_NAME, [$($PARAM),+], $DOC);+ ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+],$UDWF_FN:ident, $DOC:expr) => {+ get_or_init_udwf!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC);+ create_udwf_expr!($UDWF, $OUT_FN_NAME, [$($PARAM),+], $UDWF_FN, $DOC);};// Defines UDWF by passing a custom constructor// Defines expression API with multiple parameters- ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+], $DOC:expr, $CTOR:path) => {- get_or_init_udwf!($UDWF, $OUT_FN_NAME, $DOC, $CTOR);- create_udwf_expr!($UDWF, $OUT_FN_NAME, [$($PARAM),+], $DOC);+ ($UDWF:ident, $OUT_FN_NAME:ident, [$($PARAM:ident),+], $UDWF_FN:ident, $DOC:expr, $CTOR:path) => {+ get_or_init_udwf!($UDWF, $OUT_FN_NAME, $UDWF_FN, $DOC, $CTOR);+ create_udwf_expr!($UDWF, $OUT_FN_NAME, [$($PARAM),+], $UDWF_FN, $DOC);};}diff --git a/datafusion/functions-window/src/nth_value.rs b/datafusion/functions-window/src/nth_value.rsindex 8d37cf7e60..12b4146cc2 100644--- a/datafusion/functions-window/src/nth_value.rs+++ b/datafusion/functions-window/src/nth_value.rs@@ -45,6 +45,7 @@ define_udwf_and_expr!(First,first_value,[arg],+ first_value_udwf,"Returns the first value in the window frame",NthValue::first);@@ -52,12 +53,14 @@ define_udwf_and_expr!(Last,last_value,[arg],+ last_value_udwf,"Returns the last value in the window frame",NthValue::last);get_or_init_udwf!(NthValue,nth_value,+ nth_value_udwf,"Returns the nth value in the window frame",NthValue::nth);diff --git a/datafusion/functions-window/src/ntile.rs b/datafusion/functions-window/src/ntile.rsindex 21ce2795b4..1f9b2344e5 100644--- a/datafusion/functions-window/src/ntile.rs+++ b/datafusion/functions-window/src/ntile.rs@@ -40,6 +40,7 @@ define_udwf_and_expr!(Ntile,ntile,[arg],+ ntile_udwf,"Integer ranging from 1 to the argument value, dividing the partition as equally as possible.");diff --git a/datafusion/functions-window/src/rank.rs b/datafusion/functions-window/src/rank.rsindex 9d5af64eb9..ee8546703b 100644--- a/datafusion/functions-window/src/rank.rs+++ b/datafusion/functions-window/src/rank.rs@@ -44,6 +44,7 @@ use std::sync::{Arc, LazyLock};define_udwf_and_expr!(Rank,rank,+ rank_udwf,"Returns rank of the current row with gaps. Same as `row_number` of its first peer",Rank::basic);@@ -51,6 +52,7 @@ define_udwf_and_expr!(define_udwf_and_expr!(DenseRank,dense_rank,+ dense_rank_udwf,"Returns rank of the current row without gaps. This function counts peer groups",Rank::dense_rank);@@ -58,6 +60,7 @@ define_udwf_and_expr!(define_udwf_and_expr!(PercentRank,percent_rank,+ percent_rank_udwf,"Returns the relative rank of the current row: (rank - 1) / (total rows - 1)",Rank::percent_rank);diff --git a/datafusion/functions-window/src/row_number.rs b/datafusion/functions-window/src/row_number.rsindex d7d298cece..cd60e51def 100644--- a/datafusion/functions-window/src/row_number.rs+++ b/datafusion/functions-window/src/row_number.rs@@ -40,6 +40,7 @@ use std::sync::Arc;define_udwf_and_expr!(RowNumber,row_number,+ row_number_udwf,"Returns a unique row number for each row in window partition beginning at 1.");diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.tomlindex 7a52441477..d6cb212737 100644--- a/datafusion/physical-expr/Cargo.toml+++ b/datafusion/physical-expr/Cargo.toml@@ -65,7 +65,6 @@ arrow = { workspace = true, features = ["test_utils"] }criterion = { workspace = true }datafusion-functions = { workspace = true }insta = { workspace = true }-paste = { workspace = true }rand = { workspace = true }rstest = { workspace = true }diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rsindex c727c8fa5f..c78bbe999e 100644--- a/datafusion/physical-expr/src/expressions/negative.rs+++ b/datafusion/physical-expr/src/expressions/negative.rs@@ -214,10 +214,9 @@ mod tests {use datafusion_common::{DataFusionError, ScalarValue};use datafusion_physical_expr_common::physical_expr::fmt_sql;- use paste::paste;macro_rules! test_array_negative_op {- ($DATA_TY:tt, $($VALUE:expr),* ) => {+ ($DATA_TY:tt, $ARRAY_TY:ty, $($VALUE:expr),* ) => {let schema = Schema::new(vec![Field::new("a", DataType::$DATA_TY, true)]);let expr = negative(col("a", &schema)?, &schema)?;assert_eq!(expr.data_type(&schema)?, DataType::$DATA_TY);@@ -230,8 +229,8 @@ mod tests {)+arr.push(None);arr_expected.push(None);- let input = paste!{[<$DATA_TY Array>]::from(arr)};- let expected = &paste!{[<$DATA_TY Array>]::from(arr_expected)};+ let input = <$ARRAY_TY>::from(arr);+ let expected = &<$ARRAY_TY>::from(arr_expected);let batch =RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(input)])?;let result = expr.evaluate(&batch)?.into_array(batch.num_rows()).expect("Failed to convert to array");@@ -243,12 +242,12 @@ mod tests {#[test]fn array_negative_op() -> Result<()> {- test_array_negative_op!(Int8, 2i8, 1i8);- test_array_negative_op!(Int16, 234i16, 123i16);- test_array_negative_op!(Int32, 2345i32, 1234i32);- test_array_negative_op!(Int64, 23456i64, 12345i64);- test_array_negative_op!(Float32, 2345.0f32, 1234.0f32);- test_array_negative_op!(Float64, 23456.0f64, 12345.0f64);+ test_array_negative_op!(Int8, Int8Array, 2i8, 1i8);+ test_array_negative_op!(Int16, Int16Array, 234i16, 123i16);+ test_array_negative_op!(Int32, Int32Array, 2345i32, 1234i32);+ test_array_negative_op!(Int64, Int64Array, 23456i64, 12345i64);+ test_array_negative_op!(Float32, Float32Array, 2345.0f32, 1234.0f32);+ test_array_negative_op!(Float64, Float64Array, 23456.0f64, 12345.0f64);Ok(())}diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.tomlindex b7338cb764..cc299ce507 100644--- a/datafusion/sql/Cargo.toml+++ b/datafusion/sql/Cargo.toml@@ -73,5 +73,4 @@ datafusion-functions-window = { workspace = true }env_logger = { workspace = true }insta = { workspace = true }itertools = { workspace = true }-paste = { workspace = true }rstest = { workspace = true }diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex cd42b68949..79d2bd6ad8 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -1333,46 +1333,42 @@ mod tests {}macro_rules! test_stack_overflow {- ($num_expr:expr) => {- paste::item! {- #[test]- fn [<test_stack_overflow_ $num_expr>]() {- let schema = DFSchema::empty();- let mut planner_context = PlannerContext::default();-- let expr_str = (0..$num_expr)- .map(|i| format!("column1 = 'value{:?}'", i))- .collect::<Vec<String>>()- .join(" OR ");-- let dialect = GenericDialect{};- let mut parser = Parser::new(&dialect)- .try_with_sql(expr_str.as_str())- .unwrap();- let sql_expr = parser.parse_expr().unwrap();-- let context_provider = TestContextProvider::new();- let sql_to_rel = SqlToRel::new(&context_provider);-- // Should not stack overflow- sql_to_rel.sql_expr_to_logical_expr(- sql_expr,- &schema,- &mut planner_context,- ).unwrap();- }+ ($name:ident, $num_expr:expr) => {+ #[test]+ fn $name() {+ let schema = DFSchema::empty();+ let mut planner_context = PlannerContext::default();++ let expr_str = (0..$num_expr)+ .map(|i| format!("column1 = 'value{:?}'", i))+ .collect::<Vec<String>>()+ .join(" OR ");++ let dialect = GenericDialect {};+ let mut parser = Parser::new(&dialect)+ .try_with_sql(expr_str.as_str())+ .unwrap();+ let sql_expr = parser.parse_expr().unwrap();++ let context_provider = TestContextProvider::new();+ let sql_to_rel = SqlToRel::new(&context_provider);++ // Should not stack overflow+ sql_to_rel+ .sql_expr_to_logical_expr(sql_expr, &schema, &mut planner_context)+ .unwrap();}};}- test_stack_overflow!(64);- test_stack_overflow!(128);- test_stack_overflow!(256);- test_stack_overflow!(512);- test_stack_overflow!(1024);- test_stack_overflow!(2048);- test_stack_overflow!(4096);- test_stack_overflow!(8192);+ test_stack_overflow!(test_stack_overflow_64, 64);+ test_stack_overflow!(test_stack_overflow_128, 128);+ test_stack_overflow!(test_stack_overflow_256, 256);+ test_stack_overflow!(test_stack_overflow_512, 512);+ test_stack_overflow!(test_stack_overflow_1024, 1024);+ test_stack_overflow!(test_stack_overflow_2048, 2048);+ test_stack_overflow!(test_stack_overflow_4096, 4096);+ test_stack_overflow!(test_stack_overflow_8192, 8192);#[test]fn test_sql_to_expr_with_alias() {let schema = DFSchema::empty();