Apache DataFusion SQL Query Engine
Update datafusion-core crate to Rust 2024 edition (#19332)
Tim Saucer775277ae3cef
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex d2ecd34886..b63a3f9ccf 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -22,7 +22,7 @@ keywords = ["arrow", "query", "sql"]include = ["benches/*.rs", "src/**/*.rs", "Cargo.toml", "LICENSE.txt", "NOTICE.txt"]readme = "../../README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/core/benches/data_utils/mod.rs b/datafusion/core/benches/data_utils/mod.rsindex ff2a4e2474..630bc05660 100644--- a/datafusion/core/benches/data_utils/mod.rs+++ b/datafusion/core/benches/data_utils/mod.rs@@ -18,9 +18,9 @@//! This module provides the in-memory table for more realistic benchmarking.use arrow::array::{- builder::{Int64Builder, StringBuilder},ArrayRef, Float32Array, Float64Array, RecordBatch, StringArray, StringViewBuilder,UInt64Array,+ builder::{Int64Builder, StringBuilder},};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::datasource::MemTable;diff --git a/datafusion/core/benches/distinct_query_sql.rs b/datafusion/core/benches/distinct_query_sql.rsindex 1eb1524df8..0e638e293d 100644--- a/datafusion/core/benches/distinct_query_sql.rs+++ b/datafusion/core/benches/distinct_query_sql.rs@@ -24,10 +24,10 @@ mod data_utils;use crate::criterion::Criterion;use data_utils::{create_table_provider, make_data};use datafusion::execution::context::SessionContext;-use datafusion::physical_plan::{collect, ExecutionPlan};+use datafusion::physical_plan::{ExecutionPlan, collect};use datafusion::{datasource::MemTable, error::Result};-use datafusion_execution::config::SessionConfig;use datafusion_execution::TaskContext;+use datafusion_execution::config::SessionConfig;use parking_lot::Mutex;use std::hint::black_box;diff --git a/datafusion/core/benches/filter_query_sql.rs b/datafusion/core/benches/filter_query_sql.rsindex 16905e0f96..3b80518d32 100644--- a/datafusion/core/benches/filter_query_sql.rs+++ b/datafusion/core/benches/filter_query_sql.rs@@ -20,7 +20,7 @@ use arrow::{datatypes::{DataType, Field, Schema},record_batch::RecordBatch,};-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::prelude::SessionContext;use datafusion::{datasource::MemTable, error::Result};use futures::executor::block_on;diff --git a/datafusion/core/benches/map_query_sql.rs b/datafusion/core/benches/map_query_sql.rsindex 9e3d7a2717..67904197bc 100644--- a/datafusion/core/benches/map_query_sql.rs+++ b/datafusion/core/benches/map_query_sql.rs@@ -20,10 +20,10 @@ use std::hint::black_box;use std::sync::Arc;use arrow::array::{ArrayRef, Int32Array, RecordBatch};-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use parking_lot::Mutex;-use rand::prelude::ThreadRng;use rand::Rng;+use rand::prelude::ThreadRng;use tokio::runtime::Runtime;use datafusion::prelude::SessionContext;diff --git a/datafusion/core/benches/parquet_query_sql.rs b/datafusion/core/benches/parquet_query_sql.rsindex e2b3810480..e44524127b 100644--- a/datafusion/core/benches/parquet_query_sql.rs+++ b/datafusion/core/benches/parquet_query_sql.rs@@ -23,14 +23,14 @@ use arrow::datatypes::{SchemaRef,};use arrow::record_batch::RecordBatch;-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::instant::Instant;use futures::stream::StreamExt;use parquet::arrow::ArrowWriter;use parquet::file::properties::{WriterProperties, WriterVersion};-use rand::distr::uniform::SampleUniform;use rand::distr::Alphanumeric;+use rand::distr::uniform::SampleUniform;use rand::prelude::*;use rand::rng;use std::fs::File;diff --git a/datafusion/core/benches/physical_plan.rs b/datafusion/core/benches/physical_plan.rsindex 782c29a809..e6763b4761 100644--- a/datafusion/core/benches/physical_plan.rs+++ b/datafusion/core/benches/physical_plan.rs@@ -32,7 +32,7 @@ use tokio::runtime::Runtime;use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion::physical_plan::{collect,- expressions::{col, PhysicalSortExpr},+ expressions::{PhysicalSortExpr, col},};use datafusion::prelude::SessionContext;use datafusion_datasource::memory::MemorySourceConfig;diff --git a/datafusion/core/benches/preserve_file_partitioning.rs b/datafusion/core/benches/preserve_file_partitioning.rsindex 3a218ea55d..17ebca52cd 100644--- a/datafusion/core/benches/preserve_file_partitioning.rs+++ b/datafusion/core/benches/preserve_file_partitioning.rs@@ -38,8 +38,8 @@ use arrow::array::{ArrayRef, Float64Array, StringArray, TimestampMillisecondArrause arrow::datatypes::{DataType, Field, Schema, TimeUnit};use arrow::record_batch::RecordBatch;use arrow::util::pretty::pretty_format_batches;-use criterion::{criterion_group, criterion_main, Criterion};-use datafusion::prelude::{col, ParquetReadOptions, SessionConfig, SessionContext};+use criterion::{Criterion, criterion_group, criterion_main};+use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext, col};use datafusion_expr::SortExpr;use parquet::arrow::ArrowWriter;use parquet::file::properties::WriterProperties;diff --git a/datafusion/core/benches/push_down_filter.rs b/datafusion/core/benches/push_down_filter.rsindex 139fb12c30..3c2199c708 100644--- a/datafusion/core/benches/push_down_filter.rs+++ b/datafusion/core/benches/push_down_filter.rs@@ -18,16 +18,16 @@use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema};use bytes::{BufMut, BytesMut};-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::config::ConfigOptions;use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_execution::object_store::ObjectStoreUrl;-use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;use datafusion_physical_plan::ExecutionPlan;+use object_store::ObjectStore;use object_store::memory::InMemory;use object_store::path::Path;-use object_store::ObjectStore;use parquet::arrow::ArrowWriter;use std::sync::Arc;diff --git a/datafusion/core/benches/scalar.rs b/datafusion/core/benches/scalar.rsindex 540f7212e9..d06ed3f28b 100644--- a/datafusion/core/benches/scalar.rs+++ b/datafusion/core/benches/scalar.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::scalar::ScalarValue;fn criterion_benchmark(c: &mut Criterion) {diff --git a/datafusion/core/benches/sort.rs b/datafusion/core/benches/sort.rsindex 276151e253..4ba57a1530 100644--- a/datafusion/core/benches/sort.rs+++ b/datafusion/core/benches/sort.rs@@ -78,18 +78,18 @@ use datafusion::physical_plan::sorts::sort::SortExec;use datafusion::{execution::context::TaskContext,physical_plan::{+ ExecutionPlan, ExecutionPlanProperties,coalesce_partitions::CoalescePartitionsExec,- sorts::sort_preserving_merge::SortPreservingMergeExec, ExecutionPlan,- ExecutionPlanProperties,+ sorts::sort_preserving_merge::SortPreservingMergeExec,},prelude::SessionContext,};use datafusion_datasource::memory::MemorySourceConfig;-use datafusion_physical_expr::{expressions::col, PhysicalSortExpr};+use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};use datafusion_physical_expr_common::sort_expr::LexOrdering;/// Benchmarks for SortPreservingMerge stream-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use futures::StreamExt;use rand::rngs::StdRng;use rand::{Rng, SeedableRng};@@ -355,14 +355,14 @@ fn utf8_high_cardinality_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (utf8_low, utf8_low, utf8_high)fn utf8_tuple_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();+ let mut data_gen = DataGenerator::new();// need to sort by the combined key, so combine them together- let mut tuples: Vec<_> = gen+ let mut tuples: Vec<_> = data_gen.utf8_low_cardinality_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_high_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_high_cardinality_values()).collect();if sorted {@@ -388,14 +388,14 @@ fn utf8_tuple_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (utf8_view_low, utf8_view_low, utf8_view_high)fn utf8_view_tuple_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();+ let mut data_gen = DataGenerator::new();// need to sort by the combined key, so combine them together- let mut tuples: Vec<_> = gen+ let mut tuples: Vec<_> = data_gen.utf8_low_cardinality_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_high_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_high_cardinality_values()).collect();if sorted {@@ -421,15 +421,15 @@ fn utf8_view_tuple_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (f64, utf8_low, utf8_low, i64)fn mixed_tuple_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();+ let mut data_gen = DataGenerator::new();// need to sort by the combined key, so combine them together- let mut tuples: Vec<_> = gen+ let mut tuples: Vec<_> = data_gen.i64_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_low_cardinality_values())- .zip(gen.i64_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.i64_values()).collect();if sorted {@@ -459,15 +459,15 @@ fn mixed_tuple_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (f64, utf8_view_low, utf8_view_low, i64)fn mixed_tuple_with_utf8_view_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();+ let mut data_gen = DataGenerator::new();// need to sort by the combined key, so combine them together- let mut tuples: Vec<_> = gen+ let mut tuples: Vec<_> = data_gen.i64_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_low_cardinality_values())- .zip(gen.i64_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.i64_values()).collect();if sorted {@@ -497,8 +497,8 @@ fn mixed_tuple_with_utf8_view_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (utf8_dict)fn dictionary_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();- let mut values = gen.utf8_low_cardinality_values();+ let mut data_gen = DataGenerator::new();+ let mut values = data_gen.utf8_low_cardinality_values();if sorted {values.sort_unstable();}@@ -512,12 +512,12 @@ fn dictionary_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (utf8_dict, utf8_dict, utf8_dict)fn dictionary_tuple_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();- let mut tuples: Vec<_> = gen+ let mut data_gen = DataGenerator::new();+ let mut tuples: Vec<_> = data_gen.utf8_low_cardinality_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values()).collect();if sorted {@@ -543,13 +543,13 @@ fn dictionary_tuple_streams(sorted: bool) -> PartitionedBatches {/// Create a batch of (utf8_dict, utf8_dict, utf8_dict, i64)fn mixed_dictionary_tuple_streams(sorted: bool) -> PartitionedBatches {- let mut gen = DataGenerator::new();- let mut tuples: Vec<_> = gen+ let mut data_gen = DataGenerator::new();+ let mut tuples: Vec<_> = data_gen.utf8_low_cardinality_values().into_iter()- .zip(gen.utf8_low_cardinality_values())- .zip(gen.utf8_low_cardinality_values())- .zip(gen.i64_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.utf8_low_cardinality_values())+ .zip(data_gen.i64_values()).collect();if sorted {diff --git a/datafusion/core/benches/sort_limit_query_sql.rs b/datafusion/core/benches/sort_limit_query_sql.rsindex 7c8e5d730d..c18070fb77 100644--- a/datafusion/core/benches/sort_limit_query_sql.rs+++ b/datafusion/core/benches/sort_limit_query_sql.rs@@ -98,8 +98,7 @@ fn create_context() -> Arc<Mutex<SessionContext>> {ctx_holder.lock().push(Arc::new(Mutex::new(ctx)))});- let ctx = ctx_holder.lock().first().unwrap().clone();- ctx+ ctx_holder.lock().first().unwrap().clone()}fn criterion_benchmark(c: &mut Criterion) {diff --git a/datafusion/core/benches/spm.rs b/datafusion/core/benches/spm.rsindex ecc3f908d4..9db1306d2b 100644--- a/datafusion/core/benches/spm.rs+++ b/datafusion/core/benches/spm.rs@@ -20,13 +20,13 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray};use datafusion_execution::TaskContext;-use datafusion_physical_expr::expressions::col;use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::col;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;-use datafusion_physical_plan::{collect, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, collect};use criterion::async_executor::FuturesExecutor;-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion_datasource::memory::MemorySourceConfig;fn generate_spm_for_round_robin_tie_breaker(diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rsindex 027e925d4f..7cce7e0bd7 100644--- a/datafusion/core/benches/sql_planner.rs+++ b/datafusion/core/benches/sql_planner.rs@@ -31,15 +31,15 @@ use arrow::datatypes::{DataType, Field, Fields, Schema};use criterion::Bencher;use datafusion::datasource::MemTable;use datafusion::execution::context::SessionContext;-use datafusion_common::{config::Dialect, ScalarValue};+use datafusion_common::{ScalarValue, config::Dialect};use datafusion_expr::col;use rand_distr::num_traits::NumCast;use std::hint::black_box;use std::path::PathBuf;use std::sync::Arc;+use test_utils::TableDef;use test_utils::tpcds::tpcds_schemas;use test_utils::tpch::tpch_schemas;-use test_utils::TableDef;use tokio::runtime::Runtime;const BENCHMARKS_PATH_1: &str = "../../benchmarks/";@@ -242,8 +242,10 @@ fn criterion_benchmark(c: &mut Criterion) {if !PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists()&& !PathBuf::from(format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")).exists(){- panic!("benchmarks/data/hits_partitioned/ could not be loaded. Please run \- 'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark")+ panic!(+ "benchmarks/data/hits_partitioned/ could not be loaded. Please run \+ 'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark"+ )}let ctx = create_context();diff --git a/datafusion/core/benches/sql_planner_extended.rs b/datafusion/core/benches/sql_planner_extended.rsindex aff7cb4d10..adaf3e5911 100644--- a/datafusion/core/benches/sql_planner_extended.rs+++ b/datafusion/core/benches/sql_planner_extended.rs@@ -18,7 +18,7 @@use arrow::array::{ArrayRef, RecordBatch};use arrow_schema::DataType;use arrow_schema::TimeUnit::Nanosecond;-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::prelude::{DataFrame, SessionContext};use datafusion_catalog::MemTable;use datafusion_common::ScalarValue;diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rsindex 58797dfed6..0c188f7ba1 100644--- a/datafusion/core/benches/sql_query_with_io.rs+++ b/datafusion/core/benches/sql_query_with_io.rs@@ -20,7 +20,7 @@ use std::{fmt::Write, sync::Arc, time::Duration};use arrow::array::{Int64Builder, RecordBatch, UInt64Builder};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use bytes::Bytes;-use criterion::{criterion_group, criterion_main, Criterion, SamplingMode};+use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};use datafusion::{datasource::{file_format::parquet::ParquetFormat,@@ -31,13 +31,13 @@ use datafusion::{use datafusion_execution::runtime_env::RuntimeEnv;use itertools::Itertools;use object_store::{+ ObjectStore,memory::InMemory,path::Path,throttle::{ThrottleConfig, ThrottledStore},- ObjectStore,};use parquet::arrow::ArrowWriter;-use rand::{rngs::StdRng, Rng, SeedableRng};+use rand::{Rng, SeedableRng, rngs::StdRng};use tokio::runtime::Runtime;use url::Url;diff --git a/datafusion/core/benches/struct_query_sql.rs b/datafusion/core/benches/struct_query_sql.rsindex 5c7b427310..96434fc379 100644--- a/datafusion/core/benches/struct_query_sql.rs+++ b/datafusion/core/benches/struct_query_sql.rs@@ -20,7 +20,7 @@ use arrow::{datatypes::{DataType, Field, Schema},record_batch::RecordBatch,};-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use datafusion::prelude::SessionContext;use datafusion::{datasource::MemTable, error::Result};use futures::executor::block_on;diff --git a/datafusion/core/benches/topk_aggregate.rs b/datafusion/core/benches/topk_aggregate.rsindex 16e0444167..a4ae479de4 100644--- a/datafusion/core/benches/topk_aggregate.rs+++ b/datafusion/core/benches/topk_aggregate.rs@@ -18,13 +18,13 @@mod data_utils;use arrow::util::pretty::pretty_format_batches;-use criterion::{criterion_group, criterion_main, Criterion};+use criterion::{Criterion, criterion_group, criterion_main};use data_utils::make_data;-use datafusion::physical_plan::{collect, displayable, ExecutionPlan};+use datafusion::physical_plan::{ExecutionPlan, collect, displayable};use datafusion::prelude::SessionContext;use datafusion::{datasource::MemTable, error::Result};-use datafusion_execution::config::SessionConfig;use datafusion_execution::TaskContext;+use datafusion_execution::config::SessionConfig;use std::hint::black_box;use std::sync::Arc;use tokio::runtime::Runtime;@@ -46,7 +46,9 @@ async fn create_context(opts.optimizer.enable_topk_aggregation = use_topk;let ctx = SessionContext::new_with_config(cfg);let _ = ctx.register_table("traces", mem_table)?;- let sql = format!("select max(timestamp_ms) from traces group by trace_id order by max(timestamp_ms) desc limit {limit};");+ let sql = format!(+ "select max(timestamp_ms) from traces group by trace_id order by max(timestamp_ms) desc limit {limit};"+ );let df = ctx.sql(sql.as_str()).await?;let physical_plan = df.create_physical_plan().await?;let actual_phys_plan = displayable(physical_plan.as_ref()).indent(true).to_string();diff --git a/datafusion/core/src/bin/print_functions_docs.rs b/datafusion/core/src/bin/print_functions_docs.rsindex a9e6d4a30e..74a10bf079 100644--- a/datafusion/core/src/bin/print_functions_docs.rs+++ b/datafusion/core/src/bin/print_functions_docs.rs@@ -16,10 +16,10 @@// under the License.use datafusion::execution::SessionStateDefaults;-use datafusion_common::{not_impl_err, HashSet, Result};+use datafusion_common::{HashSet, Result, not_impl_err};use datafusion_expr::{- aggregate_doc_sections, scalar_doc_sections, window_doc_sections, AggregateUDF,- DocSection, Documentation, ScalarUDF, WindowUDF,+ AggregateUDF, DocSection, Documentation, ScalarUDF, WindowUDF,+ aggregate_doc_sections, scalar_doc_sections, window_doc_sections,};use itertools::Itertools;use std::env::args;@@ -255,7 +255,9 @@ fn print_docs(for f in &providers_with_no_docs {eprintln!(" - {f}");}- not_impl_err!("Some functions do not have documentation. Please implement `documentation` for: {providers_with_no_docs:?}")+ not_impl_err!(+ "Some functions do not have documentation. Please implement `documentation` for: {providers_with_no_docs:?}"+ )} else {Ok(docs)}diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 66dbd89f86..0d060db3bf 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -26,19 +26,19 @@ use crate::datasource::file_format::csv::CsvFormatFactory;use crate::datasource::file_format::format_as_file_type;use crate::datasource::file_format::json::JsonFormatFactory;use crate::datasource::{- provider_as_source, DefaultTableSource, MemTable, TableProvider,+ DefaultTableSource, MemTable, TableProvider, provider_as_source,};use crate::error::Result;-use crate::execution::context::{SessionState, TaskContext};use crate::execution::FunctionRegistry;+use crate::execution::context::{SessionState, TaskContext};use crate::logical_expr::utils::find_window_exprs;use crate::logical_expr::{- col, ident, Expr, JoinType, LogicalPlan, LogicalPlanBuilder,- LogicalPlanBuilderOptions, Partitioning, TableType,+ Expr, JoinType, LogicalPlan, LogicalPlanBuilder, LogicalPlanBuilderOptions,+ Partitioning, TableType, col, ident,};use crate::physical_plan::{- collect, collect_partitioned, execute_stream, execute_stream_partitioned,- ExecutionPlan, SendableRecordBatchStream,+ ExecutionPlan, SendableRecordBatchStream, collect, collect_partitioned,+ execute_stream, execute_stream_partitioned,};use crate::prelude::SessionContext;use std::any::Any;@@ -52,18 +52,17 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow_schema::FieldRef;use datafusion_common::config::{CsvOptions, JsonOptions};use datafusion_common::{- exec_err, internal_datafusion_err, not_impl_err, plan_datafusion_err, plan_err,- unqualified_field_not_found, Column, DFSchema, DataFusionError, ParamValues,- ScalarValue, SchemaError, TableReference, UnnestOptions,+ Column, DFSchema, DataFusionError, ParamValues, ScalarValue, SchemaError,+ TableReference, UnnestOptions, exec_err, internal_datafusion_err, not_impl_err,+ plan_datafusion_err, plan_err, unqualified_field_not_found,};use datafusion_expr::select_expr::SelectExpr;use datafusion_expr::{- case,+ ExplainOption, SortExpr, TableProviderFilterPushDown, UNNAMED_TABLE, case,dml::InsertOp,expr::{Alias, ScalarFunction},is_null, lit,utils::COUNT_STAR_EXPANSION,- ExplainOption, SortExpr, TableProviderFilterPushDown, UNNAMED_TABLE,};use datafusion_functions::core::coalesce;use datafusion_functions_aggregate::expr_fn::{diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rsindex 256a11ba69..50ee96da3d 100644--- a/datafusion/core/src/datasource/dynamic_file.rs+++ b/datafusion/core/src/datasource/dynamic_file.rs@@ -20,9 +20,9 @@use std::sync::Arc;+use crate::datasource::TableProvider;use crate::datasource::listing::ListingTableConfigExt;use crate::datasource::listing::{ListingTable, ListingTableConfig, ListingTableUrl};-use crate::datasource::TableProvider;use crate::error::Result;use crate::execution::context::SessionState;diff --git a/datafusion/core/src/datasource/empty.rs b/datafusion/core/src/datasource/empty.rsindex 77686c5eb7..5aeca92b16 100644--- a/datafusion/core/src/datasource/empty.rs+++ b/datafusion/core/src/datasource/empty.rs@@ -28,8 +28,8 @@ use datafusion_common::project_schema;use crate::datasource::{TableProvider, TableType};use crate::error::Result;use crate::logical_expr::Expr;-use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::empty::EmptyExec;/// An empty plan that is useful for testing and generating plans/// without mapping them to actual data.diff --git a/datafusion/core/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rsindex 3428d08a6a..b287b1ef3a 100644--- a/datafusion/core/src/datasource/file_format/avro.rs+++ b/datafusion/core/src/datasource/file_format/avro.rs@@ -26,20 +26,21 @@ mod tests {use crate::{datasource::file_format::test_util::scan_format, prelude::SessionContext,};- use arrow::array::{as_string_array, Array};+ use arrow::array::{Array, as_string_array};use datafusion_catalog::Session;use datafusion_common::test_util::batches_to_string;use datafusion_common::{+ Result,cast::{as_binary_array, as_boolean_array, as_float32_array, as_float64_array,as_int32_array, as_timestamp_microsecond_array,},- test_util, Result,+ test_util,};use datafusion_datasource_avro::AvroFormat;use datafusion_execution::config::SessionConfig;- use datafusion_physical_plan::{collect, ExecutionPlan};+ use datafusion_physical_plan::{ExecutionPlan, collect};use futures::StreamExt;use insta::assert_snapshot;@@ -245,7 +246,10 @@ mod tests {values.push(array.value(i));}- assert_eq!("[1235865600000000, 1235865660000000, 1238544000000000, 1238544060000000, 1233446400000000, 1233446460000000, 1230768000000000, 1230768060000000]", format!("{values:?}"));+ assert_eq!(+ "[1235865600000000, 1235865660000000, 1238544000000000, 1238544060000000, 1233446400000000, 1233446460000000, 1230768000000000, 1230768060000000]",+ format!("{values:?}")+ );Ok(())}diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rsindex 52fb8ae904..4b855ba582 100644--- a/datafusion/core/src/datasource/file_format/csv.rs+++ b/datafusion/core/src/datasource/file_format/csv.rs@@ -32,12 +32,12 @@ mod tests {use crate::prelude::{CsvReadOptions, SessionConfig, SessionContext};use arrow_schema::{DataType, Field, Schema, SchemaRef};use datafusion_catalog::Session;+ use datafusion_common::Result;use datafusion_common::cast::as_string_array;use datafusion_common::config::CsvOptions;use datafusion_common::internal_err;use datafusion_common::stats::Precision;use datafusion_common::test_util::{arrow_test_data, batches_to_string};- use datafusion_common::Result;use datafusion_datasource::decoder::{BatchDeserializer, DecoderDeserializer, DeserializerOutput,};@@ -45,7 +45,7 @@ mod tests {use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::write::BatchSerializer;use datafusion_expr::{col, lit};- use datafusion_physical_plan::{collect, ExecutionPlan};+ use datafusion_physical_plan::{ExecutionPlan, collect};use arrow::array::{Array, BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray,@@ -57,8 +57,8 @@ mod tests {use bytes::Bytes;use chrono::DateTime;use datafusion_common::parsers::CompressionTypeVariant;- use futures::stream::BoxStream;use futures::StreamExt;+ use futures::stream::BoxStream;use insta::assert_snapshot;use object_store::chunked::ChunkedStore;use object_store::local::LocalFileSystem;@@ -706,11 +706,11 @@ mod tests {let re = Regex::new(r"DataSourceExec: file_groups=\{(\d+) group").unwrap();- if let Some(captures) = re.captures(&plan) {- if let Some(match_) = captures.get(1) {- let n_partitions = match_.as_str().parse::<usize>().unwrap();- return Ok(n_partitions);- }+ if let Some(captures) = re.captures(&plan)+ && let Some(match_) = captures.get(1)+ {+ let n_partitions = match_.as_str().parse::<usize>().unwrap();+ return Ok(n_partitions);}internal_err!("query contains no DataSourceExec")@@ -944,17 +944,19 @@ mod tests {let files: Vec<_> = std::fs::read_dir(&path).unwrap().collect();assert_eq!(files.len(), 1);- assert!(files- .last()- .unwrap()- .as_ref()- .unwrap()- .path()- .file_name()- .unwrap()- .to_str()- .unwrap()- .ends_with(".csv.gz"));+ assert!(+ files+ .last()+ .unwrap()+ .as_ref()+ .unwrap()+ .path()+ .file_name()+ .unwrap()+ .to_str()+ .unwrap()+ .ends_with(".csv.gz")+ );Ok(())}@@ -983,17 +985,19 @@ mod tests {let files: Vec<_> = std::fs::read_dir(&path).unwrap().collect();assert_eq!(files.len(), 1);- assert!(files- .last()- .unwrap()- .as_ref()- .unwrap()- .path()- .file_name()- .unwrap()- .to_str()- .unwrap()- .ends_with(".csv"));+ assert!(+ files+ .last()+ .unwrap()+ .as_ref()+ .unwrap()+ .path()+ .file_name()+ .unwrap()+ .to_str()+ .unwrap()+ .ends_with(".csv")+ );Ok(())}@@ -1191,7 +1195,9 @@ mod tests {) -> Result<()> {let schema = csv_schema();let generator = CsvBatchGenerator::new(batch_size, line_count);- let mut deserializer = csv_deserializer(batch_size, &schema);++ let schema_clone = Arc::clone(&schema);+ let mut deserializer = csv_deserializer(batch_size, &schema_clone);for data in generator {deserializer.digest(data);@@ -1230,7 +1236,8 @@ mod tests {) -> Result<()> {let schema = csv_schema();let generator = CsvBatchGenerator::new(batch_size, line_count);- let mut deserializer = csv_deserializer(batch_size, &schema);+ let schema_clone = Arc::clone(&schema);+ let mut deserializer = csv_deserializer(batch_size, &schema_clone);for data in generator {deserializer.digest(data);@@ -1499,7 +1506,7 @@ mod tests {// Create a temp file with a .csv suffix so the reader accepts itlet mut tmp = tempfile::Builder::new().suffix(".csv").tempfile()?; // ensures path ends with .csv- // CSV has header "a,b,c". First data row is truncated (only "1,2"), second row is complete.+ // CSV has header "a,b,c". First data row is truncated (only "1,2"), second row is complete.write!(tmp, "a,b,c\n1,2\n3,4,5\n")?;let path = tmp.path().to_str().unwrap().to_string();diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rsindex 34d3d64f07..1d20d458ed 100644--- a/datafusion/core/src/datasource/file_format/json.rs+++ b/datafusion/core/src/datasource/file_format/json.rs@@ -36,7 +36,7 @@ mod tests {BatchDeserializer, DecoderDeserializer, DeserializerOutput,};use datafusion_datasource::file_format::FileFormat;- use datafusion_physical_plan::{collect, ExecutionPlan};+ use datafusion_physical_plan::{ExecutionPlan, collect};use arrow::compute::concat_batches;use arrow::datatypes::{DataType, Field};@@ -187,11 +187,11 @@ mod tests {let re = Regex::new(r"file_groups=\{(\d+) group").unwrap();- if let Some(captures) = re.captures(&plan) {- if let Some(match_) = captures.get(1) {- let count = match_.as_str().parse::<usize>().unwrap();- return Ok(count);- }+ if let Some(captures) = re.captures(&plan)+ && let Some(match_) = captures.get(1)+ {+ let count = match_.as_str().parse::<usize>().unwrap();+ return Ok(count);}internal_err!("Query contains no Exec: file_groups")diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rsindex 2756ea21cd..6bbb63f6a1 100644--- a/datafusion/core/src/datasource/file_format/mod.rs+++ b/datafusion/core/src/datasource/file_format/mod.rs@@ -39,9 +39,9 @@ pub(crate) mod test_util {use arrow_schema::SchemaRef;use datafusion_catalog::Session;use datafusion_common::Result;- use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::TableSchema;- use datafusion_datasource::{file_format::FileFormat, PartitionedFile};+ use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+ use datafusion_datasource::{PartitionedFile, file_format::FileFormat};use datafusion_execution::object_store::ObjectStoreUrl;use std::sync::Arc;@@ -73,15 +73,17 @@ pub(crate) mod test_util {.infer_stats(state, &store, file_schema.clone(), &meta).await?;- let file_groups = vec![vec![PartitionedFile {- object_meta: meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- }]- .into()];+ let file_groups = vec![+ vec![PartitionedFile {+ object_meta: meta,+ partition_values: vec![],+ range: None,+ statistics: None,+ extensions: None,+ metadata_size_hint: None,+ }]+ .into(),+ ];let exec = format.create_physical_plan(@@ -133,7 +135,10 @@ mod tests {.write_parquet(out_dir_url, DataFrameWriteOptions::new(), None).await.expect_err("should fail because input file does not match inferred schema");- assert_eq!(e.strip_backtrace(), "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'");+ assert_eq!(+ e.strip_backtrace(),+ "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'"+ );Ok(())}}diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rsindex 6cca0d5033..146c5f6f5f 100644--- a/datafusion/core/src/datasource/file_format/options.rs+++ b/datafusion/core/src/datasource/file_format/options.rs@@ -25,9 +25,9 @@ use crate::datasource::file_format::avro::AvroFormat;#[cfg(feature = "parquet")]use crate::datasource::file_format::parquet::ParquetFormat;+use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD;use crate::datasource::file_format::arrow::ArrowFormat;use crate::datasource::file_format::file_compression_type::FileCompressionType;-use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD;use crate::datasource::listing::ListingTableUrl;use crate::datasource::{file_format::csv::CsvFormat, listing::ListingOptions};use crate::error::Result;diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rsindex 87131e0824..050212bc6b 100644--- a/datafusion/core/src/datasource/file_format/parquet.rs+++ b/datafusion/core/src/datasource/file_format/parquet.rs@@ -107,8 +107,8 @@ pub(crate) mod test_util {mod tests {use std::fmt::{self, Display, Formatter};- use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Arc;+ use std::sync::atomic::{AtomicUsize, Ordering};use std::time::Duration;use crate::datasource::file_format::parquet::test_util::store_parquet;@@ -120,6 +120,7 @@ mod tests {use arrow::array::RecordBatch;use arrow_schema::Schema;use datafusion_catalog::Session;+ use datafusion_common::ScalarValue::Utf8;use datafusion_common::cast::{as_binary_array, as_binary_view_array, as_boolean_array, as_float32_array,as_float64_array, as_int32_array, as_timestamp_nanosecond_array,@@ -127,7 +128,6 @@ mod tests {use datafusion_common::config::{ParquetOptions, TableParquetOptions};use datafusion_common::stats::Precision;use datafusion_common::test_util::batches_to_string;- use datafusion_common::ScalarValue::Utf8;use datafusion_common::{Result, ScalarValue};use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};@@ -135,33 +135,33 @@ mod tests {use datafusion_datasource_parquet::{ParquetFormat, ParquetFormatFactory, ParquetSink,};+ use datafusion_execution::TaskContext;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::runtime_env::RuntimeEnv;- use datafusion_execution::TaskContext;use datafusion_expr::dml::InsertOp;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;- use datafusion_physical_plan::{collect, ExecutionPlan};+ use datafusion_physical_plan::{ExecutionPlan, collect};use crate::test_util::bounded_stream;use arrow::array::{- types::Int32Type, Array, ArrayRef, DictionaryArray, Int32Array, Int64Array,- StringArray,+ Array, ArrayRef, DictionaryArray, Int32Array, Int64Array, StringArray,+ types::Int32Type,};use arrow::datatypes::{DataType, Field};use async_trait::async_trait;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource_parquet::metadata::DFParquetMetadata;- use futures::stream::BoxStream;use futures::StreamExt;+ use futures::stream::BoxStream;use insta::assert_snapshot;- use object_store::local::LocalFileSystem;use object_store::ObjectMeta;+ use object_store::local::LocalFileSystem;use object_store::{- path::Path, GetOptions, GetResult, ListResult, MultipartUpload, ObjectStore,- PutMultipartOptions, PutOptions, PutPayload, PutResult,+ GetOptions, GetResult, ListResult, MultipartUpload, ObjectStore,+ PutMultipartOptions, PutOptions, PutPayload, PutResult, path::Path,};- use parquet::arrow::arrow_reader::ArrowReaderOptions;use parquet::arrow::ParquetRecordBatchStreamBuilder;+ use parquet::arrow::arrow_reader::ArrowReaderOptions;use parquet::file::metadata::{KeyValue, ParquetColumnIndex, ParquetMetaData, ParquetOffsetIndex,};@@ -930,7 +930,10 @@ mod tests {values.push(array.value(i));}- assert_eq!("[1235865600000000000, 1235865660000000000, 1238544000000000000, 1238544060000000000, 1233446400000000000, 1233446460000000000, 1230768000000000000, 1230768060000000000]", format!("{values:?}"));+ assert_eq!(+ "[1235865600000000000, 1235865660000000000, 1238544000000000000, 1238544060000000000, 1233446400000000000, 1233446460000000000, 1230768000000000000, 1230768060000000000]",+ format!("{values:?}")+ );Ok(())}diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 7db79485d1..7f957108cf 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -113,8 +113,8 @@ mod tests {use crate::prelude::*;use crate::{datasource::{- file_format::csv::CsvFormat, file_format::json::JsonFormat,- provider_as_source, DefaultTableSource, MemTable,+ DefaultTableSource, MemTable, file_format::csv::CsvFormat,+ file_format::json::JsonFormat, provider_as_source,},execution::options::ArrowReadOptions,test::{@@ -129,21 +129,20 @@ mod tests {ListingOptions, ListingTable, ListingTableConfig, SchemaSource,};use datafusion_common::{- assert_contains,+ DataFusionError, Result, ScalarValue, assert_contains,stats::Precision,test_util::{batches_to_string, datafusion_test_data},- DataFusionError, Result, ScalarValue,};+ use datafusion_datasource::ListingTableUrl;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_format::FileFormat;- use datafusion_datasource::ListingTableUrl;use datafusion_expr::dml::InsertOp;use datafusion_expr::{BinaryExpr, LogicalPlanBuilder, Operator};- use datafusion_physical_expr::expressions::binary;use datafusion_physical_expr::PhysicalSortExpr;+ use datafusion_physical_expr::expressions::binary;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::empty::EmptyExec;- use datafusion_physical_plan::{collect, ExecutionPlanProperties};+ use datafusion_physical_plan::{ExecutionPlanProperties, collect};use std::collections::HashMap;use std::io::Write;use std::sync::Arc;@@ -283,32 +282,36 @@ mod tests {// sort expr, but non column(vec![vec![col("int_col").add(lit(1)).sort(true, true)]],- Ok(vec![[PhysicalSortExpr {- expr: binary(- physical_col("int_col", &schema).unwrap(),- Operator::Plus,- physical_lit(1),- &schema,- )- .unwrap(),- options: SortOptions {- descending: false,- nulls_first: true,- },- }]- .into()]),+ Ok(vec![+ [PhysicalSortExpr {+ expr: binary(+ physical_col("int_col", &schema).unwrap(),+ Operator::Plus,+ physical_lit(1),+ &schema,+ )+ .unwrap(),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }]+ .into(),+ ]),),// ok with one column(vec![vec![col("string_col").sort(true, false)]],- Ok(vec![[PhysicalSortExpr {- expr: physical_col("string_col", &schema).unwrap(),- options: SortOptions {- descending: false,- nulls_first: false,- },- }]- .into()]),+ Ok(vec![+ [PhysicalSortExpr {+ expr: physical_col("string_col", &schema).unwrap(),+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ }]+ .into(),+ ]),),// ok with two columns, different options(@@ -316,19 +319,21 @@ mod tests {col("string_col").sort(true, false),col("int_col").sort(false, true),]],- Ok(vec![[- PhysicalSortExpr::new_default(- physical_col("string_col", &schema).unwrap(),- )- .asc()- .nulls_last(),- PhysicalSortExpr::new_default(- physical_col("int_col", &schema).unwrap(),- )- .desc()- .nulls_first(),- ]- .into()]),+ Ok(vec![+ [+ PhysicalSortExpr::new_default(+ physical_col("string_col", &schema).unwrap(),+ )+ .asc()+ .nulls_last(),+ PhysicalSortExpr::new_default(+ physical_col("int_col", &schema).unwrap(),+ )+ .desc()+ .nulls_first(),+ ]+ .into(),+ ]),),];@@ -725,8 +730,8 @@ mod tests {}#[tokio::test]- async fn test_insert_into_append_new_parquet_files_invalid_session_fails(- ) -> Result<()> {+ async fn test_insert_into_append_new_parquet_files_invalid_session_fails()+ -> Result<()> {let mut config_map: HashMap<String, String> = HashMap::new();config_map.insert("datafusion.execution.parquet.compression".into(),@@ -740,7 +745,10 @@ mod tests {).await.expect_err("Example should fail!");- assert_eq!(e.strip_backtrace(), "Invalid or Unsupported Configuration: zstd compression requires specifying a level such as zstd(4)");+ assert_eq!(+ e.strip_backtrace(),+ "Invalid or Unsupported Configuration: zstd compression requires specifying a level such as zstd(4)"+ );Ok(())}@@ -1410,7 +1418,9 @@ mod tests {];for (format, batch_size, soft_max_rows, expected_files) in test_cases {- println!("Testing insert with format: {format}, batch_size: {batch_size}, expected files: {expected_files}");+ println!(+ "Testing insert with format: {format}, batch_size: {batch_size}, expected files: {expected_files}"+ );let mut config_map = HashMap::new();config_map.insert(diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 9c5e41ace9..3ca388af0c 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -28,8 +28,8 @@ use crate::datasource::listing::{use crate::execution::context::SessionState;use arrow::datatypes::DataType;-use datafusion_common::{arrow_datafusion_err, plan_err, ToDFSchema};-use datafusion_common::{config_datafusion_err, Result};+use datafusion_common::{Result, config_datafusion_err};+use datafusion_common::{ToDFSchema, arrow_datafusion_err, plan_err};use datafusion_expr::CreateExternalTable;use async_trait::async_trait;@@ -220,9 +220,9 @@ mod tests {datasource::file_format::csv::CsvFormat, execution::context::SessionContext,test_util::parquet_test_data,};+ use datafusion_execution::cache::CacheAccessor;use datafusion_execution::cache::cache_manager::CacheManagerConfig;use datafusion_execution::cache::cache_unit::DefaultFileStatisticsCache;- use datafusion_execution::cache::CacheAccessor;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use glob::Pattern;diff --git a/datafusion/core/src/datasource/memory_test.rs b/datafusion/core/src/datasource/memory_test.rsindex c16837c73b..c7721cafb0 100644--- a/datafusion/core/src/datasource/memory_test.rs+++ b/datafusion/core/src/datasource/memory_test.rs@@ -19,7 +19,7 @@mod tests {use crate::datasource::MemTable;- use crate::datasource::{provider_as_source, DefaultTableSource};+ use crate::datasource::{DefaultTableSource, provider_as_source};use crate::physical_plan::collect;use crate::prelude::SessionContext;use arrow::array::{AsArray, Int32Array};@@ -29,8 +29,8 @@ mod tests {use arrow_schema::SchemaRef;use datafusion_catalog::TableProvider;use datafusion_common::{DataFusionError, Result};- use datafusion_expr::dml::InsertOp;use datafusion_expr::LogicalPlanBuilder;+ use datafusion_expr::dml::InsertOp;use futures::StreamExt;use std::collections::HashMap;use std::sync::Arc;@@ -329,12 +329,11 @@ mod tests {);let col = batch.column(0).as_primitive::<UInt64Type>();assert_eq!(col.len(), 1, "expected 1 row, got {}", col.len());- let val = col- .iter()++ col.iter().next().expect("had value")- .expect("expected non null");- val+ .expect("expected non null")}// Test inserting a single batch of data into a single partitiondiff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex 52564663b9..7d37ef8cf2 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -31,7 +31,7 @@ mod view_test;// backwards compatibilitypub use self::default_table_source::{- provider_as_source, source_as_provider, DefaultTableSource,+ DefaultTableSource, provider_as_source, source_as_provider,};pub use self::memory::MemTable;pub use self::view::ViewTable;@@ -53,22 +53,20 @@ pub use datafusion_physical_expr::create_ordering;mod tests {use crate::prelude::SessionContext;- use ::object_store::{path::Path, ObjectMeta};+ use ::object_store::{ObjectMeta, path::Path};use arrow::{array::Int32Array,datatypes::{DataType, Field, Schema, SchemaRef},record_batch::RecordBatch,};use datafusion_common::{- record_batch,+ Result, ScalarValue, record_batch,test_util::batches_to_sort_string,tree_node::{Transformed, TransformedResult, TreeNode},- Result, ScalarValue,};use datafusion_datasource::{- file_scan_config::FileScanConfigBuilder,+ PartitionedFile, file_scan_config::FileScanConfigBuilder,schema_adapter::DefaultSchemaAdapterFactory, source::DataSourceExec,- PartitionedFile,};use datafusion_datasource_parquet::source::ParquetSource;use datafusion_physical_expr::expressions::{Column, Literal};diff --git a/datafusion/core/src/datasource/physical_plan/avro.rs b/datafusion/core/src/datasource/physical_plan/avro.rsindex b97ab0e9ca..4f8fc6d0b9 100644--- a/datafusion/core/src/datasource/physical_plan/avro.rs+++ b/datafusion/core/src/datasource/physical_plan/avro.rs@@ -31,21 +31,21 @@ mod tests {use crate::test::object_store::local_unpartitioned_file;use arrow::datatypes::{DataType, Field, SchemaBuilder};use datafusion_common::test_util::batches_to_string;- use datafusion_common::{test_util, Result, ScalarValue};+ use datafusion_common::{Result, ScalarValue, test_util};use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::{PartitionedFile, TableSchema};- use datafusion_datasource_avro::source::AvroSource;use datafusion_datasource_avro::AvroFormat;+ use datafusion_datasource_avro::source::AvroSource;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_plan::ExecutionPlan;use datafusion_datasource::source::DataSourceExec;use futures::StreamExt;use insta::assert_snapshot;+ use object_store::ObjectStore;use object_store::chunked::ChunkedStore;use object_store::local::LocalFileSystem;- use object_store::ObjectStore;use rstest::*;use url::Url;diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rsindex 660be4faff..892ae5b586 100644--- a/datafusion/core/src/datasource/physical_plan/csv.rs+++ b/datafusion/core/src/datasource/physical_plan/csv.rs@@ -40,10 +40,10 @@ mod tests {use datafusion_common::config::CsvOptions;use datafusion_common::test_util::arrow_test_data;use datafusion_common::test_util::batches_to_string;- use datafusion_common::{assert_batches_eq, Result};+ use datafusion_common::{Result, assert_batches_eq};use datafusion_execution::config::SessionConfig;- use datafusion_physical_plan::metrics::MetricsSet;use datafusion_physical_plan::ExecutionPlan;+ use datafusion_physical_plan::metrics::MetricsSet;#[cfg(feature = "compression")]use datafusion_datasource::file_compression_type::FileCompressionType;@@ -621,7 +621,10 @@ mod tests {.collect().await.unwrap_err();- assert_eq!(e.strip_backtrace(), "Arrow error: Csv error: incorrect number of fields for line 1, expected 2 got more than 2")+ assert_eq!(+ e.strip_backtrace(),+ "Arrow error: Csv error: incorrect number of fields for line 1, expected 2 got more than 2"+ )}#[tokio::test]@@ -656,8 +659,8 @@ mod tests {}#[tokio::test]- async fn test_create_external_table_with_terminator_with_newlines_in_values(- ) -> Result<()> {+ async fn test_create_external_table_with_terminator_with_newlines_in_values()+ -> Result<()> {let ctx = SessionContext::new();ctx.sql(r#"CREATE EXTERNAL TABLE t1 (@@ -707,7 +710,10 @@ mod tests {).await.expect_err("should fail because input file does not match inferred schema");- assert_eq!(e.strip_backtrace(), "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'");+ assert_eq!(+ e.strip_backtrace(),+ "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'"+ );Ok(())}diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rsindex 3efea03302..922d13fa17 100644--- a/datafusion/core/src/datasource/physical_plan/json.rs+++ b/datafusion/core/src/datasource/physical_plan/json.rs@@ -34,9 +34,9 @@ mod tests {use crate::execution::SessionState;use crate::prelude::{CsvReadOptions, NdJsonReadOptions, SessionContext};use crate::test::partitioned_file_groups;+ use datafusion_common::Result;use datafusion_common::cast::{as_int32_array, as_int64_array, as_string_array};use datafusion_common::test_util::batches_to_string;- use datafusion_common::Result;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_format::FileFormat;use datafusion_datasource_json::JsonFormat;@@ -51,9 +51,9 @@ mod tests {use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;use insta::assert_snapshot;+ use object_store::ObjectStore;use object_store::chunked::ChunkedStore;use object_store::local::LocalFileSystem;- use object_store::ObjectStore;use rstest::*;use tempfile::TempDir;use url::Url;@@ -499,7 +499,10 @@ mod tests {.write_json(out_dir_url, DataFrameWriteOptions::new(), None).await.expect_err("should fail because input file does not match inferred schema");- assert_eq!(e.strip_backtrace(), "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'");+ assert_eq!(+ e.strip_backtrace(),+ "Arrow error: Parser error: Error while parsing value 'd' as type 'Int64' for column 0 at line 4. Row data: '[d,4]'"+ );Ok(())}diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rsindex 1ac292e260..c57b08545b 100644--- a/datafusion/core/src/datasource/physical_plan/mod.rs+++ b/datafusion/core/src/datasource/physical_plan/mod.rs@@ -43,8 +43,8 @@ pub use datafusion_datasource::file::FileSource;pub use datafusion_datasource::file_groups::FileGroup;pub use datafusion_datasource::file_groups::FileGroupPartitioner;pub use datafusion_datasource::file_scan_config::{- wrap_partition_type_in_dict, wrap_partition_value_in_dict, FileScanConfig,- FileScanConfigBuilder,+ FileScanConfig, FileScanConfigBuilder, wrap_partition_type_in_dict,+ wrap_partition_value_in_dict,};pub use datafusion_datasource::file_sink_config::*;@@ -57,10 +57,10 @@ mod tests {use std::sync::Arc;use arrow::array::{- cast::AsArray,- types::{Float32Type, Float64Type, UInt32Type},BinaryArray, BooleanArray, Float32Array, Int32Array, Int64Array, RecordBatch,StringArray, UInt64Array,+ cast::AsArray,+ types::{Float32Type, Float64Type, UInt32Type},};use arrow::datatypes::{DataType, Field, Schema};use arrow_schema::SchemaRef;diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rsindex 6ed01cde14..4613561c66 100644--- a/datafusion/core/src/datasource/physical_plan/parquet.rs+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs@@ -38,7 +38,7 @@ mod tests {use crate::prelude::{ParquetReadOptions, SessionConfig, SessionContext};use crate::test::object_store::local_unpartitioned_file;use arrow::array::{- ArrayRef, AsArray, Date64Array, Int32Array, Int64Array, Int8Array, StringArray,+ ArrayRef, AsArray, Date64Array, Int8Array, Int32Array, Int64Array, StringArray,StringViewArray, StructArray, TimestampNanosecondArray,};use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaBuilder};@@ -48,7 +48,7 @@ mod tests {use bytes::{BufMut, BytesMut};use datafusion_common::config::TableParquetOptions;use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};- use datafusion_common::{assert_contains, Result, ScalarValue};+ use datafusion_common::{Result, ScalarValue, assert_contains};use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;@@ -60,7 +60,7 @@ mod tests {DefaultParquetFileReaderFactory, ParquetFileReaderFactory, ParquetFormat,};use datafusion_execution::object_store::ObjectStoreUrl;- use datafusion_expr::{col, lit, when, Expr};+ use datafusion_expr::{Expr, col, lit, when};use datafusion_physical_expr::planner::logical2physical;use datafusion_physical_plan::analyze::AnalyzeExec;use datafusion_physical_plan::collect;@@ -1271,8 +1271,10 @@ mod tests {.with_table_schema(Arc::new(table_schema)).round_trip_to_batches(vec![batch1, batch2]).await;- assert_contains!(read.unwrap_err().to_string(),- "Cannot cast column 'c3' from 'Date64' (physical data type) to 'Int8' (logical data type)");+ assert_contains!(+ read.unwrap_err().to_string(),+ "Cannot cast column 'c3' from 'Date64' (physical data type) to 'Int8' (logical data type)"+ );}#[tokio::test]diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 5bba78e44d..a769bb01b4 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -34,12 +34,12 @@ use crate::{datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,},- datasource::{provider_as_source, MemTable, ViewTable},+ datasource::{MemTable, ViewTable, provider_as_source},error::Result,execution::{+ FunctionRegistry,options::ArrowReadOptions,runtime_env::{RuntimeEnv, RuntimeEnvBuilder},- FunctionRegistry,},logical_expr::AggregateUDF,logical_expr::ScalarUDF,@@ -59,43 +59,43 @@ pub use crate::execution::session_state::SessionState;use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;-use datafusion_catalog::memory::MemorySchemaProvider;use datafusion_catalog::MemoryCatalogProvider;+use datafusion_catalog::memory::MemorySchemaProvider;use datafusion_catalog::{DynamicFileCatalog, TableFunction, TableFunctionImpl, UrlTableFactory,};use datafusion_common::config::{ConfigField, ConfigOptions};use datafusion_common::metadata::ScalarAndMetadata;use datafusion_common::{+ DFSchema, DataFusionError, ParamValues, SchemaReference, TableReference,config::{ConfigExtension, TableOptions},exec_datafusion_err, exec_err, internal_datafusion_err, not_impl_err,plan_datafusion_err, plan_err,tree_node::{TreeNodeRecursion, TreeNodeVisitor},- DFSchema, DataFusionError, ParamValues, SchemaReference, TableReference,};+pub use datafusion_execution::TaskContext;use datafusion_execution::cache::cache_manager::{DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, DEFAULT_LIST_FILES_CACHE_TTL,DEFAULT_METADATA_CACHE_LIMIT,};pub use datafusion_execution::config::SessionConfig;use datafusion_execution::disk_manager::{- DiskManagerBuilder, DEFAULT_MAX_TEMP_DIRECTORY_SIZE,+ DEFAULT_MAX_TEMP_DIRECTORY_SIZE, DiskManagerBuilder,};use datafusion_execution::registry::SerializerRegistry;-pub use datafusion_execution::TaskContext;pub use datafusion_expr::execution_props::ExecutionProps;#[cfg(feature = "sql")]use datafusion_expr::planner::RelationPlanner;use datafusion_expr::simplify::SimplifyContext;use datafusion_expr::{+ Expr, UserDefinedLogicalNode, WindowUDF,expr_rewriter::FunctionRewrite,logical_plan::{DdlStatement, Statement},planner::ExprPlanner,- Expr, UserDefinedLogicalNode, WindowUDF,};+use datafusion_optimizer::Analyzer;use datafusion_optimizer::analyzer::type_coercion::TypeCoercion;use datafusion_optimizer::simplify_expressions::ExprSimplifier;-use datafusion_optimizer::Analyzer;use datafusion_optimizer::{AnalyzerRule, OptimizerRule};use datafusion_session::SessionStore;@@ -695,7 +695,7 @@ impl SessionContext {match ddl {DdlStatement::CreateExternalTable(cmd) => {(Box::pin(async move { self.create_external_table(&cmd).await })- as std::pin::Pin<Box<dyn futures::Future<Output = _> + Send>>)+ as std::pin::Pin<Box<dyn Future<Output = _> + Send>>).await}DdlStatement::CreateMemoryTable(cmd) => {@@ -1272,7 +1272,9 @@ impl SessionContext {match unit {"m" if minutes.is_none() && seconds.is_none() => minutes = Some(number),"s" if seconds.is_none() => seconds = Some(number),- _ => plan_err!("Invalid duration, unit must be either 'm' (minutes), or 's' (seconds), and be in the correct order")?,+ _ => plan_err!(+ "Invalid duration, unit must be either 'm' (minutes), or 's' (seconds), and be in the correct order"+ )?,}}@@ -1320,13 +1322,12 @@ impl SessionContext {.and_then(|c| c.schema(&resolved.schema))};- if let Some(schema) = maybe_schema {- if let Some(table_provider) = schema.table(&table).await? {- if table_provider.table_type() == table_type {- schema.deregister_table(&table)?;- return Ok(true);- }- }+ if let Some(schema) = maybe_schema+ && let Some(table_provider) = schema.table(&table).await?+ && table_provider.table_type() == table_type+ {+ schema.deregister_table(&table)?;+ return Ok(true);}Ok(false)@@ -1342,7 +1343,7 @@ impl SessionContext {_ => {return Err(DataFusionError::Configuration("Function factory has not been configured".to_string(),- ))+ ));}}};diff --git a/datafusion/core/src/execution/context/parquet.rs b/datafusion/core/src/execution/context/parquet.rsindex 731f7e59ec..823dc946ea 100644--- a/datafusion/core/src/execution/context/parquet.rs+++ b/datafusion/core/src/execution/context/parquet.rs@@ -113,7 +113,7 @@ mod tests {};use datafusion_execution::config::SessionConfig;- use tempfile::{tempdir, TempDir};+ use tempfile::{TempDir, tempdir};#[tokio::test]async fn read_with_glob_path() -> Result<()> {@@ -355,7 +355,9 @@ mod tests {let expected_path = binding[0].as_str();assert_eq!(read_df.unwrap_err().strip_backtrace(),- format!("Execution error: File path '{expected_path}' does not match the expected extension '.parquet'")+ format!(+ "Execution error: File path '{expected_path}' does not match the expected extension '.parquet'"+ ));// Read the dataframe from 'output3.parquet.snappy.parquet' with the correct file extension.diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 2466f9544b..6a9ebcdf51 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -27,14 +27,14 @@ use crate::catalog::{CatalogProviderList, SchemaProvider, TableProviderFactory};use crate::datasource::file_format::FileFormatFactory;#[cfg(feature = "sql")]use crate::datasource::provider_as_source;-use crate::execution::context::{EmptySerializerRegistry, FunctionFactory, QueryPlanner};use crate::execution::SessionStateDefaults;+use crate::execution::context::{EmptySerializerRegistry, FunctionFactory, QueryPlanner};use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner};use arrow_schema::{DataType, FieldRef};+use datafusion_catalog::MemoryCatalogProviderList;use datafusion_catalog::information_schema::{- InformationSchemaProvider, INFORMATION_SCHEMA,+ INFORMATION_SCHEMA, InformationSchemaProvider,};-use datafusion_catalog::MemoryCatalogProviderList;use datafusion_catalog::{TableFunction, TableFunctionImpl};use datafusion_common::alias::AliasGenerator;#[cfg(feature = "sql")]@@ -43,12 +43,14 @@ use datafusion_common::config::{ConfigExtension, ConfigOptions, TableOptions};use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan};use datafusion_common::tree_node::TreeNode;use datafusion_common::{- config_err, exec_err, plan_datafusion_err, DFSchema, DataFusionError,- ResolvedTableReference, TableReference,+ DFSchema, DataFusionError, ResolvedTableReference, TableReference, config_err,+ exec_err, plan_datafusion_err,};+use datafusion_execution::TaskContext;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnv;-use datafusion_execution::TaskContext;+#[cfg(feature = "sql")]+use datafusion_expr::TableSource;use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::expr_rewriter::FunctionRewrite;use datafusion_expr::planner::ExprPlanner;@@ -56,8 +58,6 @@ use datafusion_expr::planner::ExprPlanner;use datafusion_expr::planner::{RelationPlanner, TypePlanner};use datafusion_expr::registry::{FunctionRegistry, SerializerRegistry};use datafusion_expr::simplify::SimplifyInfo;-#[cfg(feature = "sql")]-use datafusion_expr::TableSource;use datafusion_expr::{AggregateUDF, Explain, Expr, ExprSchemable, LogicalPlan, ScalarUDF, WindowUDF,};@@ -67,8 +67,8 @@ use datafusion_optimizer::{};use datafusion_physical_expr::create_physical_expr;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;use datafusion_physical_plan::ExecutionPlan;use datafusion_session::Session;#[cfg(feature = "sql")]@@ -504,10 +504,10 @@ impl SessionState {let resolved = self.resolve_table_ref(reference);if let Entry::Vacant(v) = provider.tables.entry(resolved) {let resolved = v.key();- if let Ok(schema) = self.schema_for_ref(resolved.clone()) {- if let Some(table) = schema.table(&resolved.table).await? {- v.insert(provider_as_source(table));- }+ if let Ok(schema) = self.schema_for_ref(resolved.clone())+ && let Some(table) = schema.table(&resolved.table).await?+ {+ v.insert(provider_as_source(table));}}}@@ -840,10 +840,18 @@ impl SessionState {overwrite: bool,) -> Result<(), DataFusionError> {let ext = file_format.get_ext().to_lowercase();- match (self.file_formats.entry(ext.clone()), overwrite){- (Entry::Vacant(e), _) => {e.insert(file_format);},- (Entry::Occupied(mut e), true) => {e.insert(file_format);},- (Entry::Occupied(_), false) => return config_err!("File type already registered for extension {ext}. Set overwrite to true to replace this extension."),+ match (self.file_formats.entry(ext.clone()), overwrite) {+ (Entry::Vacant(e), _) => {+ e.insert(file_format);+ }+ (Entry::Occupied(mut e), true) => {+ e.insert(file_format);+ }+ (Entry::Occupied(_), false) => {+ return config_err!(+ "File type already registered for extension {ext}. Set overwrite to true to replace this extension."+ );+ }};Ok(())}@@ -1865,7 +1873,7 @@ impl ContextProvider for SessionContextProvider<'_> {}fn get_variable_type(&self, variable_names: &[String]) -> Option<DataType> {- use datafusion_expr::var_provider::{is_system_variables, VarType};+ use datafusion_expr::var_provider::{VarType, is_system_variables};if variable_names.is_empty() {return None;@@ -2162,9 +2170,9 @@ mod tests {use super::{SessionContextProvider, SessionStateBuilder};use crate::common::assert_contains;use crate::config::ConfigOptions;+ use crate::datasource::MemTable;use crate::datasource::empty::EmptyTable;use crate::datasource::provider_as_source;- use crate::datasource::MemTable;use crate::execution::context::SessionState;use crate::logical_expr::planner::ExprPlanner;use crate::logical_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF};@@ -2174,13 +2182,13 @@ mod tests {use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};use arrow::datatypes::{DataType, Field, Schema};use datafusion_catalog::MemoryCatalogProviderList;- use datafusion_common::config::Dialect;use datafusion_common::DFSchema;use datafusion_common::Result;+ use datafusion_common::config::Dialect;use datafusion_execution::config::SessionConfig;use datafusion_expr::Expr;- use datafusion_optimizer::optimizer::OptimizerRule;use datafusion_optimizer::Optimizer;+ use datafusion_optimizer::optimizer::OptimizerRule;use datafusion_physical_plan::display::DisplayableExecutionPlan;use datafusion_sql::planner::{PlannerContext, SqlToRel};use std::collections::HashMap;@@ -2287,13 +2295,15 @@ mod tests {.table_exist("employee");assert!(is_exist);let new_state = SessionStateBuilder::new_from_existing(session_state).build();- assert!(new_state- .catalog_list()- .catalog(default_catalog.as_str())- .unwrap()- .schema(default_schema.as_str())- .unwrap()- .table_exist("employee"));+ assert!(+ new_state+ .catalog_list()+ .catalog(default_catalog.as_str())+ .unwrap()+ .schema(default_schema.as_str())+ .unwrap()+ .table_exist("employee")+ );// if `with_create_default_catalog_and_schema` is disabled, the new one shouldn't create default catalog and schemalet disable_create_default =@@ -2301,10 +2311,12 @@ mod tests {let without_default_state = SessionStateBuilder::new().with_config(disable_create_default).build();- assert!(without_default_state- .catalog_list()- .catalog(&default_catalog)- .is_none());+ assert!(+ without_default_state+ .catalog_list()+ .catalog(&default_catalog)+ .is_none()+ );let new_state =SessionStateBuilder::new_from_existing(without_default_state).build();assert!(new_state.catalog_list().catalog(&default_catalog).is_none());diff --git a/datafusion/core/src/execution/session_state_defaults.rs b/datafusion/core/src/execution/session_state_defaults.rsindex bc7fd6bb17..721710d4e0 100644--- a/datafusion/core/src/execution/session_state_defaults.rs+++ b/datafusion/core/src/execution/session_state_defaults.rs@@ -17,6 +17,7 @@use crate::catalog::listing_schema::ListingSchemaProvider;use crate::catalog::{CatalogProvider, TableProviderFactory};+use crate::datasource::file_format::FileFormatFactory;use crate::datasource::file_format::arrow::ArrowFormatFactory;#[cfg(feature = "avro")]use crate::datasource::file_format::avro::AvroFormatFactory;@@ -24,7 +25,6 @@ use crate::datasource::file_format::csv::CsvFormatFactory;use crate::datasource::file_format::json::JsonFormatFactory;#[cfg(feature = "parquet")]use crate::datasource::file_format::parquet::ParquetFormatFactory;-use crate::datasource::file_format::FileFormatFactory;use crate::datasource::provider::DefaultTableFactory;use crate::execution::context::SessionState;#[cfg(feature = "nested_expressions")]diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 3e216c619b..9eaf1403e5 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -24,7 +24,7 @@ use std::sync::Arc;use crate::datasource::file_format::file_type_to_format;use crate::datasource::listing::ListingTableUrl;use crate::datasource::physical_plan::FileSinkConfig;-use crate::datasource::{source_as_provider, DefaultTableSource};+use crate::datasource::{DefaultTableSource, source_as_provider};use crate::error::{DataFusionError, Result};use crate::execution::context::{ExecutionProps, SessionState};use crate::logical_expr::utils::generate_sort_key;@@ -52,12 +52,12 @@ use crate::physical_plan::union::UnionExec;use crate::physical_plan::unnest::UnnestExec;use crate::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};use crate::physical_plan::{- displayable, windows, ExecutionPlan, ExecutionPlanProperties, InputOrderMode,- Partitioning, PhysicalExpr, WindowExpr,+ ExecutionPlan, ExecutionPlanProperties, InputOrderMode, Partitioning, PhysicalExpr,+ WindowExpr, displayable, windows,};use crate::schema_equivalence::schema_satisfied_by;-use arrow::array::{builder::StringBuilder, RecordBatch};+use arrow::array::{RecordBatch, builder::StringBuilder};use arrow::compute::SortOptions;use arrow::datatypes::Schema;use arrow_schema::Field;@@ -66,18 +66,18 @@ use datafusion_common::display::ToStringifiedPlan;use datafusion_common::format::ExplainAnalyzeLevel;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};use datafusion_common::{- assert_eq_or_internal_err, assert_or_internal_err, TableReference,+ DFSchema, ScalarValue, exec_err, internal_datafusion_err, internal_err, not_impl_err,+ plan_err,};use datafusion_common::{- exec_err, internal_datafusion_err, internal_err, not_impl_err, plan_err, DFSchema,- ScalarValue,+ TableReference, assert_eq_or_internal_err, assert_or_internal_err,};use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::memory::MemorySourceConfig;use datafusion_expr::dml::{CopyTo, InsertOp};use datafusion_expr::expr::{- physical_name, AggregateFunction, AggregateFunctionParams, Alias, GroupingSet,- NullTreatment, WindowFunction, WindowFunctionParams,+ AggregateFunction, AggregateFunctionParams, Alias, GroupingSet, NullTreatment,+ WindowFunction, WindowFunctionParams, physical_name,};use datafusion_expr::expr_rewriter::unnormalize_cols;use datafusion_expr::logical_plan::builder::wrap_projection_for_join_if_necessary;@@ -90,7 +90,7 @@ use datafusion_expr::{use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::{- create_physical_sort_exprs, LexOrdering, PhysicalSortExpr,+ LexOrdering, PhysicalSortExpr, create_physical_sort_exprs,};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::empty::EmptyExec;@@ -104,7 +104,7 @@ use datafusion_physical_plan::unnest::ListUnnest;use async_trait::async_trait;use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper};use futures::{StreamExt, TryStreamExt};-use itertools::{multiunzip, Itertools};+use itertools::{Itertools, multiunzip};use log::debug;use tokio::sync::Mutex;@@ -528,12 +528,22 @@ impl DefaultPhysicalPlanner {let keep_partition_by_columns = match source_option_tuples.get("execution.keep_partition_by_columns")- .map(|v| v.trim()) {- None => session_state.config().options().execution.keep_partition_by_columns,+ .map(|v| v.trim())+ {+ None => {+ session_state+ .config()+ .options()+ .execution+ .keep_partition_by_columns+ }Some("true") => true,Some("false") => false,- Some(value) =>- return Err(DataFusionError::Configuration(format!("provided value for 'execution.keep_partition_by_columns' was not recognized: \"{value}\""))),+ Some(value) => {+ return Err(DataFusionError::Configuration(format!(+ "provided value for 'execution.keep_partition_by_columns' was not recognized: \"{value}\""+ )));+ }};let sink_format = file_type_to_format(file_type)?@@ -601,8 +611,8 @@ impl DefaultPhysicalPlanner {let get_sort_keys = |expr: &Expr| match expr {Expr::WindowFunction(window_fun) => {let WindowFunctionParams {- ref partition_by,- ref order_by,+ partition_by,+ order_by,..} = &window_fun.as_ref().params;generate_sort_key(partition_by, order_by)@@ -612,8 +622,8 @@ impl DefaultPhysicalPlanner {match &**expr {Expr::WindowFunction(window_fun) => {let WindowFunctionParams {- ref partition_by,- ref order_by,+ partition_by,+ order_by,..} = &window_fun.as_ref().params;generate_sort_key(partition_by, order_by)@@ -626,11 +636,11 @@ impl DefaultPhysicalPlanner {let sort_keys = get_sort_keys(&window_expr[0])?;if window_expr.len() > 1 {debug_assert!(- window_expr[1..]- .iter()- .all(|expr| get_sort_keys(expr).unwrap() == sort_keys),- "all window expressions shall have the same sort keys, as guaranteed by logical planning"- );+ window_expr[1..]+ .iter()+ .all(|expr| get_sort_keys(expr).unwrap() == sort_keys),+ "all window expressions shall have the same sort keys, as guaranteed by logical planning"+ );}let logical_schema = node.schema();@@ -737,10 +747,10 @@ impl DefaultPhysicalPlanner {));}}- return internal_err!("Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences- .iter()- .map(|s| format!("\n\t- {s}"))- .join(""));+ return internal_err!(+ "Physical input schema should be the same as the one converted from logical input schema. Differences: {}",+ differences.iter().map(|s| format!("\n\t- {s}")).join("")+ );}let groups = self.create_grouping_physical_expr(@@ -800,7 +810,7 @@ impl DefaultPhysicalPlanner {_ => {return internal_err!("Unexpected result from try_plan_async_exprs"- )+ );}}}@@ -898,7 +908,7 @@ impl DefaultPhysicalPlanner {_ => {return internal_err!("Unexpected result from try_plan_async_exprs"- )+ );}};@@ -1489,19 +1499,24 @@ impl DefaultPhysicalPlanner {}let plan = match maybe_plan {- Some(v) => Ok(v),- _ => plan_err!("No installed planner was able to convert the custom node to an execution plan: {:?}", node)- }?;+ Some(v) => Ok(v),+ _ => plan_err!(+ "No installed planner was able to convert the custom node to an execution plan: {:?}",+ node+ ),+ }?;// Ensure the ExecutionPlan's schema matches the// declared logical schema to catch and warn about// logic errors when creating user defined plans.if !node.schema().matches_arrow_schema(&plan.schema()) {return plan_err!(- "Extension planner for {:?} created an ExecutionPlan with mismatched schema. \+ "Extension planner for {:?} created an ExecutionPlan with mismatched schema. \LogicalPlan schema: {:?}, ExecutionPlan schema: {:?}",- node, node.schema(), plan.schema()- );+ node,+ node.schema(),+ plan.schema()+ );} else {plan}@@ -1528,17 +1543,17 @@ impl DefaultPhysicalPlanner {LogicalPlan::Explain(_) => {return internal_err!("Unsupported logical plan: Explain must be root of the plan"- )+ );}LogicalPlan::Distinct(_) => {return internal_err!("Unsupported logical plan: Distinct should be replaced to Aggregate"- )+ );}LogicalPlan::Analyze(_) => {return internal_err!("Unsupported logical plan: Analyze must be root of the plan"- )+ );}};Ok(exec_node)@@ -1884,9 +1899,10 @@ pub fn create_window_expr_with_name(if !is_window_frame_bound_valid(window_frame) {return plan_err!(- "Invalid window frame: start bound ({}) cannot be larger than end bound ({})",- window_frame.start_bound, window_frame.end_bound- );+ "Invalid window frame: start bound ({}) cannot be larger than end bound ({})",+ window_frame.start_bound,+ window_frame.end_bound+ );}let window_frame = Arc::new(window_frame.clone());@@ -2546,7 +2562,8 @@ impl<'a> OptimizationInvariantChecker<'a> {if self.rule.schema_check()&& !is_allowed_schema_change(previous_schema.as_ref(), plan.schema().as_ref()){- internal_err!("PhysicalOptimizer rule '{}' failed. Schema mismatch. Expected original schema: {:?}, got new schema: {:?}",+ internal_err!(+ "PhysicalOptimizer rule '{}' failed. Schema mismatch. Expected original schema: {:?}, got new schema: {:?}",self.rule.name(),previous_schema,plan.schema()@@ -2641,11 +2658,11 @@ mod tests {use std::ops::{BitAnd, Not};use super::*;- use crate::datasource::file_format::options::CsvReadOptions;use crate::datasource::MemTable;+ use crate::datasource::file_format::options::CsvReadOptions;use crate::physical_plan::{- expressions, DisplayAs, DisplayFormatType, PlanProperties,- SendableRecordBatchStream,+ DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,+ expressions,};use crate::prelude::{SessionConfig, SessionContext};use crate::test_util::{scan_empty, scan_empty_with_partitions};@@ -2656,12 +2673,12 @@ mod tests {use arrow_schema::SchemaRef;use datafusion_common::config::ConfigOptions;use datafusion_common::{- assert_contains, DFSchemaRef, TableReference, ToDFSchema as _,+ DFSchemaRef, TableReference, ToDFSchema as _, assert_contains,};- use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_execution::TaskContext;+ use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_expr::builder::subquery_alias;- use datafusion_expr::{col, lit, LogicalPlanBuilder, UserDefinedLogicalNodeCore};+ use datafusion_expr::{LogicalPlanBuilder, UserDefinedLogicalNodeCore, col, lit};use datafusion_functions_aggregate::count::count_all;use datafusion_functions_aggregate::expr_fn::sum;use datafusion_physical_expr::EquivalenceProperties;@@ -3063,8 +3080,7 @@ mod tests {.create_physical_plan(&logical_plan, &session_state).await;- let expected_error =- "No installed planner was able to convert the custom node to an execution plan: NoOp";+ let expected_error = "No installed planner was able to convert the custom node to an execution plan: NoOp";match plan {Ok(_) => panic!("Expected planning failure"),Err(e) => assert!(@@ -3321,18 +3337,27 @@ mod tests {if let Some(plan) = plan.as_any().downcast_ref::<ExplainExec>() {let stringified_plans = plan.stringified_plans();assert!(stringified_plans.len() >= 4);- assert!(stringified_plans- .iter()- .any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan)));- assert!(stringified_plans- .iter()- .any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan)));- assert!(stringified_plans- .iter()- .any(|p| matches!(p.plan_type, PlanType::OptimizedPhysicalPlan { .. })));- assert!(stringified_plans- .iter()- .any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan)));+ assert!(+ stringified_plans+ .iter()+ .any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan))+ );+ assert!(+ stringified_plans+ .iter()+ .any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan))+ );+ assert!(+ stringified_plans.iter().any(|p| matches!(+ p.plan_type,+ PlanType::OptimizedPhysicalPlan { .. }+ ))+ );+ assert!(+ stringified_plans+ .iter()+ .any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan))+ );} else {panic!("Plan was not an explain plan: {}",@@ -3699,8 +3724,12 @@ digraph {}fn check_invariants(&self, check: InvariantLevel) -> Result<()> {match check {- InvariantLevel::Always => plan_err!("extension node failed it's user-defined always-invariant check"),- InvariantLevel::Executable => panic!("the OptimizationInvariantChecker should not be checking for executableness"),+ InvariantLevel::Always => plan_err!(+ "extension node failed it's user-defined always-invariant check"+ ),+ InvariantLevel::Executable => panic!(+ "the OptimizationInvariantChecker should not be checking for executableness"+ ),}}fn schema(&self) -> SchemaRef {@@ -3784,9 +3813,11 @@ digraph {let expected_err = OptimizationInvariantChecker::new(&rule).check(&failing_node, &ok_plan.schema()).unwrap_err();- assert!(expected_err- .to_string()- .contains("extension node failed it's user-defined always-invariant check"));+ assert!(+ expected_err.to_string().contains(+ "extension node failed it's user-defined always-invariant check"+ )+ );// Test: should fail when descendent extension node failslet failing_node: Arc<dyn ExecutionPlan> = Arc::new(InvariantFailsExtensionNode);@@ -3797,9 +3828,11 @@ digraph {let expected_err = OptimizationInvariantChecker::new(&rule).check(&invalid_plan, &ok_plan.schema()).unwrap_err();- assert!(expected_err- .to_string()- .contains("extension node failed it's user-defined always-invariant check"));+ assert!(+ expected_err.to_string().contains(+ "extension node failed it's user-defined always-invariant check"+ )+ );Ok(())}@@ -4118,9 +4151,10 @@ digraph {async fn test_aggregate_schema_mismatch_field_metadata() {let logical_schema =Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));- let physical_schema =- Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)- .with_metadata(HashMap::from([("key".into(), "value".into())]))]));+ let physical_schema = Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false)+ .with_metadata(HashMap::from([("key".into(), "value".into())])),+ ]));let err = plan_with_schemas(logical_schema,diff --git a/datafusion/core/src/prelude.rs b/datafusion/core/src/prelude.rsindex d723620d32..50e4a2649c 100644--- a/datafusion/core/src/prelude.rs+++ b/datafusion/core/src/prelude.rs@@ -34,10 +34,10 @@ pub use crate::execution::options::{pub use datafusion_common::Column;pub use datafusion_expr::{+ Expr,expr_fn::*,lit, lit_timestamp_nano,logical_plan::{JoinType, Partitioning},- Expr,};pub use datafusion_functions::expr_fn::*;#[cfg(feature = "nested_expressions")]diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rsindex b0ff3eb3ae..717182f1d3 100644--- a/datafusion/core/src/test/mod.rs+++ b/datafusion/core/src/test/mod.rs@@ -25,9 +25,9 @@ use std::io::{BufReader, BufWriter};use std::path::Path;use std::sync::Arc;+use crate::datasource::file_format::FileFormat;use crate::datasource::file_format::csv::CsvFormat;use crate::datasource::file_format::file_compression_type::FileCompressionType;-use crate::datasource::file_format::FileFormat;use crate::datasource::physical_plan::CsvSource;use crate::datasource::{MemTable, TableProvider};@@ -42,21 +42,21 @@ use arrow::datatypes::{DataType, Field, Schema};use arrow::record_batch::RecordBatch;#[cfg(feature = "compression")]use datafusion_common::DataFusionError;-use datafusion_datasource::source::DataSourceExec;use datafusion_datasource::TableSchema;+use datafusion_datasource::source::DataSourceExec;-#[cfg(feature = "compression")]-use bzip2::write::BzEncoder;#[cfg(feature = "compression")]use bzip2::Compression as BzCompression;+#[cfg(feature = "compression")]+use bzip2::write::BzEncoder;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource_csv::partitioned_csv_config;#[cfg(feature = "compression")]-use flate2::write::GzEncoder;-#[cfg(feature = "compression")]use flate2::Compression as GzCompression;#[cfg(feature = "compression")]+use flate2::write::GzEncoder;+#[cfg(feature = "compression")]use liblzma::write::XzEncoder;use object_store::local_unpartitioned_file;#[cfg(feature = "compression")]diff --git a/datafusion/core/src/test/object_store.rs b/datafusion/core/src/test/object_store.rsindex d31c271997..a0438e3d74 100644--- a/datafusion/core/src/test/object_store.rs+++ b/datafusion/core/src/test/object_store.rs@@ -20,20 +20,20 @@use crate::{execution::{context::SessionState, session_state::SessionStateBuilder},object_store::{- memory::InMemory, path::Path, Error, GetOptions, GetResult, ListResult,- MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions,- PutPayload, PutResult,+ Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,+ ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,+ memory::InMemory, path::Path,},prelude::SessionContext,};-use futures::{stream::BoxStream, FutureExt};+use futures::{FutureExt, stream::BoxStream};use std::{fmt::{Debug, Display, Formatter},sync::Arc,};use tokio::{sync::Barrier,- time::{timeout, Duration},+ time::{Duration, timeout},};use url::Url;diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rsindex b5213cee3f..44e884c23a 100644--- a/datafusion/core/src/test_util/parquet.rs+++ b/datafusion/core/src/test_util/parquet.rs@@ -32,15 +32,15 @@ use crate::logical_expr::execution_props::ExecutionProps;use crate::logical_expr::simplify::SimplifyContext;use crate::optimizer::simplify_expressions::ExprSimplifier;use crate::physical_expr::create_physical_expr;+use crate::physical_plan::ExecutionPlan;use crate::physical_plan::filter::FilterExec;use crate::physical_plan::metrics::MetricsSet;-use crate::physical_plan::ExecutionPlan;use crate::prelude::{Expr, SessionConfig, SessionContext};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;-use object_store::path::Path;use object_store::ObjectMeta;+use object_store::path::Path;use parquet::arrow::ArrowWriter;use parquet::file::properties::WriterProperties;@@ -203,13 +203,12 @@ impl TestParquetFile {/// Recursively searches for DataSourceExec and returns the metrics/// on the first one it findspub fn parquet_metrics(plan: &Arc<dyn ExecutionPlan>) -> Option<MetricsSet> {- if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {- if data_source_exec+ if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && data_source_exec.downcast_to_file_source::<ParquetSource>().is_some()- {- return data_source_exec.metrics();- }+ {+ return data_source_exec.metrics();}for child in plan.children() {diff --git a/datafusion/core/tests/catalog/memory.rs b/datafusion/core/tests/catalog/memory.rsindex 06ed141b2e..5258f3bf97 100644--- a/datafusion/core/tests/catalog/memory.rs+++ b/datafusion/core/tests/catalog/memory.rs@@ -116,10 +116,12 @@ async fn test_mem_provider() {assert!(provider.deregister_table(table_name).unwrap().is_none());let test_table = EmptyTable::new(Arc::new(Schema::empty()));// register table successfully- assert!(provider- .register_table(table_name.to_string(), Arc::new(test_table))- .unwrap()- .is_none());+ assert!(+ provider+ .register_table(table_name.to_string(), Arc::new(test_table))+ .unwrap()+ .is_none()+ );assert!(provider.table_exist(table_name));let other_table = EmptyTable::new(Arc::new(Schema::empty()));let result = provider.register_table(table_name.to_string(), Arc::new(other_table));diff --git a/datafusion/core/tests/catalog_listing/pruned_partition_list.rs b/datafusion/core/tests/catalog_listing/pruned_partition_list.rsindex 3cdaa3bb9b..f4782ee13c 100644--- a/datafusion/core/tests/catalog_listing/pruned_partition_list.rs+++ b/datafusion/core/tests/catalog_listing/pruned_partition_list.rs@@ -19,7 +19,7 @@ use std::sync::Arc;use arrow_schema::DataType;use futures::{FutureExt, StreamExt as _, TryStreamExt as _};-use object_store::{memory::InMemory, path::Path, ObjectStore as _};+use object_store::{ObjectStore as _, memory::InMemory, path::Path};use datafusion::execution::SessionStateBuilder;use datafusion_catalog_listing::helpers::{@@ -27,7 +27,7 @@ use datafusion_catalog_listing::helpers::{};use datafusion_common::ScalarValue;use datafusion_datasource::ListingTableUrl;-use datafusion_expr::{col, lit, Expr};+use datafusion_expr::{Expr, col, lit};use datafusion_session::Session;#[tokio::test]diff --git a/datafusion/core/tests/config_from_env.rs b/datafusion/core/tests/config_from_env.rsindex 976597c8a9..6375d4e25d 100644--- a/datafusion/core/tests/config_from_env.rs+++ b/datafusion/core/tests/config_from_env.rs@@ -20,35 +20,43 @@ use std::env;#[test]fn from_env() {- // Note: these must be a single test to avoid interference from concurrent execution- let env_key = "DATAFUSION_OPTIMIZER_FILTER_NULL_JOIN_KEYS";- // valid testing in different cases- for bool_option in ["true", "TRUE", "True", "tRUe"] {- env::set_var(env_key, bool_option);- let config = ConfigOptions::from_env().unwrap();- env::remove_var(env_key);- assert!(config.optimizer.filter_null_join_keys);- }+ unsafe {+ // Note: these must be a single test to avoid interference from concurrent execution+ let env_key = "DATAFUSION_OPTIMIZER_FILTER_NULL_JOIN_KEYS";+ // valid testing in different cases+ for bool_option in ["true", "TRUE", "True", "tRUe"] {+ env::set_var(env_key, bool_option);+ let config = ConfigOptions::from_env().unwrap();+ env::remove_var(env_key);+ assert!(config.optimizer.filter_null_join_keys);+ }- // invalid testing- env::set_var(env_key, "ttruee");- let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();- assert_eq!(err, "Error parsing 'ttruee' as bool\ncaused by\nExternal error: provided string was not `true` or `false`");- env::remove_var(env_key);+ // invalid testing+ env::set_var(env_key, "ttruee");+ let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();+ assert_eq!(+ err,+ "Error parsing 'ttruee' as bool\ncaused by\nExternal error: provided string was not `true` or `false`"+ );+ env::remove_var(env_key);- let env_key = "DATAFUSION_EXECUTION_BATCH_SIZE";+ let env_key = "DATAFUSION_EXECUTION_BATCH_SIZE";- // for valid testing- env::set_var(env_key, "4096");- let config = ConfigOptions::from_env().unwrap();- assert_eq!(config.execution.batch_size, 4096);+ // for valid testing+ env::set_var(env_key, "4096");+ let config = ConfigOptions::from_env().unwrap();+ assert_eq!(config.execution.batch_size, 4096);- // for invalid testing- env::set_var(env_key, "abc");- let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();- assert_eq!(err, "Error parsing 'abc' as usize\ncaused by\nExternal error: invalid digit found in string");+ // for invalid testing+ env::set_var(env_key, "abc");+ let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();+ assert_eq!(+ err,+ "Error parsing 'abc' as usize\ncaused by\nExternal error: invalid digit found in string"+ );- env::remove_var(env_key);- let config = ConfigOptions::from_env().unwrap();- assert_eq!(config.execution.batch_size, 8192); // set to its default value+ env::remove_var(env_key);+ let config = ConfigOptions::from_env().unwrap();+ assert_eq!(config.execution.batch_size, 8192); // set to its default value+ }}diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex 44da7d4e62..7b6a3c5fbe 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -28,11 +28,11 @@ use datafusion::datasource::{TableProvider, TableType};use datafusion::error::Result;use datafusion::execution::context::{SessionContext, TaskContext};use datafusion::logical_expr::{- col, Expr, LogicalPlan, LogicalPlanBuilder, TableScan, UNNAMED_TABLE,+ Expr, LogicalPlan, LogicalPlanBuilder, TableScan, UNNAMED_TABLE, col,};use datafusion::physical_plan::{- collect, ColumnStatistics, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,- RecordBatchStream, SendableRecordBatchStream, Statistics,+ ColumnStatistics, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,+ RecordBatchStream, SendableRecordBatchStream, Statistics, collect,};use datafusion::scalar::ScalarValue;use datafusion_catalog::Session;@@ -40,9 +40,9 @@ use datafusion_common::cast::as_primitive_array;use datafusion_common::project_schema;use datafusion_common::stats::Precision;use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_plan::PlanProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;-use datafusion_physical_plan::PlanProperties;use async_trait::async_trait;use futures::stream::Stream;diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex f4e905e1ed..ca1eaa1f95 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -35,7 +35,7 @@ use datafusion::prelude::*;use datafusion::scalar::ScalarValue;use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;-use datafusion_common::{internal_err, not_impl_err, DataFusionError};+use datafusion_common::{DataFusionError, internal_err, not_impl_err};use datafusion_expr::expr::{BinaryExpr, Cast};use datafusion_functions_aggregate::expr_fn::count;use datafusion_physical_expr::EquivalenceProperties;diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rsindex 56cdd78d70..014f356cd6 100644--- a/datafusion/core/tests/dataframe/dataframe_functions.rs+++ b/datafusion/core/tests/dataframe/dataframe_functions.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{types::Int32Type, ListArray};+use arrow::array::{ListArray, types::Int32Type};use arrow::datatypes::SchemaRef;use arrow::datatypes::{DataType, Field, Schema};use arrow::{@@ -31,7 +31,7 @@ use datafusion::prelude::*;use datafusion_common::test_util::batches_to_string;use datafusion_common::{DFSchema, ScalarValue};use datafusion_expr::expr::Alias;-use datafusion_expr::{table_scan, ExprSchemable, LogicalPlanBuilder};+use datafusion_expr::{ExprSchemable, LogicalPlanBuilder, table_scan};use datafusion_functions_aggregate::expr_fn::{approx_median, approx_percentile_cont};use datafusion_functions_nested::map::map;use insta::assert_snapshot;diff --git a/datafusion/core/tests/dataframe/describe.rs b/datafusion/core/tests/dataframe/describe.rsindex 9bd69dfa72..c61fe4fed1 100644--- a/datafusion/core/tests/dataframe/describe.rs+++ b/datafusion/core/tests/dataframe/describe.rs@@ -17,7 +17,7 @@use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_common::test_util::batches_to_string;-use datafusion_common::{test_util::parquet_test_data, Result};+use datafusion_common::{Result, test_util::parquet_test_data};use insta::assert_snapshot;#[tokio::test]diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 6eb9d9743d..c87e9fa7ab 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -20,10 +20,10 @@ mod dataframe_functions;mod describe;use arrow::array::{- record_batch, Array, ArrayRef, BooleanArray, DictionaryArray, FixedSizeListArray,- FixedSizeListBuilder, Float32Array, Float64Array, Int32Array, Int32Builder,- Int8Array, LargeListArray, ListArray, ListBuilder, RecordBatch, StringArray,- StringBuilder, StructBuilder, UInt32Array, UInt32Builder, UnionArray,+ Array, ArrayRef, BooleanArray, DictionaryArray, FixedSizeListArray,+ FixedSizeListBuilder, Float32Array, Float64Array, Int8Array, Int32Array,+ Int32Builder, LargeListArray, ListArray, ListBuilder, RecordBatch, StringArray,+ StringBuilder, StructBuilder, UInt32Array, UInt32Builder, UnionArray, record_batch,};use arrow::buffer::ScalarBuffer;use arrow::datatypes::{@@ -66,8 +66,8 @@ use datafusion::test_util::{use datafusion_catalog::TableProvider;use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};use datafusion_common::{- assert_contains, internal_datafusion_err, Constraint, Constraints, DFSchema,- DataFusionError, ScalarValue, SchemaError, TableReference, UnnestOptions,+ Constraint, Constraints, DFSchema, DataFusionError, ScalarValue, SchemaError,+ TableReference, UnnestOptions, assert_contains, internal_datafusion_err,};use datafusion_common_runtime::SpawnedTask;use datafusion_datasource::file_format::format_as_file_type;@@ -76,21 +76,21 @@ use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_expr::expr::{GroupingSet, NullTreatment, Sort, WindowFunction};use datafusion_expr::var_provider::{VarProvider, VarType};use datafusion_expr::{- cast, col, create_udf, exists, in_subquery, lit, out_ref_col, placeholder,- scalar_subquery, when, wildcard, Expr, ExprFunctionExt, ExprSchemable, LogicalPlan,- LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, TableType, WindowFrame,- WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,+ Expr, ExprFunctionExt, ExprSchemable, LogicalPlan, LogicalPlanBuilder,+ ScalarFunctionImplementation, SortExpr, TableType, WindowFrame, WindowFrameBound,+ WindowFrameUnits, WindowFunctionDefinition, cast, col, create_udf, exists,+ in_subquery, lit, out_ref_col, placeholder, scalar_subquery, when, wildcard,};+use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::aggregate::AggregateExprBuilder;use datafusion_physical_expr::expressions::Column;-use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};use datafusion_physical_plan::empty::EmptyExec;-use datafusion_physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties};+use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable};use datafusion::error::Result as DataFusionResult;use datafusion_functions_window::expr_fn::lag;@@ -2234,12 +2234,14 @@ async fn row_writer_resize_test() -> Result<()> {let data = RecordBatch::try_new(schema,- vec![- Arc::new(StringArray::from(vec![- Some("2a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),- Some("3a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"),- ]))- ],+ vec![Arc::new(StringArray::from(vec![+ Some(+ "2a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",+ ),+ Some(+ "3a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800",+ ),+ ]))],)?;let ctx = SessionContext::new();@@ -3114,15 +3116,17 @@ async fn test_count_wildcard_on_window() -> Result<()> {let df_results = ctx.table("t1").await?- .select(vec![count_all_window()- .order_by(vec![Sort::new(col("a"), false, true)])- .window_frame(WindowFrame::new_bounds(- WindowFrameUnits::Range,- WindowFrameBound::Preceding(ScalarValue::UInt32(Some(6))),- WindowFrameBound::Following(ScalarValue::UInt32(Some(2))),- ))- .build()- .unwrap()])?+ .select(vec![+ count_all_window()+ .order_by(vec![Sort::new(col("a"), false, true)])+ .window_frame(WindowFrame::new_bounds(+ WindowFrameUnits::Range,+ WindowFrameBound::Preceding(ScalarValue::UInt32(Some(6))),+ WindowFrameBound::Following(ScalarValue::UInt32(Some(2))),+ ))+ .build()+ .unwrap(),+ ])?.explain(false, false)?.collect().await?;@@ -3150,8 +3154,8 @@ async fn test_count_wildcard_on_window() -> Result<()> {#[tokio::test]// Test with `repartition_sorts` disabled, causing a full resort of the data-async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false(-) -> Result<()> {+async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false()+-> Result<()> {assert_snapshot!(union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(false).await?,@r"@@ -3168,8 +3172,8 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti#[tokio::test]// Test with `repartition_sorts` enabled to preserve pre-sorted partitions and avoid resorting-async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true(-) -> Result<()> {+async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true()+-> Result<()> {assert_snapshot!(union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(true).await?,@r#"@@ -6306,7 +6310,10 @@ async fn test_insert_into_checking() -> Result<()> {.await.unwrap_err();- assert_contains!(e.to_string(), "Inserting query schema mismatch: Expected table field 'a' with type Int64, but got 'column1' with type Utf8");+ assert_contains!(+ e.to_string(),+ "Inserting query schema mismatch: Expected table field 'a' with type Int64, but got 'column1' with type Utf8"+ );Ok(())}@@ -6438,7 +6445,10 @@ async fn test_insert_into_casting_support() -> Result<()> {.await.unwrap_err();- assert_contains!(e.to_string(), "Inserting query schema mismatch: Expected table field 'a' with type Float16, but got 'a' with type Utf8.");+ assert_contains!(+ e.to_string(),+ "Inserting query schema mismatch: Expected table field 'a' with type Float16, but got 'a' with type Utf8."+ );// Testing case2:// Inserting query schema mismatch: Expected table field 'a' with type Utf8View, but got 'a' with type Utf8.diff --git a/datafusion/core/tests/execution/coop.rs b/datafusion/core/tests/execution/coop.rsindex fb9495da50..2de0b95d0c 100644--- a/datafusion/core/tests/execution/coop.rs+++ b/datafusion/core/tests/execution/coop.rs@@ -22,25 +22,25 @@ use datafusion::common::NullEquality;use datafusion::functions_aggregate::sum;use datafusion::physical_expr::aggregate::AggregateExprBuilder;use datafusion::physical_plan;+use datafusion::physical_plan::ExecutionPlan;use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};use datafusion::physical_plan::execution_plan::Boundedness;-use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::SessionContext;-use datafusion_common::{exec_datafusion_err, DataFusionError, JoinType, ScalarValue};+use datafusion_common::{DataFusionError, JoinType, ScalarValue, exec_datafusion_err};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr_common::operator::Operator;use datafusion_expr_common::operator::Operator::{Divide, Eq, Gt, Modulo};use datafusion_functions_aggregate::min_max;+use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::expressions::{- binary, col, lit, BinaryExpr, Column, Literal,+ BinaryExpr, Column, Literal, binary, col, lit,};-use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_optimizer::ensure_coop::EnsureCooperative;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::ensure_coop::EnsureCooperative;use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion_physical_plan::coop::make_cooperative;use datafusion_physical_plan::filter::FilterExec;@@ -136,7 +136,7 @@ fn make_lazy_exec_with_range(};// Instantiate the generator with the batch and limit- let gen = RangeBatchGenerator {+ let batch_gen = RangeBatchGenerator {schema: Arc::clone(&schema),boundedness,value_range: range,@@ -145,7 +145,7 @@ fn make_lazy_exec_with_range(};// Wrap the generator in a trait object behind Arc<RwLock<_>>- let generator: Arc<RwLock<dyn LazyBatchGenerator>> = Arc::new(RwLock::new(gen));+ let generator: Arc<RwLock<dyn LazyBatchGenerator>> = Arc::new(RwLock::new(batch_gen));// Create a LazyMemoryExec with one partition using our generatorlet mut exec = LazyMemoryExec::try_new(schema, vec![generator]).unwrap();diff --git a/datafusion/core/tests/execution/datasource_split.rs b/datafusion/core/tests/execution/datasource_split.rsindex 0b90c6f326..370249cd80 100644--- a/datafusion/core/tests/execution/datasource_split.rs+++ b/datafusion/core/tests/execution/datasource_split.rs@@ -22,7 +22,7 @@ use arrow::{};use datafusion_datasource::memory::MemorySourceConfig;use datafusion_execution::TaskContext;-use datafusion_physical_plan::{common::collect, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, common::collect};use std::sync::Arc;/// Helper function to create a memory source with the given batch size and collect all batchesdiff --git a/datafusion/core/tests/execution/logical_plan.rs b/datafusion/core/tests/execution/logical_plan.rsindex ef2e263f2c..3eaa3fb2ed 100644--- a/datafusion/core/tests/execution/logical_plan.rs+++ b/datafusion/core/tests/execution/logical_plan.rs@@ -20,7 +20,7 @@use arrow::array::Int64Array;use arrow::datatypes::{DataType, Field, Schema};-use datafusion::datasource::{provider_as_source, ViewTable};+use datafusion::datasource::{ViewTable, provider_as_source};use datafusion::execution::session_state::SessionStateBuilder;use datafusion_common::{Column, DFSchema, DFSchemaRef, Result, ScalarValue, Spans};use datafusion_execution::TaskContext;diff --git a/datafusion/core/tests/expr_api/mod.rs b/datafusion/core/tests/expr_api/mod.rsindex ad2b866844..90c1b96749 100644--- a/datafusion/core/tests/expr_api/mod.rs+++ b/datafusion/core/tests/expr_api/mod.rs@@ -16,17 +16,17 @@// under the License.use arrow::array::{- builder::{ListBuilder, StringBuilder},ArrayRef, Int64Array, RecordBatch, StringArray, StructArray,+ builder::{ListBuilder, StringBuilder},};use arrow::datatypes::{DataType, Field};use arrow::util::pretty::{pretty_format_batches, pretty_format_columns};use datafusion::prelude::*;use datafusion_common::{DFSchema, ScalarValue};+use datafusion_expr::ExprFunctionExt;use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::expr::NullTreatment;use datafusion_expr::simplify::SimplifyContext;-use datafusion_expr::ExprFunctionExt;use datafusion_functions::core::expr_ext::FieldAccessor;use datafusion_functions_aggregate::first_last::first_value_udaf;use datafusion_functions_aggregate::sum::sum_udaf;diff --git a/datafusion/core/tests/expr_api/parse_sql_expr.rs b/datafusion/core/tests/expr_api/parse_sql_expr.rsindex 92c1820432..b0d8b3a349 100644--- a/datafusion/core/tests/expr_api/parse_sql_expr.rs+++ b/datafusion/core/tests/expr_api/parse_sql_expr.rs@@ -19,9 +19,9 @@ use arrow::datatypes::{DataType, Field, Schema};use datafusion::prelude::{CsvReadOptions, SessionContext};use datafusion_common::DFSchema;use datafusion_common::{DFSchemaRef, Result, ToDFSchema};+use datafusion_expr::Expr;use datafusion_expr::col;use datafusion_expr::lit;-use datafusion_expr::Expr;use datafusion_sql::unparser::Unparser;/// A schema like:///diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rsindex 46c36c6abd..470e10b592 100644--- a/datafusion/core/tests/expr_api/simplification.rs+++ b/datafusion/core/tests/expr_api/simplification.rs@@ -24,15 +24,15 @@ use arrow::array::{ArrayRef, Int32Array};use arrow::datatypes::{DataType, Field, Schema};use chrono::{DateTime, TimeZone, Utc};use datafusion::{error::Result, execution::context::ExecutionProps, prelude::*};-use datafusion_common::cast::as_int32_array;use datafusion_common::ScalarValue;+use datafusion_common::cast::as_int32_array;use datafusion_common::{DFSchemaRef, ToDFSchema};use datafusion_expr::expr::ScalarFunction;use datafusion_expr::logical_plan::builder::table_scan_with_filters;use datafusion_expr::simplify::SimplifyInfo;use datafusion_expr::{- table_scan, Cast, ColumnarValue, ExprSchemable, LogicalPlan, LogicalPlanBuilder,- ScalarUDF, Volatility,+ Cast, ColumnarValue, ExprSchemable, LogicalPlan, LogicalPlanBuilder, ScalarUDF,+ Volatility, table_scan,};use datafusion_functions::math;use datafusion_optimizer::optimizer::Optimizer;diff --git a/datafusion/core/tests/fifo/mod.rs b/datafusion/core/tests/fifo/mod.rsindex 141a3f3b75..36cc769417 100644--- a/datafusion/core/tests/fifo/mod.rs+++ b/datafusion/core/tests/fifo/mod.rs@@ -22,21 +22,21 @@mod unix_test {use std::fs::File;use std::path::PathBuf;- use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;+ use std::sync::atomic::{AtomicBool, Ordering};use std::time::Duration;use arrow::array::Array;use arrow::csv::ReaderBuilder;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};- use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use datafusion::datasource::TableProvider;+ use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use datafusion::{prelude::{CsvReadOptions, SessionConfig, SessionContext},test_util::{aggr_test_schema, arrow_test_data},};use datafusion_common::instant::Instant;- use datafusion_common::{exec_err, Result};+ use datafusion_common::{Result, exec_err};use datafusion_expr::SortExpr;use futures::StreamExt;@@ -44,7 +44,7 @@ mod unix_test {use nix::unistd;use tempfile::TempDir;use tokio::io::AsyncWriteExt;- use tokio::task::{spawn_blocking, JoinHandle};+ use tokio::task::{JoinHandle, spawn_blocking};/// Makes a TableProvider for a fifo filefn fifo_table(diff --git a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rsindex 4e04da26f7..97d1db5728 100644--- a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs@@ -24,37 +24,37 @@ use crate::fuzz_cases::aggregation_fuzzer::{};use arrow::array::{- types::Int64Type, Array, ArrayRef, AsArray, Int32Array, Int64Array, RecordBatch,- StringArray,+ Array, ArrayRef, AsArray, Int32Array, Int64Array, RecordBatch, StringArray,+ types::Int64Type,};use arrow::compute::concat_batches;use arrow::datatypes::DataType;use arrow::util::pretty::pretty_format_batches;use arrow_schema::{Field, Schema, SchemaRef};+use datafusion::datasource::MemTable;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;-use datafusion::datasource::MemTable;use datafusion::prelude::{DataFrame, SessionConfig, SessionContext};use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};use datafusion_common::{HashMap, Result};use datafusion_common_runtime::JoinSet;use datafusion_functions_aggregate::sum::sum_udaf;-use datafusion_physical_expr::expressions::{col, lit, Column};use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::{Column, col, lit};use datafusion_physical_plan::InputOrderMode;-use test_utils::{add_empty_batches, StringBatchGenerator};+use test_utils::{StringBatchGenerator, add_empty_batches};+use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::FairSpillPool;use datafusion_execution::runtime_env::RuntimeEnvBuilder;-use datafusion_execution::TaskContext;use datafusion_physical_expr::aggregate::AggregateExprBuilder;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};use datafusion_physical_plan::metrics::MetricValue;-use datafusion_physical_plan::{collect, displayable, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, collect, displayable};use rand::rngs::StdRng;-use rand::{random, rng, Rng, SeedableRng};+use rand::{Rng, SeedableRng, random, rng};// ========================================================================// The new aggregation fuzz tests based on [`AggregationFuzzer`]@@ -326,15 +326,14 @@ async fn run_aggregate_test(input1: Vec<RecordBatch>, group_by_columns: Vec<&str.unwrap(),);- let aggregate_expr =- vec![- AggregateExprBuilder::new(sum_udaf(), vec![col("d", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("sum1")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(sum_udaf(), vec![col("d", &schema).unwrap()])+ .schema(Arc::clone(&schema))+ .alias("sum1")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];let expr = group_by_columns.iter().map(|elem| (col(elem, &schema).unwrap(), (*elem).to_string()))@@ -650,7 +649,9 @@ pub(crate) fn assert_spill_count_metric(if expect_spill && spill_count == 0 {panic!("Expected spill but SpillCount metric not found or SpillCount was 0.");} else if !expect_spill && spill_count > 0 {- panic!("Expected no spill but found SpillCount metric with value greater than 0.");+ panic!(+ "Expected no spill but found SpillCount metric with value greater than 0."+ );}spill_countdiff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rsindex fa8ea0b31c..bf71053d6c 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs@@ -25,7 +25,7 @@ use datafusion_catalog::TableProvider;use datafusion_common::ScalarValue;use datafusion_common::{error::Result, utils::get_available_parallelism};use datafusion_expr::col;-use rand::{rng, Rng};+use rand::{Rng, rng};use crate::fuzz_cases::aggregation_fuzzer::data_generator::Dataset;diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rsindex aaf2d1b9ba..e49cffa89b 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs@@ -18,7 +18,7 @@use arrow::array::RecordBatch;use arrow::datatypes::DataType;use datafusion_common::Result;-use datafusion_physical_expr::{expressions::col, PhysicalSortExpr};+use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::sorts::sort::sort_batch;use test_utils::stagger_batch;@@ -209,8 +209,8 @@ mod test {sort_keys_set: vec![vec!["b".to_string()]],};- let mut gen = DatasetGenerator::new(config);- let datasets = gen.generate().unwrap();+ let mut data_gen = DatasetGenerator::new(config);+ let datasets = data_gen.generate().unwrap();// Should Generate 2 datasetsassert_eq!(datasets.len(), 2);diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rsindex 1a8ef278cc..430762b1c2 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs@@ -19,9 +19,9 @@ use std::sync::Arc;use arrow::array::RecordBatch;use arrow::util::pretty::pretty_format_batches;-use datafusion_common::{internal_datafusion_err, Result};+use datafusion_common::{Result, internal_datafusion_err};use datafusion_common_runtime::JoinSet;-use rand::{rng, Rng};+use rand::{Rng, rng};use crate::fuzz_cases::aggregation_fuzzer::query_builder::QueryBuilder;use crate::fuzz_cases::aggregation_fuzzer::{diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rsindex 766e2bedd7..0d04e98536 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs@@ -17,7 +17,7 @@use std::{collections::HashSet, str::FromStr};-use rand::{rng, seq::SliceRandom, Rng};+use rand::{Rng, rng, seq::SliceRandom};/// Random aggregate query builder///diff --git a/datafusion/core/tests/fuzz_cases/distinct_count_string_fuzz.rs b/datafusion/core/tests/fuzz_cases/distinct_count_string_fuzz.rsindex 3049631d4b..92adda200d 100644--- a/datafusion/core/tests/fuzz_cases/distinct_count_string_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/distinct_count_string_fuzz.rs@@ -19,7 +19,7 @@use std::sync::Arc;-use arrow::array::{cast::AsArray, Array, OffsetSizeTrait, RecordBatch};+use arrow::array::{Array, OffsetSizeTrait, RecordBatch, cast::AsArray};use datafusion::datasource::MemTable;use datafusion_common_runtime::JoinSet;diff --git a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rsindex 171839b390..a57095066e 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs@@ -16,19 +16,19 @@// under the License.use crate::fuzz_cases::equivalence::utils::{- create_random_schema, create_test_params, create_test_schema_2,+ TestScalarUDF, create_random_schema, create_test_params, create_test_schema_2,generate_table_for_eq_properties, generate_table_for_orderings,- is_table_same_after_sort, TestScalarUDF,+ is_table_same_after_sort,};use arrow::compute::SortOptions;-use datafusion_common::config::ConfigOptions;use datafusion_common::Result;+use datafusion_common::config::ConfigOptions;use datafusion_expr::{Operator, ScalarUDF};+use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::equivalence::{convert_to_orderings, convert_to_sort_exprs,};-use datafusion_physical_expr::expressions::{col, BinaryExpr};-use datafusion_physical_expr::ScalarFunctionExpr;+use datafusion_physical_expr::expressions::{BinaryExpr, col};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;diff --git a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs b/datafusion/core/tests/fuzz_cases/equivalence/projection.rsindex a72a1558b2..2f67e211ce 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/projection.rs@@ -16,15 +16,15 @@// under the License.use crate::fuzz_cases::equivalence::utils::{- apply_projection, create_random_schema, generate_table_for_eq_properties,- is_table_same_after_sort, TestScalarUDF,+ TestScalarUDF, apply_projection, create_random_schema,+ generate_table_for_eq_properties, is_table_same_after_sort,};use arrow::compute::SortOptions;-use datafusion_common::config::ConfigOptions;use datafusion_common::Result;+use datafusion_common::config::ConfigOptions;use datafusion_expr::{Operator, ScalarUDF};use datafusion_physical_expr::equivalence::ProjectionMapping;-use datafusion_physical_expr::expressions::{col, BinaryExpr};+use datafusion_physical_expr::expressions::{BinaryExpr, col};use datafusion_physical_expr::{PhysicalExprRef, ScalarFunctionExpr};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};diff --git a/datafusion/core/tests/fuzz_cases/equivalence/properties.rs b/datafusion/core/tests/fuzz_cases/equivalence/properties.rsindex 382c4da943..1490eb08a0 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/properties.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/properties.rs@@ -18,13 +18,13 @@use std::sync::Arc;use crate::fuzz_cases::equivalence::utils::{- create_random_schema, generate_table_for_eq_properties, is_table_same_after_sort,- TestScalarUDF,+ TestScalarUDF, create_random_schema, generate_table_for_eq_properties,+ is_table_same_after_sort,};use datafusion_common::Result;use datafusion_expr::{Operator, ScalarUDF};-use datafusion_physical_expr::expressions::{col, BinaryExpr};+use datafusion_physical_expr::expressions::{BinaryExpr, col};use datafusion_physical_expr::{LexOrdering, ScalarFunctionExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;diff --git a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs b/datafusion/core/tests/fuzz_cases/equivalence/utils.rsindex be35ddca8f..580a226721 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/utils.rs@@ -20,21 +20,21 @@ use std::cmp::Ordering;use std::sync::Arc;use arrow::array::{ArrayRef, Float32Array, Float64Array, RecordBatch, UInt32Array};-use arrow::compute::{lexsort_to_indices, take_record_batch, SortColumn, SortOptions};+use arrow::compute::{SortColumn, SortOptions, lexsort_to_indices, take_record_batch};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion_common::utils::{compare_rows, get_row_at_idx};-use datafusion_common::{exec_err, internal_datafusion_err, plan_err, Result};+use datafusion_common::{Result, exec_err, internal_datafusion_err, plan_err};use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,};use datafusion_physical_expr::equivalence::{- convert_to_orderings, EquivalenceClass, ProjectionMapping,+ EquivalenceClass, ProjectionMapping, convert_to_orderings,};use datafusion_physical_expr::{ConstExpr, EquivalenceProperties};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_plan::expressions::{col, Column};+use datafusion_physical_plan::expressions::{Column, col};use itertools::izip;use rand::prelude::*;diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rsindex 81fb3b3b08..ce422494db 100644--- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs@@ -38,8 +38,8 @@ use datafusion::physical_plan::joins::{};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::{NullEquality, ScalarValue};-use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::PhysicalExprRef;+use datafusion_physical_expr::expressions::Literal;use itertools::Itertools;use rand::Rng;@@ -921,7 +921,9 @@ impl JoinFuzzTestCase {std::fs::remove_dir_all(fuzz_debug).unwrap_or(());std::fs::create_dir_all(fuzz_debug).unwrap();let out_dir_name = &format!("{fuzz_debug}/batch_size_{batch_size}");- println!("Test result data mismatch found. HJ rows {hj_rows}, SMJ rows {smj_rows}, NLJ rows {nlj_rows}");+ println!(+ "Test result data mismatch found. HJ rows {hj_rows}, SMJ rows {smj_rows}, NLJ rows {nlj_rows}"+ );println!("The debug is ON. Input data will be saved to {out_dir_name}");Self::save_partitioned_batches_as_parquet(@@ -972,14 +974,18 @@ impl JoinFuzzTestCase {}if join_tests.contains(&NljHj) {- let err_msg_rowcnt = format!("NestedLoopJoinExec and HashJoinExec produced different row counts, batch_size: {batch_size}");+ let err_msg_rowcnt = format!(+ "NestedLoopJoinExec and HashJoinExec produced different row counts, batch_size: {batch_size}"+ );assert_eq!(nlj_rows, hj_rows, "{}", err_msg_rowcnt.as_str());if nlj_rows == 0 && hj_rows == 0 {// both joins returned no rows, skip content comparisoncontinue;}- let err_msg_contents = format!("NestedLoopJoinExec and HashJoinExec produced different results, batch_size: {batch_size}");+ let err_msg_contents = format!(+ "NestedLoopJoinExec and HashJoinExec produced different results, batch_size: {batch_size}"+ );// row level compare if any of joins returns the result// the reason is different formatting when there is no rowsfor (i, (nlj_line, hj_line)) in nlj_formatted_sorted@@ -997,10 +1003,16 @@ impl JoinFuzzTestCase {}if join_tests.contains(&HjSmj) {- let err_msg_row_cnt = format!("HashJoinExec and SortMergeJoinExec produced different row counts, batch_size: {}", &batch_size);+ let err_msg_row_cnt = format!(+ "HashJoinExec and SortMergeJoinExec produced different row counts, batch_size: {}",+ &batch_size+ );assert_eq!(hj_rows, smj_rows, "{}", err_msg_row_cnt.as_str());- let err_msg_contents = format!("SortMergeJoinExec and HashJoinExec produced different results, batch_size: {}", &batch_size);+ let err_msg_contents = format!(+ "SortMergeJoinExec and HashJoinExec produced different results, batch_size: {}",+ &batch_size+ );// row level compare if any of joins returns the result// the reason is different formatting when there is no rowsif smj_rows > 0 || hj_rows > 0 {diff --git a/datafusion/core/tests/fuzz_cases/limit_fuzz.rs b/datafusion/core/tests/fuzz_cases/limit_fuzz.rsindex 4c5ebf0402..1c5741e7a2 100644--- a/datafusion/core/tests/fuzz_cases/limit_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/limit_fuzz.rs@@ -24,7 +24,7 @@ use arrow::util::pretty::pretty_format_batches;use datafusion::datasource::MemTable;use datafusion::prelude::SessionContext;use datafusion_common::assert_contains;-use rand::{rng, Rng};+use rand::{Rng, rng};use std::sync::Arc;use test_utils::stagger_batch;diff --git a/datafusion/core/tests/fuzz_cases/merge_fuzz.rs b/datafusion/core/tests/fuzz_cases/merge_fuzz.rsindex b92dec64e3..59430a98cc 100644--- a/datafusion/core/tests/fuzz_cases/merge_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/merge_fuzz.rs@@ -27,7 +27,7 @@ use arrow::{use datafusion::datasource::memory::MemorySourceConfig;use datafusion::physical_plan::{collect,- expressions::{col, PhysicalSortExpr},+ expressions::{PhysicalSortExpr, col},sorts::sort_preserving_merge::SortPreservingMergeExec,};use datafusion::prelude::{SessionConfig, SessionContext};diff --git a/datafusion/core/tests/fuzz_cases/pruning.rs b/datafusion/core/tests/fuzz_cases/pruning.rsindex 51ec8f03e5..8a84e4c5d1 100644--- a/datafusion/core/tests/fuzz_cases/pruning.rs+++ b/datafusion/core/tests/fuzz_cases/pruning.rs@@ -29,9 +29,9 @@ use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_expr::PhysicalExpr;-use datafusion_physical_plan::{collect, filter::FilterExec, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, collect, filter::FilterExec};use itertools::Itertools;-use object_store::{memory::InMemory, path::Path, ObjectStore, PutPayload};+use object_store::{ObjectStore, PutPayload, memory::InMemory, path::Path};use parquet::{arrow::ArrowWriter,file::properties::{EnabledStatistics, WriterProperties},diff --git a/datafusion/core/tests/fuzz_cases/record_batch_generator.rs b/datafusion/core/tests/fuzz_cases/record_batch_generator.rsindex 2e6bd107b3..22b145f509 100644--- a/datafusion/core/tests/fuzz_cases/record_batch_generator.rs+++ b/datafusion/core/tests/fuzz_cases/record_batch_generator.rs@@ -19,23 +19,23 @@ use std::sync::Arc;use arrow::array::{ArrayRef, DictionaryArray, PrimitiveArray, RecordBatch};use arrow::datatypes::{- ArrowPrimitiveType, BooleanType, DataType, Date32Type, Date64Type, Decimal128Type,- Decimal256Type, Decimal32Type, Decimal64Type, DurationMicrosecondType,+ ArrowPrimitiveType, BooleanType, DataType, Date32Type, Date64Type, Decimal32Type,+ Decimal64Type, Decimal128Type, Decimal256Type, DurationMicrosecondType,DurationMillisecondType, DurationNanosecondType, DurationSecondType, Field,- Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType,Schema, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,- TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type,- UInt8Type,+ TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,+ UInt64Type,};use arrow_schema::{- DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION,- DECIMAL256_MAX_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE,- DECIMAL64_MAX_PRECISION, DECIMAL64_MAX_SCALE,+ DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE, DECIMAL64_MAX_PRECISION,+ DECIMAL64_MAX_SCALE, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE,+ DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE,};-use datafusion_common::{arrow_datafusion_err, Result};-use rand::{rng, rngs::StdRng, Rng, SeedableRng};+use datafusion_common::{Result, arrow_datafusion_err};+use rand::{Rng, SeedableRng, rng, rngs::StdRng};use test_utils::array_gen::{BinaryArrayGenerator, BooleanArrayGenerator, DecimalArrayGenerator,PrimitiveArrayGenerator, StringArrayGenerator,diff --git a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_fuzz.rsindex 28d28a6622..0d8a066d43 100644--- a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/sort_fuzz.rs@@ -20,7 +20,7 @@use std::sync::Arc;use arrow::{- array::{as_string_array, ArrayRef, Int32Array, StringArray},+ array::{ArrayRef, Int32Array, StringArray, as_string_array},compute::SortOptions,record_batch::RecordBatch,};@@ -28,7 +28,7 @@ use datafusion::datasource::memory::MemorySourceConfig;use datafusion::execution::runtime_env::RuntimeEnvBuilder;use datafusion::physical_plan::expressions::PhysicalSortExpr;use datafusion::physical_plan::sorts::sort::SortExec;-use datafusion::physical_plan::{collect, ExecutionPlan};+use datafusion::physical_plan::{ExecutionPlan, collect};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::cast::as_int32_array;use datafusion_execution::memory_pool::GreedyMemoryPool;diff --git a/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rsindex 99b20790fc..c424a31427 100644--- a/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs@@ -20,34 +20,33 @@ mod sp_repartition_fuzz_tests {use std::sync::Arc;use arrow::array::{ArrayRef, Int64Array, RecordBatch, UInt64Array};- use arrow::compute::{concat_batches, lexsort, SortColumn, SortOptions};+ use arrow::compute::{SortColumn, SortOptions, concat_batches, lexsort};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;use datafusion::physical_plan::{- collect,+ ExecutionPlan, Partitioning, collect,metrics::{BaselineMetrics, ExecutionPlanMetricsSet},repartition::RepartitionExec,sorts::sort_preserving_merge::SortPreservingMergeExec,sorts::streaming_merge::StreamingMergeBuilder,stream::RecordBatchStreamAdapter,- ExecutionPlan, Partitioning,};use datafusion::prelude::SessionContext;use datafusion_common::Result;use datafusion_execution::{config::SessionConfig, memory_pool::MemoryConsumer};+ use datafusion_physical_expr::ConstExpr;use datafusion_physical_expr::equivalence::{EquivalenceClass, EquivalenceProperties,};- use datafusion_physical_expr::expressions::{col, Column};- use datafusion_physical_expr::ConstExpr;+ use datafusion_physical_expr::expressions::{Column, col};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use test_utils::add_empty_batches;use itertools::izip;- use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng};+ use rand::{Rng, SeedableRng, rngs::StdRng, seq::SliceRandom};// Generate a schema which consists of 6 columns (a, b, c, d, e, f)fn create_test_schema() -> Result<SchemaRef> {diff --git a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rsindex 9c7e4f1b84..376306f3e0 100644--- a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs@@ -24,7 +24,7 @@ use arrow::array::RecordBatch;use arrow_schema::SchemaRef;use datafusion::datasource::MemTable;use datafusion::prelude::{SessionConfig, SessionContext};-use datafusion_common::{human_readable_size, instant::Instant, Result};+use datafusion_common::{Result, human_readable_size, instant::Instant};use datafusion_execution::disk_manager::DiskManagerBuilder;use datafusion_execution::memory_pool::{MemoryPool, UnboundedMemoryPool};use datafusion_expr::display_schema;@@ -32,14 +32,14 @@ use datafusion_physical_plan::spill::get_record_batch_memory_size;use std::time::Duration;use datafusion_execution::{memory_pool::FairSpillPool, runtime_env::RuntimeEnvBuilder};-use rand::prelude::IndexedRandom;use rand::Rng;-use rand::{rngs::StdRng, SeedableRng};+use rand::prelude::IndexedRandom;+use rand::{SeedableRng, rngs::StdRng};use crate::fuzz_cases::aggregation_fuzzer::check_equality_of_batches;use super::aggregation_fuzzer::ColumnDescr;-use super::record_batch_generator::{get_supported_types_columns, RecordBatchGenerator};+use super::record_batch_generator::{RecordBatchGenerator, get_supported_types_columns};/// Entry point for executing the sort query fuzzer.///@@ -175,16 +175,16 @@ impl SortQueryFuzzer {n_round: usize,n_query: usize,) -> bool {- if let Some(time_limit) = self.time_limit {- if Instant::now().duration_since(start_time) > time_limit {- println!(- "[SortQueryFuzzer] Time limit reached: {} queries ({} random configs each) in {} rounds",- n_round * self.queries_per_round + n_query,- self.config_variations_per_query,- n_round- );- return true;- }+ if let Some(time_limit) = self.time_limit+ && Instant::now().duration_since(start_time) > time_limit+ {+ println!(+ "[SortQueryFuzzer] Time limit reached: {} queries ({} random configs each) in {} rounds",+ n_round * self.queries_per_round + n_query,+ self.config_variations_per_query,+ n_round+ );+ return true;}false}diff --git a/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs b/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rsindex 6ea0182c25..16481516e0 100644--- a/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs+++ b/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs@@ -27,9 +27,9 @@ use arrow::{array::StringArray, compute::SortOptions, record_batch::RecordBatch}use arrow_schema::{DataType, Field, Schema};use datafusion::common::Result;use datafusion::execution::runtime_env::RuntimeEnvBuilder;+use datafusion::physical_plan::ExecutionPlan;use datafusion::physical_plan::expressions::PhysicalSortExpr;use datafusion::physical_plan::sorts::sort::SortExec;-use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::SessionConfig;use datafusion_common::units::{KB, MB};use datafusion_execution::memory_pool::{@@ -38,7 +38,7 @@ use datafusion_execution::memory_pool::{use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_functions_aggregate::array_agg::array_agg_udaf;use datafusion_physical_expr::aggregate::AggregateExprBuilder;-use datafusion_physical_expr::expressions::{col, Column};+use datafusion_physical_expr::expressions::{Column, col};use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,@@ -80,9 +80,9 @@ async fn test_sort_with_limited_memory() -> Result<()> {let total_spill_files_size = spill_count * record_batch_size;assert!(- total_spill_files_size > pool_size,- "Total spill files size {total_spill_files_size} should be greater than pool size {pool_size}",- );+ total_spill_files_size > pool_size,+ "Total spill files size {total_spill_files_size} should be greater than pool size {pool_size}",+ );Ok(())}@@ -126,8 +126,8 @@ async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch() ->}#[tokio::test]-async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch_and_changing_memory_reservation(-) -> Result<()> {+async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch_and_changing_memory_reservation()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {@@ -164,8 +164,8 @@ async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch_and_c}#[tokio::test]-async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch_and_take_all_memory(-) -> Result<()> {+async fn test_sort_with_limited_memory_and_different_sizes_of_record_batch_and_take_all_memory()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {@@ -356,16 +356,16 @@ async fn test_aggregate_with_high_cardinality_with_limited_memory() -> Result<()let total_spill_files_size = spill_count * record_batch_size;assert!(- total_spill_files_size > pool_size,- "Total spill files size {total_spill_files_size} should be greater than pool size {pool_size}",- );+ total_spill_files_size > pool_size,+ "Total spill files size {total_spill_files_size} should be greater than pool size {pool_size}",+ );Ok(())}#[tokio::test]-async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch(-) -> Result<()> {+async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {@@ -398,8 +398,8 @@ async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_}#[tokio::test]-async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch_and_changing_memory_reservation(-) -> Result<()> {+async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch_and_changing_memory_reservation()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {@@ -432,8 +432,8 @@ async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_}#[tokio::test]-async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch_and_take_all_memory(-) -> Result<()> {+async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_sizes_of_record_batch_and_take_all_memory()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {@@ -466,8 +466,8 @@ async fn test_aggregate_with_high_cardinality_with_limited_memory_and_different_}#[tokio::test]-async fn test_aggregate_with_high_cardinality_with_limited_memory_and_large_record_batch(-) -> Result<()> {+async fn test_aggregate_with_high_cardinality_with_limited_memory_and_large_record_batch()+-> Result<()> {let record_batch_size = 8192;let pool_size = 2 * MB as usize;let task_ctx = {diff --git a/datafusion/core/tests/fuzz_cases/window_fuzz.rs b/datafusion/core/tests/fuzz_cases/window_fuzz.rsindex 65a41d39d3..2ecfcd84ab 100644--- a/datafusion/core/tests/fuzz_cases/window_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/window_fuzz.rs@@ -18,19 +18,19 @@use std::sync::Arc;use arrow::array::{ArrayRef, Int32Array, StringArray};-use arrow::compute::{concat_batches, SortOptions};+use arrow::compute::{SortOptions, concat_batches};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use arrow::util::pretty::pretty_format_batches;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;use datafusion::functions_window::row_number::row_number_udwf;+use datafusion::physical_plan::InputOrderMode::{Linear, PartiallySorted, Sorted};use datafusion::physical_plan::sorts::sort::SortExec;use datafusion::physical_plan::windows::{- create_window_expr, schema_add_window_field, BoundedWindowAggExec, WindowAggExec,+ BoundedWindowAggExec, WindowAggExec, create_window_expr, schema_add_window_field,};-use datafusion::physical_plan::InputOrderMode::{Linear, PartiallySorted, Sorted};-use datafusion::physical_plan::{collect, InputOrderMode};+use datafusion::physical_plan::{InputOrderMode, collect};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::HashMap;use datafusion_common::{Result, ScalarValue};@@ -445,14 +445,14 @@ fn get_random_function(let fn_name = window_fn_map.keys().collect::<Vec<_>>()[rand_fn_idx];let (window_fn, args) = window_fn_map.values().collect::<Vec<_>>()[rand_fn_idx];let mut args = args.clone();- if let WindowFunctionDefinition::AggregateUDF(udf) = window_fn {- if !args.is_empty() {- // Do type coercion first argument- let a = args[0].clone();- let dt = a.return_field(schema.as_ref()).unwrap();- let coerced = fields_with_aggregate_udf(&[dt], udf).unwrap();- args[0] = cast(a, schema, coerced[0].data_type().clone()).unwrap();- }+ if let WindowFunctionDefinition::AggregateUDF(udf) = window_fn+ && !args.is_empty()+ {+ // Do type coercion first argument+ let a = args[0].clone();+ let dt = a.return_field(schema.as_ref()).unwrap();+ let coerced = fields_with_aggregate_udf(&[dt], udf).unwrap();+ args[0] = cast(a, schema, coerced[0].data_type().clone()).unwrap();}(window_fn.clone(), args, (*fn_name).to_string())@@ -569,10 +569,11 @@ fn convert_bound_to_current_row_if_applicable() {match bound {WindowFrameBound::Preceding(value) | WindowFrameBound::Following(value) => {- if let Ok(zero) = ScalarValue::new_zero(&value.data_type()) {- if value == &zero && rng.random_range(0..2) == 0 {- *bound = WindowFrameBound::CurrentRow;- }+ if let Ok(zero) = ScalarValue::new_zero(&value.data_type())+ && value == &zero+ && rng.random_range(0..2) == 0+ {+ *bound = WindowFrameBound::CurrentRow;}}_ => {}@@ -644,10 +645,8 @@ async fn run_window_test() as _;// Table is ordered according to ORDER BY a, b, c In linear test we use PARTITION BY b, ORDER BY a// For WindowAggExec to produce correct result it need table to be ordered by b,a. Hence add a sort.- if is_linear {- if let Some(ordering) = LexOrdering::new(sort_keys) {- exec1 = Arc::new(SortExec::new(ordering, exec1)) as _;- }+ if is_linear && let Some(ordering) = LexOrdering::new(sort_keys) {+ exec1 = Arc::new(SortExec::new(ordering, exec1)) as _;}let extended_schema = schema_add_window_field(&args, &schema, &window_fn, &fn_name)?;@@ -699,7 +698,9 @@ async fn run_window_test(// BoundedWindowAggExec should produce more chunk than the usual WindowAggExec.// Otherwise it means that we cannot generate result in running mode.- let err_msg = format!("Inconsistent result for window_frame: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, random_seed: {random_seed:?}, search_mode: {search_mode:?}, partition_by_columns:{partition_by_columns:?}, orderby_columns: {orderby_columns:?}");+ let err_msg = format!(+ "Inconsistent result for window_frame: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, random_seed: {random_seed:?}, search_mode: {search_mode:?}, partition_by_columns:{partition_by_columns:?}, orderby_columns: {orderby_columns:?}"+ );// Below check makes sure that, streaming execution generates more chunks than the bulk execution.// Since algorithms and operators works on sliding windows in the streaming execution.// However, in the current test setup for some random generated window frame clauses: It is not guaranteed@@ -731,8 +732,12 @@ async fn run_window_test(.enumerate(){if !usual_line.eq(running_line) {- println!("Inconsistent result for window_frame at line:{i:?}: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, pb_cols:{partition_by_columns:?}, ob_cols:{orderby_columns:?}, search_mode:{search_mode:?}");- println!("--------usual_formatted_sorted----------------running_formatted_sorted--------");+ println!(+ "Inconsistent result for window_frame at line:{i:?}: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, pb_cols:{partition_by_columns:?}, ob_cols:{orderby_columns:?}, search_mode:{search_mode:?}"+ );+ println!(+ "--------usual_formatted_sorted----------------running_formatted_sorted--------"+ );for (line1, line2) inusual_formatted_sorted.iter().zip(running_formatted_sorted){diff --git a/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs b/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rsindex 899c9f88d8..2c9fae20c8 100644--- a/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs+++ b/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs@@ -16,10 +16,10 @@// under the License.use datafusion_common_runtime::SpawnedTask;-use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};-use tokio::time::{interval, Duration};+use tokio::time::{Duration, interval};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::human_readable_size;@@ -38,7 +38,7 @@ use datafusion_execution::{memory_pool::FairSpillPool, runtime_env::RuntimeEnvBuasync fn measure_max_rss<F, Fut, T>(f: F) -> (T, usize)whereF: FnOnce() -> Fut,- Fut: std::future::Future<Output = T>,+ Fut: Future<Output = T>,{// Initialize system informationlet mut system = System::new_all();diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex 5d8a1d2418..c28d23ba06 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -39,19 +39,19 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;use datafusion::physical_plan::streaming::PartitionStream;use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream};use datafusion::prelude::{SessionConfig, SessionContext};-use datafusion_catalog::streaming::StreamingTable;use datafusion_catalog::Session;-use datafusion_common::{assert_contains, Result};+use datafusion_catalog::streaming::StreamingTable;+use datafusion_common::{Result, assert_contains};+use datafusion_execution::TaskContext;use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};use datafusion_execution::memory_pool::{FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool,};use datafusion_execution::runtime_env::RuntimeEnv;-use datafusion_execution::TaskContext;use datafusion_expr::{Expr, TableType};use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_optimizer::join_selection::JoinSelection;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::join_selection::JoinSelection;use datafusion_physical_plan::collect as collect_batches;use datafusion_physical_plan::common::collect;use datafusion_physical_plan::spill::get_record_batch_memory_size;@@ -604,8 +604,8 @@ async fn test_disk_spill_limit_reached() -> Result<()> {let err = df.collect().await.unwrap_err();assert_contains!(- err.to_string(),- "The used disk space during the spilling process has exceeded the allowable limit"+ err.to_string(),+ "The used disk space during the spilling process has exceeded the allowable limit");Ok(())@@ -977,11 +977,13 @@ impl Scenario {descending: false,nulls_first: false,};- let sort_information = vec![[- PhysicalSortExpr::new(col("a", &schema).unwrap(), options),- PhysicalSortExpr::new(col("b", &schema).unwrap(), options),- ]- .into()];+ let sort_information = vec![+ [+ PhysicalSortExpr::new(col("a", &schema).unwrap(), options),+ PhysicalSortExpr::new(col("b", &schema).unwrap(), options),+ ]+ .into(),+ ];let table = SortedTableProvider::new(batches, sort_information);Arc::new(table)@@ -1057,7 +1059,7 @@ fn make_dict_batches() -> Vec<RecordBatch> {let batch_size = 50;let mut i = 0;- let gen = std::iter::from_fn(move || {+ let batch_gen = std::iter::from_fn(move || {// create values like// 0000000001// 0000000002@@ -1080,7 +1082,7 @@ fn make_dict_batches() -> Vec<RecordBatch> {let num_batches = 5;- let batches: Vec<_> = gen.take(num_batches).collect();+ let batches: Vec<_> = batch_gen.take(num_batches).collect();batches.iter().enumerate().for_each(|(i, batch)| {println!("Dict batch[{i}] size is: {}", batch.get_array_memory_size());diff --git a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs b/datafusion/core/tests/memory_limit/repartition_mem_limit.rsindex a7af2f01d1..b21bffebaf 100644--- a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs+++ b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs@@ -25,7 +25,7 @@ use datafusion::{use datafusion_catalog::MemTable;use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_execution::runtime_env::RuntimeEnvBuilder;-use datafusion_physical_plan::{repartition::RepartitionExec, ExecutionPlanProperties};+use datafusion_physical_plan::{ExecutionPlanProperties, repartition::RepartitionExec};use futures::TryStreamExt;use itertools::Itertools;@@ -45,11 +45,14 @@ async fn test_repartition_memory_limit() {.with_batch_size(32).with_target_partitions(2);let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime));- let batches = vec![RecordBatch::try_from_iter(vec![(- "c1",- Arc::new(Int32Array::from_iter_values((0..10).cycle().take(100_000))) as ArrayRef,- )])- .unwrap()];+ let batches = vec![+ RecordBatch::try_from_iter(vec![(+ "c1",+ Arc::new(Int32Array::from_iter_values((0..10).cycle().take(100_000)))+ as ArrayRef,+ )])+ .unwrap(),+ ];let table = Arc::new(MemTable::try_new(batches[0].schema(), vec![batches]).unwrap());ctx.register_table("t", table).unwrap();let plan = ctxdiff --git a/datafusion/core/tests/optimizer/mod.rs b/datafusion/core/tests/optimizer/mod.rsindex b288706a54..6466e9ad96 100644--- a/datafusion/core/tests/optimizer/mod.rs+++ b/datafusion/core/tests/optimizer/mod.rs@@ -28,11 +28,11 @@ use arrow::datatypes::{};use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::TransformedResult;-use datafusion_common::{plan_err, DFSchema, Result, ScalarValue, TableReference};+use datafusion_common::{DFSchema, Result, ScalarValue, TableReference, plan_err};use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};use datafusion_expr::{- col, lit, AggregateUDF, BinaryExpr, Expr, ExprSchemable, LogicalPlan, Operator,- ScalarUDF, TableSource, WindowUDF,+ AggregateUDF, BinaryExpr, Expr, ExprSchemable, LogicalPlan, Operator, ScalarUDF,+ TableSource, WindowUDF, col, lit,};use datafusion_functions::core::expr_ext::FieldAccessor;use datafusion_optimizer::analyzer::Analyzer;diff --git a/datafusion/core/tests/parquet/custom_reader.rs b/datafusion/core/tests/parquet/custom_reader.rsindex 0a147d15a6..31ec6efd19 100644--- a/datafusion/core/tests/parquet/custom_reader.rs+++ b/datafusion/core/tests/parquet/custom_reader.rs@@ -20,7 +20,7 @@ use std::ops::Range;use std::sync::Arc;use std::time::SystemTime;-use arrow::array::{ArrayRef, Int64Array, Int8Array, StringArray};+use arrow::array::{ArrayRef, Int8Array, Int64Array, StringArray};use arrow::datatypes::{Field, Schema, SchemaBuilder};use arrow::record_batch::RecordBatch;use datafusion::datasource::listing::PartitionedFile;@@ -31,8 +31,8 @@ use datafusion::datasource::physical_plan::{use datafusion::physical_plan::collect;use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;use datafusion::prelude::SessionContext;-use datafusion_common::test_util::batches_to_sort_string;use datafusion_common::Result;+use datafusion_common::test_util::batches_to_sort_string;use bytes::Bytes;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;@@ -44,9 +44,9 @@ use insta::assert_snapshot;use object_store::memory::InMemory;use object_store::path::Path;use object_store::{ObjectMeta, ObjectStore};+use parquet::arrow::ArrowWriter;use parquet::arrow::arrow_reader::ArrowReaderOptions;use parquet::arrow::async_reader::AsyncFileReader;-use parquet::arrow::ArrowWriter;use parquet::errors::ParquetError;use parquet::file::metadata::ParquetMetaData;diff --git a/datafusion/core/tests/parquet/encryption.rs b/datafusion/core/tests/parquet/encryption.rsindex 82d8e61d9a..8b3170e367 100644--- a/datafusion/core/tests/parquet/encryption.rs+++ b/datafusion/core/tests/parquet/encryption.rs@@ -25,11 +25,11 @@ use datafusion::dataframe::DataFrameWriteOptions;use datafusion::datasource::listing::ListingOptions;use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_common::config::{EncryptionFactoryOptions, TableParquetOptions};-use datafusion_common::{assert_batches_sorted_eq, exec_datafusion_err, DataFusionError};+use datafusion_common::{DataFusionError, assert_batches_sorted_eq, exec_datafusion_err};use datafusion_datasource_parquet::ParquetFormat;use datafusion_execution::parquet_encryption::EncryptionFactory;-use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};use parquet::arrow::ArrowWriter;+use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};use parquet::encryption::decrypt::FileDecryptionProperties;use parquet::encryption::encrypt::FileEncryptionProperties;use parquet::file::column_crypto_metadata::ColumnCryptoMetaData;diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 9022b25356..0c02c8fe52 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -21,7 +21,7 @@ use std::path::Path;use std::sync::Arc;use crate::parquet::utils::MetricsFinder;-use crate::parquet::{create_data_batch, Scenario};+use crate::parquet::{Scenario, create_data_batch};use arrow::datatypes::SchemaRef;use arrow::util::pretty::pretty_format_batches;@@ -29,17 +29,17 @@ use datafusion::common::Result;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::physical_plan::ParquetSource;use datafusion::prelude::SessionContext;-use datafusion_common::{assert_contains, DFSchema};+use datafusion_common::{DFSchema, assert_contains};use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};use datafusion_execution::object_store::ObjectStoreUrl;-use datafusion_expr::{col, lit, Expr};-use datafusion_physical_plan::metrics::{MetricValue, MetricsSet};+use datafusion_expr::{Expr, col, lit};use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::metrics::{MetricValue, MetricsSet};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;-use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::arrow::ArrowWriter;+use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::file::properties::WriterProperties;use tempfile::NamedTempFile;diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rsindex 942c1e29d4..fdefdafa00 100644--- a/datafusion/core/tests/parquet/file_statistics.rs+++ b/datafusion/core/tests/parquet/file_statistics.rs@@ -18,30 +18,30 @@use std::fs;use std::sync::Arc;+use datafusion::datasource::TableProvider;use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};use datafusion::datasource::source::DataSourceExec;-use datafusion::datasource::TableProvider;use datafusion::execution::context::SessionState;use datafusion::execution::session_state::SessionStateBuilder;use datafusion::prelude::SessionContext;-use datafusion_common::stats::Precision;use datafusion_common::DFSchema;+use datafusion_common::stats::Precision;+use datafusion_execution::cache::DefaultListFilesCache;use datafusion_execution::cache::cache_manager::CacheManagerConfig;use datafusion_execution::cache::cache_unit::DefaultFileStatisticsCache;-use datafusion_execution::cache::DefaultListFilesCache;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnvBuilder;-use datafusion_expr::{col, lit, Expr};+use datafusion_expr::{Expr, col, lit};use datafusion::datasource::physical_plan::FileScanConfig;use datafusion_common::config::ConfigOptions;-use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;use datafusion_physical_optimizer::PhysicalOptimizerRule;-use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::filter::FilterExec;use tempfile::tempdir;#[tokio::test]diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rsindex 1d64669fad..e3a191ee9a 100644--- a/datafusion/core/tests/parquet/filter_pushdown.rs+++ b/datafusion/core/tests/parquet/filter_pushdown.rs@@ -31,7 +31,7 @@ use arrow::record_batch::RecordBatch;use datafusion::physical_plan::collect;use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};use datafusion::prelude::{- col, lit, lit_timestamp_nano, Expr, ParquetReadOptions, SessionContext,+ Expr, ParquetReadOptions, SessionContext, col, lit, lit_timestamp_nano,};use datafusion::test_util::parquet::{ParquetScanOptions, TestParquetFile};use datafusion_expr::utils::{conjunction, disjunction, split_conjunction};@@ -645,8 +645,8 @@ async fn predicate_cache_pushdown_default() -> datafusion_common::Result<()> {}#[tokio::test]-async fn predicate_cache_pushdown_default_selections_only(-) -> datafusion_common::Result<()> {+async fn predicate_cache_pushdown_default_selections_only()+-> datafusion_common::Result<()> {let mut config = SessionConfig::new();config.options_mut().execution.parquet.pushdown_filters = true;// forcing filter selections minimizes the number of rows read from the cachediff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex aa2a687d29..0a0478dd03 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -19,12 +19,12 @@use crate::parquet::utils::MetricsFinder;use arrow::{array::{- make_array, Array, ArrayRef, BinaryArray, Date32Array, Date64Array,- Decimal128Array, DictionaryArray, FixedSizeBinaryArray, Float64Array, Int16Array,- Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray,- StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,- TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,- UInt64Array, UInt8Array,+ Array, ArrayRef, BinaryArray, Date32Array, Date64Array, Decimal128Array,+ DictionaryArray, FixedSizeBinaryArray, Float64Array, Int8Array, Int16Array,+ Int32Array, Int64Array, LargeBinaryArray, LargeStringArray, StringArray,+ TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,+ TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array,+ make_array,},datatypes::{DataType, Field, Schema},record_batch::RecordBatch,@@ -32,7 +32,7 @@ use arrow::{};use chrono::{Datelike, Duration, TimeDelta};use datafusion::{- datasource::{provider_as_source, TableProvider},+ datasource::{TableProvider, provider_as_source},physical_plan::metrics::MetricsSet,prelude::{ParquetReadOptions, SessionConfig, SessionContext},};@@ -147,15 +147,14 @@ impl TestOutput {for metric in self.parquet_metrics.iter() {let metric = metric.as_ref();- if metric.value().name() == metric_name {- if let MetricValue::PruningMetrics {+ if metric.value().name() == metric_name+ && let MetricValue::PruningMetrics {pruning_metrics, ..} = metric.value()- {- total_pruned += pruning_metrics.pruned();- total_matched += pruning_metrics.matched();- found = true;- }+ {+ total_pruned += pruning_metrics.pruned();+ total_matched += pruning_metrics.matched();+ found = true;}}diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rsindex fb2a196b0a..17392974b6 100644--- a/datafusion/core/tests/parquet/page_pruning.rs+++ b/datafusion/core/tests/parquet/page_pruning.rs@@ -21,25 +21,25 @@ use crate::parquet::Unit::Page;use crate::parquet::{ContextWithParquet, Scenario};use arrow::array::RecordBatch;-use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::file_format::FileFormat;+use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::object_store::ObjectStoreUrl;use datafusion::datasource::physical_plan::ParquetSource;use datafusion::datasource::source::DataSourceExec;use datafusion::execution::context::SessionState;-use datafusion::physical_plan::metrics::MetricValue;use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::metrics::MetricValue;use datafusion::prelude::SessionContext;use datafusion_common::{ScalarValue, ToDFSchema};use datafusion_expr::execution_props::ExecutionProps;-use datafusion_expr::{col, lit, Expr};+use datafusion_expr::{Expr, col, lit};use datafusion_physical_expr::create_physical_expr;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use futures::StreamExt;-use object_store::path::Path;use object_store::ObjectMeta;+use object_store::path::Path;async fn get_parquet_exec(state: &SessionState,diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/schema_adapter.rsindex 3c12862f38..7e2240cf6b 100644--- a/datafusion/core/tests/parquet/schema_adapter.rs+++ b/datafusion/core/tests/parquet/schema_adapter.rs@@ -17,7 +17,7 @@use std::sync::Arc;-use arrow::array::{record_batch, RecordBatch};+use arrow::array::{RecordBatch, record_batch};use arrow_schema::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion::assert_batches_eq;@@ -26,18 +26,18 @@ use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingTableConfigExt,};use datafusion::prelude::{SessionConfig, SessionContext};-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::DataFusionError;use datafusion_common::ScalarValue;+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_datasource::ListingTableUrl;use datafusion_execution::object_store::ObjectStoreUrl;-use datafusion_physical_expr::expressions::{self, Column};use datafusion_physical_expr::PhysicalExpr;+use datafusion_physical_expr::expressions::{self, Column};use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapter, DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter,PhysicalExprAdapterFactory,};-use object_store::{memory::InMemory, path::Path, ObjectStore};+use object_store::{ObjectStore, memory::InMemory, path::Path};use parquet::arrow::ArrowWriter;async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &str) {diff --git a/datafusion/core/tests/parquet/schema_coercion.rs b/datafusion/core/tests/parquet/schema_coercion.rsindex e8666f0759..6f7e2e328d 100644--- a/datafusion/core/tests/parquet/schema_coercion.rs+++ b/datafusion/core/tests/parquet/schema_coercion.rs@@ -18,16 +18,16 @@use std::sync::Arc;use arrow::array::{- types::Int32Type, ArrayRef, DictionaryArray, Float32Array, Int64Array, RecordBatch,- StringArray,+ ArrayRef, DictionaryArray, Float32Array, Int64Array, RecordBatch, StringArray,+ types::Int32Type,};use arrow::datatypes::{DataType, Field, Schema};use datafusion::datasource::physical_plan::ParquetSource;use datafusion::physical_plan::collect;use datafusion::prelude::SessionContext;use datafusion::test::object_store::local_unpartitioned_file;-use datafusion_common::test_util::batches_to_sort_string;use datafusion_common::Result;+use datafusion_common::test_util::batches_to_sort_string;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;diff --git a/datafusion/core/tests/parquet/utils.rs b/datafusion/core/tests/parquet/utils.rsindex 24b6cadc14..e5e0026ec1 100644--- a/datafusion/core/tests/parquet/utils.rs+++ b/datafusion/core/tests/parquet/utils.rs@@ -20,7 +20,7 @@use datafusion::datasource::physical_plan::ParquetSource;use datafusion::datasource::source::DataSourceExec;use datafusion_physical_plan::metrics::MetricsSet;-use datafusion_physical_plan::{accept, ExecutionPlan, ExecutionPlanVisitor};+use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanVisitor, accept};/// Find the metrics from the first DataSourceExec encountered in the plan#[derive(Debug)]@@ -47,13 +47,12 @@ impl MetricsFinder {impl ExecutionPlanVisitor for MetricsFinder {type Error = std::convert::Infallible;fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {- if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {- if data_source_exec+ if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && data_source_exec.downcast_to_file_source::<ParquetSource>().is_some()- {- self.metrics = data_source_exec.metrics();- }+ {+ self.metrics = data_source_exec.metrics();}// stop searching once we have found the metricsOk(self.metrics.is_none())diff --git a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rsindex a79d743cb2..1fdc0ae6c7 100644--- a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs@@ -24,14 +24,15 @@ use arrow::datatypes::{DataType, Field, Schema};use arrow::record_batch::RecordBatch;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;+use datafusion_common::Result;use datafusion_common::cast::as_int64_array;use datafusion_common::config::ConfigOptions;-use datafusion_common::Result;use datafusion_execution::TaskContext;use datafusion_expr::Operator;use datafusion_physical_expr::expressions::{self, cast};-use datafusion_physical_optimizer::aggregate_statistics::AggregateStatistics;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::aggregate_statistics::AggregateStatistics;+use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::aggregates::AggregateExec;use datafusion_physical_plan::aggregates::AggregateMode;use datafusion_physical_plan::aggregates::PhysicalGroupBy;@@ -39,7 +40,6 @@ use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::common;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::projection::ProjectionExec;-use datafusion_physical_plan::ExecutionPlan;/// Mock data using a MemorySourceConfig which has an exact count statisticfn mock_data() -> Result<Arc<DataSourceExec>> {diff --git a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rsindex 9c76f6ab6f..e73f531807 100644--- a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs+++ b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs@@ -29,18 +29,18 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;use datafusion_functions_aggregate::count::count_udaf;use datafusion_functions_aggregate::sum::sum_udaf;+use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};use datafusion_physical_expr::expressions::{col, lit};-use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;+use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};use datafusion_physical_plan::displayable;use datafusion_physical_plan::repartition::RepartitionExec;-use datafusion_physical_plan::ExecutionPlan;/// Runs the CombinePartialFinalAggregate optimizer and asserts the plan against the expectedmacro_rules! assert_optimized {diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex a3d9a1e407..04dffced31 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -26,33 +26,33 @@ use crate::physical_optimizer::test_utils::{sort_preserving_merge_exec, union_exec,};-use arrow::array::{RecordBatch, UInt64Array, UInt8Array};+use arrow::array::{RecordBatch, UInt8Array, UInt64Array};use arrow::compute::SortOptions;use arrow_schema::{DataType, Field, Schema, SchemaRef};use datafusion::config::ConfigOptions;+use datafusion::datasource::MemTable;use datafusion::datasource::file_format::file_compression_type::FileCompressionType;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::object_store::ObjectStoreUrl;use datafusion::datasource::physical_plan::{CsvSource, ParquetSource};use datafusion::datasource::source::DataSourceExec;-use datafusion::datasource::MemTable;use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_common::ScalarValue;use datafusion_common::config::CsvOptions;use datafusion_common::error::Result;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::ScalarValue;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_expr::{JoinType, Operator};-use datafusion_physical_expr::expressions::{binary, lit, BinaryExpr, Column, Literal};+use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal, binary, lit};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements, PhysicalSortExpr,};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::enforce_distribution::*;use datafusion_physical_optimizer::enforce_sorting::EnforceSorting;use datafusion_physical_optimizer::output_requirements::OutputRequirements;-use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};@@ -67,8 +67,8 @@ use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::{- displayable, DisplayAs, DisplayFormatType, ExecutionPlanProperties, PlanProperties,- Statistics,+ DisplayAs, DisplayFormatType, ExecutionPlanProperties, PlanProperties, Statistics,+ displayable,};use insta::Settings;diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 3f7fa62d68..ca99b17be0 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -19,14 +19,15 @@ use std::sync::Arc;use crate::memory_limit::DummyStreamPartition;use crate::physical_optimizer::test_utils::{- aggregate_exec, bounded_window_exec, bounded_window_exec_with_partition,- check_integrity, coalesce_batches_exec, coalesce_partitions_exec, create_test_schema,- create_test_schema2, create_test_schema3, filter_exec, global_limit_exec,- hash_join_exec, local_limit_exec, memory_exec, parquet_exec, parquet_exec_with_sort,- projection_exec, repartition_exec, sort_exec, sort_exec_with_fetch, sort_expr,- sort_expr_options, sort_merge_join_exec, sort_preserving_merge_exec,+ RequirementsTestExec, aggregate_exec, bounded_window_exec,+ bounded_window_exec_with_partition, check_integrity, coalesce_batches_exec,+ coalesce_partitions_exec, create_test_schema, create_test_schema2,+ create_test_schema3, filter_exec, global_limit_exec, hash_join_exec,+ local_limit_exec, memory_exec, parquet_exec, parquet_exec_with_sort, projection_exec,+ repartition_exec, sort_exec, sort_exec_with_fetch, sort_expr, sort_expr_options,+ sort_merge_join_exec, sort_preserving_merge_exec,sort_preserving_merge_exec_with_fetch, spr_repartition_exec, stream_exec_ordered,- union_exec, RequirementsTestExec,+ union_exec,};use arrow::compute::SortOptions;@@ -65,7 +66,7 @@ use datafusion_execution::TaskContext;use datafusion_catalog::streaming::StreamingTable;use futures::StreamExt;-use insta::{assert_snapshot, Settings};+use insta::{Settings, assert_snapshot};/// Create a sorted Csv execfn csv_exec_sorted(@@ -366,8 +367,8 @@ async fn test_union_inputs_different_sorted2() -> Result<()> {#[tokio::test]// Test with `repartition_sorts` enabled to preserve pre-sorted partitions and avoid resorting-async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true(-) -> Result<()> {+async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true()+-> Result<()> {assert_snapshot!(union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(true).await?,@r"@@ -392,8 +393,8 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti#[tokio::test]// Test with `repartition_sorts` disabled, causing a full resort of the data-async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false(-) -> Result<()> {+async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false()+-> Result<()> {assert_snapshot!(union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(false).await?,@r"@@ -770,8 +771,8 @@ async fn test_soft_hard_requirements_remove_soft_requirement() -> Result<()> {}#[tokio::test]-async fn test_soft_hard_requirements_remove_soft_requirement_without_pushdowns(-) -> Result<()> {+async fn test_soft_hard_requirements_remove_soft_requirement_without_pushdowns()+-> Result<()> {let schema = create_test_schema()?;let source = parquet_exec(schema.clone());let ordering = [sort_expr_options(@@ -1069,8 +1070,8 @@ async fn test_soft_hard_requirements_multiple_sorts() -> Result<()> {}#[tokio::test]-async fn test_soft_hard_requirements_with_multiple_soft_requirements_and_output_requirement(-) -> Result<()> {+async fn test_soft_hard_requirements_with_multiple_soft_requirements_and_output_requirement()+-> Result<()> {let schema = create_test_schema()?;let source = parquet_exec(schema.clone());let ordering = [sort_expr_options(diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting_monotonicity.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting_monotonicity.rsindex ef233e2229..de7611ff21 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting_monotonicity.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting_monotonicity.rs@@ -31,7 +31,7 @@ use datafusion_physical_expr::expressions::col;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::windows::{- create_window_expr, BoundedWindowAggExec, WindowAggExec,+ BoundedWindowAggExec, WindowAggExec, create_window_expr,};use datafusion_physical_plan::{ExecutionPlan, InputOrderMode};use insta::assert_snapshot;diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rsindex 0956974cfc..f480de71d6 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs@@ -18,7 +18,7 @@use std::sync::{Arc, LazyLock};use arrow::{- array::{record_batch, Float64Array, Int32Array, RecordBatch, StringArray},+ array::{Float64Array, Int32Array, RecordBatch, StringArray, record_batch},datatypes::{DataType, Field, Schema, SchemaRef},util::pretty::pretty_format_batches,};@@ -27,8 +27,8 @@ use datafusion::{assert_batches_eq,logical_expr::Operator,physical_plan::{- expressions::{BinaryExpr, Column, Literal},PhysicalExpr,+ expressions::{BinaryExpr, Column, Literal},},prelude::{ParquetReadOptions, SessionConfig, SessionContext},scalar::ScalarValue,@@ -36,7 +36,7 @@ use datafusion::{use datafusion_catalog::memory::DataSourceExec;use datafusion_common::config::ConfigOptions;use datafusion_datasource::{- file_groups::FileGroup, file_scan_config::FileScanConfigBuilder, PartitionedFile,+ PartitionedFile, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder,};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::ScalarUDF;@@ -45,15 +45,16 @@ use datafusion_functions_aggregate::{count::count_udaf,min_max::{max_udaf, min_udaf},};+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, expressions::col};use datafusion_physical_expr::{- aggregate::{AggregateExprBuilder, AggregateFunctionExpr},Partitioning, ScalarFunctionExpr,+ aggregate::{AggregateExprBuilder, AggregateFunctionExpr},};-use datafusion_physical_expr::{expressions::col, LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::{- filter_pushdown::FilterPushdown, PhysicalOptimizerRule,+ PhysicalOptimizerRule, filter_pushdown::FilterPushdown,};use datafusion_physical_plan::{+ ExecutionPlan,aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},coalesce_batches::CoalesceBatchesExec,coalesce_partitions::CoalescePartitionsExec,@@ -61,14 +62,13 @@ use datafusion_physical_plan::{filter::FilterExec,repartition::RepartitionExec,sorts::sort::SortExec,- ExecutionPlan,};use datafusion_physical_plan::union::UnionExec;use futures::StreamExt;-use object_store::{memory::InMemory, ObjectStore};+use object_store::{ObjectStore, memory::InMemory};use regex::Regex;-use util::{format_plan_for_test, OptimizationTest, TestNode, TestScanBuilder};+use util::{OptimizationTest, TestNode, TestScanBuilder, format_plan_for_test};use crate::physical_optimizer::filter_pushdown::util::TestSource;@@ -182,12 +182,14 @@ async fn test_dynamic_filter_pushdown_through_hash_join_with_topk() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8View, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8View, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0])+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8View, false),@@ -199,12 +201,14 @@ async fn test_dynamic_filter_pushdown_through_hash_join_with_topk() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("d", Utf8, ["aa", "ab", "ac", "ad"]),- ("e", Utf8View, ["ba", "bb", "bc", "bd"]),- ("f", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("d", Utf8, ["aa", "ab", "ac", "ad"]),+ ("e", Utf8View, ["ba", "bb", "bc", "bd"]),+ ("f", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Utf8, false),Field::new("e", DataType::Utf8View, false),@@ -299,12 +303,14 @@ async fn test_static_filter_pushdown_through_hash_join() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8View, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8View, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0])+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8View, false),@@ -316,12 +322,14 @@ async fn test_static_filter_pushdown_through_hash_join() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("d", Utf8, ["aa", "ab", "ac", "ad"]),- ("e", Utf8View, ["ba", "bb", "bc", "bd"]),- ("f", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("d", Utf8, ["aa", "ab", "ac", "ad"]),+ ("e", Utf8View, ["ba", "bb", "bc", "bd"]),+ ("f", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Utf8, false),Field::new("e", DataType::Utf8View, false),@@ -562,15 +570,14 @@ fn test_pushdown_through_aggregates_on_grouping_columns() {FilterExec::try_new(col_lit_predicate("a", "foo", &schema()), coalesce).unwrap(),);- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];let group_by = PhysicalGroupBy::new_single(vec![(col("a", &schema()).unwrap(), "a".to_string()),(col("b", &schema()).unwrap(), "b".to_string()),@@ -975,12 +982,14 @@ async fn test_hashjoin_dynamic_filter_pushdown() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -992,12 +1001,14 @@ async fn test_hashjoin_dynamic_filter_pushdown() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1143,12 +1154,14 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {// +---------------+------------------------------------------------------------+// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1160,12 +1173,14 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1363,12 +1378,14 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {use datafusion_common::JoinType;use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1380,12 +1397,14 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1542,10 +1561,9 @@ async fn test_nested_hashjoin_dynamic_filter_pushdown() {// Create test data for three tables: t1, t2, t3// t1: small table with limited values (will be build side of outer join)- let t1_batches =- vec![- record_batch!(("a", Utf8, ["aa", "ab"]), ("x", Float64, [1.0, 2.0])).unwrap(),- ];+ let t1_batches = vec![+ record_batch!(("a", Utf8, ["aa", "ab"]), ("x", Float64, [1.0, 2.0])).unwrap(),+ ];let t1_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("x", DataType::Float64, false),@@ -1556,12 +1574,14 @@ async fn test_nested_hashjoin_dynamic_filter_pushdown() {.build();// t2: larger table (will be probe side of inner join, build side of outer join)- let t2_batches = vec![record_batch!(- ("b", Utf8, ["aa", "ab", "ac", "ad", "ae"]),- ("c", Utf8, ["ca", "cb", "cc", "cd", "ce"]),- ("y", Float64, [1.0, 2.0, 3.0, 4.0, 5.0])- )- .unwrap()];+ let t2_batches = vec![+ record_batch!(+ ("b", Utf8, ["aa", "ab", "ac", "ad", "ae"]),+ ("c", Utf8, ["ca", "cb", "cc", "cd", "ce"]),+ ("y", Float64, [1.0, 2.0, 3.0, 4.0, 5.0])+ )+ .unwrap(),+ ];let t2_schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Utf8, false),Field::new("c", DataType::Utf8, false),@@ -1573,11 +1593,13 @@ async fn test_nested_hashjoin_dynamic_filter_pushdown() {.build();// t3: largest table (will be probe side of inner join)- let t3_batches = vec![record_batch!(- ("d", Utf8, ["ca", "cb", "cc", "cd", "ce", "cf", "cg", "ch"]),- ("z", Float64, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])- )- .unwrap()];+ let t3_batches = vec![+ record_batch!(+ ("d", Utf8, ["ca", "cb", "cc", "cd", "ce", "cf", "cg", "ch"]),+ ("z", Float64, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])+ )+ .unwrap(),+ ];let t3_schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Utf8, false),Field::new("z", DataType::Float64, false),@@ -1689,12 +1711,14 @@ async fn test_hashjoin_parent_filter_pushdown() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0])+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -1706,12 +1730,14 @@ async fn test_hashjoin_parent_filter_pushdown() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("d", Utf8, ["aa", "ab", "ac", "ad"]),- ("e", Utf8, ["ba", "bb", "bc", "bd"]),- ("f", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("d", Utf8, ["aa", "ab", "ac", "ad"]),+ ("e", Utf8, ["ba", "bb", "bc", "bd"]),+ ("f", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Utf8, false),Field::new("e", DataType::Utf8, false),@@ -2112,11 +2138,9 @@ async fn test_aggregate_dynamic_filter_min_max_different_columns() {Field::new("a", DataType::Int32, true),Field::new("b", DataType::Int32, true),]));- let batches =- vec![- record_batch!(("a", Int32, [5, 1, 3, 8]), ("b", Int32, [7, 2, 4, 9]))- .unwrap(),- ];+ let batches = vec![+ record_batch!(("a", Int32, [5, 1, 3, 8]), ("b", Int32, [7, 2, 4, 9])).unwrap(),+ ];let min_expr =AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])@@ -2151,12 +2175,14 @@ async fn test_aggregate_dynamic_filter_multiple_mixed_expressions() {Field::new("b", DataType::Int32, true),Field::new("c", DataType::Int32, true),]));- let batches = vec![record_batch!(- ("a", Int32, [5, 1, 3, 8]),- ("b", Int32, [10, 4, 6, 12]),- ("c", Int32, [100, 70, 90, 110])- )- .unwrap()];+ let batches = vec![+ record_batch!(+ ("a", Int32, [5, 1, 3, 8]),+ ("b", Int32, [10, 4, 6, 12]),+ ("c", Int32, [100, 70, 90, 110])+ )+ .unwrap(),+ ];let min_a = AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()]).schema(Arc::clone(&schema))@@ -2328,10 +2354,9 @@ async fn test_aggregate_filter_pushdown() {// when the filter references grouping columns// Simulates: SELECT a, COUNT(b) FROM table WHERE a = 'x' GROUP BY a- let batches =- vec![- record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),- ];+ let batches = vec![+ record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),+ ];let scan = TestScanBuilder::new(schema()).with_support(true)@@ -2392,10 +2417,9 @@ async fn test_no_pushdown_filter_on_aggregate_result() {// SELECT a, COUNT(b) as cnt FROM table GROUP BY a HAVING cnt > 5// The filter on 'cnt' cannot be pushed down because it's an aggregate result- let batches =- vec![- record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),- ];+ let batches = vec![+ record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),+ ];let scan = TestScanBuilder::new(schema()).with_support(true)@@ -2464,15 +2488,14 @@ fn test_pushdown_filter_on_non_first_grouping_column() {// The filter is on 'b' (second grouping column), should push downlet scan = TestScanBuilder::new(schema()).with_support(true).build();- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];let group_by = PhysicalGroupBy::new_single(vec![(col("a", &schema()).unwrap(), "a".to_string()),@@ -2515,15 +2538,14 @@ fn test_no_pushdown_grouping_sets_filter_on_missing_column() {// Test that filters on columns missing from some grouping sets are NOT pushed throughlet scan = TestScanBuilder::new(schema()).with_support(true).build();- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];// Create GROUPING SETS with (a, b) and (b)let group_by = PhysicalGroupBy::new(@@ -2586,15 +2608,14 @@ fn test_pushdown_grouping_sets_filter_on_common_column() {// Test that filters on columns present in ALL grouping sets ARE pushed throughlet scan = TestScanBuilder::new(schema()).with_support(true).build();- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];// Create GROUPING SETS with (a, b) and (b)let group_by = PhysicalGroupBy::new(@@ -2658,15 +2679,14 @@ fn test_pushdown_with_empty_group_by() {// There are no grouping columns, so the filter should still push downlet scan = TestScanBuilder::new(schema()).with_support(true).build();- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];// Empty GROUP BY - no grouping columnslet group_by = PhysicalGroupBy::new_single(vec![]);@@ -2718,15 +2738,14 @@ fn test_pushdown_with_computed_grouping_key() {)) as Arc<dyn PhysicalExpr>;let filter = Arc::new(FilterExec::try_new(predicate, scan).unwrap());- let aggregate_expr =- vec![- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()])- .schema(schema())- .alias("cnt")- .build()- .map(Arc::new)- .unwrap(),- ];+ let aggregate_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];let c_plus_one = Arc::new(BinaryExpr::new(col("c", &schema()).unwrap(),@@ -2786,11 +2805,13 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() {.build();// Create probe side with some data- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac"]),- ("b", Utf8, ["ba", "bb", "bc"])- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac"]),+ ("b", Utf8, ["ba", "bb", "bc"])+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -3055,12 +3076,14 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with limited values- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -3072,12 +3095,14 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -3220,12 +3245,14 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {use datafusion_common::JoinType;use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};- let build_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -3237,12 +3264,14 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {.build();// Create probe side with more values- let probe_batches = vec![record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -3369,12 +3398,14 @@ async fn test_hashjoin_hash_table_pushdown_integer_keys() {use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};// Create build side with integer keys- let build_batches = vec![record_batch!(- ("id1", Int32, [1, 2]),- ("id2", Int32, [10, 20]),- ("value", Float64, [100.0, 200.0])- )- .unwrap()];+ let build_batches = vec![+ record_batch!(+ ("id1", Int32, [1, 2]),+ ("id2", Int32, [10, 20]),+ ("value", Float64, [100.0, 200.0])+ )+ .unwrap(),+ ];let build_side_schema = Arc::new(Schema::new(vec![Field::new("id1", DataType::Int32, false),Field::new("id2", DataType::Int32, false),@@ -3386,12 +3417,14 @@ async fn test_hashjoin_hash_table_pushdown_integer_keys() {.build();// Create probe side with more integer rows- let probe_batches = vec![record_batch!(- ("id1", Int32, [1, 2, 3, 4]),- ("id2", Int32, [10, 20, 30, 40]),- ("data", Utf8, ["a", "b", "c", "d"])- )- .unwrap()];+ let probe_batches = vec![+ record_batch!(+ ("id1", Int32, [1, 2, 3, 4]),+ ("id2", Int32, [10, 20, 30, 40]),+ ("data", Utf8, ["a", "b", "c", "d"])+ )+ .unwrap(),+ ];let probe_side_schema = Arc::new(Schema::new(vec![Field::new("id1", DataType::Int32, false),Field::new("id2", DataType::Int32, false),diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rsindex c32f7b2d0b..93e928b51a 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs@@ -18,26 +18,25 @@use arrow::datatypes::SchemaRef;use arrow::{array::RecordBatch, compute::concat_batches};use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};-use datafusion_common::{config::ConfigOptions, internal_err, Result};+use datafusion_common::{Result, config::ConfigOptions, internal_err};use datafusion_datasource::{- file::FileSource, file_scan_config::FileScanConfig,+ PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,file_scan_config::FileScanConfigBuilder, file_stream::FileOpenFuture,file_stream::FileOpener, schema_adapter::DefaultSchemaAdapterFactory,- schema_adapter::SchemaAdapterFactory, source::DataSourceExec, PartitionedFile,+ schema_adapter::SchemaAdapterFactory, source::DataSourceExec,};use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::filter::batch_filter;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPhase, PushedDown};use datafusion_physical_plan::{- displayable,+ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, displayable,filter::FilterExec,filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription,FilterPushdownPropagation,},metrics::ExecutionPlanMetricsSet,- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,};use futures::StreamExt;use futures::{FutureExt, Stream};diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex f9d3a04546..37bcefd418 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -26,27 +26,27 @@ use std::{use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;-use datafusion_common::{stats::Precision, ColumnStatistics, JoinType, ScalarValue};+use datafusion_common::{ColumnStatistics, JoinType, ScalarValue, stats::Precision};use datafusion_common::{JoinSide, NullEquality};use datafusion_common::{Result, Statistics};use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};use datafusion_expr::Operator;+use datafusion_physical_expr::PhysicalExprRef;use datafusion_physical_expr::expressions::col;use datafusion_physical_expr::expressions::{BinaryExpr, Column, NegativeExpr};use datafusion_physical_expr::intervals::utils::check_support;-use datafusion_physical_expr::PhysicalExprRef;use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};-use datafusion_physical_optimizer::join_selection::JoinSelection;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::join_selection::JoinSelection;+use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_physical_plan::displayable;use datafusion_physical_plan::joins::utils::ColumnIndex;use datafusion_physical_plan::joins::utils::JoinFilter;use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode};use datafusion_physical_plan::projection::ProjectionExec;-use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_physical_plan::{- execution_plan::{Boundedness, EmissionType},DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ execution_plan::{Boundedness, EmissionType},};use futures::Stream;@@ -949,10 +949,10 @@ impl Stream for UnboundedStream {mut self: Pin<&mut Self>,_cx: &mut Context<'_>,) -> Poll<Option<Self::Item>> {- if let Some(val) = self.batch_produce {- if val <= self.count {- return Poll::Ready(None);- }+ if let Some(val) = self.batch_produce+ && val <= self.count+ {+ return Poll::Ready(None);}self.count += 1;Poll::Ready(Some(Ok(self.batch.clone())))@@ -1088,9 +1088,10 @@ pub struct StatisticsExec {impl StatisticsExec {pub fn new(stats: Statistics, schema: Schema) -> Self {assert_eq!(- stats.column_statistics.len(), schema.fields().len(),- "if defined, the column statistics vector length should be the number of fields"- );+ stats.column_statistics.len(),+ schema.fields().len(),+ "if defined, the column statistics vector length should be the number of fields"+ );let cache = Self::compute_properties(Arc::new(schema.clone()));Self {stats,diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rsindex 9d172db246..b32a9bbd25 100644--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs@@ -27,16 +27,16 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_expr::Operator;-use datafusion_physical_expr::expressions::{col, lit, BinaryExpr};use datafusion_physical_expr::Partitioning;+use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::repartition::RepartitionExec;-use datafusion_physical_plan::{get_plan_string, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, get_plan_string};fn create_schema() -> SchemaRef {Arc::new(Schema::new(vec![@@ -96,51 +96,51 @@ fn transforms_streaming_table_exec_into_fetching_version_when_skip_is_zero() ->let initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);let after_optimize =LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=5"- ];+ "StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=5",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())}#[test]-fn transforms_streaming_table_exec_into_fetching_version_and_keeps_the_global_limit_when_skip_is_nonzero(-) -> Result<()> {+fn transforms_streaming_table_exec_into_fetching_version_and_keeps_the_global_limit_when_skip_is_nonzero()+-> Result<()> {let schema = create_schema();let streaming_table = stream_exec(&schema);let global_limit = global_limit_exec(streaming_table, 2, Some(5));let initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=2, fetch=5",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=2, fetch=5",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);let after_optimize =LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "GlobalLimitExec: skip=2, fetch=5",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=7"- ];+ "GlobalLimitExec: skip=2, fetch=5",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=7",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())}#[test]-fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limit(-) -> Result<()> {+fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limit()+-> Result<()> {let schema = create_schema();let streaming_table = stream_exec(&schema);let repartition = repartition_exec(streaming_table)?;@@ -152,14 +152,14 @@ fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limilet initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=5",- " CoalesceBatchesExec: target_batch_size=8192",- " FilterExec: c3@2 > 0",- " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " CoalescePartitionsExec",+ " LocalLimitExec: fetch=5",+ " CoalesceBatchesExec: target_batch_size=8192",+ " FilterExec: c3@2 > 0",+ " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);let after_optimize =@@ -170,8 +170,8 @@ fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limi" CoalesceBatchesExec: target_batch_size=8192, fetch=5"," FilterExec: c3@2 > 0"," RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())@@ -187,29 +187,29 @@ fn pushes_global_limit_exec_through_projection_exec() -> Result<()> {let initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " FilterExec: c3@2 > 0",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " FilterExec: c3@2 > 0",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);let after_optimize =LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " FilterExec: c3@2 > 0, fetch=5",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " FilterExec: c3@2 > 0, fetch=5",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())}#[test]-fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batches_exec_into_fetching_version(-) -> Result<()> {+fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batches_exec_into_fetching_version()+-> Result<()> {let schema = create_schema();let streaming_table = stream_exec(&schema);let coalesce_batches = coalesce_batches_exec(streaming_table, 8192);@@ -218,11 +218,11 @@ fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batclet initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " CoalesceBatchesExec: target_batch_size=8192",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " CoalesceBatchesExec: target_batch_size=8192",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);@@ -230,10 +230,10 @@ fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batcLimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " CoalesceBatchesExec: target_batch_size=8192, fetch=5",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " CoalesceBatchesExec: target_batch_size=8192, fetch=5",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())@@ -257,14 +257,14 @@ fn pushes_global_limit_into_multiple_fetch_plans() -> Result<()> {let initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " SortPreservingMergeExec: [c1@0 ASC]",- " SortExec: expr=[c1@0 ASC], preserve_partitioning=[false]",- " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " CoalesceBatchesExec: target_batch_size=8192",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " SortPreservingMergeExec: [c1@0 ASC]",+ " SortExec: expr=[c1@0 ASC], preserve_partitioning=[false]",+ " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",+ " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " CoalesceBatchesExec: target_batch_size=8192",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);@@ -272,13 +272,13 @@ fn pushes_global_limit_into_multiple_fetch_plans() -> Result<()> {LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "SortPreservingMergeExec: [c1@0 ASC], fetch=5",- " SortExec: TopK(fetch=5), expr=[c1@0 ASC], preserve_partitioning=[false]",- " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",- " CoalesceBatchesExec: target_batch_size=8192",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "SortPreservingMergeExec: [c1@0 ASC], fetch=5",+ " SortExec: TopK(fetch=5), expr=[c1@0 ASC], preserve_partitioning=[false]",+ " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",+ " ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",+ " CoalesceBatchesExec: target_batch_size=8192",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())@@ -296,23 +296,23 @@ fn keeps_pushed_local_limit_exec_when_there_are_multiple_input_partitions() -> Rlet initial = get_plan_string(&global_limit);let expected_initial = [- "GlobalLimitExec: skip=0, fetch=5",- " CoalescePartitionsExec",- " FilterExec: c3@2 > 0",- " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "GlobalLimitExec: skip=0, fetch=5",+ " CoalescePartitionsExec",+ " FilterExec: c3@2 > 0",+ " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(initial, expected_initial);let after_optimize =LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = [- "CoalescePartitionsExec: fetch=5",- " FilterExec: c3@2 > 0, fetch=5",- " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",- " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true"- ];+ "CoalescePartitionsExec: fetch=5",+ " FilterExec: c3@2 > 0, fetch=5",+ " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1",+ " StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true",+ ];assert_eq!(get_plan_string(&after_optimize), expected);Ok(())diff --git a/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs b/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rsindex ad15d68034..c523b4a752 100644--- a/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs+++ b/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs@@ -21,8 +21,8 @@ use insta::assert_snapshot;use std::sync::Arc;use crate::physical_optimizer::test_utils::{- build_group_by, get_optimized_plan, mock_data, parquet_exec_with_sort, schema,- TestAggregate,+ TestAggregate, build_group_by, get_optimized_plan, mock_data, parquet_exec_with_sort,+ schema,};use arrow::datatypes::DataType;@@ -34,10 +34,10 @@ use datafusion_expr::Operator;use datafusion_physical_expr::expressions::{self, cast, col};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::{+ ExecutionPlan,aggregates::{AggregateExec, AggregateMode},collect,limit::{GlobalLimitExec, LocalLimitExec},- ExecutionPlan,};async fn run_plan_and_format(plan: Arc<dyn ExecutionPlan>) -> Result<String> {diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 1846473e10..468d25e0e5 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -25,16 +25,16 @@ mod test {use datafusion::datasource::listing::ListingTable;use datafusion::prelude::SessionContext;use datafusion_catalog::TableProvider;- use datafusion_common::stats::Precision;use datafusion_common::Result;+ use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};- use datafusion_execution::config::SessionConfig;use datafusion_execution::TaskContext;+ use datafusion_execution::config::SessionConfig;use datafusion_expr_common::operator::Operator;use datafusion_functions_aggregate::count::count_udaf;- use datafusion_physical_expr::aggregate::AggregateExprBuilder;- use datafusion_physical_expr::expressions::{binary, col, lit, Column};use datafusion_physical_expr::Partitioning;+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;+ use datafusion_physical_expr::expressions::{Column, binary, col, lit};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::aggregates::{@@ -53,8 +53,8 @@ mod test {use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::union::{InterleaveExec, UnionExec};use datafusion_physical_plan::{- execute_stream_partitioned, get_plan_string, ExecutionPlan,- ExecutionPlanProperties,+ ExecutionPlan, ExecutionPlanProperties, execute_stream_partitioned,+ get_plan_string,};use futures::TryStreamExt;@@ -766,11 +766,13 @@ mod test {),]);- let aggr_expr = vec![AggregateExprBuilder::new(count_udaf(), vec![lit(1)])- .schema(Arc::clone(&scan_schema))- .alias(String::from("COUNT(c)"))- .build()- .map(Arc::new)?];+ let aggr_expr = vec![+ AggregateExprBuilder::new(count_udaf(), vec![lit(1)])+ .schema(Arc::clone(&scan_schema))+ .alias(String::from("COUNT(c)"))+ .build()+ .map(Arc::new)?,+ ];let aggregate_exec_partial: Arc<dyn ExecutionPlan> =Arc::new(AggregateExec::try_new(@@ -1067,9 +1069,11 @@ mod test {let result = repartition.partition_statistics(Some(2));assert!(result.is_err());let error = result.unwrap_err();- assert!(error- .to_string()- .contains("RepartitionExec invalid partition 2 (expected less than 2)"));+ assert!(+ error+ .to_string()+ .contains("RepartitionExec invalid partition 2 (expected less than 2)")+ );let partitions = execute_stream_partitioned(repartition.clone(),diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex 80f4fbc305..89564cb9ed 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -26,8 +26,8 @@ use datafusion::datasource::physical_plan::CsvSource;use datafusion::datasource::source::DataSourceExec;use datafusion_common::config::{ConfigOptions, CsvOptions};use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};-use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::TableSchema;+use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::{@@ -35,16 +35,16 @@ use datafusion_expr::{};use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_physical_expr::expressions::{- binary, cast, col, BinaryExpr, CaseExpr, CastExpr, Column, Literal, NegativeExpr,+ BinaryExpr, CaseExpr, CastExpr, Column, Literal, NegativeExpr, binary, cast, col,};use datafusion_physical_expr::{Distribution, Partitioning, ScalarFunctionExpr};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{OrderingRequirements, PhysicalSortExpr, PhysicalSortRequirement,};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;use datafusion_physical_optimizer::projection_pushdown::ProjectionPushdown;-use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};@@ -52,13 +52,13 @@ use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode, StreamJoinPartitionMode,SymmetricHashJoinExec,};-use datafusion_physical_plan::projection::{update_expr, ProjectionExec, ProjectionExpr};+use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr, update_expr};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};use datafusion_physical_plan::union::UnionExec;-use datafusion_physical_plan::{displayable, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, displayable};use insta::assert_snapshot;use itertools::Itertools;@@ -230,9 +230,11 @@ fn test_update_matching_exprs() -> Result<()> {.map(|(expr, alias)| ProjectionExpr::new(expr.clone(), alias.clone())).collect();for (expr, expected_expr) in exprs.into_iter().zip(expected_exprs.into_iter()) {- assert!(update_expr(&expr, &child_exprs, true)?- .unwrap()- .eq(&expected_expr));+ assert!(+ update_expr(&expr, &child_exprs, true)?+ .unwrap()+ .eq(&expected_expr)+ );}Ok(())@@ -369,9 +371,11 @@ fn test_update_projected_exprs() -> Result<()> {.map(|(expr, alias)| ProjectionExpr::new(expr.clone(), alias.clone())).collect();for (expr, expected_expr) in exprs.into_iter().zip(expected_exprs.into_iter()) {- assert!(update_expr(&expr, &proj_exprs, false)?- .unwrap()- .eq(&expected_expr));+ assert!(+ update_expr(&expr, &proj_exprs, false)?+ .unwrap()+ .eq(&expected_expr)+ );}Ok(())@@ -814,10 +818,11 @@ fn test_output_req_after_projection() -> Result<()> {.required_input_distribution()[0].clone(){- assert!(vec- .iter()- .zip(expected_distribution)- .all(|(actual, expected)| actual.eq(&expected)));+ assert!(+ vec.iter()+ .zip(expected_distribution)+ .all(|(actual, expected)| actual.eq(&expected))+ );} else {panic!("Expected HashPartitioned distribution!");};diff --git a/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs b/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rsindex 8fe35fc307..d93081f5ce 100644--- a/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs+++ b/datafusion/core/tests/physical_optimizer/replace_with_order_preserving_variants.rs@@ -50,8 +50,8 @@ use datafusion_physical_plan::{collect, displayable, ExecutionPlan, Partitioning,};-use object_store::memory::InMemory;use object_store::ObjectStore;+use object_store::memory::InMemory;use rstest::rstest;use url::Url;@@ -138,7 +138,8 @@ impl ReplaceTest {assert!(res.is_ok(),"Some errors occurred while executing the optimized physical plan: {:?}\nPlan: {}",- res.unwrap_err(), optimized_plan_string+ res.unwrap_err(),+ optimized_plan_string);}@@ -1248,7 +1249,10 @@ fn test_plan_with_order_preserving_variants_preserves_fetch() -> Result<()> {)],);let res = plan_with_order_preserving_variants(requirements, false, true, Some(15));- assert_contains!(res.unwrap_err().to_string(), "CoalescePartitionsExec fetch [10] should be greater than or equal to SortExec fetch [15]");+ assert_contains!(+ res.unwrap_err().to_string(),+ "CoalescePartitionsExec fetch [10] should be greater than or equal to SortExec fetch [15]"+ );// Test sort is without fetch, expected to get the fetch value from the coalescedlet requirements = OrderPreservationContext::new(diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rsindex f46147de1b..5aa3925e27 100644--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs@@ -30,13 +30,13 @@ use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTabuse datafusion::prelude::{CsvReadOptions, SessionContext};use datafusion_common::config::ConfigOptions;use datafusion_common::{JoinType, Result, ScalarValue};-use datafusion_physical_expr::expressions::{col, Literal};use datafusion_physical_expr::Partitioning;+use datafusion_physical_expr::expressions::{Literal, col};use datafusion_physical_expr_common::sort_expr::LexOrdering;-use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;use datafusion_physical_plan::repartition::RepartitionExec;-use datafusion_physical_plan::{displayable, ExecutionPlan};+use datafusion_physical_plan::{ExecutionPlan, displayable};use async_trait::async_trait;diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex ef3f23b7cb..1561ddf440 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -45,8 +45,8 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements, PhysicalSortExpr,};-use datafusion_physical_optimizer::limited_distinct_aggregation::LimitedDistinctAggregation;use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::limited_distinct_aggregation::LimitedDistinctAggregation;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};@@ -63,10 +63,10 @@ use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeEuse datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};use datafusion_physical_plan::tree_node::PlanContext;use datafusion_physical_plan::union::UnionExec;-use datafusion_physical_plan::windows::{create_window_expr, BoundedWindowAggExec};+use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};use datafusion_physical_plan::{- displayable, DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode,- Partitioning, PlanProperties,+ DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,+ PlanProperties, displayable,};/// Create a non sorted parquet exec@@ -462,10 +462,11 @@ impl ExecutionPlan for RequirementsTestExec {}fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {- vec![self- .required_input_ordering- .as_ref()- .map(|ordering| OrderingRequirements::from(ordering.clone()))]+ vec![+ self.required_input_ordering+ .as_ref()+ .map(|ordering| OrderingRequirements::from(ordering.clone())),+ ]}fn maintains_input_order(&self) -> Vec<bool> {diff --git a/datafusion/core/tests/physical_optimizer/window_optimize.rs b/datafusion/core/tests/physical_optimizer/window_optimize.rsindex fc1e6444d7..796f6b6259 100644--- a/datafusion/core/tests/physical_optimizer/window_optimize.rs+++ b/datafusion/core/tests/physical_optimizer/window_optimize.rs@@ -26,10 +26,10 @@ mod test {use datafusion_expr::WindowFrame;use datafusion_functions_aggregate::count::count_udaf;use datafusion_physical_expr::aggregate::AggregateExprBuilder;- use datafusion_physical_expr::expressions::{col, Column};+ use datafusion_physical_expr::expressions::{Column, col};use datafusion_physical_expr::window::PlainAggregateWindowExpr;use datafusion_physical_plan::windows::BoundedWindowAggExec;- use datafusion_physical_plan::{common, ExecutionPlan, InputOrderMode};+ use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, common};use std::sync::Arc;/// Test case for <https://github.com/apache/datafusion/issues/16308>diff --git a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rsindex 2a0c133312..01242ff41f 100644--- a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs+++ b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs@@ -32,18 +32,18 @@ use datafusion::datasource::physical_plan::{use datafusion::logical_expr::{col, lit};use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::SessionContext;+use datafusion_common::ColumnStatistics;use datafusion_common::config::CsvOptions;use datafusion_common::record_batch;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::ColumnStatistics;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::schema_adapter::{SchemaAdapter, SchemaAdapterFactory, SchemaMapper,};use datafusion::assert_batches_eq;-use datafusion_datasource::source::DataSourceExec;use datafusion_datasource::TableSchema;+use datafusion_datasource::source::DataSourceExec;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::Expr;use datafusion_physical_expr::expressions::Column;@@ -51,7 +51,7 @@ use datafusion_physical_expr::planner::logical2physical;use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-use object_store::{memory::InMemory, path::Path, ObjectStore};+use object_store::{ObjectStore, memory::InMemory, path::Path};use parquet::arrow::ArrowWriter;async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &str) {diff --git a/datafusion/core/tests/sql/aggregates/mod.rs b/datafusion/core/tests/sql/aggregates/mod.rsindex 321c158628..ede40d5c4c 100644--- a/datafusion/core/tests/sql/aggregates/mod.rs+++ b/datafusion/core/tests/sql/aggregates/mod.rs@@ -20,15 +20,15 @@use super::*;use arrow::{array::{- types::UInt32Type, Decimal128Array, DictionaryArray, DurationNanosecondArray,- Int32Array, LargeBinaryArray, StringArray, TimestampMicrosecondArray,- UInt16Array, UInt32Array, UInt64Array, UInt8Array,+ Decimal128Array, DictionaryArray, DurationNanosecondArray, Int32Array,+ LargeBinaryArray, StringArray, TimestampMicrosecondArray, UInt8Array,+ UInt16Array, UInt32Array, UInt64Array, types::UInt32Type,},datatypes::{DataType, Field, Schema, TimeUnit},record_batch::RecordBatch,};use datafusion::{- common::{test_util::batches_to_string, Result},+ common::{Result, test_util::batches_to_string},execution::{config::SessionConfig, context::SessionContext},};use datafusion_catalog::MemTable;@@ -959,8 +959,8 @@ impl FuzzTimestampTestData {}/// Sets up test contexts for fuzz table with timestamps and both single and multiple partitions-pub async fn setup_fuzz_timestamp_test_contexts(-) -> Result<(SessionContext, SessionContext)> {+pub async fn setup_fuzz_timestamp_test_contexts()+-> Result<(SessionContext, SessionContext)> {let test_data = FuzzTimestampTestData::new();// Single partition contextdiff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex b0a8aa08f2..201bcf178b 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -847,8 +847,7 @@ async fn csv_explain_analyze_order_by() {// Ensure that the ordering is not optimized away from the plan// https://github.com/apache/datafusion/issues/6379- let needle =- "SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=100, elapsed_compute";+ let needle = "SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=100, elapsed_compute";assert_contains!(&formatted, needle);}@@ -888,7 +887,7 @@ async fn parquet_explain_analyze() {(i_file < i_rowgroup_stat)&& (i_rowgroup_stat < i_rowgroup_bloomfilter)&& (i_rowgroup_bloomfilter < i_page),- "The parquet pruning metrics should be displayed in an order of: file range -> row group statistics -> row group bloom filter -> page index."+ "The parquet pruning metrics should be displayed in an order of: file range -> row group statistics -> row group bloom filter -> page index.");}diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rsindex 5ce3865d65..c7f02e4868 100644--- a/datafusion/core/tests/sql/joins.rs+++ b/datafusion/core/tests/sql/joins.rs@@ -38,14 +38,16 @@ async fn join_change_in_planner() -> Result<()> {Field::new("a2", DataType::UInt32, false),]));// Specify the ordering:- let file_sort_order = vec![[col("a1")]- .into_iter()- .map(|e| {- let ascending = true;- let nulls_first = false;- e.sort(ascending, nulls_first)- })- .collect::<Vec<_>>()];+ let file_sort_order = vec![+ [col("a1")]+ .into_iter()+ .map(|e| {+ let ascending = true;+ let nulls_first = false;+ e.sort(ascending, nulls_first)+ })+ .collect::<Vec<_>>(),+ ];register_unbounded_file_with_ordering(&ctx,schema.clone(),@@ -95,14 +97,16 @@ async fn join_no_order_on_filter() -> Result<()> {Field::new("a3", DataType::UInt32, false),]));// Specify the ordering:- let file_sort_order = vec![[col("a1")]- .into_iter()- .map(|e| {- let ascending = true;- let nulls_first = false;- e.sort(ascending, nulls_first)- })- .collect::<Vec<_>>()];+ let file_sort_order = vec![+ [col("a1")]+ .into_iter()+ .map(|e| {+ let ascending = true;+ let nulls_first = false;+ e.sort(ascending, nulls_first)+ })+ .collect::<Vec<_>>(),+ ];register_unbounded_file_with_ordering(&ctx,schema.clone(),@@ -202,7 +206,10 @@ async fn join_change_in_planner_without_sort_not_allowed() -> Result<()> {match df.create_physical_plan().await {Ok(_) => panic!("Expecting error."),Err(e) => {- assert_eq!(e.strip_backtrace(), "SanityCheckPlan\ncaused by\nError during planning: Join operation cannot operate on a non-prunable stream without enabling the 'allow_symmetric_joins_without_pruning' configuration flag")+ assert_eq!(+ e.strip_backtrace(),+ "SanityCheckPlan\ncaused by\nError during planning: Join operation cannot operate on a non-prunable stream without enabling the 'allow_symmetric_joins_without_pruning' configuration flag"+ )}}Ok(())diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rsindex 426ec213b3..3052ccf2b6 100644--- a/datafusion/core/tests/sql/mod.rs+++ b/datafusion/core/tests/sql/mod.rs@@ -24,10 +24,10 @@ use arrow::{use datafusion::error::Result;use datafusion::logical_expr::{Aggregate, LogicalPlan, TableScan};-use datafusion::physical_plan::collect;-use datafusion::physical_plan::metrics::MetricValue;use datafusion::physical_plan::ExecutionPlan;use datafusion::physical_plan::ExecutionPlanVisitor;+use datafusion::physical_plan::collect;+use datafusion::physical_plan::metrics::MetricValue;use datafusion::prelude::*;use datafusion::test_util;use datafusion::{execution::context::SessionContext, physical_plan::displayable};@@ -335,8 +335,7 @@ async fn nyc() -> Result<()> {match &optimized_plan {LogicalPlan::Aggregate(Aggregate { input, .. }) => match input.as_ref() {LogicalPlan::TableScan(TableScan {- ref projected_schema,- ..+ projected_schema, ..}) => {assert_eq!(2, projected_schema.fields().len());assert_eq!(projected_schema.field(0).name(), "passenger_count");diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rsindex 3ee4e37589..c6f920584d 100644--- a/datafusion/core/tests/sql/path_partition.rs+++ b/datafusion/core/tests/sql/path_partition.rs@@ -35,9 +35,9 @@ use datafusion::{test_util::{self, arrow_test_data, parquet_test_data},};use datafusion_catalog::TableProvider;+use datafusion_common::ScalarValue;use datafusion_common::stats::Precision;use datafusion_common::test_util::batches_to_sort_string;-use datafusion_common::ScalarValue;use datafusion_execution::config::SessionConfig;use async_trait::async_trait;@@ -45,11 +45,11 @@ use bytes::Bytes;use chrono::{TimeZone, Utc};use futures::stream::{self, BoxStream};use insta::assert_snapshot;+use object_store::{Attributes, MultipartUpload, PutMultipartOptions, PutPayload};use object_store::{- path::Path, GetOptions, GetResult, GetResultPayload, ListResult, ObjectMeta,- ObjectStore, PutOptions, PutResult,+ GetOptions, GetResult, GetResultPayload, ListResult, ObjectMeta, ObjectStore,+ PutOptions, PutResult, path::Path,};-use object_store::{Attributes, MultipartUpload, PutMultipartOptions, PutPayload};use url::Url;#[tokio::test]diff --git a/datafusion/core/tests/sql/runtime_config.rs b/datafusion/core/tests/sql/runtime_config.rsindex d6dc698399..d85892c254 100644--- a/datafusion/core/tests/sql/runtime_config.rs+++ b/datafusion/core/tests/sql/runtime_config.rs@@ -23,8 +23,8 @@ use std::time::Duration;use datafusion::execution::context::SessionContext;use datafusion::execution::context::TaskContext;use datafusion::prelude::SessionConfig;-use datafusion_execution::cache::cache_manager::CacheManagerConfig;use datafusion_execution::cache::DefaultListFilesCache;+use datafusion_execution::cache::cache_manager::CacheManagerConfig;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_physical_plan::common::collect;diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rsindex 5a51451461..6126793145 100644--- a/datafusion/core/tests/sql/select.rs+++ b/datafusion/core/tests/sql/select.rs@@ -18,7 +18,7 @@use std::collections::HashMap;use super::*;-use datafusion_common::{metadata::ScalarAndMetadata, ParamValues, ScalarValue};+use datafusion_common::{ParamValues, ScalarValue, metadata::ScalarAndMetadata};use insta::assert_snapshot;#[tokio::test]diff --git a/datafusion/core/tests/tracing/asserting_tracer.rs b/datafusion/core/tests/tracing/asserting_tracer.rsindex 292e066e5f..700f9f3308 100644--- a/datafusion/core/tests/tracing/asserting_tracer.rs+++ b/datafusion/core/tests/tracing/asserting_tracer.rs@@ -21,7 +21,7 @@ use std::ops::Deref;use std::sync::{Arc, LazyLock};use datafusion_common::{HashMap, HashSet};-use datafusion_common_runtime::{set_join_set_tracer, JoinSetTracer};+use datafusion_common_runtime::{JoinSetTracer, set_join_set_tracer};use futures::future::BoxFuture;use tokio::sync::{Mutex, MutexGuard};diff --git a/datafusion/core/tests/tracing/traceable_object_store.rs b/datafusion/core/tests/tracing/traceable_object_store.rsindex 60ef1cc5d6..00aa4ea3f3 100644--- a/datafusion/core/tests/tracing/traceable_object_store.rs+++ b/datafusion/core/tests/tracing/traceable_object_store.rs@@ -20,8 +20,8 @@use crate::tracing::asserting_tracer::assert_traceability;use futures::stream::BoxStream;use object_store::{- path::Path, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,- ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,+ GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,+ PutMultipartOptions, PutOptions, PutPayload, PutResult, path::Path,};use std::fmt::{Debug, Display, Formatter};use std::sync::Arc;diff --git a/datafusion/core/tests/user_defined/expr_planner.rs b/datafusion/core/tests/user_defined/expr_planner.rsindex 07d289cab0..c2558fbba2 100644--- a/datafusion/core/tests/user_defined/expr_planner.rs+++ b/datafusion/core/tests/user_defined/expr_planner.rs@@ -26,9 +26,9 @@ use datafusion::logical_expr::Operator;use datafusion::prelude::*;use datafusion::sql::sqlparser::ast::BinaryOperator;use datafusion_common::ScalarValue;+use datafusion_expr::BinaryExpr;use datafusion_expr::expr::Alias;use datafusion_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr};-use datafusion_expr::BinaryExpr;#[derive(Debug)]struct MyCustomPlanner;diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rsindex e0a3e98604..7ad00dece1 100644--- a/datafusion/core/tests/user_defined/insert_operation.rs+++ b/datafusion/core/tests/user_defined/insert_operation.rs@@ -25,12 +25,12 @@ use datafusion::{};use datafusion_catalog::{Session, TableProvider};use datafusion_common::config::Dialect;-use datafusion_expr::{dml::InsertOp, Expr, TableType};+use datafusion_expr::{Expr, TableType, dml::InsertOp};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::SchedulingType;use datafusion_physical_plan::{- execution_plan::{Boundedness, EmissionType},DisplayAs, ExecutionPlan, PlanProperties,+ execution_plan::{Boundedness, EmissionType},};#[tokio::test]diff --git a/datafusion/core/tests/user_defined/relation_planner.rs b/datafusion/core/tests/user_defined/relation_planner.rsindex dc06c783db..e1c7a26184 100644--- a/datafusion/core/tests/user_defined/relation_planner.rs+++ b/datafusion/core/tests/user_defined/relation_planner.rs@@ -25,11 +25,11 @@ use datafusion::catalog::memory::MemTable;use datafusion::common::test_util::batches_to_string;use datafusion::prelude::*;use datafusion_common::{Result, ScalarValue};+use datafusion_expr::Expr;use datafusion_expr::logical_plan::builder::LogicalPlanBuilder;use datafusion_expr::planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,};-use datafusion_expr::Expr;use datafusion_sql::sqlparser::ast::TableFactor;use insta::assert_snapshot;diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rsindex 28a78feed3..ca13b104a5 100644--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs@@ -23,13 +23,13 @@ use std::collections::HashMap;use std::hash::{Hash, Hasher};use std::mem::{size_of, size_of_val};use std::sync::{- atomic::{AtomicBool, Ordering},Arc,+ atomic::{AtomicBool, Ordering},};use arrow::array::{- record_batch, types::UInt64Type, Array, AsArray, Int32Array, PrimitiveArray,- StringArray, StructArray, UInt64Array,+ Array, AsArray, Int32Array, PrimitiveArray, StringArray, StructArray, UInt64Array,+ record_batch, types::UInt64Type,};use arrow::datatypes::{Fields, Schema};use arrow_schema::FieldRef;@@ -56,8 +56,8 @@ use datafusion_common::{cast::as_primitive_array, exec_err};use datafusion_expr::expr::WindowFunction;use datafusion_expr::{- col, create_udaf, function::AccumulatorArgs, AggregateUDFImpl, Expr,- GroupsAccumulator, LogicalPlanBuilder, SimpleAggregateUDF, WindowFunctionDefinition,+ AggregateUDFImpl, Expr, GroupsAccumulator, LogicalPlanBuilder, SimpleAggregateUDF,+ WindowFunctionDefinition, col, create_udaf, function::AccumulatorArgs,};use datafusion_functions_aggregate::average::AvgAccumulator;@@ -164,7 +164,10 @@ async fn test_udaf_as_window_with_frame_without_retract_batch() {let sql = "SELECT time_sum(time) OVER(ORDER BY time ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) as time_sum from t";// Note if this query ever does start workinglet err = execute(&ctx, sql).await.unwrap_err();- assert_contains!(err.to_string(), "This feature is not implemented: Aggregate can not be used as a sliding accumulator because `retract_batch` is not implemented: time_sum(t.time) ORDER BY [t.time ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING");+ assert_contains!(+ err.to_string(),+ "This feature is not implemented: Aggregate can not be used as a sliding accumulator because `retract_batch` is not implemented: time_sum(t.time) ORDER BY [t.time ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+ );}/// Basic query for with a udaf returning a structure@@ -329,9 +332,10 @@ async fn case_sensitive_identifiers_user_defined_aggregates() -> Result<()> {// doesn't work as it was registered as non lowercaselet err = ctx.sql("SELECT MY_AVG(i) FROM t").await.unwrap_err();- assert!(err- .to_string()- .contains("Error during planning: Invalid function \'my_avg\'"));+ assert!(+ err.to_string()+ .contains("Error during planning: Invalid function \'my_avg\'")+ );// Can call it if you put quoteslet result = ctx@@ -761,11 +765,11 @@ impl Accumulator for FirstSelector {// Update the actual valuesfor (value, time) in v.iter().zip(t.iter()) {- if let (Some(time), Some(value)) = (time, value) {- if time < self.time {- self.value = value;- self.time = time;- }+ if let (Some(time), Some(value)) = (time, value)+ && time < self.time+ {+ self.value = value;+ self.time = time;}}diff --git a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rsindex 5b9585170a..168d81fc6b 100644--- a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs@@ -21,7 +21,7 @@ use arrow::array::{Int32Array, RecordBatch, StringArray};use arrow::datatypes::{DataType, Field, Schema};use async_trait::async_trait;use datafusion::prelude::*;-use datafusion_common::{assert_batches_eq, Result};+use datafusion_common::{Result, assert_batches_eq};use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex 2d0e02719c..6d98d6823c 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -70,7 +70,7 @@ use arrow::{use datafusion::execution::session_state::SessionStateBuilder;use datafusion::{common::cast::as_int64_array,- common::{arrow_datafusion_err, DFSchemaRef},+ common::{DFSchemaRef, arrow_datafusion_err},error::{DataFusionError, Result},execution::{context::{QueryPlanner, SessionState, TaskContext},@@ -91,10 +91,10 @@ use datafusion::{};use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{assert_eq_or_internal_err, assert_or_internal_err, ScalarValue};+use datafusion_common::{ScalarValue, assert_eq_or_internal_err, assert_or_internal_err};use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};-use datafusion_optimizer::optimizer::ApplyOrder;use datafusion_optimizer::AnalyzerRule;+use datafusion_optimizer::optimizer::ApplyOrder;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use async_trait::async_trait;@@ -433,21 +433,21 @@ impl OptimizerRule for OptimizerMakeExtensionNodeInvalid {plan: LogicalPlan,_config: &dyn OptimizerConfig,) -> Result<Transformed<LogicalPlan>, DataFusionError> {- if let LogicalPlan::Extension(Extension { node }) = &plan {- if let Some(prev) = node.as_any().downcast_ref::<TopKPlanNode>() {- return Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(TopKPlanNode {- k: prev.k,- input: prev.input.clone(),- expr: prev.expr.clone(),- // In a real use case, this rewriter could have change the number of inputs, etc- invariant_mock: Some(InvariantMock {- should_fail_invariant: true,- kind: InvariantLevel::Always,- }),+ if let LogicalPlan::Extension(Extension { node }) = &plan+ && let Some(prev) = node.as_any().downcast_ref::<TopKPlanNode>()+ {+ return Ok(Transformed::yes(LogicalPlan::Extension(Extension {+ node: Arc::new(TopKPlanNode {+ k: prev.k,+ input: prev.input.clone(),+ expr: prev.expr.clone(),+ // In a real use case, this rewriter could have change the number of inputs, etc+ invariant_mock: Some(InvariantMock {+ should_fail_invariant: true,+ kind: InvariantLevel::Always,}),- })));- }+ }),+ })));};Ok(Transformed::no(plan))@@ -515,23 +515,18 @@ impl OptimizerRule for TopKOptimizerRule {return Ok(Transformed::no(plan));};- if let LogicalPlan::Sort(Sort {- ref expr,- ref input,- ..- }) = limit.input.as_ref()+ if let LogicalPlan::Sort(Sort { expr, input, .. }) = limit.input.as_ref()+ && expr.len() == 1{- if expr.len() == 1 {- // we found a sort with a single sort expr, replace with a a TopK- return Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(TopKPlanNode {- k: fetch,- input: input.as_ref().clone(),- expr: expr[0].clone(),- invariant_mock: self.invariant_mock.clone(),- }),- })));- }+ // we found a sort with a single sort expr, replace with a a TopK+ return Ok(Transformed::yes(LogicalPlan::Extension(Extension {+ node: Arc::new(TopKPlanNode {+ k: fetch,+ input: input.as_ref().clone(),+ expr: expr[0].clone(),+ invariant_mock: self.invariant_mock.clone(),+ }),+ })));}Ok(Transformed::no(plan))diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rsindex 1361091a4c..d0be724326 100644--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs@@ -20,11 +20,11 @@ use std::collections::HashMap;use std::hash::{Hash, Hasher};use std::sync::Arc;-use arrow::array::{as_string_array, create_array, record_batch, Int8Array, UInt64Array};use arrow::array::{- builder::BooleanBuilder, cast::AsArray, Array, ArrayRef, Float32Array, Float64Array,- Int32Array, RecordBatch, StringArray,+ Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,+ builder::BooleanBuilder, cast::AsArray,};+use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch};use arrow::compute::kernels::numeric::add;use arrow::datatypes::{DataType, Field, Schema};use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType};@@ -38,14 +38,15 @@ use datafusion_common::metadata::FieldMetadata;use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_common::utils::take_function_args;use datafusion_common::{- assert_batches_eq, assert_batches_sorted_eq, assert_contains, exec_datafusion_err,- exec_err, not_impl_err, plan_err, DFSchema, DataFusionError, Result, ScalarValue,+ DFSchema, DataFusionError, Result, ScalarValue, assert_batches_eq,+ assert_batches_sorted_eq, assert_contains, exec_datafusion_err, exec_err,+ not_impl_err, plan_err,};use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo};use datafusion_expr::{- lit_with_metadata, Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody,- LogicalPlanBuilder, OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs,- ScalarUDF, ScalarUDFImpl, Signature, Volatility,+ Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, LogicalPlanBuilder,+ OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,+ Signature, Volatility, lit_with_metadata,};use datafusion_expr_common::signature::TypeSignature;use datafusion_functions_nested::range::range_udf;@@ -426,9 +427,10 @@ async fn case_sensitive_identifiers_user_defined_functions() -> Result<()> {let err = plan_and_collect(&ctx, "SELECT MY_FUNC(i) FROM t").await.unwrap_err();- assert!(err- .to_string()- .contains("Error during planning: Invalid function \'my_func\'"));+ assert!(+ err.to_string()+ .contains("Error during planning: Invalid function \'my_func\'")+ );// Can call it if you put quoteslet result = plan_and_collect(&ctx, "SELECT \"MY_FUNC\"(i) FROM t").await?;@@ -1094,10 +1096,11 @@ async fn create_scalar_function_from_sql_statement() -> Result<()> {// Create the `better_add` function dynamically via CREATE FUNCTION statementassert!(ctx.sql(sql).await.is_ok());// try to `drop function` when sql options have allow ddl disabled- assert!(ctx- .sql_with_options("drop function better_add", options)- .await- .is_err());+ assert!(+ ctx.sql_with_options("drop function better_add", options)+ .await+ .is_err()+ );let result = ctx.sql("select better_add(2.0, 2.0)")diff --git a/datafusion/core/tests/user_defined/user_defined_table_functions.rs b/datafusion/core/tests/user_defined/user_defined_table_functions.rsindex 2c6611f382..bc3a472078 100644--- a/datafusion/core/tests/user_defined/user_defined_table_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_table_functions.rs@@ -21,17 +21,17 @@ use std::path::Path;use std::sync::Arc;use arrow::array::Int64Array;-use arrow::csv::reader::Format;use arrow::csv::ReaderBuilder;+use arrow::csv::reader::Format;use datafusion::arrow::datatypes::SchemaRef;use datafusion::arrow::record_batch::RecordBatch;use datafusion::common::test_util::batches_to_string;-use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::TableProvider;+use datafusion::datasource::memory::MemorySourceConfig;use datafusion::error::Result;use datafusion::execution::TaskContext;-use datafusion::physical_plan::{collect, ExecutionPlan};+use datafusion::physical_plan::{ExecutionPlan, collect};use datafusion::prelude::SessionContext;use datafusion_catalog::Session;use datafusion_catalog::TableFunctionImpl;@@ -205,7 +205,7 @@ impl TableFunctionImpl for SimpleCsvTableFunc {let mut filepath = String::new();for expr in exprs {match expr {- Expr::Literal(ScalarValue::Utf8(Some(ref path)), _) => {+ Expr::Literal(ScalarValue::Utf8(Some(path)), _) => {filepath.clone_from(path);}expr => new_exprs.push(expr.clone()),diff --git a/datafusion/core/tests/user_defined/user_defined_window_functions.rs b/datafusion/core/tests/user_defined/user_defined_window_functions.rsindex 33607ebc0d..f62cab523f 100644--- a/datafusion/core/tests/user_defined/user_defined_window_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_window_functions.rs@@ -19,8 +19,8 @@//! user defined window functionsuse arrow::array::{- record_batch, Array, ArrayRef, AsArray, Int64Array, RecordBatch, StringArray,- UInt64Array,+ Array, ArrayRef, AsArray, Int64Array, RecordBatch, StringArray, UInt64Array,+ record_batch,};use arrow::datatypes::{DataType, Field, Schema};use arrow_schema::FieldRef;@@ -38,8 +38,8 @@ use datafusion_functions_window_common::{expr::ExpressionArgs, field::WindowUDFFieldArgs,};use datafusion_physical_expr::{- expressions::{col, lit},PhysicalExpr,+ expressions::{col, lit},};use std::collections::HashMap;use std::hash::{Hash, Hasher};@@ -47,8 +47,8 @@ use std::{any::Any,ops::Range,sync::{- atomic::{AtomicUsize, Ordering},Arc,+ atomic::{AtomicUsize, Ordering},},};@@ -62,8 +62,7 @@ const UNBOUNDED_WINDOW_QUERY_WITH_ALIAS: &str = "SELECT x, y, val, \from t ORDER BY x, y";/// A query with a window function evaluated over a moving window-const BOUNDED_WINDOW_QUERY: &str =- "SELECT x, y, val, \+const BOUNDED_WINDOW_QUERY: &str = "SELECT x, y, val, \odd_counter(val) OVER (PARTITION BY x ORDER BY y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) \from t ORDER BY x, y";@@ -616,7 +615,9 @@ impl PartitionEvaluator for OddCounter {ranks_in_partition: &[Range<usize>],) -> Result<ArrayRef> {self.test_state.inc_evaluate_all_with_rank_called();- println!("evaluate_all_with_rank, values: {num_rows:#?}, ranks_in_partitions: {ranks_in_partition:?}");+ println!(+ "evaluate_all_with_rank, values: {num_rows:#?}, ranks_in_partitions: {ranks_in_partition:?}"+ );// when evaluating with ranks, just return the inverse rank insteadlet array: Int64Array = ranks_in_partition.iter()