Apache DataFusion SQL Query Engine
Update remaining crates to rust 2024 edition (#19361)
Tim Saucer59dcc362ca2b
diff --git a/Cargo.toml b/Cargo.tomlindex 9275f0e1bd..c9afc5fcb5 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -71,7 +71,7 @@ resolver = "2"[workspace.package]authors = ["Apache DataFusion <[email protected]>"]-edition = "2021"+edition = "2024"homepage = "https://datafusion.apache.org"license = "Apache-2.0"readme = "README.md"diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rsindex 027ca07a8b..2bc2bd4458 100644--- a/benchmarks/src/bin/external_aggr.rs+++ b/benchmarks/src/bin/external_aggr.rs@@ -33,16 +33,16 @@ use datafusion::datasource::listing::{};use datafusion::datasource::{MemTable, TableProvider};use datafusion::error::Result;+use datafusion::execution::SessionStateBuilder;use datafusion::execution::memory_pool::FairSpillPool;use datafusion::execution::runtime_env::RuntimeEnvBuilder;-use datafusion::execution::SessionStateBuilder;use datafusion::physical_plan::display::DisplayableExecutionPlan;use datafusion::physical_plan::{collect, displayable};use datafusion::prelude::*;use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult};use datafusion_common::instant::Instant;use datafusion_common::utils::get_available_parallelism;-use datafusion_common::{exec_err, DEFAULT_PARQUET_EXTENSION};+use datafusion_common::{DEFAULT_PARQUET_EXTENSION, exec_err};use datafusion_common::{human_readable_size, units};#[derive(Debug, StructOpt)]diff --git a/benchmarks/src/bin/mem_profile.rs b/benchmarks/src/bin/mem_profile.rsindex 16fc3871be..025efefe06 100644--- a/benchmarks/src/bin/mem_profile.rs+++ b/benchmarks/src/bin/mem_profile.rs@@ -199,21 +199,18 @@ fn run_query(args: &[String], results: &mut Vec<QueryResult>) -> Result<()> {// Look for lines that contain execution time / memory statswhile let Some(line) = iter.next() {- if let Some((query, duration_ms)) = parse_query_time(line) {- if let Some(next_line) = iter.peek() {- if let Some((peak_rss, peak_commit, page_faults)) =- parse_vm_line(next_line)- {- results.push(QueryResult {- query,- duration_ms,- peak_rss,- peak_commit,- page_faults,- });- break;- }- }+ if let Some((query, duration_ms)) = parse_query_time(line)+ && let Some(next_line) = iter.peek()+ && let Some((peak_rss, peak_commit, page_faults)) = parse_vm_line(next_line)+ {+ results.push(QueryResult {+ query,+ duration_ms,+ peak_rss,+ peak_commit,+ page_faults,+ });+ break;}}diff --git a/benchmarks/src/cancellation.rs b/benchmarks/src/cancellation.rsindex fcf03fbc54..1b4c04b409 100644--- a/benchmarks/src/cancellation.rs+++ b/benchmarks/src/cancellation.rs@@ -25,22 +25,22 @@ use arrow::array::Array;use arrow::datatypes::DataType;use arrow::record_batch::RecordBatch;use datafusion::common::{Result, ScalarValue};-use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::file_format::FileFormat;+use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::{ListingOptions, ListingTableUrl};-use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::execution::TaskContext;-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;+use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion::prelude::*;use datafusion_common::instant::Instant;use futures::TryStreamExt;use object_store::ObjectStore;-use parquet::arrow::async_writer::ParquetObjectWriter;use parquet::arrow::AsyncArrowWriter;+use parquet::arrow::async_writer::ParquetObjectWriter;+use rand::Rng;use rand::distr::Alphanumeric;use rand::rngs::ThreadRng;-use rand::Rng;use structopt::StructOpt;use tokio::runtime::Runtime;use tokio_util::sync::CancellationToken;diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rsindex 796b6a4dec..9036e7d950 100644--- a/benchmarks/src/clickbench.rs+++ b/benchmarks/src/clickbench.rs@@ -19,7 +19,7 @@ use std::fs;use std::io::ErrorKind;use std::path::{Path, PathBuf};-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};use datafusion::logical_expr::{ExplainFormat, ExplainOption};use datafusion::{error::{DataFusionError, Result},@@ -156,7 +156,9 @@ impl RunOpt {.any(|opt| opt.contains("prefer_existing_sort=true"));if !has_prefer_sort {- println!("ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true");+ println!(+ "ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true"+ );println!("ℹ️ to optimize queries while maintaining parallelism");}}diff --git a/benchmarks/src/h2o.rs b/benchmarks/src/h2o.rsindex be74252031..07a40447d4 100644--- a/benchmarks/src/h2o.rs+++ b/benchmarks/src/h2o.rs@@ -20,11 +20,11 @@//! - [H2O AI Benchmark](https://duckdb.org/2023/04/14/h2oai.html)//! - [Extended window function benchmark](https://duckdb.org/2024/06/26/benchmarks-over-time.html#window-functions-benchmark)-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt};+use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};use datafusion::logical_expr::{ExplainFormat, ExplainOption};use datafusion::{error::Result, prelude::SessionContext};use datafusion_common::{- exec_datafusion_err, instant::Instant, internal_err, DataFusionError, TableReference,+ DataFusionError, TableReference, exec_datafusion_err, instant::Instant, internal_err,};use std::path::{Path, PathBuf};use structopt::StructOpt;diff --git a/benchmarks/src/hj.rs b/benchmarks/src/hj.rsindex 505b322745..562047f615 100644--- a/benchmarks/src/hj.rs+++ b/benchmarks/src/hj.rs@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};use datafusion::physical_plan::execute_stream;use datafusion::{error::Result, prelude::SessionContext};use datafusion_common::instant::Instant;-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};use structopt::StructOpt;use futures::StreamExt;diff --git a/benchmarks/src/imdb/convert.rs b/benchmarks/src/imdb/convert.rsindex e7949aa715..2c4e127025 100644--- a/benchmarks/src/imdb/convert.rs+++ b/benchmarks/src/imdb/convert.rs@@ -26,8 +26,8 @@ use structopt::StructOpt;use datafusion::common::not_impl_err;-use super::get_imdb_table_schema;use super::IMDB_TABLES;+use super::get_imdb_table_schema;#[derive(Debug, StructOpt)]pub struct ConvertOpt {diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rsindex 11bd424ba6..05f1870c5d 100644--- a/benchmarks/src/imdb/run.rs+++ b/benchmarks/src/imdb/run.rs@@ -19,16 +19,16 @@ use std::path::PathBuf;use std::sync::Arc;use super::{- get_imdb_table_schema, get_query_sql, IMDB_QUERY_END_ID, IMDB_QUERY_START_ID,- IMDB_TABLES,+ IMDB_QUERY_END_ID, IMDB_QUERY_START_ID, IMDB_TABLES, get_imdb_table_schema,+ get_query_sql,};-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};use arrow::record_batch::RecordBatch;use arrow::util::pretty::{self, pretty_format_batches};+use datafusion::datasource::file_format::FileFormat;use datafusion::datasource::file_format::csv::CsvFormat;use datafusion::datasource::file_format::parquet::ParquetFormat;-use datafusion::datasource::file_format::FileFormat;use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};diff --git a/benchmarks/src/nlj.rs b/benchmarks/src/nlj.rsindex 7d1e14f694..cbf5a03fbf 100644--- a/benchmarks/src/nlj.rs+++ b/benchmarks/src/nlj.rs@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};use datafusion::physical_plan::execute_stream;use datafusion::{error::Result, prelude::SessionContext};use datafusion_common::instant::Instant;-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};use structopt::StructOpt;use futures::StreamExt;@@ -268,8 +268,8 @@ impl RunOpt {let elapsed = start.elapsed();println!(- "Query {query_name} iteration {i} returned {row_count} rows in {elapsed:?}"- );+ "Query {query_name} iteration {i} returned {row_count} rows in {elapsed:?}"+ );query_results.push(QueryResult { elapsed, row_count });}diff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rsindex 32a620a12d..53902e0930 100644--- a/benchmarks/src/smj.rs+++ b/benchmarks/src/smj.rs@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};use datafusion::physical_plan::execute_stream;use datafusion::{error::Result, prelude::SessionContext};use datafusion_common::instant::Instant;-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};use structopt::StructOpt;use futures::StreamExt;diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rsindex 09b5a676bb..2f3be76f05 100644--- a/benchmarks/src/sort_tpch.rs+++ b/benchmarks/src/sort_tpch.rs@@ -36,11 +36,11 @@ use datafusion::execution::SessionStateBuilder;use datafusion::physical_plan::display::DisplayableExecutionPlan;use datafusion::physical_plan::{displayable, execute_stream};use datafusion::prelude::*;+use datafusion_common::DEFAULT_PARQUET_EXTENSION;use datafusion_common::instant::Instant;use datafusion_common::utils::get_available_parallelism;-use datafusion_common::DEFAULT_PARQUET_EXTENSION;-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};#[derive(Debug, StructOpt)]pub struct RunOpt {diff --git a/benchmarks/src/tpcds/run.rs b/benchmarks/src/tpcds/run.rsindex 3b7e4ad135..3f579024ba 100644--- a/benchmarks/src/tpcds/run.rs+++ b/benchmarks/src/tpcds/run.rs@@ -19,7 +19,7 @@ use std::fs;use std::path::PathBuf;use std::sync::Arc;-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};use arrow::record_batch::RecordBatch;use arrow::util::pretty::{self, pretty_format_batches};@@ -34,7 +34,7 @@ use datafusion::physical_plan::{collect, displayable};use datafusion::prelude::*;use datafusion_common::instant::Instant;use datafusion_common::utils::get_available_parallelism;-use datafusion_common::{plan_err, DEFAULT_PARQUET_EXTENSION};+use datafusion_common::{DEFAULT_PARQUET_EXTENSION, plan_err};use log::info;use structopt::StructOpt;diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rsindex 84dfc53a5e..65bb9594f0 100644--- a/benchmarks/src/tpch/run.rs+++ b/benchmarks/src/tpch/run.rs@@ -19,16 +19,16 @@ use std::path::PathBuf;use std::sync::Arc;use super::{- get_query_sql, get_tbl_tpch_table_schema, get_tpch_table_schema, TPCH_QUERY_END_ID,- TPCH_QUERY_START_ID, TPCH_TABLES,+ TPCH_QUERY_END_ID, TPCH_QUERY_START_ID, TPCH_TABLES, get_query_sql,+ get_tbl_tpch_table_schema, get_tpch_table_schema,};-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};use arrow::record_batch::RecordBatch;use arrow::util::pretty::{self, pretty_format_batches};+use datafusion::datasource::file_format::FileFormat;use datafusion::datasource::file_format::csv::CsvFormat;use datafusion::datasource::file_format::parquet::ParquetFormat;-use datafusion::datasource::file_format::FileFormat;use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rsindex 6627a287df..b1d5bc99fb 100644--- a/benchmarks/src/util/options.rs+++ b/benchmarks/src/util/options.rs@@ -105,7 +105,7 @@ impl CommonOpt {return Err(DataFusionError::Configuration(format!("Invalid memory pool type: {}",self.mem_pool_type- )))+ )));}};rt_builder = rt_builderdiff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rsindex 764ea648ff..df17674e62 100644--- a/benchmarks/src/util/run.rs+++ b/benchmarks/src/util/run.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use datafusion::{error::Result, DATAFUSION_VERSION};+use datafusion::{DATAFUSION_VERSION, error::Result};use datafusion_common::utils::get_available_parallelism;use serde::{Serialize, Serializer};use serde_json::Value;diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.tomlindex b03327bf13..ea94da1895 100644--- a/datafusion-cli/Cargo.toml+++ b/datafusion-cli/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-cli"description = "Command Line Client for DataFusion query engine."readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rsindex 164c5b892a..7eff0d0b5c 100644--- a/datafusion-examples/examples/builtin_functions/function_factory.rs+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs@@ -19,7 +19,7 @@use arrow::datatypes::DataType;use datafusion::common::tree_node::{Transformed, TreeNode};-use datafusion::common::{exec_datafusion_err, exec_err, internal_err, DataFusionError};+use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err, internal_err};use datafusion::error::Result;use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionContext, SessionState,diff --git a/datafusion-examples/examples/builtin_functions/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rsindex 4c6b6c67ac..e8376cd0c9 100644--- a/datafusion-examples/examples/builtin_functions/regexp.rs+++ b/datafusion-examples/examples/builtin_functions/regexp.rs@@ -134,11 +134,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,assert_batches_eq!(&[- "+---------------------------------------------------+----------------------------------------------------+",- "| regexp_like(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_like(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",- "+---------------------------------------------------+----------------------------------------------------+",- "| true | true |",- "+---------------------------------------------------+----------------------------------------------------+",+ "+---------------------------------------------------+----------------------------------------------------+",+ "| regexp_like(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_like(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",+ "+---------------------------------------------------+----------------------------------------------------+",+ "| true | true |",+ "+---------------------------------------------------+----------------------------------------------------+",],&result);@@ -264,11 +264,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,assert_batches_eq!(&[- "+----------------------------------------------------+-----------------------------------------------------+",- "| regexp_match(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_match(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",- "+----------------------------------------------------+-----------------------------------------------------+",- "| [John Smith] | [Smith Jones] |",- "+----------------------------------------------------+-----------------------------------------------------+",+ "+----------------------------------------------------+-----------------------------------------------------+",+ "| regexp_match(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_match(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",+ "+----------------------------------------------------+-----------------------------------------------------+",+ "| [John Smith] | [Smith Jones] |",+ "+----------------------------------------------------+-----------------------------------------------------+",],&result);@@ -290,21 +290,21 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,assert_batches_eq!(&[- "+---------------------------------------------------------------------------------------------------------+",- "| regexp_replace(examples.values,examples.patterns,examples.replacement,concat(Utf8(\"g\"),examples.flags)) |",- "+---------------------------------------------------------------------------------------------------------+",- "| bbabbbc |",- "| B |",- "| aec |",- "| AbC |",- "| aBC |",- "| 4000 |",- "| xyz |",- "| München |",- "| Moscow |",- "| Koln |",- "| Today |",- "+---------------------------------------------------------------------------------------------------------+",+ "+---------------------------------------------------------------------------------------------------------+",+ "| regexp_replace(examples.values,examples.patterns,examples.replacement,concat(Utf8(\"g\"),examples.flags)) |",+ "+---------------------------------------------------------------------------------------------------------+",+ "| bbabbbc |",+ "| B |",+ "| aec |",+ "| AbC |",+ "| aBC |",+ "| 4000 |",+ "| xyz |",+ "| München |",+ "| Moscow |",+ "| Koln |",+ "| Today |",+ "+---------------------------------------------------------------------------------------------------------+",],&result);@@ -318,11 +318,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,assert_batches_eq!(&[- "+------------------------------------------------------------------------+",- "| regexp_replace(Utf8(\"foobarbaz\"),Utf8(\"b(..)\"),Utf8(\"X\\1Y\"),Utf8(\"g\")) |",- "+------------------------------------------------------------------------+",- "| fooXarYXazY |",- "+------------------------------------------------------------------------+",+ "+------------------------------------------------------------------------+",+ "| regexp_replace(Utf8(\"foobarbaz\"),Utf8(\"b(..)\"),Utf8(\"X\\1Y\"),Utf8(\"g\")) |",+ "+------------------------------------------------------------------------+",+ "| fooXarYXazY |",+ "+------------------------------------------------------------------------+",],&result);diff --git a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs b/datafusion-examples/examples/custom_data_source/csv_json_opener.rsindex 48b885839f..7b2e321362 100644--- a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs+++ b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs@@ -36,7 +36,7 @@ use datafusion::{use datafusion::datasource::physical_plan::FileScanConfigBuilder;use futures::StreamExt;-use object_store::{local::LocalFileSystem, memory::InMemory, ObjectStore};+use object_store::{ObjectStore, local::LocalFileSystem, memory::InMemory};/// This example demonstrates using the low level [`FileStream`] / [`FileOpener`] APIs to directly/// read data from (CSV/JSON) into Arrow RecordBatches.diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex ebb6c7a822..b276ae32cf 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -24,10 +24,10 @@ use std::sync::{Arc, Mutex};use std::time::Duration;use async_trait::async_trait;-use datafusion::arrow::array::{UInt64Builder, UInt8Builder};+use datafusion::arrow::array::{UInt8Builder, UInt64Builder};use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::arrow::record_batch::RecordBatch;-use datafusion::datasource::{provider_as_source, TableProvider, TableType};+use datafusion::datasource::{TableProvider, TableType, provider_as_source};use datafusion::error::Result;use datafusion::execution::context::TaskContext;use datafusion::logical_expr::LogicalPlanBuilder;@@ -35,8 +35,8 @@ use datafusion::physical_expr::EquivalenceProperties;use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion::physical_plan::memory::MemoryStream;use datafusion::physical_plan::{- project_schema, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,- PlanProperties, SendableRecordBatchStream,+ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,+ SendableRecordBatchStream, project_schema,};use datafusion::prelude::*;diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rsindex 80b88f7a38..895b6f52b6 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs@@ -19,21 +19,21 @@use std::sync::Arc;-use arrow::array::{record_batch, RecordBatch};+use arrow::array::{RecordBatch, record_batch};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::assert_batches_eq;+use datafusion::common::Result;use datafusion::common::not_impl_err;use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion::common::Result;use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};use datafusion::execution::context::SessionContext;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::parquet::arrow::ArrowWriter;-use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};use datafusion::physical_expr::PhysicalExpr;+use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};use datafusion::prelude::SessionConfig;use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,diff --git a/datafusion-examples/examples/custom_data_source/custom_file_format.rs b/datafusion-examples/examples/custom_data_source/custom_file_format.rsindex 05a34d3a83..6817beec41 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_format.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_format.rs@@ -27,13 +27,13 @@ use datafusion::{catalog::Session,common::{GetExt, Statistics},datasource::{+ MemTable,file_format::{- csv::CsvFormatFactory, file_compression_type::FileCompressionType,- FileFormat, FileFormatFactory,+ FileFormat, FileFormatFactory, csv::CsvFormatFactory,+ file_compression_type::FileCompressionType,},physical_plan::{FileScanConfig, FileSinkConfig, FileSource},table_schema::TableSchema,- MemTable,},error::Result,execution::session_state::SessionStateBuilder,diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rsindex 7381dc938a..227cea5e7c 100644--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -40,10 +40,10 @@ use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::properties::WriterProperties;use datafusion::physical_expr::PhysicalExpr;use datafusion::physical_plan::ExecutionPlan;-use datafusion::prelude::{lit, SessionConfig};+use datafusion::prelude::{SessionConfig, lit};use datafusion_physical_expr_adapter::{- replace_columns_with_literals, DefaultPhysicalExprAdapterFactory,- PhysicalExprAdapter, PhysicalExprAdapterFactory,+ DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,+ replace_columns_with_literals,};use futures::StreamExt;use object_store::memory::InMemory;@@ -136,8 +136,12 @@ pub async fn default_column_values() -> Result<()> {println!("\n=== Key Insight ===");println!("This example demonstrates how PhysicalExprAdapter works:");println!("1. Physical schema only has 'id' and 'name' columns");- println!("2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults");- println!("3. Our custom adapter uses replace_columns_with_literals to inject default values");+ println!(+ "2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults"+ );+ println!(+ "3. Our custom adapter uses replace_columns_with_literals to inject default values"+ );println!("4. Default values from metadata are cast to proper types at planning time");println!("5. The DefaultPhysicalExprAdapter handles other schema adaptations");println!("\nNote: PhysicalExprAdapter is specifically for filter predicates.");diff --git a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs b/datafusion-examples/examples/custom_data_source/file_stream_provider.rsindex b1f471b5b3..936da0a33d 100644--- a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs+++ b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs@@ -47,8 +47,8 @@ mod non_windows {use std::fs::{File, OpenOptions};use std::io::Write;use std::path::PathBuf;- use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;+ use std::sync::atomic::{AtomicBool, Ordering};use std::thread;use std::time::Duration;@@ -59,9 +59,9 @@ mod non_windows {use tempfile::TempDir;use tokio::task::JoinSet;- use datafusion::common::{exec_err, Result};- use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};+ use datafusion::common::{Result, exec_err};use datafusion::datasource::TableProvider;+ use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use datafusion::logical_expr::SortExpr;use datafusion::prelude::{SessionConfig, SessionContext};diff --git a/datafusion-examples/examples/data_io/catalog.rs b/datafusion-examples/examples/data_io/catalog.rsindex e68e48bdea..d2ddff82e3 100644--- a/datafusion-examples/examples/data_io/catalog.rs+++ b/datafusion-examples/examples/data_io/catalog.rs@@ -23,9 +23,9 @@ use datafusion::{arrow::util::pretty,catalog::{CatalogProvider, CatalogProviderList, SchemaProvider},datasource::{- file_format::{csv::CsvFormat, FileFormat},- listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},TableProvider,+ file_format::{FileFormat, csv::CsvFormat},+ listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},},error::Result,execution::context::SessionState,diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rsindex a4b0de9301..d2ffacc946 100644--- a/datafusion-examples/examples/data_io/json_shredding.rs+++ b/datafusion-examples/examples/data_io/json_shredding.rs@@ -27,7 +27,7 @@ use datafusion::assert_batches_eq;use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};-use datafusion::common::{assert_contains, exec_datafusion_err, Result};+use datafusion::common::{Result, assert_contains, exec_datafusion_err};use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};@@ -39,7 +39,7 @@ use datafusion::logical_expr::{use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::properties::WriterProperties;use datafusion::physical_expr::PhysicalExpr;-use datafusion::physical_expr::{expressions, ScalarFunctionExpr};+use datafusion::physical_expr::{ScalarFunctionExpr, expressions};use datafusion::prelude::SessionConfig;use datafusion::scalar::ScalarValue;use datafusion_physical_expr_adapter::{@@ -233,7 +233,7 @@ impl ScalarUDFImpl for JsonGetStr {_ => {return Err(exec_datafusion_err!("json_get_str first argument must be a string"- ))+ ));}};// We expect a string array that contains JSON strings@@ -249,7 +249,7 @@ impl ScalarUDFImpl for JsonGetStr {_ => {return Err(exec_datafusion_err!("json_get_str second argument must be a string array"- ))+ ));}};let values = json_array@@ -316,44 +316,43 @@ impl ShreddedJsonRewriter {expr: Arc<dyn PhysicalExpr>,physical_file_schema: &Schema,) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {- if let Some(func) = expr.as_any().downcast_ref::<ScalarFunctionExpr>() {- if func.name() == "json_get_str" && func.args().len() == 2 {- // Get the key from the first argument- if let Some(literal) = func.args()[0]+ if let Some(func) = expr.as_any().downcast_ref::<ScalarFunctionExpr>()+ && func.name() == "json_get_str"+ && func.args().len() == 2+ {+ // Get the key from the first argument+ if let Some(literal) = func.args()[0]+ .as_any()+ .downcast_ref::<expressions::Literal>()+ && let ScalarValue::Utf8(Some(field_name)) = literal.value()+ {+ // Get the column from the second argument+ if let Some(column) = func.args()[1].as_any()- .downcast_ref::<expressions::Literal>()+ .downcast_ref::<expressions::Column>(){- if let ScalarValue::Utf8(Some(field_name)) = literal.value() {- // Get the column from the second argument- if let Some(column) = func.args()[1]- .as_any()- .downcast_ref::<expressions::Column>()- {- let column_name = column.name();- // Check if there's a flat column with underscore prefix- let flat_column_name = format!("_{column_name}.{field_name}");-- if let Ok(flat_field_index) =- physical_file_schema.index_of(&flat_column_name)- {- let flat_field =- physical_file_schema.field(flat_field_index);-- if flat_field.data_type() == &DataType::Utf8 {- // Replace the whole expression with a direct column reference- let new_expr = Arc::new(expressions::Column::new(- &flat_column_name,- flat_field_index,- ))- as Arc<dyn PhysicalExpr>;-- return Ok(Transformed {- data: new_expr,- tnr: TreeNodeRecursion::Stop,- transformed: true,- });- }- }+ let column_name = column.name();+ // Check if there's a flat column with underscore prefix+ let flat_column_name = format!("_{column_name}.{field_name}");++ if let Ok(flat_field_index) =+ physical_file_schema.index_of(&flat_column_name)+ {+ let flat_field = physical_file_schema.field(flat_field_index);++ if flat_field.data_type() == &DataType::Utf8 {+ // Replace the whole expression with a direct column reference+ let new_expr = Arc::new(expressions::Column::new(+ &flat_column_name,+ flat_field_index,+ ))+ as Arc<dyn PhysicalExpr>;++ return Ok(Transformed {+ data: new_expr,+ tnr: TreeNodeRecursion::Stop,+ transformed: true,+ });}}}diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rsindex caa3be2111..3f4ebe7a92 100644--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs@@ -22,35 +22,35 @@ use std::collections::{HashMap, HashSet};use std::fs::File;use std::ops::Range;use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};use datafusion::catalog::Session;use datafusion::common::{- internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,+ DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err,};+use datafusion::datasource::TableProvider;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::physical_plan::parquet::ParquetAccessPlan;use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetFileReaderFactory, ParquetSource,};-use datafusion::datasource::TableProvider;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::logical_expr::utils::conjunction;use datafusion::logical_expr::{TableProviderFilterPushDown, TableType};+use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelector,};use datafusion::parquet::arrow::async_reader::{AsyncFileReader, ParquetObjectReader};-use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::metadata::ParquetMetaData;use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties};use datafusion::parquet::schema::types::ColumnPath;-use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};use datafusion::physical_expr::PhysicalExpr;+use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};use datafusion::physical_optimizer::pruning::PruningPredicate;-use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;use datafusion::prelude::*;use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};@@ -58,8 +58,8 @@ use arrow::datatypes::SchemaRef;use async_trait::async_trait;use bytes::Bytes;use datafusion::datasource::memory::DataSourceExec;-use futures::future::BoxFuture;use futures::FutureExt;+use futures::future::BoxFuture;use object_store::ObjectStore;use tempfile::TempDir;use url::Url;diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rsindex 88a052546b..bcaca2ed5c 100644--- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs@@ -118,11 +118,11 @@ use arrow::record_batch::RecordBatch;use arrow_schema::{DataType, Field, Schema, SchemaRef};use async_trait::async_trait;use datafusion::catalog::{Session, TableProvider};-use datafusion::common::{exec_err, HashMap, HashSet, Result};+use datafusion::common::{HashMap, HashSet, Result, exec_err};+use datafusion::datasource::TableType;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::memory::DataSourceExec;use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetSource};-use datafusion::datasource::TableType;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::logical_expr::{Operator, TableProviderFilterPushDown};use datafusion::parquet::arrow::ArrowWriter;@@ -132,7 +132,7 @@ use datafusion::parquet::file::reader::{FileReader, SerializedFileReader};use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::*;use datafusion::scalar::ScalarValue;-use std::fs::{read_dir, File};+use std::fs::{File, read_dir};use std::io::{Read, Seek, SeekFrom, Write};use std::path::{Path, PathBuf};use std::sync::Arc;@@ -419,21 +419,15 @@ impl TableProvider for DistinctIndexTable {// equality analysis or write your own custom logic.let mut target: Option<&str> = None;- if filters.len() == 1 {- if let Expr::BinaryExpr(expr) = &filters[0] {- if expr.op == Operator::Eq {- if let (- Expr::Column(c),- Expr::Literal(ScalarValue::Utf8(Some(v)), _),- ) = (&*expr.left, &*expr.right)- {- if c.name == "category" {- println!("Filtering for category: {v}");- target = Some(v);- }- }- }- }+ if filters.len() == 1+ && let Expr::BinaryExpr(expr) = &filters[0]+ && expr.op == Operator::Eq+ && let (Expr::Column(c), Expr::Literal(ScalarValue::Utf8(Some(v)), _)) =+ (&*expr.left, &*expr.right)+ && c.name == "category"+ {+ println!("Filtering for category: {v}");+ target = Some(v);}// Determine which files to scanlet files_to_scan: Vec<_> = selfdiff --git a/datafusion-examples/examples/data_io/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rsindex e3070cddde..f88ab91321 100644--- a/datafusion-examples/examples/data_io/parquet_encrypted.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs@@ -75,7 +75,9 @@ pub async fn parquet_encrypted() -> datafusion::common::Result<()> {let encrypted_parquet_df = ctx.read_parquet(tempfile_str, read_options).await?;// Show information from the dataframe- println!("\n\n===============================================================================");+ println!(+ "\n\n==============================================================================="+ );println!("Encrypted Parquet DataFrame:");query_dataframe(&encrypted_parquet_df).await?;diff --git a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs b/datafusion-examples/examples/data_io/parquet_exec_visitor.rsindex 925c202eac..d38fe9e171 100644--- a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs+++ b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs@@ -27,7 +27,7 @@ use datafusion::error::DataFusionError;use datafusion::execution::context::SessionContext;use datafusion::physical_plan::metrics::MetricValue;use datafusion::physical_plan::{- execute_stream, visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor,+ ExecutionPlan, ExecutionPlanVisitor, execute_stream, visit_execution_plan,};use futures::StreamExt;@@ -100,18 +100,17 @@ impl ExecutionPlanVisitor for ParquetExecVisitor {/// or `post_visit` (visit each node after its children/inputs)fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {// If needed match on a specific `ExecutionPlan` node type- if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {- if let Some((file_config, _)) =+ if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && let Some((file_config, _)) =data_source_exec.downcast_to_file_source::<ParquetSource>()- {- self.file_groups = Some(file_config.file_groups.clone());-- let metrics = match data_source_exec.metrics() {- None => return Ok(true),- Some(metrics) => metrics,- };- self.bytes_scanned = metrics.sum_by_name("bytes_scanned");- }+ {+ self.file_groups = Some(file_config.file_groups.clone());++ let metrics = match data_source_exec.metrics() {+ None => return Ok(true),+ Some(metrics) => metrics,+ };+ self.bytes_scanned = metrics.sum_by_name("bytes_scanned");}Ok(true)}diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rsindex 5f6fe3e297..2afc6c1047 100644--- a/datafusion-examples/examples/data_io/parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_index.rs@@ -27,19 +27,19 @@ use async_trait::async_trait;use datafusion::catalog::Session;use datafusion::common::pruning::PruningStatistics;use datafusion::common::{- internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,+ DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err,};+use datafusion::datasource::TableProvider;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::memory::DataSourceExec;use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetSource};-use datafusion::datasource::TableProvider;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::logical_expr::{- utils::conjunction, TableProviderFilterPushDown, TableType,+ TableProviderFilterPushDown, TableType, utils::conjunction,};use datafusion::parquet::arrow::arrow_reader::statistics::StatisticsConverter;use datafusion::parquet::arrow::{- arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter,+ ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder,};use datafusion::physical_expr::PhysicalExpr;use datafusion::physical_optimizer::pruning::PruningPredicate;@@ -52,8 +52,8 @@ use std::fs;use std::fs::{DirEntry, File};use std::ops::Range;use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};use tempfile::TempDir;use url::Url;diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rsindex 3e1568fb41..10ec26b1d5 100644--- a/datafusion-examples/examples/data_io/remote_catalog.rs+++ b/datafusion-examples/examples/data_io/remote_catalog.rs@@ -41,8 +41,8 @@ use datafusion::common::{assert_batches_eq, internal_datafusion_err, plan_err};use datafusion::datasource::memory::MemorySourceConfig;use datafusion::execution::SendableRecordBatchStream;use datafusion::logical_expr::{Expr, TableType};-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;use datafusion::prelude::{DataFrame, SessionContext};use futures::TryStreamExt;use std::any::Any;diff --git a/datafusion-examples/examples/dataframe/dataframe.rs b/datafusion-examples/examples/dataframe/dataframe.rsindex 6953170191..94653e80c8 100644--- a/datafusion-examples/examples/dataframe/dataframe.rs+++ b/datafusion-examples/examples/dataframe/dataframe.rs@@ -20,18 +20,18 @@use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StringViewArray};use datafusion::arrow::datatypes::{DataType, Field, Schema};use datafusion::catalog::MemTable;+use datafusion::common::ScalarValue;use datafusion::common::config::CsvOptions;use datafusion::common::parsers::CompressionTypeVariant;-use datafusion::common::ScalarValue;use datafusion::dataframe::DataFrameWriteOptions;use datafusion::error::Result;use datafusion::functions_aggregate::average::avg;use datafusion::functions_aggregate::min_max::max;use datafusion::prelude::*;-use std::fs::{create_dir_all, File};+use std::fs::{File, create_dir_all};use std::io::Write;use std::sync::Arc;-use tempfile::{tempdir, TempDir};+use tempfile::{TempDir, tempdir};/// This example demonstrates using DataFusion's DataFrame API///diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rsindex d179271246..48475acbb1 100644--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs@@ -30,7 +30,7 @@ use arrow::record_batch::RecordBatch;use arrow_schema::SchemaRef;use datafusion::common::record_batch;use datafusion::common::{exec_datafusion_err, internal_err};-use datafusion::datasource::{memory::MemTable, DefaultTableSource};+use datafusion::datasource::{DefaultTableSource, memory::MemTable};use datafusion::error::Result;use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};use datafusion::execution::runtime_env::RuntimeEnvBuilder;diff --git a/datafusion-examples/examples/execution_monitoring/tracing.rs b/datafusion-examples/examples/execution_monitoring/tracing.rsindex f0e5a03715..5fa759f2d5 100644--- a/datafusion-examples/examples/execution_monitoring/tracing.rs+++ b/datafusion-examples/examples/execution_monitoring/tracing.rs@@ -51,17 +51,17 @@//! 10:29:40.809 INFO main ThreadId(01) tracing: ***** WITH tracer: Non-main tasks DID inherit the `run_instrumented_query` span *****//! ```-use datafusion::common::runtime::{set_join_set_tracer, JoinSetTracer};+use datafusion::common::runtime::{JoinSetTracer, set_join_set_tracer};use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::ListingOptions;use datafusion::error::Result;use datafusion::prelude::*;use datafusion::test_util::parquet_test_data;-use futures::future::BoxFuture;use futures::FutureExt;+use futures::future::BoxFuture;use std::any::Any;use std::sync::Arc;-use tracing::{info, instrument, Instrument, Level, Span};+use tracing::{Instrument, Level, Span, info, instrument};/// Demonstrates the tracing injection feature for the DataFusion runtimepub async fn tracing() -> Result<()> {@@ -75,7 +75,9 @@ pub async fn tracing() -> Result<()> {// Run query WITHOUT tracer injection.info!("***** RUNNING WITHOUT INJECTED TRACER *****");run_instrumented_query().await?;- info!("***** WITHOUT tracer: `tokio-runtime-worker` tasks did NOT inherit the `run_instrumented_query` span *****");+ info!(+ "***** WITHOUT tracer: `tokio-runtime-worker` tasks did NOT inherit the `run_instrumented_query` span *****"+ );// Inject custom tracer so tasks run in the current span.info!("Injecting custom tracer...");@@ -84,7 +86,9 @@ pub async fn tracing() -> Result<()> {// Run query WITH tracer injection.info!("***** RUNNING WITH INJECTED TRACER *****");run_instrumented_query().await?;- info!("***** WITH tracer: `tokio-runtime-worker` tasks DID inherit the `run_instrumented_query` span *****");+ info!(+ "***** WITH tracer: `tokio-runtime-worker` tasks DID inherit the `run_instrumented_query` span *****"+ );Ok(())}diff --git a/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs b/datafusion-examples/examples/external_dependency/dataframe_to_s3.rsindex d7fb17ba57..fdb8a3c9c0 100644--- a/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs+++ b/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs@@ -21,8 +21,8 @@ use std::env;use std::sync::Arc;use datafusion::dataframe::DataFrameWriteOptions;-use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::file_format::FileFormat;+use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::ListingOptions;use datafusion::error::Result;use datafusion::prelude::*;diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.tomlindex 612a219324..f393b2971e 100644--- a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml+++ b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml@@ -18,7 +18,7 @@[package]name = "ffi_module_interface"version = "0.1.0"-edition = "2021"+edition = "2024"publish = false[dependencies]diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rsindex 88690e9297..1166eeb707 100644--- a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs+++ b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs@@ -16,11 +16,10 @@// under the License.use abi_stable::{- declare_root_module_statics,+ StableAbi, declare_root_module_statics,library::{LibraryError, RootModule},package_version_strings,sabi_types::VersionStrings,- StableAbi,};use datafusion_ffi::table_provider::FFI_TableProvider;diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.tomlindex 1f68bb1bb1..c60cd7c029 100644--- a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml+++ b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml@@ -18,7 +18,7 @@[package]name = "ffi_module_loader"version = "0.1.0"-edition = "2021"+edition = "2024"publish = false[dependencies]diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rsindex 170c7c3eff..d76af96058 100644--- a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs+++ b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs@@ -22,7 +22,7 @@ use datafusion::{prelude::SessionContext,};-use abi_stable::library::{development_utils::compute_library_path, RootModule};+use abi_stable::library::{RootModule, development_utils::compute_library_path};use datafusion::datasource::TableProvider;use ffi_module_interface::TableProviderModuleRef;diff --git a/datafusion-examples/examples/flight/server.rs b/datafusion-examples/examples/flight/server.rsindex 7f348e7d59..aad82e28b1 100644--- a/datafusion-examples/examples/flight/server.rs+++ b/datafusion-examples/examples/flight/server.rs@@ -31,9 +31,9 @@ use tonic::{Request, Response, Status, Streaming};use datafusion::prelude::*;use arrow_flight::{- flight_service_server::FlightService, flight_service_server::FlightServiceServer,Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo,HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, Ticket,+ flight_service_server::FlightService, flight_service_server::FlightServiceServer,};#[derive(Clone)]diff --git a/datafusion-examples/examples/flight/sql_server.rs b/datafusion-examples/examples/flight/sql_server.rsindex 529296fac8..435e05ffc0 100644--- a/datafusion-examples/examples/flight/sql_server.rs+++ b/datafusion-examples/examples/flight/sql_server.rs@@ -416,7 +416,9 @@ impl FlightSqlService for FlightSqlServiceImpl {) -> Result<(), Status> {let handle = std::str::from_utf8(&handle.prepared_statement_handle);if let Ok(handle) = handle {- info!("do_action_close_prepared_statement: removing plan and results for {handle}");+ info!(+ "do_action_close_prepared_statement: removing plan and results for {handle}"+ );let _ = self.remove_plan(handle);let _ = self.remove_result(handle);}diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rsindex 043f548b3b..f3910d461b 100644--- a/datafusion-examples/examples/proto/composed_extension_codec.rs+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs@@ -36,8 +36,8 @@ use std::any::Any;use std::fmt::Debug;use std::sync::Arc;-use datafusion::common::internal_err;use datafusion::common::Result;+use datafusion::common::internal_err;use datafusion::execution::TaskContext;use datafusion::physical_plan::{DisplayAs, ExecutionPlan};use datafusion::prelude::SessionContext;diff --git a/datafusion-examples/examples/query_planning/analyzer_rule.rs b/datafusion-examples/examples/query_planning/analyzer_rule.rsindex df1ea4c1d7..a86f5cdd2a 100644--- a/datafusion-examples/examples/query_planning/analyzer_rule.rs+++ b/datafusion-examples/examples/query_planning/analyzer_rule.rs@@ -18,10 +18,10 @@//! See `main.rs` for how to run it.use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};+use datafusion::common::Result;use datafusion::common::config::ConfigOptions;use datafusion::common::tree_node::{Transformed, TreeNode};-use datafusion::common::Result;-use datafusion::logical_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder};+use datafusion::logical_expr::{Expr, LogicalPlan, LogicalPlanBuilder, col, lit};use datafusion::optimizer::analyzer::AnalyzerRule;use datafusion::prelude::SessionContext;use std::sync::{Arc, Mutex};diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rsindex 47066f2ba2..47de669023 100644--- a/datafusion-examples/examples/query_planning/expr_api.rs+++ b/datafusion-examples/examples/query_planning/expr_api.rs@@ -20,7 +20,7 @@use std::collections::HashMap;use std::sync::Arc;-use arrow::array::{BooleanArray, Int32Array, Int8Array};+use arrow::array::{BooleanArray, Int8Array, Int32Array};use arrow::record_batch::RecordBatch;use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit};@@ -37,7 +37,7 @@ use datafusion::logical_expr::simplify::SimplifyContext;use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator};use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter;use datafusion::optimizer::simplify_expressions::ExprSimplifier;-use datafusion::physical_expr::{analyze, AnalysisContext, ExprBoundaries};+use datafusion::physical_expr::{AnalysisContext, ExprBoundaries, analyze};use datafusion::prelude::*;/// This example demonstrates the DataFusion [`Expr`] API.@@ -344,9 +344,11 @@ fn boundary_analysis_and_selectivity_demo() -> Result<()> {//// (a' - b' + 1) / (a - b)// (10000 - 5000 + 1) / (10000 - 1)- assert!(analysis- .selectivity- .is_some_and(|selectivity| (0.5..=0.6).contains(&selectivity)));+ assert!(+ analysis+ .selectivity+ .is_some_and(|selectivity| (0.5..=0.6).contains(&selectivity))+ );Ok(())}@@ -417,9 +419,11 @@ fn boundary_analysis_in_conjunctions_demo() -> Result<()> {//// Granted a column such as age will more likely follow a Normal distribution// as such our selectivity estimation will not be as good as it can.- assert!(analysis- .selectivity- .is_some_and(|selectivity| (0.1..=0.2).contains(&selectivity)));+ assert!(+ analysis+ .selectivity+ .is_some_and(|selectivity| (0.1..=0.2).contains(&selectivity))+ );// The above example was a good way to look at how we can derive better// interval and get a lower selectivity during boundary analysis.@@ -535,10 +539,11 @@ fn type_coercion_demo() -> Result<()> {let physical_expr =datafusion::physical_expr::create_physical_expr(&expr, &df_schema, &props)?;let e = physical_expr.evaluate(&batch).unwrap_err();- assert!(e- .find_root()- .to_string()- .contains("Invalid comparison operation: Int8 > Int32"));+ assert!(+ e.find_root()+ .to_string()+ .contains("Invalid comparison operation: Int8 > Int32")+ );// 1. Type coercion with `SessionContext::create_physical_expr` which implicitly applies type coercion before constructing the physical expr.let physical_expr =diff --git a/datafusion-examples/examples/query_planning/optimizer_rule.rs b/datafusion-examples/examples/query_planning/optimizer_rule.rsindex 34aef85c64..de9de7737a 100644--- a/datafusion-examples/examples/query_planning/optimizer_rule.rs+++ b/datafusion-examples/examples/query_planning/optimizer_rule.rs@@ -20,7 +20,7 @@use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};use arrow::datatypes::DataType;use datafusion::common::tree_node::{Transformed, TreeNode};-use datafusion::common::{assert_batches_eq, Result, ScalarValue};+use datafusion::common::{Result, ScalarValue, assert_batches_eq};use datafusion::logical_expr::{BinaryExpr, ColumnarValue, Expr, LogicalPlan, Operator, ScalarFunctionArgs,ScalarUDF, ScalarUDFImpl, Signature, Volatility,diff --git a/datafusion-examples/examples/query_planning/plan_to_sql.rs b/datafusion-examples/examples/query_planning/plan_to_sql.rsindex 95e04907d9..756cc80b8f 100644--- a/datafusion-examples/examples/query_planning/plan_to_sql.rs+++ b/datafusion-examples/examples/query_planning/plan_to_sql.rs@@ -34,7 +34,7 @@ use datafusion::sql::unparser::extension_unparser::UserDefinedLogicalNodeUnparseuse datafusion::sql::unparser::extension_unparser::{UnparseToStatementResult, UnparseWithinStatementResult,};-use datafusion::sql::unparser::{plan_to_sql, Unparser};+use datafusion::sql::unparser::{Unparser, plan_to_sql};use std::fmt;use std::sync::Arc;diff --git a/datafusion-examples/examples/relation_planner/match_recognize.rs b/datafusion-examples/examples/relation_planner/match_recognize.rsindex 7eeddb1f97..60baf9bd61 100644--- a/datafusion-examples/examples/relation_planner/match_recognize.rs+++ b/datafusion-examples/examples/relation_planner/match_recognize.rs@@ -77,11 +77,11 @@ use arrow::record_batch::RecordBatch;use datafusion::prelude::*;use datafusion_common::{DFSchemaRef, Result};use datafusion_expr::{+ Expr, UserDefinedLogicalNode,logical_plan::{Extension, InvariantLevel, LogicalPlan},planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,},- Expr, UserDefinedLogicalNode,};use datafusion_sql::sqlparser::ast::TableFactor;use insta::assert_snapshot;diff --git a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs b/datafusion-examples/examples/relation_planner/pivot_unpivot.rsindex 64158bb384..86a6cb9555 100644--- a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs+++ b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs@@ -65,14 +65,13 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Int64Array, StringArray};use arrow::record_batch::RecordBatch;use datafusion::prelude::*;-use datafusion_common::{plan_datafusion_err, Result, ScalarValue};+use datafusion_common::{Result, ScalarValue, plan_datafusion_err};use datafusion_expr::{- case, col, lit,+ Expr, case, col, lit,logical_plan::builder::LogicalPlanBuilder,planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,},- Expr,};use datafusion_sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};use insta::assert_snapshot;diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex a3ae5396ed..207fffe132 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -99,32 +99,32 @@ use futures::{ready,stream::{Stream, StreamExt},};-use rand::{rngs::StdRng, Rng, SeedableRng};+use rand::{Rng, SeedableRng, rngs::StdRng};use tonic::async_trait;use datafusion::{execution::{- context::QueryPlanner, RecordBatchStream, SendableRecordBatchStream,- SessionState, SessionStateBuilder, TaskContext,+ RecordBatchStream, SendableRecordBatchStream, SessionState, SessionStateBuilder,+ TaskContext, context::QueryPlanner,},physical_expr::EquivalenceProperties,physical_plan::{- metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput},DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput},},physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner},prelude::*,};use datafusion_common::{- internal_err, not_impl_err, plan_datafusion_err, plan_err, DFSchemaRef,- DataFusionError, Result, Statistics,+ DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,+ plan_datafusion_err, plan_err,};use datafusion_expr::{+ UserDefinedLogicalNode, UserDefinedLogicalNodeCore,logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,},- UserDefinedLogicalNode, UserDefinedLogicalNodeCore,};use datafusion_sql::sqlparser::ast::{self, TableFactor, TableSampleMethod, TableSampleUnit,@@ -341,15 +341,14 @@ impl RelationPlanner for TableSamplePlanner {};// Validate sampling method- if let Some(method) = &sample.name {- if *method != TableSampleMethod::Bernoulli- && *method != TableSampleMethod::Row- {- return not_impl_err!(- "Sampling method {} is not supported (only BERNOULLI and ROW)",- method- );- }+ if let Some(method) = &sample.name+ && *method != TableSampleMethod::Bernoulli+ && *method != TableSampleMethod::Row+ {+ return not_impl_err!(+ "Sampling method {} is not supported (only BERNOULLI and ROW)",+ method+ );}// Offset sampling (ClickHouse-style) not supporteddiff --git a/datafusion-examples/examples/sql_ops/analysis.rs b/datafusion-examples/examples/sql_ops/analysis.rsindex 266c650772..4243a29278 100644--- a/datafusion-examples/examples/sql_ops/analysis.rs+++ b/datafusion-examples/examples/sql_ops/analysis.rs@@ -25,8 +25,8 @@use std::sync::Arc;-use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion::common::Result;+use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion::logical_expr::LogicalPlan;use datafusion::{datasource::MemTable,diff --git a/datafusion-examples/examples/sql_ops/frontend.rs b/datafusion-examples/examples/sql_ops/frontend.rsindex 2cfb9de05c..025fe47e75 100644--- a/datafusion-examples/examples/sql_ops/frontend.rs+++ b/datafusion-examples/examples/sql_ops/frontend.rs@@ -18,7 +18,7 @@//! See `main.rs` for how to run it.use arrow::datatypes::{DataType, Field, Schema, SchemaRef};-use datafusion::common::{plan_err, TableReference};+use datafusion::common::{TableReference, plan_err};use datafusion::config::ConfigOptions;use datafusion::error::Result;use datafusion::logical_expr::{diff --git a/datafusion-examples/examples/sql_ops/query.rs b/datafusion-examples/examples/sql_ops/query.rsindex 3e052f0823..90d0c3ca34 100644--- a/datafusion-examples/examples/sql_ops/query.rs+++ b/datafusion-examples/examples/sql_ops/query.rs@@ -17,13 +17,13 @@//! See `main.rs` for how to run it.-use datafusion::arrow::array::{UInt64Array, UInt8Array};+use datafusion::arrow::array::{UInt8Array, UInt64Array};use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::arrow::record_batch::RecordBatch;use datafusion::common::{assert_batches_eq, exec_datafusion_err};+use datafusion::datasource::MemTable;use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::ListingOptions;-use datafusion::datasource::MemTable;use datafusion::error::{DataFusionError, Result};use datafusion::prelude::*;use object_store::local::LocalFileSystem;@@ -153,7 +153,8 @@ async fn query_parquet() -> Result<()> {"| 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |","+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",],- &results);+ &results+ );// Second example were we temporarily move into the test data's parent directory and// simulate a relative path, this requires registering an ObjectStore.@@ -202,7 +203,8 @@ async fn query_parquet() -> Result<()> {"| 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |","+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",],- &results);+ &results+ );// Reset the current directorystd::env::set_current_dir(cur_dir)?;diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rsindex b4916e5bd0..fbb9e65248 100644--- a/datafusion-examples/examples/udf/advanced_udaf.rs+++ b/datafusion-examples/examples/udf/advanced_udaf.rs@@ -28,13 +28,13 @@ use arrow::array::{use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type};use arrow::record_batch::RecordBatch;use arrow_schema::FieldRef;-use datafusion::common::{cast::as_float64_array, ScalarValue};+use datafusion::common::{ScalarValue, cast::as_float64_array};use datafusion::error::Result;use datafusion::logical_expr::{+ Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature,expr::AggregateFunction,function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},simplify::SimplifyInfo,- Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature,};use datafusion::prelude::*;diff --git a/datafusion-examples/examples/udf/advanced_udf.rs b/datafusion-examples/examples/udf/advanced_udf.rsindex 7459501a1c..a00a7e7df4 100644--- a/datafusion-examples/examples/udf/advanced_udf.rs+++ b/datafusion-examples/examples/udf/advanced_udf.rs@@ -21,15 +21,15 @@ use std::any::Any;use std::sync::Arc;use arrow::array::{- new_null_array, Array, ArrayRef, AsArray, Float32Array, Float64Array,+ Array, ArrayRef, AsArray, Float32Array, Float64Array, new_null_array,};use arrow::compute;use arrow::datatypes::{DataType, Float64Type};use arrow::record_batch::RecordBatch;-use datafusion::common::{exec_err, internal_err, ScalarValue};+use datafusion::common::{ScalarValue, exec_err, internal_err};use datafusion::error::Result;-use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties};use datafusion::logical_expr::Volatility;+use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties};use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,};diff --git a/datafusion-examples/examples/udf/async_udf.rs b/datafusion-examples/examples/udf/async_udf.rsindex 18630d33a3..c31e8290cc 100644--- a/datafusion-examples/examples/udf/async_udf.rs+++ b/datafusion-examples/examples/udf/async_udf.rs@@ -93,19 +93,19 @@ pub async fn async_udf() -> Result<()> {assert_batches_eq!([- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",- "| plan_type | plan |",- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",- "| logical_plan | SubqueryAlias: a |",- "| | Filter: ask_llm(CAST(animal.name AS Utf8View), Utf8View(\"Is this animal furry?\")) |",- "| | TableScan: animal projection=[id, name] |",- "| physical_plan | FilterExec: __async_fn_0@2, projection=[id@0, name@1] |",- "| | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |",- "| | AsyncFuncExec: async_expr=[async_expr(name=__async_fn_0, expr=ask_llm(CAST(name@1 AS Utf8View), Is this animal furry?))] |",- "| | CoalesceBatchesExec: target_batch_size=8192 |",- "| | DataSourceExec: partitions=1, partition_sizes=[1] |",- "| | |",- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",+ "| plan_type | plan |",+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",+ "| logical_plan | SubqueryAlias: a |",+ "| | Filter: ask_llm(CAST(animal.name AS Utf8View), Utf8View(\"Is this animal furry?\")) |",+ "| | TableScan: animal projection=[id, name] |",+ "| physical_plan | FilterExec: __async_fn_0@2, projection=[id@0, name@1] |",+ "| | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |",+ "| | AsyncFuncExec: async_expr=[async_expr(name=__async_fn_0, expr=ask_llm(CAST(name@1 AS Utf8View), Is this animal furry?))] |",+ "| | CoalesceBatchesExec: target_batch_size=8192 |",+ "| | DataSourceExec: partitions=1, partition_sizes=[1] |",+ "| | |",+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",],&results);diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rsindex 12ee74fc52..087b8ba73a 100644--- a/datafusion-examples/examples/udf/simple_udtf.rs+++ b/datafusion-examples/examples/udf/simple_udtf.rs@@ -17,16 +17,16 @@//! See `main.rs` for how to run it.-use arrow::csv::reader::Format;use arrow::csv::ReaderBuilder;+use arrow::csv::reader::Format;use async_trait::async_trait;use datafusion::arrow::datatypes::SchemaRef;use datafusion::arrow::record_batch::RecordBatch;use datafusion::catalog::Session;use datafusion::catalog::TableFunctionImpl;-use datafusion::common::{plan_err, ScalarValue};-use datafusion::datasource::memory::MemorySourceConfig;+use datafusion::common::{ScalarValue, plan_err};use datafusion::datasource::TableProvider;+use datafusion::datasource::memory::MemorySourceConfig;use datafusion::error::Result;use datafusion::execution::context::ExecutionProps;use datafusion::logical_expr::simplify::SimplifyContext;@@ -134,8 +134,7 @@ struct LocalCsvTableFunc {}impl TableFunctionImpl for LocalCsvTableFunc {fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {- let Some(Expr::Literal(ScalarValue::Utf8(Some(ref path)), _)) = exprs.first()- else {+ let Some(Expr::Literal(ScalarValue::Utf8(Some(path)), _)) = exprs.first() else {return plan_err!("read_csv requires at least one string argument");};diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.tomlindex 388cdac71f..be1374b371 100644--- a/datafusion/catalog-listing/Cargo.toml+++ b/datafusion/catalog-listing/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-catalog-listing"description = "datafusion-catalog-listing"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/catalog/Cargo.toml b/datafusion/catalog/Cargo.tomlindex 2d9f26ea50..1009e9aee4 100644--- a/datafusion/catalog/Cargo.toml+++ b/datafusion/catalog/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-catalog"description = "datafusion-catalog"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/common-runtime/Cargo.toml b/datafusion/common-runtime/Cargo.tomlindex 65e1544601..fd9a818bcb 100644--- a/datafusion/common-runtime/Cargo.toml+++ b/datafusion/common-runtime/Cargo.toml@@ -21,7 +21,7 @@ description = "Common Runtime functionality for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex dac06de659..5c36e33c72 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -21,7 +21,7 @@ description = "Common functionality for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex b63a3f9ccf..d2ecd34886 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -22,7 +22,7 @@ keywords = ["arrow", "query", "sql"]include = ["benches/*.rs", "src/**/*.rs", "Cargo.toml", "LICENSE.txt", "NOTICE.txt"]readme = "../../README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/datasource-arrow/Cargo.toml b/datafusion/datasource-arrow/Cargo.tomlindex 002408d643..fbadc8708c 100644--- a/datafusion/datasource-arrow/Cargo.toml+++ b/datafusion/datasource-arrow/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource-arrow"description = "datafusion-datasource-arrow"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.tomlindex fec17e133c..c9299aeb10 100644--- a/datafusion/datasource-avro/Cargo.toml+++ b/datafusion/datasource-avro/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource-avro"description = "datafusion-datasource-avro"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/datasource-csv/Cargo.toml b/datafusion/datasource-csv/Cargo.tomlindex 74e5ff3300..2950925127 100644--- a/datafusion/datasource-csv/Cargo.toml+++ b/datafusion/datasource-csv/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource-csv"description = "datafusion-datasource-csv"readme = "README.md"authors.workspace = true-edition = "2024"+edition = { workspace = true }homepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.tomlindex a8984849a2..37fa8d43a0 100644--- a/datafusion/datasource-json/Cargo.toml+++ b/datafusion/datasource-json/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource-json"description = "datafusion-datasource-json"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.tomlindex 19b814fd0b..a5f6f56ac6 100644--- a/datafusion/datasource-parquet/Cargo.toml+++ b/datafusion/datasource-parquet/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource-parquet"description = "datafusion-datasource-parquet"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.tomlindex a41c4d173c..48bf30f7a4 100644--- a/datafusion/datasource/Cargo.toml+++ b/datafusion/datasource/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-datasource"description = "datafusion-datasource"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/doc/Cargo.toml b/datafusion/doc/Cargo.tomlindex 94edf70bc7..c1368c1531 100644--- a/datafusion/doc/Cargo.toml+++ b/datafusion/doc/Cargo.toml@@ -21,7 +21,7 @@ description = "Documentation module for DataFusion query engine"keywords = ["datafusion", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.tomlindex 444a060d8b..ca1fba07ca 100644--- a/datafusion/execution/Cargo.toml+++ b/datafusion/execution/Cargo.toml@@ -21,7 +21,7 @@ description = "Execution configuration support for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.tomlindex 7b13334a6a..5ee46b454e 100644--- a/datafusion/expr-common/Cargo.toml+++ b/datafusion/expr-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Logical plan and expression representation for DataFusion query ekeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.tomlindex 37de339605..75aa59595b 100644--- a/datafusion/expr/Cargo.toml+++ b/datafusion/expr/Cargo.toml@@ -21,7 +21,7 @@ description = "Logical plan and expression representation for DataFusion query ekeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/ffi/src/arrow_wrappers.rs b/datafusion/ffi/src/arrow_wrappers.rsindex df3f1e3b4c..4b91fb551b 100644--- a/datafusion/ffi/src/arrow_wrappers.rs+++ b/datafusion/ffi/src/arrow_wrappers.rs@@ -19,10 +19,10 @@ use std::sync::Arc;use abi_stable::StableAbi;use arrow::{- array::{make_array, ArrayRef},+ array::{ArrayRef, make_array},datatypes::{Schema, SchemaRef},error::ArrowError,- ffi::{from_ffi, to_ffi, FFI_ArrowArray, FFI_ArrowSchema},+ ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi, to_ffi},};use datafusion_common::{DataFusionError, ScalarValue};use log::error;@@ -38,7 +38,9 @@ impl From<SchemaRef> for WrappedSchema {let ffi_schema = match FFI_ArrowSchema::try_from(value.as_ref()) {Ok(s) => s,Err(e) => {- error!("Unable to convert DataFusion Schema to FFI_ArrowSchema in FFI_PlanProperties. {e}");+ error!(+ "Unable to convert DataFusion Schema to FFI_ArrowSchema in FFI_PlanProperties. {e}"+ );FFI_ArrowSchema::empty()}};@@ -52,7 +54,9 @@ impl From<SchemaRef> for WrappedSchema {/// empty schema.#[cfg(not(tarpaulin_include))]fn catch_df_schema_error(e: &ArrowError) -> Schema {- error!("Unable to convert from FFI_ArrowSchema to DataFusion Schema in FFI_PlanProperties. {e}");+ error!(+ "Unable to convert from FFI_ArrowSchema to DataFusion Schema in FFI_PlanProperties. {e}"+ );Schema::empty()}diff --git a/datafusion/ffi/src/catalog_provider.rs b/datafusion/ffi/src/catalog_provider.rsindex 8c3e4cbc15..fe3cce1065 100644--- a/datafusion/ffi/src/catalog_provider.rs+++ b/datafusion/ffi/src/catalog_provider.rs@@ -18,8 +18,8 @@use std::{any::Any, ffi::c_void, sync::Arc};use abi_stable::{- std_types::{ROption, RResult, RString, RVec},StableAbi,+ std_types::{ROption, RResult, RString, RVec},};use datafusion::catalog::{CatalogProvider, SchemaProvider};use tokio::runtime::Handle;@@ -88,31 +88,39 @@ struct ProviderPrivateData {impl FFI_CatalogProvider {unsafe fn inner(&self) -> &Arc<dyn CatalogProvider + Send> {- let private_data = self.private_data as *const ProviderPrivateData;- &(*private_data).provider+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ &(*private_data).provider+ }}unsafe fn runtime(&self) -> Option<Handle> {- let private_data = self.private_data as *const ProviderPrivateData;- (*private_data).runtime.clone()+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ (*private_data).runtime.clone()+ }}}unsafe extern "C" fn schema_names_fn_wrapper(provider: &FFI_CatalogProvider,) -> RVec<RString> {- let names = provider.inner().schema_names();- names.into_iter().map(|s| s.into()).collect()+ unsafe {+ let names = provider.inner().schema_names();+ names.into_iter().map(|s| s.into()).collect()+ }}unsafe extern "C" fn schema_fn_wrapper(provider: &FFI_CatalogProvider,name: RString,) -> ROption<FFI_SchemaProvider> {- let maybe_schema = provider.inner().schema(name.as_str());- maybe_schema- .map(|schema| FFI_SchemaProvider::new(schema, provider.runtime()))- .into()+ unsafe {+ let maybe_schema = provider.inner().schema(name.as_str());+ maybe_schema+ .map(|schema| FFI_SchemaProvider::new(schema, provider.runtime()))+ .into()+ }}unsafe extern "C" fn register_schema_fn_wrapper(@@ -120,16 +128,18 @@ unsafe extern "C" fn register_schema_fn_wrapper(name: RString,schema: &FFI_SchemaProvider,) -> FFIResult<ROption<FFI_SchemaProvider>> {- let runtime = provider.runtime();- let provider = provider.inner();- let schema: Arc<dyn SchemaProvider + Send> = schema.into();+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();+ let schema: Arc<dyn SchemaProvider + Send> = schema.into();- let returned_schema =- rresult_return!(provider.register_schema(name.as_str(), schema))- .map(|schema| FFI_SchemaProvider::new(schema, runtime))- .into();+ let returned_schema =+ rresult_return!(provider.register_schema(name.as_str(), schema))+ .map(|schema| FFI_SchemaProvider::new(schema, runtime))+ .into();- RResult::ROk(returned_schema)+ RResult::ROk(returned_schema)+ }}unsafe extern "C" fn deregister_schema_fn_wrapper(@@ -137,47 +147,54 @@ unsafe extern "C" fn deregister_schema_fn_wrapper(name: RString,cascade: bool,) -> FFIResult<ROption<FFI_SchemaProvider>> {- let runtime = provider.runtime();- let provider = provider.inner();-- let maybe_schema =- rresult_return!(provider.deregister_schema(name.as_str(), cascade));-- RResult::ROk(- maybe_schema- .map(|schema| FFI_SchemaProvider::new(schema, runtime))- .into(),- )+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();++ let maybe_schema =+ rresult_return!(provider.deregister_schema(name.as_str(), cascade));++ RResult::ROk(+ maybe_schema+ .map(|schema| FFI_SchemaProvider::new(schema, runtime))+ .into(),+ )+ }}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_CatalogProvider) {- debug_assert!(!provider.private_data.is_null());- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);- drop(private_data);- provider.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!provider.private_data.is_null());+ let private_data =+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);+ drop(private_data);+ provider.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_CatalogProvider,) -> FFI_CatalogProvider {- let old_private_data = provider.private_data as *const ProviderPrivateData;- let runtime = (*old_private_data).runtime.clone();-- let private_data = Box::into_raw(Box::new(ProviderPrivateData {- provider: Arc::clone(&(*old_private_data).provider),- runtime,- })) as *mut c_void;-- FFI_CatalogProvider {- schema_names: schema_names_fn_wrapper,- schema: schema_fn_wrapper,- register_schema: register_schema_fn_wrapper,- deregister_schema: deregister_schema_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ let old_private_data = provider.private_data as *const ProviderPrivateData;+ let runtime = (*old_private_data).runtime.clone();++ let private_data = Box::into_raw(Box::new(ProviderPrivateData {+ provider: Arc::clone(&(*old_private_data).provider),+ runtime,+ })) as *mut c_void;++ FFI_CatalogProvider {+ schema_names: schema_names_fn_wrapper,+ schema: schema_fn_wrapper,+ register_schema: register_schema_fn_wrapper,+ deregister_schema: deregister_schema_fn_wrapper,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -307,11 +324,13 @@ mod tests {let prior_schema = Arc::new(MemorySchemaProvider::new());let catalog = Arc::new(MemoryCatalogProvider::new());- assert!(catalog- .as_ref()- .register_schema("prior_schema", prior_schema)- .unwrap()- .is_none());+ assert!(+ catalog+ .as_ref()+ .register_schema("prior_schema", prior_schema)+ .unwrap()+ .is_none()+ );let mut ffi_catalog = FFI_CatalogProvider::new(catalog, None);ffi_catalog.library_marker_id = crate::mock_foreign_marker_id;@@ -360,17 +379,21 @@ mod tests {// Verify local libraries can be downcast to their originallet foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();- assert!(foreign_catalog- .as_any()- .downcast_ref::<MemoryCatalogProvider>()- .is_some());+ assert!(+ foreign_catalog+ .as_any()+ .downcast_ref::<MemoryCatalogProvider>()+ .is_some()+ );// Verify different library markers generate foreign providersffi_catalog.library_marker_id = crate::mock_foreign_marker_id;let foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();- assert!(foreign_catalog- .as_any()- .downcast_ref::<ForeignCatalogProvider>()- .is_some());+ assert!(+ foreign_catalog+ .as_any()+ .downcast_ref::<ForeignCatalogProvider>()+ .is_some()+ );}}diff --git a/datafusion/ffi/src/catalog_provider_list.rs b/datafusion/ffi/src/catalog_provider_list.rsindex 102e9e4ef5..a119dd9485 100644--- a/datafusion/ffi/src/catalog_provider_list.rs+++ b/datafusion/ffi/src/catalog_provider_list.rs@@ -18,8 +18,8 @@use std::{any::Any, ffi::c_void, sync::Arc};use abi_stable::{- std_types::{ROption, RString, RVec},StableAbi,+ std_types::{ROption, RString, RVec},};use datafusion::catalog::{CatalogProvider, CatalogProviderList};use tokio::runtime::Handle;@@ -75,21 +75,27 @@ struct ProviderPrivateData {impl FFI_CatalogProviderList {unsafe fn inner(&self) -> &Arc<dyn CatalogProviderList + Send> {- let private_data = self.private_data as *const ProviderPrivateData;- &(*private_data).provider+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ &(*private_data).provider+ }}unsafe fn runtime(&self) -> Option<Handle> {- let private_data = self.private_data as *const ProviderPrivateData;- (*private_data).runtime.clone()+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ (*private_data).runtime.clone()+ }}}unsafe extern "C" fn catalog_names_fn_wrapper(provider: &FFI_CatalogProviderList,) -> RVec<RString> {- let names = provider.inner().catalog_names();- names.into_iter().map(|s| s.into()).collect()+ unsafe {+ let names = provider.inner().catalog_names();+ names.into_iter().map(|s| s.into()).collect()+ }}unsafe extern "C" fn register_catalog_fn_wrapper(@@ -97,55 +103,64 @@ unsafe extern "C" fn register_catalog_fn_wrapper(name: RString,catalog: &FFI_CatalogProvider,) -> ROption<FFI_CatalogProvider> {- let runtime = provider.runtime();- let provider = provider.inner();- let catalog: Arc<dyn CatalogProvider + Send> = catalog.into();-- provider- .register_catalog(name.into(), catalog)- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))- .into()+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();+ let catalog: Arc<dyn CatalogProvider + Send> = catalog.into();++ provider+ .register_catalog(name.into(), catalog)+ .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))+ .into()+ }}unsafe extern "C" fn catalog_fn_wrapper(provider: &FFI_CatalogProviderList,name: RString,) -> ROption<FFI_CatalogProvider> {- let runtime = provider.runtime();- let provider = provider.inner();- provider- .catalog(name.as_str())- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))- .into()+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();+ provider+ .catalog(name.as_str())+ .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))+ .into()+ }}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_CatalogProviderList) {- debug_assert!(!provider.private_data.is_null());- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);- drop(private_data);- provider.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!provider.private_data.is_null());+ let private_data =+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);+ drop(private_data);+ provider.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_CatalogProviderList,) -> FFI_CatalogProviderList {- let old_private_data = provider.private_data as *const ProviderPrivateData;- let runtime = (*old_private_data).runtime.clone();-- let private_data = Box::into_raw(Box::new(ProviderPrivateData {- provider: Arc::clone(&(*old_private_data).provider),- runtime,- })) as *mut c_void;-- FFI_CatalogProviderList {- register_catalog: register_catalog_fn_wrapper,- catalog_names: catalog_names_fn_wrapper,- catalog: catalog_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ let old_private_data = provider.private_data as *const ProviderPrivateData;+ let runtime = (*old_private_data).runtime.clone();++ let private_data = Box::into_raw(Box::new(ProviderPrivateData {+ provider: Arc::clone(&(*old_private_data).provider),+ runtime,+ })) as *mut c_void;++ FFI_CatalogProviderList {+ register_catalog: register_catalog_fn_wrapper,+ catalog_names: catalog_names_fn_wrapper,+ catalog: catalog_fn_wrapper,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -257,10 +272,12 @@ mod tests {let prior_catalog = Arc::new(MemoryCatalogProvider::new());let catalog_list = Arc::new(MemoryCatalogProviderList::new());- assert!(catalog_list- .as_ref()- .register_catalog("prior_catalog".to_owned(), prior_catalog)- .is_none());+ assert!(+ catalog_list+ .as_ref()+ .register_catalog("prior_catalog".to_owned(), prior_catalog)+ .is_none()+ );let mut ffi_catalog_list = FFI_CatalogProviderList::new(catalog_list, None);ffi_catalog_list.library_marker_id = crate::mock_foreign_marker_id;@@ -306,18 +323,22 @@ mod tests {// Verify local libraries can be downcast to their originallet foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =(&ffi_catalog_list).into();- assert!(foreign_catalog_list- .as_any()- .downcast_ref::<MemoryCatalogProviderList>()- .is_some());+ assert!(+ foreign_catalog_list+ .as_any()+ .downcast_ref::<MemoryCatalogProviderList>()+ .is_some()+ );// Verify different library markers generate foreign providersffi_catalog_list.library_marker_id = crate::mock_foreign_marker_id;let foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =(&ffi_catalog_list).into();- assert!(foreign_catalog_list- .as_any()- .downcast_ref::<ForeignCatalogProviderList>()- .is_some());+ assert!(+ foreign_catalog_list+ .as_any()+ .downcast_ref::<ForeignCatalogProviderList>()+ .is_some()+ );}}diff --git a/datafusion/ffi/src/execution/task_ctx.rs b/datafusion/ffi/src/execution/task_ctx.rsindex 602e8dec8e..999a33bc1a 100644--- a/datafusion/ffi/src/execution/task_ctx.rs+++ b/datafusion/ffi/src/execution/task_ctx.rs@@ -18,12 +18,12 @@use std::ffi::c_void;use std::sync::Arc;+use abi_stable::StableAbi;use abi_stable::pmr::ROption;use abi_stable::std_types::{RHashMap, RString};-use abi_stable::StableAbi;+use datafusion_execution::TaskContext;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnv;-use datafusion_execution::TaskContext;use datafusion_expr::{AggregateUDF, AggregateUDFImpl, ScalarUDF, ScalarUDFImpl, WindowUDF, WindowUDFImpl,};@@ -76,66 +76,84 @@ struct TaskContextPrivateData {impl FFI_TaskContext {unsafe fn inner(&self) -> &Arc<TaskContext> {- let private_data = self.private_data as *const TaskContextPrivateData;- &(*private_data).ctx+ unsafe {+ let private_data = self.private_data as *const TaskContextPrivateData;+ &(*private_data).ctx+ }}}unsafe extern "C" fn session_id_fn_wrapper(ctx: &FFI_TaskContext) -> RString {- let ctx = ctx.inner();- ctx.session_id().into()+ unsafe {+ let ctx = ctx.inner();+ ctx.session_id().into()+ }}unsafe extern "C" fn task_id_fn_wrapper(ctx: &FFI_TaskContext) -> ROption<RString> {- let ctx = ctx.inner();- ctx.task_id().map(|s| s.as_str().into()).into()+ unsafe {+ let ctx = ctx.inner();+ ctx.task_id().map(|s| s.as_str().into()).into()+ }}unsafe extern "C" fn session_config_fn_wrapper(ctx: &FFI_TaskContext,) -> FFI_SessionConfig {- let ctx = ctx.inner();- ctx.session_config().into()+ unsafe {+ let ctx = ctx.inner();+ ctx.session_config().into()+ }}unsafe extern "C" fn scalar_functions_fn_wrapper(ctx: &FFI_TaskContext,) -> RHashMap<RString, FFI_ScalarUDF> {- let ctx = ctx.inner();- ctx.scalar_functions()- .iter()- .map(|(name, udf)| (name.to_owned().into(), Arc::clone(udf).into()))- .collect()+ unsafe {+ let ctx = ctx.inner();+ ctx.scalar_functions()+ .iter()+ .map(|(name, udf)| (name.to_owned().into(), Arc::clone(udf).into()))+ .collect()+ }}unsafe extern "C" fn aggregate_functions_fn_wrapper(ctx: &FFI_TaskContext,) -> RHashMap<RString, FFI_AggregateUDF> {- let ctx = ctx.inner();- ctx.aggregate_functions()- .iter()- .map(|(name, udaf)| {- (- name.to_owned().into(),- FFI_AggregateUDF::from(Arc::clone(udaf)),- )- })- .collect()+ unsafe {+ let ctx = ctx.inner();+ ctx.aggregate_functions()+ .iter()+ .map(|(name, udaf)| {+ (+ name.to_owned().into(),+ FFI_AggregateUDF::from(Arc::clone(udaf)),+ )+ })+ .collect()+ }}unsafe extern "C" fn window_functions_fn_wrapper(ctx: &FFI_TaskContext,) -> RHashMap<RString, FFI_WindowUDF> {- let ctx = ctx.inner();- ctx.window_functions()- .iter()- .map(|(name, udf)| (name.to_owned().into(), FFI_WindowUDF::from(Arc::clone(udf))))- .collect()+ unsafe {+ let ctx = ctx.inner();+ ctx.window_functions()+ .iter()+ .map(|(name, udf)| {+ (name.to_owned().into(), FFI_WindowUDF::from(Arc::clone(udf)))+ })+ .collect()+ }}unsafe extern "C" fn release_fn_wrapper(ctx: &mut FFI_TaskContext) {- let private_data = Box::from_raw(ctx.private_data as *mut TaskContextPrivateData);- drop(private_data);+ unsafe {+ let private_data = Box::from_raw(ctx.private_data as *mut TaskContextPrivateData);+ drop(private_data);+ }}impl Drop for FFI_TaskContext {diff --git a/datafusion/ffi/src/execution/task_ctx_provider.rs b/datafusion/ffi/src/execution/task_ctx_provider.rsindex b9861a68d0..e04080c0bc 100644--- a/datafusion/ffi/src/execution/task_ctx_provider.rs+++ b/datafusion/ffi/src/execution/task_ctx_provider.rs@@ -19,7 +19,7 @@ use std::ffi::c_void;use std::sync::{Arc, Weak};use abi_stable::StableAbi;-use datafusion_common::{ffi_datafusion_err, DataFusionError};+use datafusion_common::{DataFusionError, ffi_datafusion_err};use datafusion_execution::{TaskContext, TaskContextProvider};use crate::execution::task_ctx::FFI_TaskContext;@@ -67,44 +67,54 @@ struct TaskContextProviderPrivateData {impl FFI_TaskContextProvider {unsafe fn inner(&self) -> Option<Arc<TaskContext>> {- let private_data = self.private_data as *const TaskContextProviderPrivateData;- (*private_data).ctx.upgrade().map(|ctx| ctx.task_ctx())+ unsafe {+ let private_data = self.private_data as *const TaskContextProviderPrivateData;+ (*private_data).ctx.upgrade().map(|ctx| ctx.task_ctx())+ }}}unsafe extern "C" fn task_ctx_fn_wrapper(ctx_provider: &FFI_TaskContextProvider,) -> FFIResult<FFI_TaskContext> {- rresult!(ctx_provider- .inner()- .map(FFI_TaskContext::from)- .ok_or_else(|| {- ffi_datafusion_err!(- "TaskContextProvider went out of scope over FFI boundary."- )- }))+ unsafe {+ rresult!(+ ctx_provider+ .inner()+ .map(FFI_TaskContext::from)+ .ok_or_else(|| {+ ffi_datafusion_err!(+ "TaskContextProvider went out of scope over FFI boundary."+ )+ })+ )+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_TaskContextProvider,) -> FFI_TaskContextProvider {- let private_data = provider.private_data as *const TaskContextProviderPrivateData;- let ctx = Weak::clone(&(*private_data).ctx);+ unsafe {+ let private_data = provider.private_data as *const TaskContextProviderPrivateData;+ let ctx = Weak::clone(&(*private_data).ctx);- let private_data = Box::new(TaskContextProviderPrivateData { ctx });+ let private_data = Box::new(TaskContextProviderPrivateData { ctx });- FFI_TaskContextProvider {- task_ctx: task_ctx_fn_wrapper,- release: release_fn_wrapper,- clone: clone_fn_wrapper,- private_data: Box::into_raw(private_data) as *mut c_void,- library_marker_id: crate::get_library_marker_id,+ FFI_TaskContextProvider {+ task_ctx: task_ctx_fn_wrapper,+ release: release_fn_wrapper,+ clone: clone_fn_wrapper,+ private_data: Box::into_raw(private_data) as *mut c_void,+ library_marker_id: crate::get_library_marker_id,+ }}}unsafe extern "C" fn release_fn_wrapper(ctx: &mut FFI_TaskContextProvider) {- let private_data =- Box::from_raw(ctx.private_data as *mut TaskContextProviderPrivateData);- drop(private_data);+ unsafe {+ let private_data =+ Box::from_raw(ctx.private_data as *mut TaskContextProviderPrivateData);+ drop(private_data);+ }}impl Drop for FFI_TaskContextProvider {fn drop(&mut self) {diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex 8ee67d7e18..d8ef1c9827 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -18,8 +18,8 @@use std::{ffi::c_void, pin::Pin, sync::Arc};use abi_stable::{- std_types::{RString, RVec},StableAbi,+ std_types::{RString, RVec},};use datafusion::{error::DataFusionError,@@ -98,34 +98,43 @@ unsafe extern "C" fn properties_fn_wrapper(unsafe extern "C" fn children_fn_wrapper(plan: &FFI_ExecutionPlan,) -> RVec<FFI_ExecutionPlan> {- let private_data = plan.private_data as *const ExecutionPlanPrivateData;- let plan = &(*private_data).plan;- let ctx = &(*private_data).context;- let runtime = &(*private_data).runtime;-- let children: Vec<_> = plan- .children()- .into_iter()- .map(|child| {- FFI_ExecutionPlan::new(Arc::clone(child), Arc::clone(ctx), runtime.clone())- })- .collect();-- children.into()+ unsafe {+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;+ let plan = &(*private_data).plan;+ let ctx = &(*private_data).context;+ let runtime = &(*private_data).runtime;++ let children: Vec<_> = plan+ .children()+ .into_iter()+ .map(|child| {+ FFI_ExecutionPlan::new(+ Arc::clone(child),+ Arc::clone(ctx),+ runtime.clone(),+ )+ })+ .collect();++ children.into()+ }}unsafe extern "C" fn execute_fn_wrapper(plan: &FFI_ExecutionPlan,partition: usize,) -> FFIResult<FFI_RecordBatchStream> {- let private_data = plan.private_data as *const ExecutionPlanPrivateData;- let plan = &(*private_data).plan;- let ctx = &(*private_data).context;- let runtime = (*private_data).runtime.clone();-- rresult!(plan- .execute(partition, Arc::clone(ctx))- .map(|rbs| FFI_RecordBatchStream::new(rbs, runtime)))+ unsafe {+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;+ let plan = &(*private_data).plan;+ let ctx = &(*private_data).context;+ let runtime = (*private_data).runtime.clone();++ rresult!(+ plan.execute(partition, Arc::clone(ctx))+ .map(|rbs| FFI_RecordBatchStream::new(rbs, runtime))+ )+ }}unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> RString {@@ -133,21 +142,26 @@ unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> RString {}unsafe extern "C" fn release_fn_wrapper(plan: &mut FFI_ExecutionPlan) {- debug_assert!(!plan.private_data.is_null());- let private_data = Box::from_raw(plan.private_data as *mut ExecutionPlanPrivateData);- drop(private_data);- plan.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!plan.private_data.is_null());+ let private_data =+ Box::from_raw(plan.private_data as *mut ExecutionPlanPrivateData);+ drop(private_data);+ plan.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(plan: &FFI_ExecutionPlan) -> FFI_ExecutionPlan {- let private_data = plan.private_data as *const ExecutionPlanPrivateData;- let plan_data = &(*private_data);-- FFI_ExecutionPlan::new(- Arc::clone(&plan_data.plan),- Arc::clone(&plan_data.context),- plan_data.runtime.clone(),- )+ unsafe {+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;+ let plan_data = &(*private_data);++ FFI_ExecutionPlan::new(+ Arc::clone(&plan_data.plan),+ Arc::clone(&plan_data.context),+ plan_data.runtime.clone(),+ )+ }}impl Clone for FFI_ExecutionPlan {@@ -306,8 +320,8 @@ pub(crate) mod tests {use arrow::datatypes::{DataType, Field, Schema};use datafusion::{physical_plan::{- execution_plan::{Boundedness, EmissionType},Partitioning,+ execution_plan::{Boundedness, EmissionType},},prelude::SessionContext,};@@ -468,9 +482,11 @@ pub(crate) mod tests {// Verify different library markers generate foreign providersffi_plan.library_marker_id = crate::mock_foreign_marker_id;let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into().unwrap();- assert!(foreign_plan- .as_any()- .downcast_ref::<ForeignExecutionPlan>()- .is_some());+ assert!(+ foreign_plan+ .as_any()+ .downcast_ref::<ForeignExecutionPlan>()+ .is_some()+ );}}diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rsindex 720d83ce9a..abd07d0a27 100644--- a/datafusion/ffi/src/physical_expr/mod.rs+++ b/datafusion/ffi/src/physical_expr/mod.rs@@ -27,18 +27,18 @@ use std::{};use abi_stable::{- std_types::{ROption, RResult, RString, RVec},StableAbi,+ std_types::{ROption, RResult, RString, RVec},};use arrow::{array::{ArrayRef, BooleanArray, RecordBatch},datatypes::SchemaRef,};-use arrow_schema::{ffi::FFI_ArrowSchema, DataType, Field, FieldRef, Schema};-use datafusion_common::{ffi_datafusion_err, Result};+use arrow_schema::{DataType, Field, FieldRef, Schema, ffi::FFI_ArrowSchema};+use datafusion_common::{Result, ffi_datafusion_err};use datafusion_expr::{- interval_arithmetic::Interval, sort_properties::ExprProperties,- statistics::Distribution, ColumnarValue,+ ColumnarValue, interval_arithmetic::Interval, sort_properties::ExprProperties,+ statistics::Distribution,};use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;@@ -191,10 +191,11 @@ unsafe extern "C" fn evaluate_fn_wrapper(batch: WrappedArray,) -> FFIResult<FFI_ColumnarValue> {let batch = rresult_return!(wrapped_array_to_record_batch(batch));- rresult!(expr- .inner()- .evaluate(&batch)- .and_then(FFI_ColumnarValue::try_from))+ rresult!(+ expr.inner()+ .evaluate(&batch)+ .and_then(FFI_ColumnarValue::try_from)+ )}unsafe extern "C" fn return_field_fn_wrapper(@@ -203,10 +204,11 @@ unsafe extern "C" fn return_field_fn_wrapper() -> FFIResult<WrappedSchema> {let expr = expr.inner();let schema: SchemaRef = input_schema.into();- rresult!(expr- .return_field(&schema)- .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(Into::into))- .map(WrappedSchema))+ rresult!(+ expr.return_field(&schema)+ .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(Into::into))+ .map(WrappedSchema)+ )}unsafe extern "C" fn evaluate_selection_fn_wrapper(@@ -216,14 +218,17 @@ unsafe extern "C" fn evaluate_selection_fn_wrapper() -> FFIResult<FFI_ColumnarValue> {let batch = rresult_return!(wrapped_array_to_record_batch(batch));let selection: ArrayRef = rresult_return!(selection.try_into());- let selection = rresult_return!(selection- .as_any()- .downcast_ref::<BooleanArray>()- .ok_or(ffi_datafusion_err!("Unexpected selection array type")));- rresult!(expr- .inner()- .evaluate_selection(&batch, selection)- .and_then(FFI_ColumnarValue::try_from))+ let selection = rresult_return!(+ selection+ .as_any()+ .downcast_ref::<BooleanArray>()+ .ok_or(ffi_datafusion_err!("Unexpected selection array type"))+ );+ rresult!(+ expr.inner()+ .evaluate_selection(&batch, selection)+ .and_then(FFI_ColumnarValue::try_from)+ )}unsafe extern "C" fn children_fn_wrapper(@@ -251,15 +256,18 @@ unsafe extern "C" fn evaluate_bounds_fn_wrapper(children: RVec<FFI_Interval>,) -> FFIResult<FFI_Interval> {let expr = expr.inner();- let children = rresult_return!(children- .into_iter()- .map(Interval::try_from)- .collect::<Result<Vec<_>>>());+ let children = rresult_return!(+ children+ .into_iter()+ .map(Interval::try_from)+ .collect::<Result<Vec<_>>>()+ );let children_borrowed = children.iter().collect::<Vec<_>>();- rresult!(expr- .evaluate_bounds(&children_borrowed)- .and_then(FFI_Interval::try_from))+ rresult!(+ expr.evaluate_bounds(&children_borrowed)+ .and_then(FFI_Interval::try_from)+ )}unsafe extern "C" fn propagate_constraints_fn_wrapper(@@ -269,21 +277,25 @@ unsafe extern "C" fn propagate_constraints_fn_wrapper() -> FFIResult<ROption<RVec<FFI_Interval>>> {let expr = expr.inner();let interval = rresult_return!(Interval::try_from(interval));- let children = rresult_return!(children- .into_iter()- .map(Interval::try_from)- .collect::<Result<Vec<_>>>());+ let children = rresult_return!(+ children+ .into_iter()+ .map(Interval::try_from)+ .collect::<Result<Vec<_>>>()+ );let children_borrowed = children.iter().collect::<Vec<_>>();let result =rresult_return!(expr.propagate_constraints(&interval, &children_borrowed));- let result = rresult_return!(result- .map(|intervals| intervals- .into_iter()- .map(FFI_Interval::try_from)- .collect::<Result<RVec<_>>>())- .transpose());+ let result = rresult_return!(+ result+ .map(|intervals| intervals+ .into_iter()+ .map(FFI_Interval::try_from)+ .collect::<Result<RVec<_>>>())+ .transpose()+ );RResult::ROk(result.into())}@@ -293,14 +305,17 @@ unsafe extern "C" fn evaluate_statistics_fn_wrapper(children: RVec<FFI_Distribution>,) -> FFIResult<FFI_Distribution> {let expr = expr.inner();- let children = rresult_return!(children- .into_iter()- .map(Distribution::try_from)- .collect::<Result<Vec<_>>>());+ let children = rresult_return!(+ children+ .into_iter()+ .map(Distribution::try_from)+ .collect::<Result<Vec<_>>>()+ );let children_borrowed = children.iter().collect::<Vec<_>>();- rresult!(expr- .evaluate_statistics(&children_borrowed)- .and_then(|dist| FFI_Distribution::try_from(&dist)))+ rresult!(+ expr.evaluate_statistics(&children_borrowed)+ .and_then(|dist| FFI_Distribution::try_from(&dist))+ )}unsafe extern "C" fn propagate_statistics_fn_wrapper(@@ -310,19 +325,23 @@ unsafe extern "C" fn propagate_statistics_fn_wrapper() -> FFIResult<ROption<RVec<FFI_Distribution>>> {let expr = expr.inner();let parent = rresult_return!(Distribution::try_from(parent));- let children = rresult_return!(children- .into_iter()- .map(Distribution::try_from)- .collect::<Result<Vec<_>>>());+ let children = rresult_return!(+ children+ .into_iter()+ .map(Distribution::try_from)+ .collect::<Result<Vec<_>>>()+ );let children_borrowed = children.iter().collect::<Vec<_>>();let result = rresult_return!(expr.propagate_statistics(&parent, &children_borrowed));- let result = rresult_return!(result- .map(|dists| dists- .iter()- .map(FFI_Distribution::try_from)- .collect::<Result<RVec<_>>>())- .transpose());+ let result = rresult_return!(+ result+ .map(|dists| dists+ .iter()+ .map(FFI_Distribution::try_from)+ .collect::<Result<RVec<_>>>())+ .transpose()+ );RResult::ROk(result.into())}@@ -332,13 +351,16 @@ unsafe extern "C" fn get_properties_fn_wrapper(children: RVec<FFI_ExprProperties>,) -> FFIResult<FFI_ExprProperties> {let expr = expr.inner();- let children = rresult_return!(children- .into_iter()- .map(ExprProperties::try_from)- .collect::<Result<Vec<_>>>());- rresult!(expr- .get_properties(&children)- .and_then(|p| FFI_ExprProperties::try_from(&p)))+ let children = rresult_return!(+ children+ .into_iter()+ .map(ExprProperties::try_from)+ .collect::<Result<Vec<_>>>()+ );+ rresult!(+ expr.get_properties(&children)+ .and_then(|p| FFI_ExprProperties::try_from(&p))+ )}unsafe extern "C" fn fmt_sql_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFIResult<RString> {@@ -351,9 +373,10 @@ unsafe extern "C" fn snapshot_fn_wrapper(expr: &FFI_PhysicalExpr,) -> FFIResult<ROption<FFI_PhysicalExpr>> {let expr = expr.inner();- rresult!(expr- .snapshot()- .map(|snapshot| snapshot.map(FFI_PhysicalExpr::from).into()))+ rresult!(+ expr.snapshot()+ .map(|snapshot| snapshot.map(FFI_PhysicalExpr::from).into())+ )}unsafe extern "C" fn snapshot_generation_fn_wrapper(expr: &FFI_PhysicalExpr) -> u64 {@@ -378,43 +401,48 @@ unsafe extern "C" fn hash_fn_wrapper(expr: &FFI_PhysicalExpr) -> u64 {}unsafe extern "C" fn release_fn_wrapper(expr: &mut FFI_PhysicalExpr) {- debug_assert!(!expr.private_data.is_null());- let private_data = Box::from_raw(expr.private_data as *mut PhysicalExprPrivateData);- drop(private_data);- expr.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!expr.private_data.is_null());+ let private_data =+ Box::from_raw(expr.private_data as *mut PhysicalExprPrivateData);+ drop(private_data);+ expr.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFI_PhysicalExpr {- let old_private_data = expr.private_data as *const PhysicalExprPrivateData;-- let private_data = Box::into_raw(Box::new(PhysicalExprPrivateData {- expr: Arc::clone(&(*old_private_data).expr),- })) as *mut c_void;-- FFI_PhysicalExpr {- data_type: data_type_fn_wrapper,- nullable: nullable_fn_wrapper,- evaluate: evaluate_fn_wrapper,- return_field: return_field_fn_wrapper,- evaluate_selection: evaluate_selection_fn_wrapper,- children: children_fn_wrapper,- new_with_children: new_with_children_fn_wrapper,- evaluate_bounds: evaluate_bounds_fn_wrapper,- propagate_constraints: propagate_constraints_fn_wrapper,- evaluate_statistics: evaluate_statistics_fn_wrapper,- propagate_statistics: propagate_statistics_fn_wrapper,- get_properties: get_properties_fn_wrapper,- fmt_sql: fmt_sql_fn_wrapper,- snapshot: snapshot_fn_wrapper,- snapshot_generation: snapshot_generation_fn_wrapper,- is_volatile_node: is_volatile_node_fn_wrapper,- display: display_fn_wrapper,- hash: hash_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ let old_private_data = expr.private_data as *const PhysicalExprPrivateData;++ let private_data = Box::into_raw(Box::new(PhysicalExprPrivateData {+ expr: Arc::clone(&(*old_private_data).expr),+ })) as *mut c_void;++ FFI_PhysicalExpr {+ data_type: data_type_fn_wrapper,+ nullable: nullable_fn_wrapper,+ evaluate: evaluate_fn_wrapper,+ return_field: return_field_fn_wrapper,+ evaluate_selection: evaluate_selection_fn_wrapper,+ children: children_fn_wrapper,+ new_with_children: new_with_children_fn_wrapper,+ evaluate_bounds: evaluate_bounds_fn_wrapper,+ propagate_constraints: propagate_constraints_fn_wrapper,+ evaluate_statistics: evaluate_statistics_fn_wrapper,+ propagate_statistics: propagate_statistics_fn_wrapper,+ get_properties: get_properties_fn_wrapper,+ fmt_sql: fmt_sql_fn_wrapper,+ snapshot: snapshot_fn_wrapper,+ snapshot_generation: snapshot_generation_fn_wrapper,+ is_volatile_node: is_volatile_node_fn_wrapper,+ display: display_fn_wrapper,+ hash: hash_fn_wrapper,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -559,8 +587,13 @@ impl PhysicalExpr for ForeignPhysicalExpr {) -> Result<Arc<dyn PhysicalExpr>> {unsafe {let children = children.into_iter().map(FFI_PhysicalExpr::from).collect();- df_result!((self.expr.new_with_children)(&self.expr, &children)- .map(|expr| <Arc<dyn PhysicalExpr>>::from(&expr)))+ df_result!(+ (self.expr.new_with_children)(&self.expr, &children).map(|expr| <Arc<+ dyn PhysicalExpr,+ >>::from(+ &expr+ ))+ )}}@@ -709,11 +742,11 @@ mod tests {sync::Arc,};- use arrow::array::{record_batch, BooleanArray, RecordBatch};- use datafusion_common::{tree_node::DynTreeNode, DataFusionError, ScalarValue};+ use arrow::array::{BooleanArray, RecordBatch, record_batch};+ use datafusion_common::{DataFusionError, ScalarValue, tree_node::DynTreeNode};use datafusion_expr::{interval_arithmetic::Interval, statistics::Distribution};use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};- use datafusion_physical_expr_common::physical_expr::{fmt_sql, PhysicalExpr};+ use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};use crate::physical_expr::FFI_PhysicalExpr;diff --git a/datafusion/ffi/src/physical_expr/partitioning.rs b/datafusion/ffi/src/physical_expr/partitioning.rsindex 32ca1f4835..fb16705e68 100644--- a/datafusion/ffi/src/physical_expr/partitioning.rs+++ b/datafusion/ffi/src/physical_expr/partitioning.rs@@ -17,7 +17,7 @@use std::sync::Arc;-use abi_stable::{std_types::RVec, StableAbi};+use abi_stable::{StableAbi, std_types::RVec};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -70,7 +70,7 @@ impl From<&FFI_Partitioning> for Partitioning {#[cfg(test)]mod tests {- use datafusion_physical_expr::{expressions::lit, Partitioning};+ use datafusion_physical_expr::{Partitioning, expressions::lit};use crate::physical_expr::partitioning::FFI_Partitioning;diff --git a/datafusion/ffi/src/plan_properties.rs b/datafusion/ffi/src/plan_properties.rsindex 40ac82c39d..430b34c984 100644--- a/datafusion/ffi/src/plan_properties.rs+++ b/datafusion/ffi/src/plan_properties.rs@@ -18,24 +18,24 @@use std::{ffi::c_void, sync::Arc};use abi_stable::{- std_types::{RResult::ROk, RVec},StableAbi,+ std_types::{RResult::ROk, RVec},};use arrow::datatypes::SchemaRef;use datafusion::{error::{DataFusionError, Result},physical_expr::EquivalenceProperties,physical_plan::{- execution_plan::{Boundedness, EmissionType},PlanProperties,+ execution_plan::{Boundedness, EmissionType},},prelude::SessionContext,};use datafusion_proto::{physical_plan::{+ DefaultPhysicalExtensionCodec,from_proto::{parse_physical_sort_exprs, parse_protobuf_partitioning},to_proto::{serialize_partitioning, serialize_physical_sort_exprs},- DefaultPhysicalExtensionCodec,},protobuf::{Partitioning, PhysicalSortExprNodeCollection},};@@ -141,11 +141,13 @@ unsafe extern "C" fn schema_fn_wrapper(properties: &FFI_PlanProperties) -> Wrapp}unsafe extern "C" fn release_fn_wrapper(props: &mut FFI_PlanProperties) {- debug_assert!(!props.private_data.is_null());- let private_data =- Box::from_raw(props.private_data as *mut PlanPropertiesPrivateData);- drop(private_data);- props.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!props.private_data.is_null());+ let private_data =+ Box::from_raw(props.private_data as *mut PlanPropertiesPrivateData);+ drop(private_data);+ props.private_data = std::ptr::null_mut();+ }}impl Drop for FFI_PlanProperties {diff --git a/datafusion/ffi/src/proto/logical_extension_codec.rs b/datafusion/ffi/src/proto/logical_extension_codec.rsindex 928a7c5bdb..3f79cfc732 100644--- a/datafusion/ffi/src/proto/logical_extension_codec.rs+++ b/datafusion/ffi/src/proto/logical_extension_codec.rs@@ -18,12 +18,12 @@use std::ffi::c_void;use std::sync::Arc;-use abi_stable::std_types::{RResult, RSlice, RStr, RVec};use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RSlice, RStr, RVec};use arrow::datatypes::SchemaRef;use datafusion_catalog::TableProvider;use datafusion_common::error::Result;-use datafusion_common::{not_impl_err, TableReference};+use datafusion_common::{TableReference, not_impl_err};use datafusion_datasource::file_format::FileFormatFactory;use datafusion_execution::TaskContext;use datafusion_expr::{@@ -261,9 +261,11 @@ unsafe extern "C" fn try_encode_udwf_fn_wrapper(}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_LogicalExtensionCodec) {- let private_data =- Box::from_raw(provider.private_data as *mut LogicalExtensionCodecPrivateData);- drop(private_data);+ unsafe {+ let private_data =+ Box::from_raw(provider.private_data as *mut LogicalExtensionCodecPrivateData);+ drop(private_data);+ }}unsafe extern "C" fn clone_fn_wrapper(@@ -472,7 +474,7 @@ mod tests {use arrow::array::record_batch;use arrow_schema::{DataType, Field, Schema, SchemaRef};use datafusion_catalog::{MemTable, TableProvider};- use datafusion_common::{exec_err, Result, TableReference};+ use datafusion_common::{Result, TableReference, exec_err};use datafusion_datasource::file_format::FileFormatFactory;use datafusion_execution::TaskContext;use datafusion_expr::ptr_eq::arc_ptr_eq;diff --git a/datafusion/ffi/src/proto/physical_extension_codec.rs b/datafusion/ffi/src/proto/physical_extension_codec.rsindex 8f3667400e..89a9a2cead 100644--- a/datafusion/ffi/src/proto/physical_extension_codec.rs+++ b/datafusion/ffi/src/proto/physical_extension_codec.rs@@ -18,8 +18,8 @@use std::ffi::c_void;use std::sync::Arc;-use abi_stable::std_types::{RResult, RSlice, RStr, RVec};use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RSlice, RStr, RVec};use datafusion_common::error::Result;use datafusion_execution::TaskContext;use datafusion_expr::{@@ -242,9 +242,12 @@ unsafe extern "C" fn try_encode_udwf_fn_wrapper(}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_PhysicalExtensionCodec) {- let private_data =- Box::from_raw(provider.private_data as *mut PhysicalExtensionCodecPrivateData);- drop(private_data);+ unsafe {+ let private_data = Box::from_raw(+ provider.private_data as *mut PhysicalExtensionCodecPrivateData,+ );+ drop(private_data);+ }}unsafe extern "C" fn clone_fn_wrapper(@@ -412,7 +415,7 @@ pub(crate) mod tests {use std::sync::Arc;use arrow_schema::{DataType, Field, Schema};- use datafusion_common::{exec_err, Result};+ use datafusion_common::{Result, exec_err};use datafusion_execution::TaskContext;use datafusion_expr::ptr_eq::arc_ptr_eq;use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF, WindowUDFImpl};diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rsindex b515125f9a..d84b710c6f 100644--- a/datafusion/ffi/src/record_batch_stream.rs+++ b/datafusion/ffi/src/record_batch_stream.rs@@ -18,12 +18,12 @@use std::{ffi::c_void, task::Poll};use abi_stable::{- std_types::{ROption, RResult},StableAbi,+ std_types::{ROption, RResult},};use arrow::array::{Array, RecordBatch};use arrow::{- array::{make_array, StructArray},+ array::{StructArray, make_array},ffi::{from_ffi, to_ffi},};use async_ffi::{ContextExt, FfiContext, FfiPoll};@@ -95,18 +95,22 @@ impl FFI_RecordBatchStream {unsafe impl Send for FFI_RecordBatchStream {}unsafe extern "C" fn schema_fn_wrapper(stream: &FFI_RecordBatchStream) -> WrappedSchema {- let private_data = stream.private_data as *const RecordBatchStreamPrivateData;- let stream = &(*private_data).rbs;+ unsafe {+ let private_data = stream.private_data as *const RecordBatchStreamPrivateData;+ let stream = &(*private_data).rbs;- (*stream).schema().into()+ (*stream).schema().into()+ }}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_RecordBatchStream) {- debug_assert!(!provider.private_data.is_null());- let private_data =- Box::from_raw(provider.private_data as *mut RecordBatchStreamPrivateData);- drop(private_data);- provider.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!provider.private_data.is_null());+ let private_data =+ Box::from_raw(provider.private_data as *mut RecordBatchStreamPrivateData);+ drop(private_data);+ provider.private_data = std::ptr::null_mut();+ }}pub(crate) fn record_batch_to_wrapped_array(@@ -138,18 +142,20 @@ unsafe extern "C" fn poll_next_fn_wrapper(stream: &FFI_RecordBatchStream,cx: &mut FfiContext,) -> FfiPoll<ROption<FFIResult<WrappedArray>>> {- let private_data = stream.private_data as *mut RecordBatchStreamPrivateData;- let stream = &mut (*private_data).rbs;+ unsafe {+ let private_data = stream.private_data as *mut RecordBatchStreamPrivateData;+ let stream = &mut (*private_data).rbs;- let _guard = (*private_data).runtime.as_ref().map(|rt| rt.enter());+ let _guard = (*private_data).runtime.as_ref().map(|rt| rt.enter());- let poll_result = cx.with_context(|std_cx| {- (*stream)- .try_poll_next_unpin(std_cx)- .map(maybe_record_batch_to_wrapped_stream)- });+ let poll_result = cx.with_context(|std_cx| {+ (*stream)+ .try_poll_next_unpin(std_cx)+ .map(maybe_record_batch_to_wrapped_stream)+ });- poll_result.into()+ poll_result.into()+ }}impl RecordBatchStream for FFI_RecordBatchStream {diff --git a/datafusion/ffi/src/schema_provider.rs b/datafusion/ffi/src/schema_provider.rsindex eafdb2e694..cea41ae30f 100644--- a/datafusion/ffi/src/schema_provider.rs+++ b/datafusion/ffi/src/schema_provider.rs@@ -18,8 +18,8 @@use std::{any::Any, ffi::c_void, sync::Arc};use abi_stable::{- std_types::{ROption, RResult, RString, RVec},StableAbi,+ std_types::{ROption, RResult, RString, RVec},};use async_ffi::{FfiFuture, FutureExt};use async_trait::async_trait;@@ -97,40 +97,48 @@ struct ProviderPrivateData {impl FFI_SchemaProvider {unsafe fn inner(&self) -> &Arc<dyn SchemaProvider + Send> {- let private_data = self.private_data as *const ProviderPrivateData;- &(*private_data).provider+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ &(*private_data).provider+ }}unsafe fn runtime(&self) -> Option<Handle> {- let private_data = self.private_data as *const ProviderPrivateData;- (*private_data).runtime.clone()+ unsafe {+ let private_data = self.private_data as *const ProviderPrivateData;+ (*private_data).runtime.clone()+ }}}unsafe extern "C" fn table_names_fn_wrapper(provider: &FFI_SchemaProvider,) -> RVec<RString> {- let provider = provider.inner();+ unsafe {+ let provider = provider.inner();- let table_names = provider.table_names();- table_names.into_iter().map(|s| s.into()).collect()+ let table_names = provider.table_names();+ table_names.into_iter().map(|s| s.into()).collect()+ }}unsafe extern "C" fn table_fn_wrapper(provider: &FFI_SchemaProvider,name: RString,) -> FfiFuture<FFIResult<ROption<FFI_TableProvider>>> {- let runtime = provider.runtime();- let provider = Arc::clone(provider.inner());+ unsafe {+ let runtime = provider.runtime();+ let provider = Arc::clone(provider.inner());- async move {- let table = rresult_return!(provider.table(name.as_str()).await)- .map(|t| FFI_TableProvider::new(t, true, runtime))- .into();+ async move {+ let table = rresult_return!(provider.table(name.as_str()).await)+ .map(|t| FFI_TableProvider::new(t, true, runtime))+ .into();- RResult::ROk(table)+ RResult::ROk(table)+ }+ .into_ffi()}- .into_ffi()}unsafe extern "C" fn register_table_fn_wrapper(@@ -138,67 +146,76 @@ unsafe extern "C" fn register_table_fn_wrapper(name: RString,table: FFI_TableProvider,) -> FFIResult<ROption<FFI_TableProvider>> {- let runtime = provider.runtime();- let provider = provider.inner();+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();- let table = Arc::new(ForeignTableProvider(table));+ let table = Arc::new(ForeignTableProvider(table));- let returned_table = rresult_return!(provider.register_table(name.into(), table))- .map(|t| FFI_TableProvider::new(t, true, runtime));+ let returned_table = rresult_return!(provider.register_table(name.into(), table))+ .map(|t| FFI_TableProvider::new(t, true, runtime));- RResult::ROk(returned_table.into())+ RResult::ROk(returned_table.into())+ }}unsafe extern "C" fn deregister_table_fn_wrapper(provider: &FFI_SchemaProvider,name: RString,) -> FFIResult<ROption<FFI_TableProvider>> {- let runtime = provider.runtime();- let provider = provider.inner();+ unsafe {+ let runtime = provider.runtime();+ let provider = provider.inner();- let returned_table = rresult_return!(provider.deregister_table(name.as_str()))- .map(|t| FFI_TableProvider::new(t, true, runtime));+ let returned_table = rresult_return!(provider.deregister_table(name.as_str()))+ .map(|t| FFI_TableProvider::new(t, true, runtime));- RResult::ROk(returned_table.into())+ RResult::ROk(returned_table.into())+ }}unsafe extern "C" fn table_exist_fn_wrapper(provider: &FFI_SchemaProvider,name: RString,) -> bool {- provider.inner().table_exist(name.as_str())+ unsafe { provider.inner().table_exist(name.as_str()) }}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_SchemaProvider) {- debug_assert!(!provider.private_data.is_null());- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);- drop(private_data);- provider.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!provider.private_data.is_null());+ let private_data =+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);+ drop(private_data);+ provider.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_SchemaProvider,) -> FFI_SchemaProvider {- let old_private_data = provider.private_data as *const ProviderPrivateData;- let runtime = (*old_private_data).runtime.clone();-- let private_data = Box::into_raw(Box::new(ProviderPrivateData {- provider: Arc::clone(&(*old_private_data).provider),- runtime,- })) as *mut c_void;-- FFI_SchemaProvider {- owner_name: provider.owner_name.clone(),- table_names: table_names_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,- table: table_fn_wrapper,- register_table: register_table_fn_wrapper,- deregister_table: deregister_table_fn_wrapper,- table_exist: table_exist_fn_wrapper,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ let old_private_data = provider.private_data as *const ProviderPrivateData;+ let runtime = (*old_private_data).runtime.clone();++ let private_data = Box::into_raw(Box::new(ProviderPrivateData {+ provider: Arc::clone(&(*old_private_data).provider),+ runtime,+ })) as *mut c_void;++ FFI_SchemaProvider {+ owner_name: provider.owner_name.clone(),+ table_names: table_names_fn_wrapper,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,+ table: table_fn_wrapper,+ register_table: register_table_fn_wrapper,+ deregister_table: deregister_table_fn_wrapper,+ table_exist: table_exist_fn_wrapper,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -342,11 +359,13 @@ mod tests {#[tokio::test]async fn test_round_trip_ffi_schema_provider() {let schema_provider = Arc::new(MemorySchemaProvider::new());- assert!(schema_provider- .as_ref()- .register_table("prior_table".to_string(), empty_table())- .unwrap()- .is_none());+ assert!(+ schema_provider+ .as_ref()+ .register_table("prior_table".to_string(), empty_table())+ .unwrap()+ .is_none()+ );let mut ffi_schema_provider = FFI_SchemaProvider::new(schema_provider, None);ffi_schema_provider.library_marker_id = crate::mock_foreign_marker_id;@@ -403,17 +422,21 @@ mod tests {// Verify local libraries can be downcast to their originallet foreign_schema: Arc<dyn SchemaProvider + Send> = (&ffi_schema).into();- assert!(foreign_schema- .as_any()- .downcast_ref::<MemorySchemaProvider>()- .is_some());+ assert!(+ foreign_schema+ .as_any()+ .downcast_ref::<MemorySchemaProvider>()+ .is_some()+ );// Verify different library markers generate foreign providersffi_schema.library_marker_id = crate::mock_foreign_marker_id;let foreign_schema: Arc<dyn SchemaProvider + Send> = (&ffi_schema).into();- assert!(foreign_schema- .as_any()- .downcast_ref::<ForeignSchemaProvider>()- .is_some());+ assert!(+ foreign_schema+ .as_any()+ .downcast_ref::<ForeignSchemaProvider>()+ .is_some()+ );}}diff --git a/datafusion/ffi/src/session/config.rs b/datafusion/ffi/src/session/config.rsindex 5fbd6175e0..da026bbd8c 100644--- a/datafusion/ffi/src/session/config.rs+++ b/datafusion/ffi/src/session/config.rs@@ -18,8 +18,8 @@use std::collections::HashMap;use std::ffi::c_void;-use abi_stable::std_types::{RHashMap, RString};use abi_stable::StableAbi;+use abi_stable::std_types::{RHashMap, RString};use datafusion_common::error::{DataFusionError, Result};use datafusion_execution::config::SessionConfig;@@ -84,25 +84,29 @@ unsafe extern "C" fn config_options_fn_wrapper(}unsafe extern "C" fn release_fn_wrapper(config: &mut FFI_SessionConfig) {- debug_assert!(!config.private_data.is_null());- let private_data =- Box::from_raw(config.private_data as *mut SessionConfigPrivateData);- drop(private_data);- config.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!config.private_data.is_null());+ let private_data =+ Box::from_raw(config.private_data as *mut SessionConfigPrivateData);+ drop(private_data);+ config.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(config: &FFI_SessionConfig) -> FFI_SessionConfig {- let old_private_data = config.private_data as *mut SessionConfigPrivateData;- let old_config = (*old_private_data).config.clone();+ unsafe {+ let old_private_data = config.private_data as *mut SessionConfigPrivateData;+ let old_config = (*old_private_data).config.clone();- let private_data = Box::new(SessionConfigPrivateData { config: old_config });+ let private_data = Box::new(SessionConfigPrivateData { config: old_config });- FFI_SessionConfig {- config_options: config_options_fn_wrapper,- private_data: Box::into_raw(private_data) as *mut c_void,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- library_marker_id: crate::get_library_marker_id,+ FFI_SessionConfig {+ config_options: config_options_fn_wrapper,+ private_data: Box::into_raw(private_data) as *mut c_void,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ library_marker_id: crate::get_library_marker_id,+ }}}diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rsindex dea892d432..1694f46dc7 100644--- a/datafusion/ffi/src/session/mod.rs+++ b/datafusion/ffi/src/session/mod.rs@@ -20,17 +20,17 @@ use std::collections::HashMap;use std::ffi::c_void;use std::sync::Arc;-use abi_stable::std_types::{RHashMap, RResult, RStr, RString, RVec};use abi_stable::StableAbi;-use arrow_schema::ffi::FFI_ArrowSchema;+use abi_stable::std_types::{RHashMap, RResult, RStr, RString, RVec};use arrow_schema::SchemaRef;+use arrow_schema::ffi::FFI_ArrowSchema;use async_ffi::{FfiFuture, FutureExt};use async_trait::async_trait;use datafusion_common::config::{ConfigOptions, TableOptions};use datafusion_common::{DFSchema, DataFusionError};+use datafusion_execution::TaskContext;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnv;-use datafusion_execution::TaskContext;use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::{AggregateUDF, AggregateUDFImpl, Expr, LogicalPlan, ScalarUDF, ScalarUDFImpl,@@ -39,9 +39,9 @@ use datafusion_expr::{use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_plan::ExecutionPlan;use datafusion_proto::bytes::{logical_plan_from_bytes, logical_plan_to_bytes};+use datafusion_proto::logical_plan::LogicalExtensionCodec;use datafusion_proto::logical_plan::from_proto::parse_expr;use datafusion_proto::logical_plan::to_proto::serialize_expr;-use datafusion_proto::logical_plan::LogicalExtensionCodec;use datafusion_proto::protobuf::LogicalExprNode;use datafusion_session::Session;use prost::Message;@@ -142,8 +142,10 @@ impl FFI_SessionRef {}unsafe fn runtime(&self) -> &Option<Handle> {- let private_data = self.private_data as *const SessionPrivateData;- &(*private_data).runtime+ unsafe {+ let private_data = self.private_data as *const SessionPrivateData;+ &(*private_data).runtime+ }}}@@ -161,22 +163,26 @@ unsafe extern "C" fn create_physical_plan_fn_wrapper(session: &FFI_SessionRef,logical_plan_serialized: RVec<u8>,) -> FfiFuture<FFIResult<FFI_ExecutionPlan>> {- let runtime = session.runtime().clone();- let session = session.clone();- async move {- let session = session.inner();- let task_ctx = session.task_ctx();-- let logical_plan = rresult_return!(logical_plan_from_bytes(- logical_plan_serialized.as_slice(),- task_ctx.as_ref(),- ));-- let physical_plan = session.create_physical_plan(&logical_plan).await;-- rresult!(physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, task_ctx, runtime)))+ unsafe {+ let runtime = session.runtime().clone();+ let session = session.clone();+ async move {+ let session = session.inner();+ let task_ctx = session.task_ctx();++ let logical_plan = rresult_return!(logical_plan_from_bytes(+ logical_plan_serialized.as_slice(),+ task_ctx.as_ref(),+ ));++ let physical_plan = session.create_physical_plan(&logical_plan).await;++ rresult!(+ physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, task_ctx, runtime))+ )+ }+ .into_ffi()}- .into_ffi()}unsafe extern "C" fn create_physical_expr_fn_wrapper(@@ -267,36 +273,41 @@ unsafe extern "C" fn task_ctx_fn_wrapper(session: &FFI_SessionRef) -> FFI_TaskCo}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_SessionRef) {- let private_data = Box::from_raw(provider.private_data as *mut SessionPrivateData);- drop(private_data);+ unsafe {+ let private_data =+ Box::from_raw(provider.private_data as *mut SessionPrivateData);+ drop(private_data);+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_SessionRef) -> FFI_SessionRef {- let old_private_data = provider.private_data as *const SessionPrivateData;-- let private_data = Box::into_raw(Box::new(SessionPrivateData {- session: (*old_private_data).session,- runtime: (*old_private_data).runtime.clone(),- })) as *mut c_void;-- FFI_SessionRef {- session_id: session_id_fn_wrapper,- config: config_fn_wrapper,- create_physical_plan: create_physical_plan_fn_wrapper,- create_physical_expr: create_physical_expr_fn_wrapper,- scalar_functions: scalar_functions_fn_wrapper,- aggregate_functions: aggregate_functions_fn_wrapper,- window_functions: window_functions_fn_wrapper,- table_options: table_options_fn_wrapper,- default_table_options: default_table_options_fn_wrapper,- task_ctx: task_ctx_fn_wrapper,- logical_codec: provider.logical_codec.clone(),-- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ let old_private_data = provider.private_data as *const SessionPrivateData;++ let private_data = Box::into_raw(Box::new(SessionPrivateData {+ session: (*old_private_data).session,+ runtime: (*old_private_data).runtime.clone(),+ })) as *mut c_void;++ FFI_SessionRef {+ session_id: session_id_fn_wrapper,+ config: config_fn_wrapper,+ create_physical_plan: create_physical_plan_fn_wrapper,+ create_physical_expr: create_physical_expr_fn_wrapper,+ scalar_functions: scalar_functions_fn_wrapper,+ aggregate_functions: aggregate_functions_fn_wrapper,+ window_functions: window_functions_fn_wrapper,+ table_options: table_options_fn_wrapper,+ default_table_options: default_table_options_fn_wrapper,+ task_ctx: task_ctx_fn_wrapper,+ logical_codec: provider.logical_codec.clone(),++ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -532,7 +543,9 @@ impl Session for ForeignSession {}fn table_options_mut(&mut self) -> &mut TableOptions {- log::warn!("Mutating table options is not supported via FFI. Changes will not have an effect.");+ log::warn!(+ "Mutating table options is not supported via FFI. Changes will not have an effect."+ );&mut self.table_options}@@ -577,7 +590,10 @@ mod tests {let logical_plan = LogicalPlan::default();let physical_plan = foreign_session.create_physical_plan(&logical_plan).await?;- assert_eq!(format!("{physical_plan:?}"), "EmptyExec { schema: Schema { fields: [], metadata: {} }, partitions: 1, cache: PlanProperties { eq_properties: EquivalenceProperties { eq_group: EquivalenceGroup { map: {}, classes: [] }, oeq_class: OrderingEquivalenceClass { orderings: [] }, oeq_cache: OrderingEquivalenceCache { normal_cls: OrderingEquivalenceClass { orderings: [] }, leading_map: {} }, constraints: Constraints { inner: [] }, schema: Schema { fields: [], metadata: {} } }, partitioning: UnknownPartitioning(1), emission_type: Incremental, boundedness: Bounded, evaluation_type: Lazy, scheduling_type: Cooperative, output_ordering: None } }");+ assert_eq!(+ format!("{physical_plan:?}"),+ "EmptyExec { schema: Schema { fields: [], metadata: {} }, partitions: 1, cache: PlanProperties { eq_properties: EquivalenceProperties { eq_group: EquivalenceGroup { map: {}, classes: [] }, oeq_class: OrderingEquivalenceClass { orderings: [] }, oeq_cache: OrderingEquivalenceCache { normal_cls: OrderingEquivalenceClass { orderings: [] }, leading_map: {} }, constraints: Constraints { inner: [] }, schema: Schema { fields: [], metadata: {} } }, partitioning: UnknownPartitioning(1), emission_type: Incremental, boundedness: Bounded, evaluation_type: Lazy, scheduling_type: Cooperative, output_ordering: None } }"+ );assert_eq!(format!("{:?}", foreign_session.default_table_options()),diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rsindex 01672bb69e..8929d02b0e 100644--- a/datafusion/ffi/src/table_provider.rs+++ b/datafusion/ffi/src/table_provider.rs@@ -18,8 +18,8 @@use std::{any::Any, ffi::c_void, sync::Arc};use abi_stable::{- std_types::{ROption, RResult, RVec},StableAbi,+ std_types::{ROption, RResult, RVec},};use arrow::datatypes::SchemaRef;use async_ffi::{FfiFuture, FutureExt};@@ -28,14 +28,14 @@ use datafusion::{catalog::{Session, TableProvider},datasource::TableType,error::DataFusionError,- execution::{session_state::SessionStateBuilder, TaskContext},- logical_expr::{logical_plan::dml::InsertOp, TableProviderFilterPushDown},+ execution::{TaskContext, session_state::SessionStateBuilder},+ logical_expr::{TableProviderFilterPushDown, logical_plan::dml::InsertOp},physical_plan::ExecutionPlan,prelude::{Expr, SessionContext},};use datafusion_proto::{logical_plan::{- from_proto::parse_exprs, to_proto::serialize_exprs, DefaultLogicalExtensionCodec,+ DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,},protobuf::LogicalExprList,};@@ -319,10 +319,13 @@ unsafe extern "C" fn insert_into_fn_wrapper(}unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_TableProvider) {- debug_assert!(!provider.private_data.is_null());- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);- drop(private_data);- provider.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!provider.private_data.is_null());+ let private_data =+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);+ drop(private_data);+ provider.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_TableProvider) -> FFI_TableProvider {@@ -469,7 +472,7 @@ impl TableProvider for ForeignTableProvider {return Ok(vec![TableProviderFilterPushDown::Unsupported;filters.len()- ])+ ]);}};@@ -647,18 +650,22 @@ mod tests {// Verify local libraries can be downcast to their originallet foreign_table: Arc<dyn TableProvider> = (&ffi_table).into();- assert!(foreign_table- .as_any()- .downcast_ref::<datafusion::datasource::MemTable>()- .is_some());+ assert!(+ foreign_table+ .as_any()+ .downcast_ref::<datafusion::datasource::MemTable>()+ .is_some()+ );// Verify different library markers generate foreign providersffi_table.library_marker_id = crate::mock_foreign_marker_id;let foreign_table: Arc<dyn TableProvider> = (&ffi_table).into();- assert!(foreign_table- .as_any()- .downcast_ref::<ForeignTableProvider>()- .is_some());+ assert!(+ foreign_table+ .as_any()+ .downcast_ref::<ForeignTableProvider>()+ .is_some()+ );Ok(())}diff --git a/datafusion/ffi/src/tests/catalog.rs b/datafusion/ffi/src/tests/catalog.rsindex b6efbdf726..de012659f2 100644--- a/datafusion/ffi/src/tests/catalog.rs+++ b/datafusion/ffi/src/tests/catalog.rs@@ -163,7 +163,9 @@ impl CatalogProvider for FixedCatalogProvider {schema: Arc<dyn SchemaProvider>,) -> Result<Option<Arc<dyn SchemaProvider>>> {if !["apple", "banana", "cherry", "date"].contains(&name) {- return exec_err!("FixedCatalogProvider only provides four schemas: apple, banana, cherry, date");+ return exec_err!(+ "FixedCatalogProvider only provides four schemas: apple, banana, cherry, date"+ );}self.inner.register_schema(name, schema)@@ -222,7 +224,9 @@ impl CatalogProviderList for FixedCatalogProviderList {catalog: Arc<dyn CatalogProvider>,) -> Option<Arc<dyn CatalogProvider>> {if !["blue", "red", "green", "yellow"].contains(&name.as_str()) {- log::warn!("FixedCatalogProviderList only provides four catalogs: blue, red, green, yellow");+ log::warn!(+ "FixedCatalogProviderList only provides four catalogs: blue, red, green, yellow"+ );return None;}diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex d9b4a61579..dfb4756107 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -18,12 +18,11 @@use std::sync::Arc;use abi_stable::{- declare_root_module_statics, export_root_module,+ StableAbi, declare_root_module_statics, export_root_module,library::{LibraryError, RootModule},package_version_strings,prefix_type::PrefixTypeTrait,sabi_types::VersionStrings,- StableAbi,};use catalog::create_catalog_provider;diff --git a/datafusion/ffi/src/udaf/accumulator.rs b/datafusion/ffi/src/udaf/accumulator.rsindex c0ab0bdd5b..924371538c 100644--- a/datafusion/ffi/src/udaf/accumulator.rs+++ b/datafusion/ffi/src/udaf/accumulator.rs@@ -16,8 +16,8 @@// under the License.use abi_stable::{- std_types::{RResult, RVec},StableAbi,+ std_types::{RResult, RVec},};use arrow::{array::ArrayRef, error::ArrowError};use datafusion::{@@ -86,14 +86,18 @@ pub struct AccumulatorPrivateData {impl FFI_Accumulator {#[inline]unsafe fn inner_mut(&mut self) -> &mut Box<dyn Accumulator> {- let private_data = self.private_data as *mut AccumulatorPrivateData;- &mut (*private_data).accumulator+ unsafe {+ let private_data = self.private_data as *mut AccumulatorPrivateData;+ &mut (*private_data).accumulator+ }}#[inline]unsafe fn inner(&self) -> &dyn Accumulator {- let private_data = self.private_data as *const AccumulatorPrivateData;- (*private_data).accumulator.deref()+ unsafe {+ let private_data = self.private_data as *const AccumulatorPrivateData;+ (*private_data).accumulator.deref()+ }}}@@ -101,87 +105,101 @@ unsafe extern "C" fn update_batch_fn_wrapper(accumulator: &mut FFI_Accumulator,values: RVec<WrappedArray>,) -> FFIResult<()> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let values_arrays = values- .into_iter()- .map(|v| v.try_into().map_err(DataFusionError::from))- .collect::<Result<Vec<ArrayRef>>>();- let values_arrays = rresult_return!(values_arrays);+ let values_arrays = values+ .into_iter()+ .map(|v| v.try_into().map_err(DataFusionError::from))+ .collect::<Result<Vec<ArrayRef>>>();+ let values_arrays = rresult_return!(values_arrays);- rresult!(accumulator.update_batch(&values_arrays))+ rresult!(accumulator.update_batch(&values_arrays))+ }}unsafe extern "C" fn evaluate_fn_wrapper(accumulator: &mut FFI_Accumulator,) -> FFIResult<RVec<u8>> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let scalar_result = rresult_return!(accumulator.evaluate());- let proto_result: datafusion_proto::protobuf::ScalarValue =- rresult_return!((&scalar_result).try_into());+ let scalar_result = rresult_return!(accumulator.evaluate());+ let proto_result: datafusion_proto::protobuf::ScalarValue =+ rresult_return!((&scalar_result).try_into());- RResult::ROk(proto_result.encode_to_vec().into())+ RResult::ROk(proto_result.encode_to_vec().into())+ }}unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_Accumulator) -> usize {- accumulator.inner().size()+ unsafe { accumulator.inner().size() }}unsafe extern "C" fn state_fn_wrapper(accumulator: &mut FFI_Accumulator,) -> FFIResult<RVec<RVec<u8>>> {- let accumulator = accumulator.inner_mut();-- let state = rresult_return!(accumulator.state());- let state = state- .into_iter()- .map(|state_val| {- datafusion_proto::protobuf::ScalarValue::try_from(&state_val)- .map_err(DataFusionError::from)- .map(|v| RVec::from(v.encode_to_vec()))- })- .collect::<Result<Vec<_>>>()- .map(|state_vec| state_vec.into());-- rresult!(state)+ unsafe {+ let accumulator = accumulator.inner_mut();++ let state = rresult_return!(accumulator.state());+ let state = state+ .into_iter()+ .map(|state_val| {+ datafusion_proto::protobuf::ScalarValue::try_from(&state_val)+ .map_err(DataFusionError::from)+ .map(|v| RVec::from(v.encode_to_vec()))+ })+ .collect::<Result<Vec<_>>>()+ .map(|state_vec| state_vec.into());++ rresult!(state)+ }}unsafe extern "C" fn merge_batch_fn_wrapper(accumulator: &mut FFI_Accumulator,states: RVec<WrappedArray>,) -> FFIResult<()> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let states = rresult_return!(states- .into_iter()- .map(|state| ArrayRef::try_from(state).map_err(DataFusionError::from))- .collect::<Result<Vec<_>>>());+ let states = rresult_return!(+ states+ .into_iter()+ .map(|state| ArrayRef::try_from(state).map_err(DataFusionError::from))+ .collect::<Result<Vec<_>>>()+ );- rresult!(accumulator.merge_batch(&states))+ rresult!(accumulator.merge_batch(&states))+ }}unsafe extern "C" fn retract_batch_fn_wrapper(accumulator: &mut FFI_Accumulator,values: RVec<WrappedArray>,) -> FFIResult<()> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let values_arrays = values- .into_iter()- .map(|v| v.try_into().map_err(DataFusionError::from))- .collect::<Result<Vec<ArrayRef>>>();- let values_arrays = rresult_return!(values_arrays);+ let values_arrays = values+ .into_iter()+ .map(|v| v.try_into().map_err(DataFusionError::from))+ .collect::<Result<Vec<ArrayRef>>>();+ let values_arrays = rresult_return!(values_arrays);- rresult!(accumulator.retract_batch(&values_arrays))+ rresult!(accumulator.retract_batch(&values_arrays))+ }}unsafe extern "C" fn release_fn_wrapper(accumulator: &mut FFI_Accumulator) {- if !accumulator.private_data.is_null() {- let private_data =- Box::from_raw(accumulator.private_data as *mut AccumulatorPrivateData);- drop(private_data);- accumulator.private_data = null_mut();+ unsafe {+ if !accumulator.private_data.is_null() {+ let private_data =+ Box::from_raw(accumulator.private_data as *mut AccumulatorPrivateData);+ drop(private_data);+ accumulator.private_data = null_mut();+ }}}@@ -323,7 +341,7 @@ impl Accumulator for ForeignAccumulator {#[cfg(test)]mod tests {use super::{FFI_Accumulator, ForeignAccumulator};- use arrow::array::{make_array, Array};+ use arrow::array::{Array, make_array};use datafusion::{common::create_array, error::Result,functions_aggregate::average::AvgAccumulator, logical_expr::Accumulator,diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rsindex d79ecba14c..a5cc370436 100644--- a/datafusion/ffi/src/udaf/accumulator_args.rs+++ b/datafusion/ffi/src/udaf/accumulator_args.rs@@ -19,8 +19,8 @@ use std::sync::Arc;use crate::arrow_wrappers::WrappedSchema;use abi_stable::{- std_types::{RString, RVec},StableAbi,+ std_types::{RString, RVec},};use arrow::{datatypes::Schema, ffi::FFI_ArrowSchema};use arrow_schema::FieldRef;@@ -33,9 +33,9 @@ use datafusion::{use datafusion_common::ffi_datafusion_err;use datafusion_proto::{physical_plan::{+ DefaultPhysicalExtensionCodec,from_proto::{parse_physical_exprs, parse_physical_sort_exprs},to_proto::{serialize_physical_exprs, serialize_physical_sort_exprs},- DefaultPhysicalExtensionCodec,},protobuf::PhysicalAggregateExprNode,};diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rsindex 69113d8128..403c09fb51 100644--- a/datafusion/ffi/src/udaf/groups_accumulator.rs+++ b/datafusion/ffi/src/udaf/groups_accumulator.rs@@ -21,8 +21,8 @@ use crate::{df_result, rresult, rresult_return,};use abi_stable::{- std_types::{ROption, RVec},StableAbi,+ std_types::{ROption, RVec},};use arrow::{array::{Array, ArrayRef, BooleanArray},@@ -100,14 +100,18 @@ pub struct GroupsAccumulatorPrivateData {impl FFI_GroupsAccumulator {#[inline]unsafe fn inner_mut(&mut self) -> &mut Box<dyn GroupsAccumulator> {- let private_data = self.private_data as *mut GroupsAccumulatorPrivateData;- &mut (*private_data).accumulator+ unsafe {+ let private_data = self.private_data as *mut GroupsAccumulatorPrivateData;+ &mut (*private_data).accumulator+ }}#[inline]unsafe fn inner(&self) -> &dyn GroupsAccumulator {- let private_data = self.private_data as *const GroupsAccumulatorPrivateData;- (*private_data).accumulator.deref()+ unsafe {+ let private_data = self.private_data as *const GroupsAccumulatorPrivateData;+ (*private_data).accumulator.deref()+ }}}@@ -137,46 +141,56 @@ unsafe extern "C" fn update_batch_fn_wrapper(opt_filter: ROption<WrappedArray>,total_num_groups: usize,) -> FFIResult<()> {- let accumulator = accumulator.inner_mut();- let values = rresult_return!(process_values(values));- let group_indices: Vec<usize> = group_indices.into_iter().collect();- let opt_filter = rresult_return!(process_opt_filter(opt_filter));-- rresult!(accumulator.update_batch(- &values,- &group_indices,- opt_filter.as_ref(),- total_num_groups- ))+ unsafe {+ let accumulator = accumulator.inner_mut();+ let values = rresult_return!(process_values(values));+ let group_indices: Vec<usize> = group_indices.into_iter().collect();+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));++ rresult!(accumulator.update_batch(+ &values,+ &group_indices,+ opt_filter.as_ref(),+ total_num_groups+ ))+ }}unsafe extern "C" fn evaluate_fn_wrapper(accumulator: &mut FFI_GroupsAccumulator,emit_to: FFI_EmitTo,) -> FFIResult<WrappedArray> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let result = rresult_return!(accumulator.evaluate(emit_to.into()));+ let result = rresult_return!(accumulator.evaluate(emit_to.into()));- rresult!(WrappedArray::try_from(&result))+ rresult!(WrappedArray::try_from(&result))+ }}unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_GroupsAccumulator) -> usize {- let accumulator = accumulator.inner();- accumulator.size()+ unsafe {+ let accumulator = accumulator.inner();+ accumulator.size()+ }}unsafe extern "C" fn state_fn_wrapper(accumulator: &mut FFI_GroupsAccumulator,emit_to: FFI_EmitTo,) -> FFIResult<RVec<WrappedArray>> {- let accumulator = accumulator.inner_mut();+ unsafe {+ let accumulator = accumulator.inner_mut();- let state = rresult_return!(accumulator.state(emit_to.into()));- rresult!(state- .into_iter()- .map(|arr| WrappedArray::try_from(&arr).map_err(DataFusionError::from))- .collect::<Result<RVec<_>>>())+ let state = rresult_return!(accumulator.state(emit_to.into()));+ rresult!(+ state+ .into_iter()+ .map(|arr| WrappedArray::try_from(&arr).map_err(DataFusionError::from))+ .collect::<Result<RVec<_>>>()+ )+ }}unsafe extern "C" fn merge_batch_fn_wrapper(@@ -186,17 +200,19 @@ unsafe extern "C" fn merge_batch_fn_wrapper(opt_filter: ROption<WrappedArray>,total_num_groups: usize,) -> FFIResult<()> {- let accumulator = accumulator.inner_mut();- let values = rresult_return!(process_values(values));- let group_indices: Vec<usize> = group_indices.into_iter().collect();- let opt_filter = rresult_return!(process_opt_filter(opt_filter));-- rresult!(accumulator.merge_batch(- &values,- &group_indices,- opt_filter.as_ref(),- total_num_groups- ))+ unsafe {+ let accumulator = accumulator.inner_mut();+ let values = rresult_return!(process_values(values));+ let group_indices: Vec<usize> = group_indices.into_iter().collect();+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));++ rresult!(accumulator.merge_batch(+ &values,+ &group_indices,+ opt_filter.as_ref(),+ total_num_groups+ ))+ }}unsafe extern "C" fn convert_to_state_fn_wrapper(@@ -204,24 +220,31 @@ unsafe extern "C" fn convert_to_state_fn_wrapper(values: RVec<WrappedArray>,opt_filter: ROption<WrappedArray>,) -> FFIResult<RVec<WrappedArray>> {- let accumulator = accumulator.inner();- let values = rresult_return!(process_values(values));- let opt_filter = rresult_return!(process_opt_filter(opt_filter));- let state =- rresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref()));-- rresult!(state- .iter()- .map(|arr| WrappedArray::try_from(arr).map_err(DataFusionError::from))- .collect::<Result<RVec<_>>>())+ unsafe {+ let accumulator = accumulator.inner();+ let values = rresult_return!(process_values(values));+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));+ let state =+ rresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref()));++ rresult!(+ state+ .iter()+ .map(|arr| WrappedArray::try_from(arr).map_err(DataFusionError::from))+ .collect::<Result<RVec<_>>>()+ )+ }}unsafe extern "C" fn release_fn_wrapper(accumulator: &mut FFI_GroupsAccumulator) {- if !accumulator.private_data.is_null() {- let private_data =- Box::from_raw(accumulator.private_data as *mut GroupsAccumulatorPrivateData);- drop(private_data);- accumulator.private_data = null_mut();+ unsafe {+ if !accumulator.private_data.is_null() {+ let private_data = Box::from_raw(+ accumulator.private_data as *mut GroupsAccumulatorPrivateData,+ );+ drop(private_data);+ accumulator.private_data = null_mut();+ }}}@@ -446,7 +469,7 @@ impl From<FFI_EmitTo> for EmitTo {#[cfg(test)]mod tests {use super::{FFI_EmitTo, FFI_GroupsAccumulator, ForeignGroupsAccumulator};- use arrow::array::{make_array, Array, BooleanArray};+ use arrow::array::{Array, BooleanArray, make_array};use datafusion::functions_aggregate::stddev::StddevGroupsAccumulator;use datafusion::{common::create_array,diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rsindex 838c3f909b..352b804338 100644--- a/datafusion/ffi/src/udaf/mod.rs+++ b/datafusion/ffi/src/udaf/mod.rs@@ -16,8 +16,8 @@// under the License.use abi_stable::{- std_types::{ROption, RResult, RStr, RString, RVec},StableAbi,+ std_types::{ROption, RResult, RStr, RString, RVec},};use accumulator::FFI_Accumulator;use accumulator_args::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};@@ -27,10 +27,10 @@ use arrow_schema::FieldRef;use datafusion::{error::DataFusionError,logical_expr::{+ Accumulator, GroupsAccumulator,function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},type_coercion::functions::fields_with_aggregate_udf,utils::AggregateOrderSensitivity,- Accumulator, GroupsAccumulator,},};use datafusion::{@@ -44,7 +44,7 @@ use std::hash::{Hash, Hasher};use std::{ffi::c_void, sync::Arc};use crate::util::{- rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped, FFIResult,+ FFIResult, rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped,};use crate::{arrow_wrappers::WrappedSchema,@@ -163,8 +163,10 @@ pub struct AggregateUDFPrivateData {impl FFI_AggregateUDF {unsafe fn inner(&self) -> &Arc<AggregateUDF> {- let private_data = self.private_data as *const AggregateUDFPrivateData;- &(*private_data).udaf+ unsafe {+ let private_data = self.private_data as *const AggregateUDFPrivateData;+ &(*private_data).udaf+ }}}@@ -172,87 +174,104 @@ unsafe extern "C" fn return_field_fn_wrapper(udaf: &FFI_AggregateUDF,arg_fields: RVec<WrappedSchema>,) -> FFIResult<WrappedSchema> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let arg_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields));+ let arg_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields));- let return_field = udaf- .return_field(&arg_fields)- .and_then(|v| {- FFI_ArrowSchema::try_from(v.as_ref()).map_err(DataFusionError::from)- })- .map(WrappedSchema);+ let return_field = udaf+ .return_field(&arg_fields)+ .and_then(|v| {+ FFI_ArrowSchema::try_from(v.as_ref()).map_err(DataFusionError::from)+ })+ .map(WrappedSchema);- rresult!(return_field)+ rresult!(return_field)+ }}unsafe extern "C" fn accumulator_fn_wrapper(udaf: &FFI_AggregateUDF,args: FFI_AccumulatorArgs,) -> FFIResult<FFI_Accumulator> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));- rresult!(udaf- .accumulator(accumulator_args.into())- .map(FFI_Accumulator::from))+ rresult!(+ udaf.accumulator(accumulator_args.into())+ .map(FFI_Accumulator::from)+ )+ }}unsafe extern "C" fn create_sliding_accumulator_fn_wrapper(udaf: &FFI_AggregateUDF,args: FFI_AccumulatorArgs,) -> FFIResult<FFI_Accumulator> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));- rresult!(udaf- .create_sliding_accumulator(accumulator_args.into())- .map(FFI_Accumulator::from))+ rresult!(+ udaf.create_sliding_accumulator(accumulator_args.into())+ .map(FFI_Accumulator::from)+ )+ }}unsafe extern "C" fn create_groups_accumulator_fn_wrapper(udaf: &FFI_AggregateUDF,args: FFI_AccumulatorArgs,) -> FFIResult<FFI_GroupsAccumulator> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));- rresult!(udaf- .create_groups_accumulator(accumulator_args.into())- .map(FFI_GroupsAccumulator::from))+ rresult!(+ udaf.create_groups_accumulator(accumulator_args.into())+ .map(FFI_GroupsAccumulator::from)+ )+ }}unsafe extern "C" fn groups_accumulator_supported_fn_wrapper(udaf: &FFI_AggregateUDF,args: FFI_AccumulatorArgs,) -> bool {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- ForeignAccumulatorArgs::try_from(args)- .map(|a| udaf.groups_accumulator_supported((&a).into()))- .unwrap_or_else(|e| {- log::warn!("Unable to parse accumulator args. {e}");- false- })+ ForeignAccumulatorArgs::try_from(args)+ .map(|a| udaf.groups_accumulator_supported((&a).into()))+ .unwrap_or_else(|e| {+ log::warn!("Unable to parse accumulator args. {e}");+ false+ })+ }}unsafe extern "C" fn with_beneficial_ordering_fn_wrapper(udaf: &FFI_AggregateUDF,beneficial_ordering: bool,) -> FFIResult<ROption<FFI_AggregateUDF>> {- let udaf = udaf.inner().as_ref().clone();+ unsafe {+ let udaf = udaf.inner().as_ref().clone();- let result = rresult_return!(udaf.with_beneficial_ordering(beneficial_ordering));- let result = rresult_return!(result- .map(|func| func.with_beneficial_ordering(beneficial_ordering))- .transpose())- .flatten()- .map(|func| FFI_AggregateUDF::from(Arc::new(func)));+ let result = rresult_return!(udaf.with_beneficial_ordering(beneficial_ordering));+ let result = rresult_return!(+ result+ .map(|func| func.with_beneficial_ordering(beneficial_ordering))+ .transpose()+ )+ .flatten()+ .map(|func| FFI_AggregateUDF::from(Arc::new(func)));- RResult::ROk(result.into())+ RResult::ROk(result.into())+ }}unsafe extern "C" fn state_fields_fn_wrapper(@@ -263,79 +282,93 @@ unsafe extern "C" fn state_fields_fn_wrapper(ordering_fields: RVec<RVec<u8>>,is_distinct: bool,) -> FFIResult<RVec<RVec<u8>>> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let input_fields = &rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));- let return_field = rresult_return!(Field::try_from(&return_field.0)).into();+ let input_fields = &rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));+ let return_field = rresult_return!(Field::try_from(&return_field.0)).into();- let ordering_fields = &rresult_return!(ordering_fields- .into_iter()- .map(|field_bytes| datafusion_proto_common::Field::decode(field_bytes.as_ref()))- .collect::<std::result::Result<Vec<_>, DecodeError>>());+ let ordering_fields = &rresult_return!(+ ordering_fields+ .into_iter()+ .map(|field_bytes| datafusion_proto_common::Field::decode(+ field_bytes.as_ref()+ ))+ .collect::<std::result::Result<Vec<_>, DecodeError>>()+ );++ let ordering_fields =+ &rresult_return!(parse_proto_fields_to_fields(ordering_fields))+ .into_iter()+ .map(Arc::new)+ .collect::<Vec<_>>();++ let args = StateFieldsArgs {+ name: name.as_str(),+ input_fields,+ return_field,+ ordering_fields,+ is_distinct,+ };- let ordering_fields = &rresult_return!(parse_proto_fields_to_fields(ordering_fields))+ let state_fields = rresult_return!(udaf.state_fields(args));+ let state_fields = rresult_return!(+ state_fields+ .iter()+ .map(|f| f.as_ref())+ .map(datafusion_proto::protobuf::Field::try_from)+ .map(|v| v.map_err(DataFusionError::from))+ .collect::<Result<Vec<_>>>()+ ).into_iter()- .map(Arc::new)- .collect::<Vec<_>>();-- let args = StateFieldsArgs {- name: name.as_str(),- input_fields,- return_field,- ordering_fields,- is_distinct,- };+ .map(|field| field.encode_to_vec().into())+ .collect();- let state_fields = rresult_return!(udaf.state_fields(args));- let state_fields = rresult_return!(state_fields- .iter()- .map(|f| f.as_ref())- .map(datafusion_proto::protobuf::Field::try_from)- .map(|v| v.map_err(DataFusionError::from))- .collect::<Result<Vec<_>>>())- .into_iter()- .map(|field| field.encode_to_vec().into())- .collect();-- RResult::ROk(state_fields)+ RResult::ROk(state_fields)+ }}unsafe extern "C" fn order_sensitivity_fn_wrapper(udaf: &FFI_AggregateUDF,) -> FFI_AggregateOrderSensitivity {- udaf.inner().order_sensitivity().into()+ unsafe { udaf.inner().order_sensitivity().into() }}unsafe extern "C" fn coerce_types_fn_wrapper(udaf: &FFI_AggregateUDF,arg_types: RVec<WrappedSchema>,) -> FFIResult<RVec<WrappedSchema>> {- let udaf = udaf.inner();+ unsafe {+ let udaf = udaf.inner();- let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types));+ let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types));- let arg_fields = arg_types- .iter()- .map(|dt| Field::new("f", dt.clone(), true))- .map(Arc::new)- .collect::<Vec<_>>();- let return_types = rresult_return!(fields_with_aggregate_udf(&arg_fields, udaf))- .into_iter()- .map(|f| f.data_type().to_owned())- .collect::<Vec<_>>();+ let arg_fields = arg_types+ .iter()+ .map(|dt| Field::new("f", dt.clone(), true))+ .map(Arc::new)+ .collect::<Vec<_>>();+ let return_types = rresult_return!(fields_with_aggregate_udf(&arg_fields, udaf))+ .into_iter()+ .map(|f| f.data_type().to_owned())+ .collect::<Vec<_>>();- rresult!(vec_datatype_to_rvec_wrapped(&return_types))+ rresult!(vec_datatype_to_rvec_wrapped(&return_types))+ }}unsafe extern "C" fn release_fn_wrapper(udaf: &mut FFI_AggregateUDF) {- debug_assert!(!udaf.private_data.is_null());- let private_data = Box::from_raw(udaf.private_data as *mut AggregateUDFPrivateData);- drop(private_data);- udaf.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!udaf.private_data.is_null());+ let private_data =+ Box::from_raw(udaf.private_data as *mut AggregateUDFPrivateData);+ drop(private_data);+ udaf.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(udaf: &FFI_AggregateUDF) -> FFI_AggregateUDF {- Arc::clone(udaf.inner()).into()+ unsafe { Arc::clone(udaf.inner()).into() }}impl Clone for FFI_AggregateUDF {@@ -835,10 +868,12 @@ mod tests {// Verify different library markers generate foreign providersffi_udaf.library_marker_id = crate::mock_foreign_marker_id;let foreign_udaf: Arc<dyn AggregateUDFImpl> = (&ffi_udaf).into();- assert!(foreign_udaf- .as_any()- .downcast_ref::<ForeignAggregateUDF>()- .is_some());+ assert!(+ foreign_udaf+ .as_any()+ .downcast_ref::<ForeignAggregateUDF>()+ .is_some()+ );Ok(())}diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rsindex 7ed3105e63..6df0e0bca5 100644--- a/datafusion/ffi/src/udf/mod.rs+++ b/datafusion/ffi/src/udf/mod.rs@@ -23,14 +23,14 @@ use crate::{volatility::FFI_Volatility,};use abi_stable::{- std_types::{RResult, RString, RVec},StableAbi,+ std_types::{RResult, RString, RVec},};use arrow::datatypes::{DataType, Field};use arrow::{array::ArrayRef,error::ArrowError,- ffi::{from_ffi, to_ffi, FFI_ArrowSchema},+ ffi::{FFI_ArrowSchema, from_ffi, to_ffi},};use arrow_schema::FieldRef;use datafusion::config::ConfigOptions;@@ -163,61 +163,68 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper(number_rows: usize,return_field: WrappedSchema,) -> FFIResult<WrappedArray> {- let args = args- .into_iter()- .map(|arr| {- from_ffi(arr.array, &arr.schema.0)- .map(|v| ColumnarValue::Array(arrow::array::make_array(v)))- })- .collect::<std::result::Result<_, _>>();+ unsafe {+ let args = args+ .into_iter()+ .map(|arr| {+ from_ffi(arr.array, &arr.schema.0)+ .map(|v| ColumnarValue::Array(arrow::array::make_array(v)))+ })+ .collect::<std::result::Result<_, _>>();- let args = rresult_return!(args);- let return_field = rresult_return!(Field::try_from(&return_field.0)).into();+ let args = rresult_return!(args);+ let return_field = rresult_return!(Field::try_from(&return_field.0)).into();- let arg_fields = arg_fields- .into_iter()- .map(|wrapped_field| {- Field::try_from(&wrapped_field.0)- .map(Arc::new)- .map_err(DataFusionError::from)- })- .collect::<Result<Vec<FieldRef>>>();- let arg_fields = rresult_return!(arg_fields);-- let args = ScalarFunctionArgs {- args,- arg_fields,- number_rows,- return_field,- // TODO: pass config options: https://github.com/apache/datafusion/issues/17035- config_options: Arc::new(ConfigOptions::default()),- };-- let result = rresult_return!(udf- .inner()- .invoke_with_args(args)- .and_then(|r| r.to_array(number_rows)));+ let arg_fields = arg_fields+ .into_iter()+ .map(|wrapped_field| {+ Field::try_from(&wrapped_field.0)+ .map(Arc::new)+ .map_err(DataFusionError::from)+ })+ .collect::<Result<Vec<FieldRef>>>();+ let arg_fields = rresult_return!(arg_fields);- let (result_array, result_schema) = rresult_return!(to_ffi(&result.to_data()));+ let args = ScalarFunctionArgs {+ args,+ arg_fields,+ number_rows,+ return_field,+ // TODO: pass config options: https://github.com/apache/datafusion/issues/17035+ config_options: Arc::new(ConfigOptions::default()),+ };- RResult::ROk(WrappedArray {- array: result_array,- schema: WrappedSchema(result_schema),- })+ let result = rresult_return!(+ udf.inner()+ .invoke_with_args(args)+ .and_then(|r| r.to_array(number_rows))+ );++ let (result_array, result_schema) = rresult_return!(to_ffi(&result.to_data()));++ RResult::ROk(WrappedArray {+ array: result_array,+ schema: WrappedSchema(result_schema),+ })+ }}unsafe extern "C" fn release_fn_wrapper(udf: &mut FFI_ScalarUDF) {- debug_assert!(!udf.private_data.is_null());- let private_data = Box::from_raw(udf.private_data as *mut ScalarUDFPrivateData);- drop(private_data);- udf.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!udf.private_data.is_null());+ let private_data = Box::from_raw(udf.private_data as *mut ScalarUDFPrivateData);+ drop(private_data);+ udf.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(udf: &FFI_ScalarUDF) -> FFI_ScalarUDF {- let private_data = udf.private_data as *const ScalarUDFPrivateData;- let udf_data = &(*private_data);+ unsafe {+ let private_data = udf.private_data as *const ScalarUDFPrivateData;+ let udf_data = &(*private_data);- Arc::clone(&udf_data.udf).into()+ Arc::clone(&udf_data.udf).into()+ }}impl Clone for FFI_ScalarUDF {@@ -460,10 +467,12 @@ mod tests {// Verify different library markers generate foreign providersffi_udf.library_marker_id = crate::mock_foreign_marker_id;let foreign_udf: Arc<dyn ScalarUDFImpl> = (&ffi_udf).into();- assert!(foreign_udf- .as_any()- .downcast_ref::<ForeignScalarUDF>()- .is_some());+ assert!(+ foreign_udf+ .as_any()+ .downcast_ref::<ForeignScalarUDF>()+ .is_some()+ );Ok(())}diff --git a/datafusion/ffi/src/udf/return_type_args.rs b/datafusion/ffi/src/udf/return_type_args.rsindex c9396b7971..2803558e56 100644--- a/datafusion/ffi/src/udf/return_type_args.rs+++ b/datafusion/ffi/src/udf/return_type_args.rs@@ -16,8 +16,8 @@// under the License.use abi_stable::{- std_types::{ROption, RVec},StableAbi,+ std_types::{ROption, RVec},};use arrow_schema::FieldRef;use datafusion::{diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rsindex f154295958..ccecee2217 100644--- a/datafusion/ffi/src/udtf.rs+++ b/datafusion/ffi/src/udtf.rs@@ -18,8 +18,8 @@use std::{ffi::c_void, sync::Arc};use abi_stable::{- std_types::{RResult, RVec},StableAbi,+ std_types::{RResult, RVec},};use datafusion::error::Result;@@ -29,7 +29,7 @@ use datafusion::{};use datafusion_proto::{logical_plan::{- from_proto::parse_exprs, to_proto::serialize_exprs, DefaultLogicalExtensionCodec,+ DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,},protobuf::LogicalExprList,};@@ -106,10 +106,13 @@ unsafe extern "C" fn call_fn_wrapper(}unsafe extern "C" fn release_fn_wrapper(udtf: &mut FFI_TableFunction) {- debug_assert!(!udtf.private_data.is_null());- let private_data = Box::from_raw(udtf.private_data as *mut TableFunctionPrivateData);- drop(private_data);- udtf.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!udtf.private_data.is_null());+ let private_data =+ Box::from_raw(udtf.private_data as *mut TableFunctionPrivateData);+ drop(private_data);+ udtf.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(udtf: &FFI_TableFunction) -> FFI_TableFunction {@@ -206,7 +209,7 @@ mod tests {use super::*;use arrow::{array::{- record_batch, ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array,+ ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array, record_batch,},datatypes::{DataType, Field, Schema},};diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rsindex 7a65a4784f..f0e871fb42 100644--- a/datafusion/ffi/src/udwf/mod.rs+++ b/datafusion/ffi/src/udwf/mod.rs@@ -19,8 +19,8 @@ use std::ffi::c_void;use std::hash::{Hash, Hasher};use std::sync::Arc;-use abi_stable::std_types::{ROption, RResult, RString, RVec};use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RString, RVec};use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Schema, SchemaRef};use arrow_schema::{Field, FieldRef};@@ -43,8 +43,8 @@ mod range;use crate::arrow_wrappers::WrappedSchema;use crate::util::{- rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref,- vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped, FFIResult,+ FFIResult, rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref,+ vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped,};use crate::volatility::FFI_Volatility;use crate::{df_result, rresult, rresult_return};@@ -112,8 +112,10 @@ pub struct WindowUDFPrivateData {impl FFI_WindowUDF {unsafe fn inner(&self) -> &Arc<WindowUDF> {- let private_data = self.private_data as *const WindowUDFPrivateData;- &(*private_data).udf+ unsafe {+ let private_data = self.private_data as *const WindowUDFPrivateData;+ &(*private_data).udf+ }}}@@ -121,13 +123,16 @@ unsafe extern "C" fn partition_evaluator_fn_wrapper(udwf: &FFI_WindowUDF,args: FFI_PartitionEvaluatorArgs,) -> FFIResult<FFI_PartitionEvaluator> {- let inner = udwf.inner();+ unsafe {+ let inner = udwf.inner();- let args = rresult_return!(ForeignPartitionEvaluatorArgs::try_from(args));+ let args = rresult_return!(ForeignPartitionEvaluatorArgs::try_from(args));- let evaluator = rresult_return!(inner.partition_evaluator_factory((&args).into()));+ let evaluator =+ rresult_return!(inner.partition_evaluator_factory((&args).into()));- RResult::ROk(evaluator.into())+ RResult::ROk(evaluator.into())+ }}unsafe extern "C" fn field_fn_wrapper(@@ -135,71 +140,79 @@ unsafe extern "C" fn field_fn_wrapper(input_fields: RVec<WrappedSchema>,display_name: RString,) -> FFIResult<WrappedSchema> {- let inner = udwf.inner();+ unsafe {+ let inner = udwf.inner();- let input_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));+ let input_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));- let field = rresult_return!(inner.field(WindowUDFFieldArgs::new(- &input_fields,- display_name.as_str()- )));+ let field = rresult_return!(inner.field(WindowUDFFieldArgs::new(+ &input_fields,+ display_name.as_str()+ )));- let schema = Arc::new(Schema::new(vec![field]));+ let schema = Arc::new(Schema::new(vec![field]));- RResult::ROk(WrappedSchema::from(schema))+ RResult::ROk(WrappedSchema::from(schema))+ }}unsafe extern "C" fn coerce_types_fn_wrapper(udwf: &FFI_WindowUDF,arg_types: RVec<WrappedSchema>,) -> FFIResult<RVec<WrappedSchema>> {- let inner = udwf.inner();-- let arg_fields = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types))- .into_iter()- .map(|dt| Field::new("f", dt, false))- .map(Arc::new)- .collect::<Vec<_>>();-- let return_fields = rresult_return!(fields_with_window_udf(&arg_fields, inner));- let return_types = return_fields- .into_iter()- .map(|f| f.data_type().to_owned())- .collect::<Vec<_>>();-- rresult!(vec_datatype_to_rvec_wrapped(&return_types))+ unsafe {+ let inner = udwf.inner();++ let arg_fields = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types))+ .into_iter()+ .map(|dt| Field::new("f", dt, false))+ .map(Arc::new)+ .collect::<Vec<_>>();++ let return_fields = rresult_return!(fields_with_window_udf(&arg_fields, inner));+ let return_types = return_fields+ .into_iter()+ .map(|f| f.data_type().to_owned())+ .collect::<Vec<_>>();++ rresult!(vec_datatype_to_rvec_wrapped(&return_types))+ }}unsafe extern "C" fn release_fn_wrapper(udwf: &mut FFI_WindowUDF) {- debug_assert!(!udwf.private_data.is_null());- let private_data = Box::from_raw(udwf.private_data as *mut WindowUDFPrivateData);- drop(private_data);- udwf.private_data = std::ptr::null_mut();+ unsafe {+ debug_assert!(!udwf.private_data.is_null());+ let private_data = Box::from_raw(udwf.private_data as *mut WindowUDFPrivateData);+ drop(private_data);+ udwf.private_data = std::ptr::null_mut();+ }}unsafe extern "C" fn clone_fn_wrapper(udwf: &FFI_WindowUDF) -> FFI_WindowUDF {- // let private_data = udf.private_data as *const WindowUDFPrivateData;- // let udf_data = &(*private_data);-- // let private_data = Box::new(WindowUDFPrivateData {- // udf: Arc::clone(&udf_data.udf),- // });- let private_data = Box::new(WindowUDFPrivateData {- udf: Arc::clone(udwf.inner()),- });-- FFI_WindowUDF {- name: udwf.name.clone(),- aliases: udwf.aliases.clone(),- volatility: udwf.volatility.clone(),- partition_evaluator: partition_evaluator_fn_wrapper,- sort_options: udwf.sort_options.clone(),- coerce_types: coerce_types_fn_wrapper,- field: field_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- private_data: Box::into_raw(private_data) as *mut c_void,- library_marker_id: crate::get_library_marker_id,+ unsafe {+ // let private_data = udf.private_data as *const WindowUDFPrivateData;+ // let udf_data = &(*private_data);++ // let private_data = Box::new(WindowUDFPrivateData {+ // udf: Arc::clone(&udf_data.udf),+ // });+ let private_data = Box::new(WindowUDFPrivateData {+ udf: Arc::clone(udwf.inner()),+ });++ FFI_WindowUDF {+ name: udwf.name.clone(),+ aliases: udwf.aliases.clone(),+ volatility: udwf.volatility.clone(),+ partition_evaluator: partition_evaluator_fn_wrapper,+ sort_options: udwf.sort_options.clone(),+ coerce_types: coerce_types_fn_wrapper,+ field: field_fn_wrapper,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ private_data: Box::into_raw(private_data) as *mut c_void,+ library_marker_id: crate::get_library_marker_id,+ }}}@@ -387,10 +400,10 @@ impl From<&FFI_SortOptions> for SortOptions {mod tests {use std::sync::Arc;- use arrow::array::{create_array, ArrayRef};- use datafusion::functions_window::lead_lag::{lag_udwf, WindowShift};+ use arrow::array::{ArrayRef, create_array};+ use datafusion::functions_window::lead_lag::{WindowShift, lag_udwf};use datafusion::logical_expr::expr::Sort;- use datafusion::logical_expr::{col, ExprFunctionExt, WindowUDF, WindowUDFImpl};+ use datafusion::logical_expr::{ExprFunctionExt, WindowUDF, WindowUDFImpl, col};use datafusion::prelude::SessionContext;use crate::tests::create_record_batch;@@ -462,18 +475,22 @@ mod tests {// Verify local libraries can be downcast to their originallet foreign_udwf: Arc<dyn WindowUDFImpl> = (&ffi_udwf).into();- assert!(foreign_udwf- .as_any()- .downcast_ref::<WindowShift>()- .is_some());+ assert!(+ foreign_udwf+ .as_any()+ .downcast_ref::<WindowShift>()+ .is_some()+ );// Verify different library markers generate foreign providersffi_udwf.library_marker_id = crate::mock_foreign_marker_id;let foreign_udwf: Arc<dyn WindowUDFImpl> = (&ffi_udwf).into();- assert!(foreign_udwf- .as_any()- .downcast_ref::<ForeignWindowUDF>()- .is_some());+ assert!(+ foreign_udwf+ .as_any()+ .downcast_ref::<ForeignWindowUDF>()+ .is_some()+ );Ok(())}diff --git a/datafusion/ffi/src/udwf/partition_evaluator.rs b/datafusion/ffi/src/udwf/partition_evaluator.rsindex 57cc8f73aa..b3fa9e1ef5 100644--- a/datafusion/ffi/src/udwf/partition_evaluator.rs+++ b/datafusion/ffi/src/udwf/partition_evaluator.rs@@ -21,11 +21,11 @@ use super::range::FFI_Range;use crate::util::FFIResult;use crate::{arrow_wrappers::WrappedArray, df_result, rresult, rresult_return};use abi_stable::std_types::RResult;-use abi_stable::{std_types::RVec, StableAbi};+use abi_stable::{StableAbi, std_types::RVec};use arrow::{array::ArrayRef, error::ArrowError};use datafusion::{error::{DataFusionError, Result},- logical_expr::{window_state::WindowAggState, PartitionEvaluator},+ logical_expr::{PartitionEvaluator, window_state::WindowAggState},scalar::ScalarValue,};use prost::Message;@@ -89,13 +89,17 @@ pub struct PartitionEvaluatorPrivateData {impl FFI_PartitionEvaluator {unsafe fn inner_mut(&mut self) -> &mut Box<dyn PartitionEvaluator + 'static> {- let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;- &mut (*private_data).evaluator+ unsafe {+ let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;+ &mut (*private_data).evaluator+ }}unsafe fn inner(&self) -> &(dyn PartitionEvaluator + 'static) {- let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;- (*private_data).evaluator.as_ref()+ unsafe {+ let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;+ (*private_data).evaluator.as_ref()+ }}}@@ -104,19 +108,24 @@ unsafe extern "C" fn evaluate_all_fn_wrapper(values: RVec<WrappedArray>,num_rows: usize,) -> FFIResult<WrappedArray> {- let inner = evaluator.inner_mut();-- let values_arrays = values- .into_iter()- .map(|v| v.try_into().map_err(DataFusionError::from))- .collect::<Result<Vec<ArrayRef>>>();- let values_arrays = rresult_return!(values_arrays);-- let return_array = inner- .evaluate_all(&values_arrays, num_rows)- .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));-- rresult!(return_array)+ unsafe {+ let inner = evaluator.inner_mut();++ let values_arrays = values+ .into_iter()+ .map(|v| v.try_into().map_err(DataFusionError::from))+ .collect::<Result<Vec<ArrayRef>>>();+ let values_arrays = rresult_return!(values_arrays);++ let return_array =+ inner+ .evaluate_all(&values_arrays, num_rows)+ .and_then(|array| {+ WrappedArray::try_from(&array).map_err(DataFusionError::from)+ });++ rresult!(return_array)+ }}unsafe extern "C" fn evaluate_fn_wrapper(@@ -124,21 +133,24 @@ unsafe extern "C" fn evaluate_fn_wrapper(values: RVec<WrappedArray>,range: FFI_Range,) -> FFIResult<RVec<u8>> {- let inner = evaluator.inner_mut();-- let values_arrays = values- .into_iter()- .map(|v| v.try_into().map_err(DataFusionError::from))- .collect::<Result<Vec<ArrayRef>>>();- let values_arrays = rresult_return!(values_arrays);-- // let return_array = (inner.evaluate(&values_arrays, &range.into()));- // .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));- let scalar_result = rresult_return!(inner.evaluate(&values_arrays, &range.into()));- let proto_result: datafusion_proto::protobuf::ScalarValue =- rresult_return!((&scalar_result).try_into());-- RResult::ROk(proto_result.encode_to_vec().into())+ unsafe {+ let inner = evaluator.inner_mut();++ let values_arrays = values+ .into_iter()+ .map(|v| v.try_into().map_err(DataFusionError::from))+ .collect::<Result<Vec<ArrayRef>>>();+ let values_arrays = rresult_return!(values_arrays);++ // let return_array = (inner.evaluate(&values_arrays, &range.into()));+ // .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));+ let scalar_result =+ rresult_return!(inner.evaluate(&values_arrays, &range.into()));+ let proto_result: datafusion_proto::protobuf::ScalarValue =+ rresult_return!((&scalar_result).try_into());++ RResult::ROk(proto_result.encode_to_vec().into())+ }}unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper(@@ -146,18 +158,22 @@ unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper(num_rows: usize,ranks_in_partition: RVec<FFI_Range>,) -> FFIResult<WrappedArray> {- let inner = evaluator.inner();+ unsafe {+ let inner = evaluator.inner();- let ranks_in_partition = ranks_in_partition- .into_iter()- .map(Range::from)- .collect::<Vec<_>>();+ let ranks_in_partition = ranks_in_partition+ .into_iter()+ .map(Range::from)+ .collect::<Vec<_>>();- let return_array = inner- .evaluate_all_with_rank(num_rows, &ranks_in_partition)- .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));+ let return_array = inner+ .evaluate_all_with_rank(num_rows, &ranks_in_partition)+ .and_then(|array| {+ WrappedArray::try_from(&array).map_err(DataFusionError::from)+ });- rresult!(return_array)+ rresult!(return_array)+ }}unsafe extern "C" fn get_range_fn_wrapper(@@ -165,18 +181,23 @@ unsafe extern "C" fn get_range_fn_wrapper(idx: usize,n_rows: usize,) -> FFIResult<FFI_Range> {- let inner = evaluator.inner();- let range = inner.get_range(idx, n_rows).map(FFI_Range::from);+ unsafe {+ let inner = evaluator.inner();+ let range = inner.get_range(idx, n_rows).map(FFI_Range::from);- rresult!(range)+ rresult!(range)+ }}unsafe extern "C" fn release_fn_wrapper(evaluator: &mut FFI_PartitionEvaluator) {- if !evaluator.private_data.is_null() {- let private_data =- Box::from_raw(evaluator.private_data as *mut PartitionEvaluatorPrivateData);- drop(private_data);- evaluator.private_data = std::ptr::null_mut();+ unsafe {+ if !evaluator.private_data.is_null() {+ let private_data = Box::from_raw(+ evaluator.private_data as *mut PartitionEvaluatorPrivateData,+ );+ drop(private_data);+ evaluator.private_data = std::ptr::null_mut();+ }}}diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rsindex 93090af6f7..e272b14166 100644--- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs+++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs@@ -18,7 +18,7 @@use std::{collections::HashMap, sync::Arc};use crate::arrow_wrappers::WrappedSchema;-use abi_stable::{std_types::RVec, StableAbi};+use abi_stable::{StableAbi, std_types::RVec};use arrow::{datatypes::{DataType, Field, Schema, SchemaRef},error::ArrowError,@@ -28,14 +28,14 @@ use arrow_schema::FieldRef;use datafusion::{error::{DataFusionError, Result},logical_expr::function::PartitionEvaluatorArgs,- physical_plan::{expressions::Column, PhysicalExpr},+ physical_plan::{PhysicalExpr, expressions::Column},prelude::SessionContext,};use datafusion_common::ffi_datafusion_err;use datafusion_proto::{physical_plan::{- from_proto::parse_physical_expr, to_proto::serialize_physical_exprs,- DefaultPhysicalExtensionCodec,+ DefaultPhysicalExtensionCodec, from_proto::parse_physical_expr,+ to_proto::serialize_physical_exprs,},protobuf::PhysicalExprNode,};diff --git a/datafusion/ffi/src/util.rs b/datafusion/ffi/src/util.rsindex 330286e3d5..db6eb0552d 100644--- a/datafusion/ffi/src/util.rs+++ b/datafusion/ffi/src/util.rs@@ -180,8 +180,10 @@ pub(crate) mod tests {let returned_err_r_result = wrap_result(err_result);assert!(returned_err_r_result.is_err());- assert!(returned_err_r_result- .unwrap_err()- .starts_with(format!("FFI error: {ERROR_VALUE}").as_str()));+ assert!(+ returned_err_r_result+ .unwrap_err()+ .starts_with(format!("FFI error: {ERROR_VALUE}").as_str())+ );}}diff --git a/datafusion/ffi/tests/ffi_udaf.rs b/datafusion/ffi/tests/ffi_udaf.rsindex 23ca913fca..6aac88d494 100644--- a/datafusion/ffi/tests/ffi_udaf.rs+++ b/datafusion/ffi/tests/ffi_udaf.rs@@ -23,7 +23,7 @@ mod tests {use datafusion::common::record_batch;use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};- use datafusion::prelude::{col, SessionContext};+ use datafusion::prelude::{SessionContext, col};use std::sync::Arc;use datafusion_ffi::tests::utils::get_module;diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rsindex 5f96308355..8f63265fd7 100644--- a/datafusion/ffi/tests/ffi_udf.rs+++ b/datafusion/ffi/tests/ffi_udf.rs@@ -23,7 +23,7 @@ mod tests {use datafusion::common::record_batch;use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::{ScalarUDF, ScalarUDFImpl};- use datafusion::prelude::{col, SessionContext};+ use datafusion::prelude::{SessionContext, col};use std::sync::Arc;use datafusion_ffi::tests::create_record_batch;diff --git a/datafusion/ffi/tests/ffi_udtf.rs b/datafusion/ffi/tests/ffi_udtf.rsindex 8339b350cd..a0135a903f 100644--- a/datafusion/ffi/tests/ffi_udtf.rs+++ b/datafusion/ffi/tests/ffi_udtf.rs@@ -22,7 +22,7 @@ mod tests {use std::sync::Arc;- use arrow::array::{create_array, ArrayRef};+ use arrow::array::{ArrayRef, create_array};use datafusion::catalog::TableFunctionImpl;use datafusion::error::{DataFusionError, Result};use datafusion::prelude::SessionContext;diff --git a/datafusion/ffi/tests/ffi_udwf.rs b/datafusion/ffi/tests/ffi_udwf.rsindex e2a823cb16..01ffe59973 100644--- a/datafusion/ffi/tests/ffi_udwf.rs+++ b/datafusion/ffi/tests/ffi_udwf.rs@@ -19,10 +19,10 @@/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {- use arrow::array::{create_array, ArrayRef};+ use arrow::array::{ArrayRef, create_array};use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::expr::Sort;- use datafusion::logical_expr::{col, ExprFunctionExt, WindowUDF, WindowUDFImpl};+ use datafusion::logical_expr::{ExprFunctionExt, WindowUDF, WindowUDFImpl, col};use datafusion::prelude::SessionContext;use datafusion_ffi::tests::create_record_batch;use datafusion_ffi::tests::utils::get_module;diff --git a/datafusion/functions-aggregate-common/Cargo.toml b/datafusion/functions-aggregate-common/Cargo.tomlindex dd7a97d9c7..1d4fb29d9c 100644--- a/datafusion/functions-aggregate-common/Cargo.toml+++ b/datafusion/functions-aggregate-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Utility functions for implementing aggregate functions for the Dakeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.tomlindex 1d07c1f8a0..8f8697fef0 100644--- a/datafusion/functions-aggregate/Cargo.toml+++ b/datafusion/functions-aggregate/Cargo.toml@@ -21,7 +21,7 @@ description = "Traits and types for logical plans and expressions for DataFusionkeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.tomlindex 52cddadd0a..6b0241a10a 100644--- a/datafusion/functions-nested/Cargo.toml+++ b/datafusion/functions-nested/Cargo.toml@@ -21,7 +21,7 @@ description = "Nested Type Function packages for the DataFusion query engine"keywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/functions-table/Cargo.toml b/datafusion/functions-table/Cargo.tomlindex 8894ef8c4a..aa401fbd7d 100644--- a/datafusion/functions-table/Cargo.toml+++ b/datafusion/functions-table/Cargo.toml@@ -21,7 +21,7 @@ description = "Traits and types for logical plans and expressions for DataFusionkeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/functions-window-common/Cargo.toml b/datafusion/functions-window-common/Cargo.tomlindex d15542ffae..6af668c145 100644--- a/datafusion/functions-window-common/Cargo.toml+++ b/datafusion/functions-window-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Common functions for implementing user-defined window functions fkeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"authors = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }license = { workspace = true }repository = { workspace = true }diff --git a/datafusion/functions-window/Cargo.toml b/datafusion/functions-window/Cargo.tomlindex fbe6a34f29..42690907ae 100644--- a/datafusion/functions-window/Cargo.toml+++ b/datafusion/functions-window/Cargo.toml@@ -21,7 +21,7 @@ description = "Window function packages for the DataFusion query engine"keywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.tomlindex 443dca898b..3e832691f9 100644--- a/datafusion/functions/Cargo.toml+++ b/datafusion/functions/Cargo.toml@@ -21,7 +21,7 @@ description = "Function packages for the DataFusion query engine"keywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/macros/Cargo.toml b/datafusion/macros/Cargo.tomlindex 8e33833250..53691edf5a 100644--- a/datafusion/macros/Cargo.toml+++ b/datafusion/macros/Cargo.toml@@ -21,7 +21,7 @@ description = "Procedural macros for DataFusion query engine"keywords = ["datafusion", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.tomlindex 0fb08684cd..15d3261ca5 100644--- a/datafusion/optimizer/Cargo.toml+++ b/datafusion/optimizer/Cargo.toml@@ -21,7 +21,7 @@ description = "DataFusion Query Optimizer"keywords = ["datafusion", "query", "optimizer"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/physical-expr-adapter/Cargo.toml b/datafusion/physical-expr-adapter/Cargo.tomlindex 2436c2dbb3..03e1b1f065 100644--- a/datafusion/physical-expr-adapter/Cargo.toml+++ b/datafusion/physical-expr-adapter/Cargo.toml@@ -4,7 +4,7 @@ description = "Physical expression schema adaptation utilities for DataFusion"keywords = ["datafusion", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex df46be6f51..4602e59c42 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Common functionality of physical expression for DataFusion querykeywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.tomlindex 70cf88b5af..772a1f8f4a 100644--- a/datafusion/physical-expr/Cargo.toml+++ b/datafusion/physical-expr/Cargo.toml@@ -21,7 +21,7 @@ description = "Physical expression implementation for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/physical-optimizer/Cargo.toml b/datafusion/physical-optimizer/Cargo.tomlindex 1c94b8cd49..395da10d62 100644--- a/datafusion/physical-optimizer/Cargo.toml+++ b/datafusion/physical-optimizer/Cargo.toml@@ -21,7 +21,7 @@ description = "DataFusion Physical Optimizer"keywords = ["datafusion", "query", "optimizer"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 398dc2e793..68e67fa018 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -21,7 +21,7 @@ description = "Physical (ExecutionPlan) implementations for DataFusion query engkeywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/proto-common/Cargo.toml b/datafusion/proto-common/Cargo.tomlindex f92ee554c4..c67c8892a3 100644--- a/datafusion/proto-common/Cargo.toml+++ b/datafusion/proto-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Protobuf serialization of DataFusion common types"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex df46fb2a9f..b00bd0dcc6 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -21,7 +21,7 @@ description = "Protobuf serialization of DataFusion logical plan expressions"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/pruning/Cargo.toml b/datafusion/pruning/Cargo.tomlindex 97d9035d1e..bd898cba20 100644--- a/datafusion/pruning/Cargo.toml+++ b/datafusion/pruning/Cargo.toml@@ -3,7 +3,7 @@ name = "datafusion-pruning"description = "DataFusion Pruning Logic"readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/session/Cargo.toml b/datafusion/session/Cargo.tomlindex 3bed851425..230e26d1fc 100644--- a/datafusion/session/Cargo.toml+++ b/datafusion/session/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-session"description = "datafusion-session"readme = "README.md"authors.workspace = true-edition = "2024"+edition.workspace = truehomepage.workspace = truelicense.workspace = truerepository.workspace = truediff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex 02d747df6d..09959db41f 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -24,7 +24,7 @@ repository = { workspace = true }authors = { workspace = true }readme = "README.md"license = { workspace = true }-edition = "2024"+edition = { workspace = true }[package.metadata.docs.rs]all-features = truediff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.tomlindex f7afbe9c31..a814292a3d 100644--- a/datafusion/sql/Cargo.toml+++ b/datafusion/sql/Cargo.toml@@ -21,7 +21,7 @@ description = "DataFusion SQL Query Planner"keywords = ["datafusion", "sql", "parser", "planner"]readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/sqllogictest/bin/postgres_container.rs b/datafusion/sqllogictest/bin/postgres_container.rsindex 411562a7cc..e5addf5408 100644--- a/datafusion/sqllogictest/bin/postgres_container.rs+++ b/datafusion/sqllogictest/bin/postgres_container.rs@@ -16,19 +16,19 @@// under the License.use crate::Options;+use ContainerCommands::{FetchHost, FetchPort};use datafusion::common::Result;use log::info;use std::env::set_var;use std::future::Future;use std::sync::LazyLock;use std::{env, thread};+use testcontainers::ImageExt;use testcontainers::core::IntoContainerPort;use testcontainers::runners::AsyncRunner;-use testcontainers::ImageExt;use testcontainers_modules::postgres;use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};-use tokio::sync::{mpsc, Mutex};-use ContainerCommands::{FetchHost, FetchPort};+use tokio::sync::{Mutex, mpsc};#[derive(Debug)]pub enum ContainerCommands {@@ -86,7 +86,9 @@ pub async fn initialize_postgres_container(options: &Options) -> Result<()> {let pg_uri = format!("postgresql://postgres:postgres@{db_host}:{db_port}/test");info!("Postgres uri is {pg_uri}");- set_var("PG_URI", pg_uri);+ unsafe {+ set_var("PG_URI", pg_uri);+ }} else {// close receiverPOSTGRES_IN.rx.lock().await.close();diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rsindex 0f9f6c83dc..8037532c09 100644--- a/datafusion/sqllogictest/bin/sqllogictests.rs+++ b/datafusion/sqllogictest/bin/sqllogictests.rs@@ -18,11 +18,11 @@use clap::Parser;use datafusion::common::instant::Instant;use datafusion::common::utils::get_available_parallelism;-use datafusion::common::{exec_datafusion_err, exec_err, DataFusionError, Result};+use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};use datafusion_sqllogictest::{- df_value_validator, read_dir_recursive, setup_scratch_dir, should_skip_file,- should_skip_record, value_normalizer, CurrentlyExecutingSqlTracker, DataFusion,- DataFusionSubstraitRoundTrip, Filter, TestContext,+ CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,+ TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,+ should_skip_file, should_skip_record, value_normalizer,};use futures::stream::StreamExt;use indicatif::{@@ -32,8 +32,8 @@ use itertools::Itertools;use log::Level::Info;use log::{info, log_enabled};use sqllogictest::{- parse_file, strict_column_validator, AsyncDB, Condition, MakeConnection, Normalizer,- Record, Validator,+ AsyncDB, Condition, MakeConnection, Normalizer, Record, Validator, parse_file,+ strict_column_validator,};#[cfg(feature = "postgres")]@@ -135,7 +135,9 @@ async fn run_tests() -> Result<()> {eprintln!(" {error}");}- eprintln!("\nTemporary file check failed. Please ensure that within each test file, any scratch file created is placed under a folder with the same name as the test file (without extension).\nExample: inside `join.slt`, temporary files must be created under `.../scratch/join/`\n");+ eprintln!(+ "\nTemporary file check failed. Please ensure that within each test file, any scratch file created is placed under a folder with the same name as the test file (without extension).\nExample: inside `join.slt`, temporary files must be created under `.../scratch/join/`\n"+ );return exec_err!("sqllogictests scratch file check failed");}@@ -857,18 +859,18 @@ fn scratch_file_check(test_files: &[TestFile]) -> Result<Vec<String>> {let lines: Vec<&str> = content.lines().collect();for (line_num, line) in lines.iter().enumerate() {- if let Some(captures) = scratch_pattern.captures(line) {- if let Some(found_target) = captures.get(1) {- let found_target = found_target.as_str();- if found_target != expected_target {- errors.push(format!(- "File {}:{}: scratch target '{}' does not match file name '{}'",- test_file.path.display(),- line_num + 1,- found_target,- expected_target- ));- }+ if let Some(captures) = scratch_pattern.captures(line)+ && let Some(found_target) = captures.get(1)+ {+ let found_target = found_target.as_str();+ if found_target != expected_target {+ errors.push(format!(+ "File {}:{}: scratch target '{}' does not match file name '{}'",+ test_file.path.display(),+ line_num + 1,+ found_target,+ expected_target+ ));}}}diff --git a/datafusion/sqllogictest/src/engines/conversion.rs b/datafusion/sqllogictest/src/engines/conversion.rsindex 984845b584..633029a2de 100644--- a/datafusion/sqllogictest/src/engines/conversion.rs+++ b/datafusion/sqllogictest/src/engines/conversion.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::datatypes::{i256, Decimal128Type, Decimal256Type, DecimalType};+use arrow::datatypes::{Decimal128Type, Decimal256Type, DecimalType, i256};use bigdecimal::BigDecimal;use half::f16;use rust_decimal::prelude::*;@@ -133,7 +133,7 @@ pub(crate) fn big_decimal_to_str(value: BigDecimal, round_digits: Option<i64>) -#[cfg(test)]mod tests {use super::big_decimal_to_str;- use bigdecimal::{num_bigint::BigInt, BigDecimal};+ use bigdecimal::{BigDecimal, num_bigint::BigInt};macro_rules! assert_decimal_str_eq {($integer:expr, $scale:expr, $round_digits:expr, $expected:expr) => {diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rsindex 13c107408c..df43a9a34c 100644--- a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs@@ -18,7 +18,7 @@use std::sync::Arc;use std::{path::PathBuf, time::Duration};-use super::{error::Result, normalize, DFSqlLogicTestError};+use super::{DFSqlLogicTestError, error::Result, normalize};use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;use crate::engines::output::{DFColumnType, DFOutput};use crate::is_spark_path;diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rsindex 086e3af6f0..d4b4377e30 100644--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs@@ -21,7 +21,7 @@ use std::{path::PathBuf, time::Duration};use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;use crate::engines::datafusion_engine::Result;use crate::engines::output::{DFColumnType, DFOutput};-use crate::{convert_batches, convert_schema_to_types, DFSqlLogicTestError};+use crate::{DFSqlLogicTestError, convert_batches, convert_schema_to_types};use arrow::record_batch::RecordBatch;use async_trait::async_trait;use datafusion::logical_expr::LogicalPlan;diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rsindex 8a6f44de43..ee2987db07 100644--- a/datafusion/sqllogictest/src/engines/mod.rs+++ b/datafusion/sqllogictest/src/engines/mod.rs@@ -22,10 +22,10 @@ mod datafusion_engine;mod datafusion_substrait_roundtrip_engine;mod output;-pub use datafusion_engine::convert_batches;-pub use datafusion_engine::convert_schema_to_types;pub use datafusion_engine::DFSqlLogicTestError;pub use datafusion_engine::DataFusion;+pub use datafusion_engine::convert_batches;+pub use datafusion_engine::convert_schema_to_types;pub use datafusion_substrait_roundtrip_engine::DataFusionSubstraitRoundTrip;pub use output::DFColumnType;pub use output::DFOutput;diff --git a/datafusion/sqllogictest/src/filters.rs b/datafusion/sqllogictest/src/filters.rsindex 44482236f7..568fa3f666 100644--- a/datafusion/sqllogictest/src/filters.rs+++ b/datafusion/sqllogictest/src/filters.rs@@ -120,10 +120,10 @@ pub fn should_skip_record<D: AsyncDB>(if !loc.file().contains(&filter.file_substring) {continue;}- if let Some(line_num) = filter.line_number {- if loc.line() != line_num {- continue;- }+ if let Some(line_num) = filter.line_number+ && loc.line() != line_num+ {+ continue;}// This filter matches both file name substring and the exact@@ -142,12 +142,11 @@ fn statement_is_skippable(statement: &Statement) -> bool {// Cannot skip SELECT INTO statements, as they can also create tables// that further test cases will use.- if let SqlStatement::Query(v) = sql_stmt.as_ref() {- if let SetExpr::Select(v) = v.body.as_ref() {- if v.into.is_some() {- return false;- }- }+ if let SqlStatement::Query(v) = sql_stmt.as_ref()+ && let SetExpr::Select(v) = v.body.as_ref()+ && v.into.is_some()+ {+ return false;}// Only SELECT and EXPLAIN statements can be skipped, as any otherdiff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rsindex d9319dbbbd..f228ee89ab 100644--- a/datafusion/sqllogictest/src/lib.rs+++ b/datafusion/sqllogictest/src/lib.rs@@ -28,14 +28,14 @@mod engines;-pub use engines::convert_batches;-pub use engines::convert_schema_to_types;pub use engines::CurrentlyExecutingSqlTracker;pub use engines::DFColumnType;pub use engines::DFOutput;pub use engines::DFSqlLogicTestError;pub use engines::DataFusion;pub use engines::DataFusionSubstraitRoundTrip;+pub use engines::convert_batches;+pub use engines::convert_schema_to_types;#[cfg(feature = "postgres")]pub use engines::Postgres;diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 110108a554..dc6dc29fdc 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -32,12 +32,12 @@ use arrow::record_batch::RecordBatch;use datafusion::catalog::{CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, Session,};-use datafusion::common::{exec_err, not_impl_err, DataFusionError, Result, ScalarValue};+use datafusion::common::{DataFusionError, Result, ScalarValue, exec_err, not_impl_err};use datafusion::functions::math::abs;use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};use datafusion::logical_expr::{- create_udf, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,- Signature, Volatility,+ ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,+ Volatility, create_udf,};use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::*;@@ -49,8 +49,8 @@ use datafusion::{use crate::is_spark_path;use async_trait::async_trait;use datafusion::common::cast::as_float64_array;-use datafusion::execution::runtime_env::RuntimeEnv;use datafusion::execution::SessionStateBuilder;+use datafusion::execution::runtime_env::RuntimeEnv;use log::info;use tempfile::TempDir;@@ -440,7 +440,9 @@ impl ScalarUDFImpl for GetMetadataUdf {let key = match &args.args[1] {ColumnarValue::Scalar(ScalarValue::Utf8(Some(k))) => k.clone(),_ => {- return exec_err!("get_metadata second argument must be a string literal")+ return exec_err!(+ "get_metadata second argument must be a string literal"+ );}};diff --git a/datafusion/sqllogictest/src/util.rs b/datafusion/sqllogictest/src/util.rsindex 487bafc4c9..6a3d3944e4 100644--- a/datafusion/sqllogictest/src/util.rs+++ b/datafusion/sqllogictest/src/util.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use datafusion::common::{exec_datafusion_err, Result};+use datafusion::common::{Result, exec_datafusion_err};use itertools::Itertools;use log::Level::Warn;use log::{info, log_enabled, warn};diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.tomlindex 67097ff522..8bfec86497 100644--- a/datafusion/substrait/Cargo.toml+++ b/datafusion/substrait/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-substrait"description = "DataFusion Substrait Producer and Consumer"readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/wasmtest/Cargo.toml b/datafusion/wasmtest/Cargo.tomlindex dfe6d84dd6..16fa9790f6 100644--- a/datafusion/wasmtest/Cargo.toml+++ b/datafusion/wasmtest/Cargo.toml@@ -20,7 +20,7 @@ name = "datafusion-wasmtest"description = "Test library to compile datafusion crates to wasm"readme = "README.md"version = { workspace = true }-edition = "2024"+edition = { workspace = true }homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/rustfmt.toml b/rustfmt.tomlindex 4522e520a4..c680d9d068 100644--- a/rustfmt.toml+++ b/rustfmt.toml@@ -15,7 +15,7 @@# specific language governing permissions and limitations# under the License.-edition = "2021"+edition = "2024"max_width = 90# ignore generated filesdiff --git a/test-utils/src/array_gen/binary.rs b/test-utils/src/array_gen/binary.rsindex 9740eeae5e..ab0530a9ab 100644--- a/test-utils/src/array_gen/binary.rs+++ b/test-utils/src/array_gen/binary.rs@@ -19,8 +19,8 @@ use arrow::array::{ArrayRef, BinaryViewArray, GenericBinaryArray, OffsetSizeTrait, UInt32Array,};use arrow::compute;-use rand::rngs::StdRng;use rand::Rng;+use rand::rngs::StdRng;/// Randomly generate binary arrayspub struct BinaryArrayGenerator {diff --git a/test-utils/src/array_gen/boolean.rs b/test-utils/src/array_gen/boolean.rsindex 004d615b4c..c9104c0b8e 100644--- a/test-utils/src/array_gen/boolean.rs+++ b/test-utils/src/array_gen/boolean.rs@@ -17,8 +17,8 @@use arrow::array::{ArrayRef, BooleanArray, BooleanBuilder, UInt32Array};use arrow::compute::take;-use rand::rngs::StdRng;use rand::Rng;+use rand::rngs::StdRng;/// Randomly generate boolean arrayspub struct BooleanArrayGenerator {diff --git a/test-utils/src/array_gen/decimal.rs b/test-utils/src/array_gen/decimal.rsindex c5ec8ac5e8..54fa2269d6 100644--- a/test-utils/src/array_gen/decimal.rs+++ b/test-utils/src/array_gen/decimal.rs@@ -17,8 +17,8 @@use arrow::array::{ArrayRef, PrimitiveArray, PrimitiveBuilder, UInt32Array};use arrow::datatypes::DecimalType;-use rand::rngs::StdRng;use rand::Rng;+use rand::rngs::StdRng;use super::random_data::RandomNativeData;diff --git a/test-utils/src/array_gen/primitive.rs b/test-utils/src/array_gen/primitive.rsindex 62a38a1b4c..5944879600 100644--- a/test-utils/src/array_gen/primitive.rs+++ b/test-utils/src/array_gen/primitive.rs@@ -17,9 +17,9 @@use arrow::array::{ArrayRef, ArrowPrimitiveType, PrimitiveArray, UInt32Array};use arrow::datatypes::DataType;-use chrono_tz::{Tz, TZ_VARIANTS};+use chrono_tz::{TZ_VARIANTS, Tz};use rand::prelude::IndexedRandom;-use rand::{rng, rngs::StdRng, Rng};+use rand::{Rng, rng, rngs::StdRng};use std::sync::Arc;use super::random_data::RandomNativeData;diff --git a/test-utils/src/array_gen/random_data.rs b/test-utils/src/array_gen/random_data.rsindex ea2b872f7d..f341d23417 100644--- a/test-utils/src/array_gen/random_data.rs+++ b/test-utils/src/array_gen/random_data.rs@@ -17,19 +17,19 @@use arrow::array::ArrowPrimitiveType;use arrow::datatypes::{- i256, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Decimal32Type,- Decimal64Type, DurationMicrosecondType, DurationMillisecondType,- DurationNanosecondType, DurationSecondType, Float32Type, Float64Type, Int16Type,- Int32Type, Int64Type, Int8Type, IntervalDayTime, IntervalDayTimeType,- IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalYearMonthType,- Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,+ Date32Type, Date64Type, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,+ DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,+ DurationSecondType, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,+ Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,+ IntervalMonthDayNanoType, IntervalYearMonthType, Time32MillisecondType,+ Time32SecondType, Time64MicrosecondType, Time64NanosecondType,TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,- TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, i256,};+use rand::Rng;use rand::distr::StandardUniform;use rand::prelude::Distribution;use rand::rngs::StdRng;-use rand::Rng;/// Generate corresponding NativeType value randomly according to/// ArrowPrimitiveType.diff --git a/test-utils/src/array_gen/string.rs b/test-utils/src/array_gen/string.rsindex 546485fd8d..896182290c 100644--- a/test-utils/src/array_gen/string.rs+++ b/test-utils/src/array_gen/string.rs@@ -18,9 +18,9 @@use arrow::array::{ArrayRef, GenericStringArray, OffsetSizeTrait, StringViewArray, UInt32Array,};+use rand::Rng;use rand::distr::StandardUniform;use rand::rngs::StdRng;-use rand::Rng;/// Randomly generate string arrayspub struct StringArrayGenerator {diff --git a/test-utils/src/string_gen.rs b/test-utils/src/string_gen.rsindex 75ed03898a..21eecc05b8 100644--- a/test-utils/src/string_gen.rs+++ b/test-utils/src/string_gen.rs@@ -19,7 +19,7 @@ use crate::array_gen::StringArrayGenerator;use crate::stagger_batch;use arrow::record_batch::RecordBatch;use rand::rngs::StdRng;-use rand::{rng, Rng, SeedableRng};+use rand::{Rng, SeedableRng, rng};/// Randomly generate stringspub struct StringBatchGenerator(StringArrayGenerator);