Skip to content

Apache DataFusion SQL Query Engine

Update remaining crates to rust 2024 edition (#19361)

Tim Saucer59dcc362ca2b
diff --git a/Cargo.toml b/Cargo.toml
index 9275f0e1bd..c9afc5fcb5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -71,7 +71,7 @@ resolver = "2"
[workspace.package]
authors = ["Apache DataFusion <[email protected]>"]
-edition = "2021"
+edition = "2024"
homepage = "https://datafusion.apache.org"
license = "Apache-2.0"
readme = "README.md"
diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rs
index 027ca07a8b..2bc2bd4458 100644
--- a/benchmarks/src/bin/external_aggr.rs
+++ b/benchmarks/src/bin/external_aggr.rs
@@ -33,16 +33,16 @@ use datafusion::datasource::listing::{
};
use datafusion::datasource::{MemTable, TableProvider};
use datafusion::error::Result;
+use datafusion::execution::SessionStateBuilder;
use datafusion::execution::memory_pool::FairSpillPool;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
-use datafusion::execution::SessionStateBuilder;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::{collect, displayable};
use datafusion::prelude::*;
use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult};
use datafusion_common::instant::Instant;
use datafusion_common::utils::get_available_parallelism;
-use datafusion_common::{exec_err, DEFAULT_PARQUET_EXTENSION};
+use datafusion_common::{DEFAULT_PARQUET_EXTENSION, exec_err};
use datafusion_common::{human_readable_size, units};
#[derive(Debug, StructOpt)]
diff --git a/benchmarks/src/bin/mem_profile.rs b/benchmarks/src/bin/mem_profile.rs
index 16fc3871be..025efefe06 100644
--- a/benchmarks/src/bin/mem_profile.rs
+++ b/benchmarks/src/bin/mem_profile.rs
@@ -199,21 +199,18 @@ fn run_query(args: &[String], results: &mut Vec<QueryResult>) -> Result<()> {
// Look for lines that contain execution time / memory stats
while let Some(line) = iter.next() {
- if let Some((query, duration_ms)) = parse_query_time(line) {
- if let Some(next_line) = iter.peek() {
- if let Some((peak_rss, peak_commit, page_faults)) =
- parse_vm_line(next_line)
- {
- results.push(QueryResult {
- query,
- duration_ms,
- peak_rss,
- peak_commit,
- page_faults,
- });
- break;
- }
- }
+ if let Some((query, duration_ms)) = parse_query_time(line)
+ && let Some(next_line) = iter.peek()
+ && let Some((peak_rss, peak_commit, page_faults)) = parse_vm_line(next_line)
+ {
+ results.push(QueryResult {
+ query,
+ duration_ms,
+ peak_rss,
+ peak_commit,
+ page_faults,
+ });
+ break;
}
}
diff --git a/benchmarks/src/cancellation.rs b/benchmarks/src/cancellation.rs
index fcf03fbc54..1b4c04b409 100644
--- a/benchmarks/src/cancellation.rs
+++ b/benchmarks/src/cancellation.rs
@@ -25,22 +25,22 @@ use arrow::array::Array;
use arrow::datatypes::DataType;
use arrow::record_batch::RecordBatch;
use datafusion::common::{Result, ScalarValue};
-use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::file_format::FileFormat;
+use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::{ListingOptions, ListingTableUrl};
-use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::execution::TaskContext;
-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::prelude::*;
use datafusion_common::instant::Instant;
use futures::TryStreamExt;
use object_store::ObjectStore;
-use parquet::arrow::async_writer::ParquetObjectWriter;
use parquet::arrow::AsyncArrowWriter;
+use parquet::arrow::async_writer::ParquetObjectWriter;
+use rand::Rng;
use rand::distr::Alphanumeric;
use rand::rngs::ThreadRng;
-use rand::Rng;
use structopt::StructOpt;
use tokio::runtime::Runtime;
use tokio_util::sync::CancellationToken;
diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rs
index 796b6a4dec..9036e7d950 100644
--- a/benchmarks/src/clickbench.rs
+++ b/benchmarks/src/clickbench.rs
@@ -19,7 +19,7 @@ use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
use datafusion::logical_expr::{ExplainFormat, ExplainOption};
use datafusion::{
error::{DataFusionError, Result},
@@ -156,7 +156,9 @@ impl RunOpt {
.any(|opt| opt.contains("prefer_existing_sort=true"));
if !has_prefer_sort {
- println!("ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true");
+ println!(
+ "ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true"
+ );
println!("ℹ️ to optimize queries while maintaining parallelism");
}
}
diff --git a/benchmarks/src/h2o.rs b/benchmarks/src/h2o.rs
index be74252031..07a40447d4 100644
--- a/benchmarks/src/h2o.rs
+++ b/benchmarks/src/h2o.rs
@@ -20,11 +20,11 @@
//! - [H2O AI Benchmark](https://duckdb.org/2023/04/14/h2oai.html)
//! - [Extended window function benchmark](https://duckdb.org/2024/06/26/benchmarks-over-time.html#window-functions-benchmark)
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt};
+use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};
use datafusion::logical_expr::{ExplainFormat, ExplainOption};
use datafusion::{error::Result, prelude::SessionContext};
use datafusion_common::{
- exec_datafusion_err, instant::Instant, internal_err, DataFusionError, TableReference,
+ DataFusionError, TableReference, exec_datafusion_err, instant::Instant, internal_err,
};
use std::path::{Path, PathBuf};
use structopt::StructOpt;
diff --git a/benchmarks/src/hj.rs b/benchmarks/src/hj.rs
index 505b322745..562047f615 100644
--- a/benchmarks/src/hj.rs
+++ b/benchmarks/src/hj.rs
@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
use datafusion::physical_plan::execute_stream;
use datafusion::{error::Result, prelude::SessionContext};
use datafusion_common::instant::Instant;
-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};
+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};
use structopt::StructOpt;
use futures::StreamExt;
diff --git a/benchmarks/src/imdb/convert.rs b/benchmarks/src/imdb/convert.rs
index e7949aa715..2c4e127025 100644
--- a/benchmarks/src/imdb/convert.rs
+++ b/benchmarks/src/imdb/convert.rs
@@ -26,8 +26,8 @@ use structopt::StructOpt;
use datafusion::common::not_impl_err;
-use super::get_imdb_table_schema;
use super::IMDB_TABLES;
+use super::get_imdb_table_schema;
#[derive(Debug, StructOpt)]
pub struct ConvertOpt {
diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs
index 11bd424ba6..05f1870c5d 100644
--- a/benchmarks/src/imdb/run.rs
+++ b/benchmarks/src/imdb/run.rs
@@ -19,16 +19,16 @@ use std::path::PathBuf;
use std::sync::Arc;
use super::{
- get_imdb_table_schema, get_query_sql, IMDB_QUERY_END_ID, IMDB_QUERY_START_ID,
- IMDB_TABLES,
+ IMDB_QUERY_END_ID, IMDB_QUERY_START_ID, IMDB_TABLES, get_imdb_table_schema,
+ get_query_sql,
};
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty::{self, pretty_format_batches};
+use datafusion::datasource::file_format::FileFormat;
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::file_format::parquet::ParquetFormat;
-use datafusion::datasource::file_format::FileFormat;
use datafusion::datasource::listing::{
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
};
diff --git a/benchmarks/src/nlj.rs b/benchmarks/src/nlj.rs
index 7d1e14f694..cbf5a03fbf 100644
--- a/benchmarks/src/nlj.rs
+++ b/benchmarks/src/nlj.rs
@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
use datafusion::physical_plan::execute_stream;
use datafusion::{error::Result, prelude::SessionContext};
use datafusion_common::instant::Instant;
-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};
+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};
use structopt::StructOpt;
use futures::StreamExt;
@@ -268,8 +268,8 @@ impl RunOpt {
let elapsed = start.elapsed();
println!(
- "Query {query_name} iteration {i} returned {row_count} rows in {elapsed:?}"
- );
+ "Query {query_name} iteration {i} returned {row_count} rows in {elapsed:?}"
+ );
query_results.push(QueryResult { elapsed, row_count });
}
diff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rs
index 32a620a12d..53902e0930 100644
--- a/benchmarks/src/smj.rs
+++ b/benchmarks/src/smj.rs
@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
use datafusion::physical_plan::execute_stream;
use datafusion::{error::Result, prelude::SessionContext};
use datafusion_common::instant::Instant;
-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};
+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};
use structopt::StructOpt;
use futures::StreamExt;
diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rs
index 09b5a676bb..2f3be76f05 100644
--- a/benchmarks/src/sort_tpch.rs
+++ b/benchmarks/src/sort_tpch.rs
@@ -36,11 +36,11 @@ use datafusion::execution::SessionStateBuilder;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
use datafusion::physical_plan::{displayable, execute_stream};
use datafusion::prelude::*;
+use datafusion_common::DEFAULT_PARQUET_EXTENSION;
use datafusion_common::instant::Instant;
use datafusion_common::utils::get_available_parallelism;
-use datafusion_common::DEFAULT_PARQUET_EXTENSION;
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
#[derive(Debug, StructOpt)]
pub struct RunOpt {
diff --git a/benchmarks/src/tpcds/run.rs b/benchmarks/src/tpcds/run.rs
index 3b7e4ad135..3f579024ba 100644
--- a/benchmarks/src/tpcds/run.rs
+++ b/benchmarks/src/tpcds/run.rs
@@ -19,7 +19,7 @@ use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty::{self, pretty_format_batches};
@@ -34,7 +34,7 @@ use datafusion::physical_plan::{collect, displayable};
use datafusion::prelude::*;
use datafusion_common::instant::Instant;
use datafusion_common::utils::get_available_parallelism;
-use datafusion_common::{plan_err, DEFAULT_PARQUET_EXTENSION};
+use datafusion_common::{DEFAULT_PARQUET_EXTENSION, plan_err};
use log::info;
use structopt::StructOpt;
diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs
index 84dfc53a5e..65bb9594f0 100644
--- a/benchmarks/src/tpch/run.rs
+++ b/benchmarks/src/tpch/run.rs
@@ -19,16 +19,16 @@ use std::path::PathBuf;
use std::sync::Arc;
use super::{
- get_query_sql, get_tbl_tpch_table_schema, get_tpch_table_schema, TPCH_QUERY_END_ID,
- TPCH_QUERY_START_ID, TPCH_TABLES,
+ TPCH_QUERY_END_ID, TPCH_QUERY_START_ID, TPCH_TABLES, get_query_sql,
+ get_tbl_tpch_table_schema, get_tpch_table_schema,
};
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty::{self, pretty_format_batches};
+use datafusion::datasource::file_format::FileFormat;
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::file_format::parquet::ParquetFormat;
-use datafusion::datasource::file_format::FileFormat;
use datafusion::datasource::listing::{
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
};
diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rs
index 6627a287df..b1d5bc99fb 100644
--- a/benchmarks/src/util/options.rs
+++ b/benchmarks/src/util/options.rs
@@ -105,7 +105,7 @@ impl CommonOpt {
return Err(DataFusionError::Configuration(format!(
"Invalid memory pool type: {}",
self.mem_pool_type
- )))
+ )));
}
};
rt_builder = rt_builder
diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rs
index 764ea648ff..df17674e62 100644
--- a/benchmarks/src/util/run.rs
+++ b/benchmarks/src/util/run.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use datafusion::{error::Result, DATAFUSION_VERSION};
+use datafusion::{DATAFUSION_VERSION, error::Result};
use datafusion_common::utils::get_available_parallelism;
use serde::{Serialize, Serializer};
use serde_json::Value;
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index b03327bf13..ea94da1895 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-cli"
description = "Command Line Client for DataFusion query engine."
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rs
index 164c5b892a..7eff0d0b5c 100644
--- a/datafusion-examples/examples/builtin_functions/function_factory.rs
+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs
@@ -19,7 +19,7 @@
use arrow::datatypes::DataType;
use datafusion::common::tree_node::{Transformed, TreeNode};
-use datafusion::common::{exec_datafusion_err, exec_err, internal_err, DataFusionError};
+use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err, internal_err};
use datafusion::error::Result;
use datafusion::execution::context::{
FunctionFactory, RegisterFunction, SessionContext, SessionState,
diff --git a/datafusion-examples/examples/builtin_functions/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rs
index 4c6b6c67ac..e8376cd0c9 100644
--- a/datafusion-examples/examples/builtin_functions/regexp.rs
+++ b/datafusion-examples/examples/builtin_functions/regexp.rs
@@ -134,11 +134,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
assert_batches_eq!(
&[
- "+---------------------------------------------------+----------------------------------------------------+",
- "| regexp_like(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_like(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",
- "+---------------------------------------------------+----------------------------------------------------+",
- "| true | true |",
- "+---------------------------------------------------+----------------------------------------------------+",
+ "+---------------------------------------------------+----------------------------------------------------+",
+ "| regexp_like(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_like(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",
+ "+---------------------------------------------------+----------------------------------------------------+",
+ "| true | true |",
+ "+---------------------------------------------------+----------------------------------------------------+",
],
&result
);
@@ -264,11 +264,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
assert_batches_eq!(
&[
- "+----------------------------------------------------+-----------------------------------------------------+",
- "| regexp_match(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_match(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",
- "+----------------------------------------------------+-----------------------------------------------------+",
- "| [John Smith] | [Smith Jones] |",
- "+----------------------------------------------------+-----------------------------------------------------+",
+ "+----------------------------------------------------+-----------------------------------------------------+",
+ "| regexp_match(Utf8(\"John Smith\"),Utf8(\"^.*Smith$\")) | regexp_match(Utf8(\"Smith Jones\"),Utf8(\"^Smith.*$\")) |",
+ "+----------------------------------------------------+-----------------------------------------------------+",
+ "| [John Smith] | [Smith Jones] |",
+ "+----------------------------------------------------+-----------------------------------------------------+",
],
&result
);
@@ -290,21 +290,21 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
assert_batches_eq!(
&[
- "+---------------------------------------------------------------------------------------------------------+",
- "| regexp_replace(examples.values,examples.patterns,examples.replacement,concat(Utf8(\"g\"),examples.flags)) |",
- "+---------------------------------------------------------------------------------------------------------+",
- "| bbabbbc |",
- "| B |",
- "| aec |",
- "| AbC |",
- "| aBC |",
- "| 4000 |",
- "| xyz |",
- "| München |",
- "| Moscow |",
- "| Koln |",
- "| Today |",
- "+---------------------------------------------------------------------------------------------------------+",
+ "+---------------------------------------------------------------------------------------------------------+",
+ "| regexp_replace(examples.values,examples.patterns,examples.replacement,concat(Utf8(\"g\"),examples.flags)) |",
+ "+---------------------------------------------------------------------------------------------------------+",
+ "| bbabbbc |",
+ "| B |",
+ "| aec |",
+ "| AbC |",
+ "| aBC |",
+ "| 4000 |",
+ "| xyz |",
+ "| München |",
+ "| Moscow |",
+ "| Koln |",
+ "| Today |",
+ "+---------------------------------------------------------------------------------------------------------+",
],
&result
);
@@ -318,11 +318,11 @@ Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
assert_batches_eq!(
&[
- "+------------------------------------------------------------------------+",
- "| regexp_replace(Utf8(\"foobarbaz\"),Utf8(\"b(..)\"),Utf8(\"X\\1Y\"),Utf8(\"g\")) |",
- "+------------------------------------------------------------------------+",
- "| fooXarYXazY |",
- "+------------------------------------------------------------------------+",
+ "+------------------------------------------------------------------------+",
+ "| regexp_replace(Utf8(\"foobarbaz\"),Utf8(\"b(..)\"),Utf8(\"X\\1Y\"),Utf8(\"g\")) |",
+ "+------------------------------------------------------------------------+",
+ "| fooXarYXazY |",
+ "+------------------------------------------------------------------------+",
],
&result
);
diff --git a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
index 48b885839f..7b2e321362 100644
--- a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
+++ b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
@@ -36,7 +36,7 @@ use datafusion::{
use datafusion::datasource::physical_plan::FileScanConfigBuilder;
use futures::StreamExt;
-use object_store::{local::LocalFileSystem, memory::InMemory, ObjectStore};
+use object_store::{ObjectStore, local::LocalFileSystem, memory::InMemory};
/// This example demonstrates using the low level [`FileStream`] / [`FileOpener`] APIs to directly
/// read data from (CSV/JSON) into Arrow RecordBatches.
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
index ebb6c7a822..b276ae32cf 100644
--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
@@ -24,10 +24,10 @@ use std::sync::{Arc, Mutex};
use std::time::Duration;
use async_trait::async_trait;
-use datafusion::arrow::array::{UInt64Builder, UInt8Builder};
+use datafusion::arrow::array::{UInt8Builder, UInt64Builder};
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::datasource::{provider_as_source, TableProvider, TableType};
+use datafusion::datasource::{TableProvider, TableType, provider_as_source};
use datafusion::error::Result;
use datafusion::execution::context::TaskContext;
use datafusion::logical_expr::LogicalPlanBuilder;
@@ -35,8 +35,8 @@ use datafusion::physical_expr::EquivalenceProperties;
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion::physical_plan::memory::MemoryStream;
use datafusion::physical_plan::{
- project_schema, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
- PlanProperties, SendableRecordBatchStream,
+ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+ SendableRecordBatchStream, project_schema,
};
use datafusion::prelude::*;
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
index 80b88f7a38..895b6f52b6 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
@@ -19,21 +19,21 @@
use std::sync::Arc;
-use arrow::array::{record_batch, RecordBatch};
+use arrow::array::{RecordBatch, record_batch};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::assert_batches_eq;
+use datafusion::common::Result;
use datafusion::common::not_impl_err;
use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion::common::Result;
use datafusion::datasource::listing::{
ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,
};
use datafusion::execution::context::SessionContext;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::parquet::arrow::ArrowWriter;
-use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};
use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};
use datafusion::prelude::SessionConfig;
use datafusion_physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_format.rs b/datafusion-examples/examples/custom_data_source/custom_file_format.rs
index 05a34d3a83..6817beec41 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_format.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_format.rs
@@ -27,13 +27,13 @@ use datafusion::{
catalog::Session,
common::{GetExt, Statistics},
datasource::{
+ MemTable,
file_format::{
- csv::CsvFormatFactory, file_compression_type::FileCompressionType,
- FileFormat, FileFormatFactory,
+ FileFormat, FileFormatFactory, csv::CsvFormatFactory,
+ file_compression_type::FileCompressionType,
},
physical_plan::{FileScanConfig, FileSinkConfig, FileSource},
table_schema::TableSchema,
- MemTable,
},
error::Result,
execution::session_state::SessionStateBuilder,
diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rs
index 7381dc938a..227cea5e7c 100644
--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs
+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs
@@ -40,10 +40,10 @@ use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::properties::WriterProperties;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::{lit, SessionConfig};
+use datafusion::prelude::{SessionConfig, lit};
use datafusion_physical_expr_adapter::{
- replace_columns_with_literals, DefaultPhysicalExprAdapterFactory,
- PhysicalExprAdapter, PhysicalExprAdapterFactory,
+ DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
+ replace_columns_with_literals,
};
use futures::StreamExt;
use object_store::memory::InMemory;
@@ -136,8 +136,12 @@ pub async fn default_column_values() -> Result<()> {
println!("\n=== Key Insight ===");
println!("This example demonstrates how PhysicalExprAdapter works:");
println!("1. Physical schema only has 'id' and 'name' columns");
- println!("2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults");
- println!("3. Our custom adapter uses replace_columns_with_literals to inject default values");
+ println!(
+ "2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults"
+ );
+ println!(
+ "3. Our custom adapter uses replace_columns_with_literals to inject default values"
+ );
println!("4. Default values from metadata are cast to proper types at planning time");
println!("5. The DefaultPhysicalExprAdapter handles other schema adaptations");
println!("\nNote: PhysicalExprAdapter is specifically for filter predicates.");
diff --git a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
index b1f471b5b3..936da0a33d 100644
--- a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
+++ b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
@@ -47,8 +47,8 @@ mod non_windows {
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::PathBuf;
- use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
+ use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use std::time::Duration;
@@ -59,9 +59,9 @@ mod non_windows {
use tempfile::TempDir;
use tokio::task::JoinSet;
- use datafusion::common::{exec_err, Result};
- use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};
+ use datafusion::common::{Result, exec_err};
use datafusion::datasource::TableProvider;
+ use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};
use datafusion::logical_expr::SortExpr;
use datafusion::prelude::{SessionConfig, SessionContext};
diff --git a/datafusion-examples/examples/data_io/catalog.rs b/datafusion-examples/examples/data_io/catalog.rs
index e68e48bdea..d2ddff82e3 100644
--- a/datafusion-examples/examples/data_io/catalog.rs
+++ b/datafusion-examples/examples/data_io/catalog.rs
@@ -23,9 +23,9 @@ use datafusion::{
arrow::util::pretty,
catalog::{CatalogProvider, CatalogProviderList, SchemaProvider},
datasource::{
- file_format::{csv::CsvFormat, FileFormat},
- listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
TableProvider,
+ file_format::{FileFormat, csv::CsvFormat},
+ listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
},
error::Result,
execution::context::SessionState,
diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rs
index a4b0de9301..d2ffacc946 100644
--- a/datafusion-examples/examples/data_io/json_shredding.rs
+++ b/datafusion-examples/examples/data_io/json_shredding.rs
@@ -27,7 +27,7 @@ use datafusion::assert_batches_eq;
use datafusion::common::tree_node::{
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
};
-use datafusion::common::{assert_contains, exec_datafusion_err, Result};
+use datafusion::common::{Result, assert_contains, exec_datafusion_err};
use datafusion::datasource::listing::{
ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,
};
@@ -39,7 +39,7 @@ use datafusion::logical_expr::{
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::properties::WriterProperties;
use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_expr::{expressions, ScalarFunctionExpr};
+use datafusion::physical_expr::{ScalarFunctionExpr, expressions};
use datafusion::prelude::SessionConfig;
use datafusion::scalar::ScalarValue;
use datafusion_physical_expr_adapter::{
@@ -233,7 +233,7 @@ impl ScalarUDFImpl for JsonGetStr {
_ => {
return Err(exec_datafusion_err!(
"json_get_str first argument must be a string"
- ))
+ ));
}
};
// We expect a string array that contains JSON strings
@@ -249,7 +249,7 @@ impl ScalarUDFImpl for JsonGetStr {
_ => {
return Err(exec_datafusion_err!(
"json_get_str second argument must be a string array"
- ))
+ ));
}
};
let values = json_array
@@ -316,44 +316,43 @@ impl ShreddedJsonRewriter {
expr: Arc<dyn PhysicalExpr>,
physical_file_schema: &Schema,
) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {
- if let Some(func) = expr.as_any().downcast_ref::<ScalarFunctionExpr>() {
- if func.name() == "json_get_str" && func.args().len() == 2 {
- // Get the key from the first argument
- if let Some(literal) = func.args()[0]
+ if let Some(func) = expr.as_any().downcast_ref::<ScalarFunctionExpr>()
+ && func.name() == "json_get_str"
+ && func.args().len() == 2
+ {
+ // Get the key from the first argument
+ if let Some(literal) = func.args()[0]
+ .as_any()
+ .downcast_ref::<expressions::Literal>()
+ && let ScalarValue::Utf8(Some(field_name)) = literal.value()
+ {
+ // Get the column from the second argument
+ if let Some(column) = func.args()[1]
.as_any()
- .downcast_ref::<expressions::Literal>()
+ .downcast_ref::<expressions::Column>()
{
- if let ScalarValue::Utf8(Some(field_name)) = literal.value() {
- // Get the column from the second argument
- if let Some(column) = func.args()[1]
- .as_any()
- .downcast_ref::<expressions::Column>()
- {
- let column_name = column.name();
- // Check if there's a flat column with underscore prefix
- let flat_column_name = format!("_{column_name}.{field_name}");
-
- if let Ok(flat_field_index) =
- physical_file_schema.index_of(&flat_column_name)
- {
- let flat_field =
- physical_file_schema.field(flat_field_index);
-
- if flat_field.data_type() == &DataType::Utf8 {
- // Replace the whole expression with a direct column reference
- let new_expr = Arc::new(expressions::Column::new(
- &flat_column_name,
- flat_field_index,
- ))
- as Arc<dyn PhysicalExpr>;
-
- return Ok(Transformed {
- data: new_expr,
- tnr: TreeNodeRecursion::Stop,
- transformed: true,
- });
- }
- }
+ let column_name = column.name();
+ // Check if there's a flat column with underscore prefix
+ let flat_column_name = format!("_{column_name}.{field_name}");
+
+ if let Ok(flat_field_index) =
+ physical_file_schema.index_of(&flat_column_name)
+ {
+ let flat_field = physical_file_schema.field(flat_field_index);
+
+ if flat_field.data_type() == &DataType::Utf8 {
+ // Replace the whole expression with a direct column reference
+ let new_expr = Arc::new(expressions::Column::new(
+ &flat_column_name,
+ flat_field_index,
+ ))
+ as Arc<dyn PhysicalExpr>;
+
+ return Ok(Transformed {
+ data: new_expr,
+ tnr: TreeNodeRecursion::Stop,
+ transformed: true,
+ });
}
}
}
diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rs
index caa3be2111..3f4ebe7a92 100644
--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs
@@ -22,35 +22,35 @@ use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::ops::Range;
use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
use datafusion::catalog::Session;
use datafusion::common::{
- internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
+ DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err,
};
+use datafusion::datasource::TableProvider;
use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::physical_plan::parquet::ParquetAccessPlan;
use datafusion::datasource::physical_plan::{
FileScanConfigBuilder, ParquetFileReaderFactory, ParquetSource,
};
-use datafusion::datasource::TableProvider;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::logical_expr::utils::conjunction;
use datafusion::logical_expr::{TableProviderFilterPushDown, TableType};
+use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::arrow::arrow_reader::{
ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelector,
};
use datafusion::parquet::arrow::async_reader::{AsyncFileReader, ParquetObjectReader};
-use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::metadata::ParquetMetaData;
use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties};
use datafusion::parquet::schema::types::ColumnPath;
-use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};
use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};
use datafusion::physical_optimizer::pruning::PruningPredicate;
-use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
use datafusion::prelude::*;
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
@@ -58,8 +58,8 @@ use arrow::datatypes::SchemaRef;
use async_trait::async_trait;
use bytes::Bytes;
use datafusion::datasource::memory::DataSourceExec;
-use futures::future::BoxFuture;
use futures::FutureExt;
+use futures::future::BoxFuture;
use object_store::ObjectStore;
use tempfile::TempDir;
use url::Url;
diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rs
index 88a052546b..bcaca2ed5c 100644
--- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs
@@ -118,11 +118,11 @@ use arrow::record_batch::RecordBatch;
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use async_trait::async_trait;
use datafusion::catalog::{Session, TableProvider};
-use datafusion::common::{exec_err, HashMap, HashSet, Result};
+use datafusion::common::{HashMap, HashSet, Result, exec_err};
+use datafusion::datasource::TableType;
use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::memory::DataSourceExec;
use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetSource};
-use datafusion::datasource::TableType;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::logical_expr::{Operator, TableProviderFilterPushDown};
use datafusion::parquet::arrow::ArrowWriter;
@@ -132,7 +132,7 @@ use datafusion::parquet::file::reader::{FileReader, SerializedFileReader};
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::*;
use datafusion::scalar::ScalarValue;
-use std::fs::{read_dir, File};
+use std::fs::{File, read_dir};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -419,21 +419,15 @@ impl TableProvider for DistinctIndexTable {
// equality analysis or write your own custom logic.
let mut target: Option<&str> = None;
- if filters.len() == 1 {
- if let Expr::BinaryExpr(expr) = &filters[0] {
- if expr.op == Operator::Eq {
- if let (
- Expr::Column(c),
- Expr::Literal(ScalarValue::Utf8(Some(v)), _),
- ) = (&*expr.left, &*expr.right)
- {
- if c.name == "category" {
- println!("Filtering for category: {v}");
- target = Some(v);
- }
- }
- }
- }
+ if filters.len() == 1
+ && let Expr::BinaryExpr(expr) = &filters[0]
+ && expr.op == Operator::Eq
+ && let (Expr::Column(c), Expr::Literal(ScalarValue::Utf8(Some(v)), _)) =
+ (&*expr.left, &*expr.right)
+ && c.name == "category"
+ {
+ println!("Filtering for category: {v}");
+ target = Some(v);
}
// Determine which files to scan
let files_to_scan: Vec<_> = self
diff --git a/datafusion-examples/examples/data_io/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rs
index e3070cddde..f88ab91321 100644
--- a/datafusion-examples/examples/data_io/parquet_encrypted.rs
+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs
@@ -75,7 +75,9 @@ pub async fn parquet_encrypted() -> datafusion::common::Result<()> {
let encrypted_parquet_df = ctx.read_parquet(tempfile_str, read_options).await?;
// Show information from the dataframe
- println!("\n\n===============================================================================");
+ println!(
+ "\n\n==============================================================================="
+ );
println!("Encrypted Parquet DataFrame:");
query_dataframe(&encrypted_parquet_df).await?;
diff --git a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
index 925c202eac..d38fe9e171 100644
--- a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
+++ b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
@@ -27,7 +27,7 @@ use datafusion::error::DataFusionError;
use datafusion::execution::context::SessionContext;
use datafusion::physical_plan::metrics::MetricValue;
use datafusion::physical_plan::{
- execute_stream, visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor,
+ ExecutionPlan, ExecutionPlanVisitor, execute_stream, visit_execution_plan,
};
use futures::StreamExt;
@@ -100,18 +100,17 @@ impl ExecutionPlanVisitor for ParquetExecVisitor {
/// or `post_visit` (visit each node after its children/inputs)
fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
// If needed match on a specific `ExecutionPlan` node type
- if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
- if let Some((file_config, _)) =
+ if let Some(data_source_exec) = plan.as_any().downcast_ref::<DataSourceExec>()
+ && let Some((file_config, _)) =
data_source_exec.downcast_to_file_source::<ParquetSource>()
- {
- self.file_groups = Some(file_config.file_groups.clone());
-
- let metrics = match data_source_exec.metrics() {
- None => return Ok(true),
- Some(metrics) => metrics,
- };
- self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
- }
+ {
+ self.file_groups = Some(file_config.file_groups.clone());
+
+ let metrics = match data_source_exec.metrics() {
+ None => return Ok(true),
+ Some(metrics) => metrics,
+ };
+ self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
}
Ok(true)
}
diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rs
index 5f6fe3e297..2afc6c1047 100644
--- a/datafusion-examples/examples/data_io/parquet_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_index.rs
@@ -27,19 +27,19 @@ use async_trait::async_trait;
use datafusion::catalog::Session;
use datafusion::common::pruning::PruningStatistics;
use datafusion::common::{
- internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
+ DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err,
};
+use datafusion::datasource::TableProvider;
use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::memory::DataSourceExec;
use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetSource};
-use datafusion::datasource::TableProvider;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::logical_expr::{
- utils::conjunction, TableProviderFilterPushDown, TableType,
+ TableProviderFilterPushDown, TableType, utils::conjunction,
};
use datafusion::parquet::arrow::arrow_reader::statistics::StatisticsConverter;
use datafusion::parquet::arrow::{
- arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter,
+ ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder,
};
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_optimizer::pruning::PruningPredicate;
@@ -52,8 +52,8 @@ use std::fs;
use std::fs::{DirEntry, File};
use std::ops::Range;
use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
use tempfile::TempDir;
use url::Url;
diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rs
index 3e1568fb41..10ec26b1d5 100644
--- a/datafusion-examples/examples/data_io/remote_catalog.rs
+++ b/datafusion-examples/examples/data_io/remote_catalog.rs
@@ -41,8 +41,8 @@ use datafusion::common::{assert_batches_eq, internal_datafusion_err, plan_err};
use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::execution::SendableRecordBatchStream;
use datafusion::logical_expr::{Expr, TableType};
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::prelude::{DataFrame, SessionContext};
use futures::TryStreamExt;
use std::any::Any;
diff --git a/datafusion-examples/examples/dataframe/dataframe.rs b/datafusion-examples/examples/dataframe/dataframe.rs
index 6953170191..94653e80c8 100644
--- a/datafusion-examples/examples/dataframe/dataframe.rs
+++ b/datafusion-examples/examples/dataframe/dataframe.rs
@@ -20,18 +20,18 @@
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StringViewArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::catalog::MemTable;
+use datafusion::common::ScalarValue;
use datafusion::common::config::CsvOptions;
use datafusion::common::parsers::CompressionTypeVariant;
-use datafusion::common::ScalarValue;
use datafusion::dataframe::DataFrameWriteOptions;
use datafusion::error::Result;
use datafusion::functions_aggregate::average::avg;
use datafusion::functions_aggregate::min_max::max;
use datafusion::prelude::*;
-use std::fs::{create_dir_all, File};
+use std::fs::{File, create_dir_all};
use std::io::Write;
use std::sync::Arc;
-use tempfile::{tempdir, TempDir};
+use tempfile::{TempDir, tempdir};
/// This example demonstrates using DataFusion's DataFrame API
///
diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
index d179271246..48475acbb1 100644
--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
@@ -30,7 +30,7 @@ use arrow::record_batch::RecordBatch;
use arrow_schema::SchemaRef;
use datafusion::common::record_batch;
use datafusion::common::{exec_datafusion_err, internal_err};
-use datafusion::datasource::{memory::MemTable, DefaultTableSource};
+use datafusion::datasource::{DefaultTableSource, memory::MemTable};
use datafusion::error::Result;
use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
diff --git a/datafusion-examples/examples/execution_monitoring/tracing.rs b/datafusion-examples/examples/execution_monitoring/tracing.rs
index f0e5a03715..5fa759f2d5 100644
--- a/datafusion-examples/examples/execution_monitoring/tracing.rs
+++ b/datafusion-examples/examples/execution_monitoring/tracing.rs
@@ -51,17 +51,17 @@
//! 10:29:40.809 INFO main ThreadId(01) tracing: ***** WITH tracer: Non-main tasks DID inherit the `run_instrumented_query` span *****
//! ```
-use datafusion::common::runtime::{set_join_set_tracer, JoinSetTracer};
+use datafusion::common::runtime::{JoinSetTracer, set_join_set_tracer};
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
use datafusion::error::Result;
use datafusion::prelude::*;
use datafusion::test_util::parquet_test_data;
-use futures::future::BoxFuture;
use futures::FutureExt;
+use futures::future::BoxFuture;
use std::any::Any;
use std::sync::Arc;
-use tracing::{info, instrument, Instrument, Level, Span};
+use tracing::{Instrument, Level, Span, info, instrument};
/// Demonstrates the tracing injection feature for the DataFusion runtime
pub async fn tracing() -> Result<()> {
@@ -75,7 +75,9 @@ pub async fn tracing() -> Result<()> {
// Run query WITHOUT tracer injection.
info!("***** RUNNING WITHOUT INJECTED TRACER *****");
run_instrumented_query().await?;
- info!("***** WITHOUT tracer: `tokio-runtime-worker` tasks did NOT inherit the `run_instrumented_query` span *****");
+ info!(
+ "***** WITHOUT tracer: `tokio-runtime-worker` tasks did NOT inherit the `run_instrumented_query` span *****"
+ );
// Inject custom tracer so tasks run in the current span.
info!("Injecting custom tracer...");
@@ -84,7 +86,9 @@ pub async fn tracing() -> Result<()> {
// Run query WITH tracer injection.
info!("***** RUNNING WITH INJECTED TRACER *****");
run_instrumented_query().await?;
- info!("***** WITH tracer: `tokio-runtime-worker` tasks DID inherit the `run_instrumented_query` span *****");
+ info!(
+ "***** WITH tracer: `tokio-runtime-worker` tasks DID inherit the `run_instrumented_query` span *****"
+ );
Ok(())
}
diff --git a/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs b/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs
index d7fb17ba57..fdb8a3c9c0 100644
--- a/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs
+++ b/datafusion-examples/examples/external_dependency/dataframe_to_s3.rs
@@ -21,8 +21,8 @@ use std::env;
use std::sync::Arc;
use datafusion::dataframe::DataFrameWriteOptions;
-use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::file_format::FileFormat;
+use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
use datafusion::error::Result;
use datafusion::prelude::*;
diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml
index 612a219324..f393b2971e 100644
--- a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml
+++ b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ffi_module_interface"
version = "0.1.0"
-edition = "2021"
+edition = "2024"
publish = false
[dependencies]
diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs
index 88690e9297..1166eeb707 100644
--- a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs
+++ b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs
@@ -16,11 +16,10 @@
// under the License.
use abi_stable::{
- declare_root_module_statics,
+ StableAbi, declare_root_module_statics,
library::{LibraryError, RootModule},
package_version_strings,
sabi_types::VersionStrings,
- StableAbi,
};
use datafusion_ffi::table_provider::FFI_TableProvider;
diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml
index 1f68bb1bb1..c60cd7c029 100644
--- a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml
+++ b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ffi_module_loader"
version = "0.1.0"
-edition = "2021"
+edition = "2024"
publish = false
[dependencies]
diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs
index 170c7c3eff..d76af96058 100644
--- a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs
+++ b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs
@@ -22,7 +22,7 @@ use datafusion::{
prelude::SessionContext,
};
-use abi_stable::library::{development_utils::compute_library_path, RootModule};
+use abi_stable::library::{RootModule, development_utils::compute_library_path};
use datafusion::datasource::TableProvider;
use ffi_module_interface::TableProviderModuleRef;
diff --git a/datafusion-examples/examples/flight/server.rs b/datafusion-examples/examples/flight/server.rs
index 7f348e7d59..aad82e28b1 100644
--- a/datafusion-examples/examples/flight/server.rs
+++ b/datafusion-examples/examples/flight/server.rs
@@ -31,9 +31,9 @@ use tonic::{Request, Response, Status, Streaming};
use datafusion::prelude::*;
use arrow_flight::{
- flight_service_server::FlightService, flight_service_server::FlightServiceServer,
Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo,
HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, Ticket,
+ flight_service_server::FlightService, flight_service_server::FlightServiceServer,
};
#[derive(Clone)]
diff --git a/datafusion-examples/examples/flight/sql_server.rs b/datafusion-examples/examples/flight/sql_server.rs
index 529296fac8..435e05ffc0 100644
--- a/datafusion-examples/examples/flight/sql_server.rs
+++ b/datafusion-examples/examples/flight/sql_server.rs
@@ -416,7 +416,9 @@ impl FlightSqlService for FlightSqlServiceImpl {
) -> Result<(), Status> {
let handle = std::str::from_utf8(&handle.prepared_statement_handle);
if let Ok(handle) = handle {
- info!("do_action_close_prepared_statement: removing plan and results for {handle}");
+ info!(
+ "do_action_close_prepared_statement: removing plan and results for {handle}"
+ );
let _ = self.remove_plan(handle);
let _ = self.remove_result(handle);
}
diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rs
index 043f548b3b..f3910d461b 100644
--- a/datafusion-examples/examples/proto/composed_extension_codec.rs
+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs
@@ -36,8 +36,8 @@ use std::any::Any;
use std::fmt::Debug;
use std::sync::Arc;
-use datafusion::common::internal_err;
use datafusion::common::Result;
+use datafusion::common::internal_err;
use datafusion::execution::TaskContext;
use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
use datafusion::prelude::SessionContext;
diff --git a/datafusion-examples/examples/query_planning/analyzer_rule.rs b/datafusion-examples/examples/query_planning/analyzer_rule.rs
index df1ea4c1d7..a86f5cdd2a 100644
--- a/datafusion-examples/examples/query_planning/analyzer_rule.rs
+++ b/datafusion-examples/examples/query_planning/analyzer_rule.rs
@@ -18,10 +18,10 @@
//! See `main.rs` for how to run it.
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
+use datafusion::common::Result;
use datafusion::common::config::ConfigOptions;
use datafusion::common::tree_node::{Transformed, TreeNode};
-use datafusion::common::Result;
-use datafusion::logical_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder};
+use datafusion::logical_expr::{Expr, LogicalPlan, LogicalPlanBuilder, col, lit};
use datafusion::optimizer::analyzer::AnalyzerRule;
use datafusion::prelude::SessionContext;
use std::sync::{Arc, Mutex};
diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rs
index 47066f2ba2..47de669023 100644
--- a/datafusion-examples/examples/query_planning/expr_api.rs
+++ b/datafusion-examples/examples/query_planning/expr_api.rs
@@ -20,7 +20,7 @@
use std::collections::HashMap;
use std::sync::Arc;
-use arrow::array::{BooleanArray, Int32Array, Int8Array};
+use arrow::array::{BooleanArray, Int8Array, Int32Array};
use arrow::record_batch::RecordBatch;
use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
@@ -37,7 +37,7 @@ use datafusion::logical_expr::simplify::SimplifyContext;
use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator};
use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter;
use datafusion::optimizer::simplify_expressions::ExprSimplifier;
-use datafusion::physical_expr::{analyze, AnalysisContext, ExprBoundaries};
+use datafusion::physical_expr::{AnalysisContext, ExprBoundaries, analyze};
use datafusion::prelude::*;
/// This example demonstrates the DataFusion [`Expr`] API.
@@ -344,9 +344,11 @@ fn boundary_analysis_and_selectivity_demo() -> Result<()> {
//
// (a' - b' + 1) / (a - b)
// (10000 - 5000 + 1) / (10000 - 1)
- assert!(analysis
- .selectivity
- .is_some_and(|selectivity| (0.5..=0.6).contains(&selectivity)));
+ assert!(
+ analysis
+ .selectivity
+ .is_some_and(|selectivity| (0.5..=0.6).contains(&selectivity))
+ );
Ok(())
}
@@ -417,9 +419,11 @@ fn boundary_analysis_in_conjunctions_demo() -> Result<()> {
//
// Granted a column such as age will more likely follow a Normal distribution
// as such our selectivity estimation will not be as good as it can.
- assert!(analysis
- .selectivity
- .is_some_and(|selectivity| (0.1..=0.2).contains(&selectivity)));
+ assert!(
+ analysis
+ .selectivity
+ .is_some_and(|selectivity| (0.1..=0.2).contains(&selectivity))
+ );
// The above example was a good way to look at how we can derive better
// interval and get a lower selectivity during boundary analysis.
@@ -535,10 +539,11 @@ fn type_coercion_demo() -> Result<()> {
let physical_expr =
datafusion::physical_expr::create_physical_expr(&expr, &df_schema, &props)?;
let e = physical_expr.evaluate(&batch).unwrap_err();
- assert!(e
- .find_root()
- .to_string()
- .contains("Invalid comparison operation: Int8 > Int32"));
+ assert!(
+ e.find_root()
+ .to_string()
+ .contains("Invalid comparison operation: Int8 > Int32")
+ );
// 1. Type coercion with `SessionContext::create_physical_expr` which implicitly applies type coercion before constructing the physical expr.
let physical_expr =
diff --git a/datafusion-examples/examples/query_planning/optimizer_rule.rs b/datafusion-examples/examples/query_planning/optimizer_rule.rs
index 34aef85c64..de9de7737a 100644
--- a/datafusion-examples/examples/query_planning/optimizer_rule.rs
+++ b/datafusion-examples/examples/query_planning/optimizer_rule.rs
@@ -20,7 +20,7 @@
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
use arrow::datatypes::DataType;
use datafusion::common::tree_node::{Transformed, TreeNode};
-use datafusion::common::{assert_batches_eq, Result, ScalarValue};
+use datafusion::common::{Result, ScalarValue, assert_batches_eq};
use datafusion::logical_expr::{
BinaryExpr, ColumnarValue, Expr, LogicalPlan, Operator, ScalarFunctionArgs,
ScalarUDF, ScalarUDFImpl, Signature, Volatility,
diff --git a/datafusion-examples/examples/query_planning/plan_to_sql.rs b/datafusion-examples/examples/query_planning/plan_to_sql.rs
index 95e04907d9..756cc80b8f 100644
--- a/datafusion-examples/examples/query_planning/plan_to_sql.rs
+++ b/datafusion-examples/examples/query_planning/plan_to_sql.rs
@@ -34,7 +34,7 @@ use datafusion::sql::unparser::extension_unparser::UserDefinedLogicalNodeUnparse
use datafusion::sql::unparser::extension_unparser::{
UnparseToStatementResult, UnparseWithinStatementResult,
};
-use datafusion::sql::unparser::{plan_to_sql, Unparser};
+use datafusion::sql::unparser::{Unparser, plan_to_sql};
use std::fmt;
use std::sync::Arc;
diff --git a/datafusion-examples/examples/relation_planner/match_recognize.rs b/datafusion-examples/examples/relation_planner/match_recognize.rs
index 7eeddb1f97..60baf9bd61 100644
--- a/datafusion-examples/examples/relation_planner/match_recognize.rs
+++ b/datafusion-examples/examples/relation_planner/match_recognize.rs
@@ -77,11 +77,11 @@ use arrow::record_batch::RecordBatch;
use datafusion::prelude::*;
use datafusion_common::{DFSchemaRef, Result};
use datafusion_expr::{
+ Expr, UserDefinedLogicalNode,
logical_plan::{Extension, InvariantLevel, LogicalPlan},
planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
},
- Expr, UserDefinedLogicalNode,
};
use datafusion_sql::sqlparser::ast::TableFactor;
use insta::assert_snapshot;
diff --git a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
index 64158bb384..86a6cb9555 100644
--- a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
+++ b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
@@ -65,14 +65,13 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, Int64Array, StringArray};
use arrow::record_batch::RecordBatch;
use datafusion::prelude::*;
-use datafusion_common::{plan_datafusion_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, plan_datafusion_err};
use datafusion_expr::{
- case, col, lit,
+ Expr, case, col, lit,
logical_plan::builder::LogicalPlanBuilder,
planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
},
- Expr,
};
use datafusion_sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};
use insta::assert_snapshot;
diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs
index a3ae5396ed..207fffe132 100644
--- a/datafusion-examples/examples/relation_planner/table_sample.rs
+++ b/datafusion-examples/examples/relation_planner/table_sample.rs
@@ -99,32 +99,32 @@ use futures::{
ready,
stream::{Stream, StreamExt},
};
-use rand::{rngs::StdRng, Rng, SeedableRng};
+use rand::{Rng, SeedableRng, rngs::StdRng};
use tonic::async_trait;
use datafusion::{
execution::{
- context::QueryPlanner, RecordBatchStream, SendableRecordBatchStream,
- SessionState, SessionStateBuilder, TaskContext,
+ RecordBatchStream, SendableRecordBatchStream, SessionState, SessionStateBuilder,
+ TaskContext, context::QueryPlanner,
},
physical_expr::EquivalenceProperties,
physical_plan::{
- metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput},
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
+ metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput},
},
physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner},
prelude::*,
};
use datafusion_common::{
- internal_err, not_impl_err, plan_datafusion_err, plan_err, DFSchemaRef,
- DataFusionError, Result, Statistics,
+ DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
+ plan_datafusion_err, plan_err,
};
use datafusion_expr::{
+ UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},
planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
},
- UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
};
use datafusion_sql::sqlparser::ast::{
self, TableFactor, TableSampleMethod, TableSampleUnit,
@@ -341,15 +341,14 @@ impl RelationPlanner for TableSamplePlanner {
};
// Validate sampling method
- if let Some(method) = &sample.name {
- if *method != TableSampleMethod::Bernoulli
- && *method != TableSampleMethod::Row
- {
- return not_impl_err!(
- "Sampling method {} is not supported (only BERNOULLI and ROW)",
- method
- );
- }
+ if let Some(method) = &sample.name
+ && *method != TableSampleMethod::Bernoulli
+ && *method != TableSampleMethod::Row
+ {
+ return not_impl_err!(
+ "Sampling method {} is not supported (only BERNOULLI and ROW)",
+ method
+ );
}
// Offset sampling (ClickHouse-style) not supported
diff --git a/datafusion-examples/examples/sql_ops/analysis.rs b/datafusion-examples/examples/sql_ops/analysis.rs
index 266c650772..4243a29278 100644
--- a/datafusion-examples/examples/sql_ops/analysis.rs
+++ b/datafusion-examples/examples/sql_ops/analysis.rs
@@ -25,8 +25,8 @@
use std::sync::Arc;
-use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion::common::Result;
+use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion::logical_expr::LogicalPlan;
use datafusion::{
datasource::MemTable,
diff --git a/datafusion-examples/examples/sql_ops/frontend.rs b/datafusion-examples/examples/sql_ops/frontend.rs
index 2cfb9de05c..025fe47e75 100644
--- a/datafusion-examples/examples/sql_ops/frontend.rs
+++ b/datafusion-examples/examples/sql_ops/frontend.rs
@@ -18,7 +18,7 @@
//! See `main.rs` for how to run it.
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion::common::{plan_err, TableReference};
+use datafusion::common::{TableReference, plan_err};
use datafusion::config::ConfigOptions;
use datafusion::error::Result;
use datafusion::logical_expr::{
diff --git a/datafusion-examples/examples/sql_ops/query.rs b/datafusion-examples/examples/sql_ops/query.rs
index 3e052f0823..90d0c3ca34 100644
--- a/datafusion-examples/examples/sql_ops/query.rs
+++ b/datafusion-examples/examples/sql_ops/query.rs
@@ -17,13 +17,13 @@
//! See `main.rs` for how to run it.
-use datafusion::arrow::array::{UInt64Array, UInt8Array};
+use datafusion::arrow::array::{UInt8Array, UInt64Array};
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::common::{assert_batches_eq, exec_datafusion_err};
+use datafusion::datasource::MemTable;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
-use datafusion::datasource::MemTable;
use datafusion::error::{DataFusionError, Result};
use datafusion::prelude::*;
use object_store::local::LocalFileSystem;
@@ -153,7 +153,8 @@ async fn query_parquet() -> Result<()> {
"| 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |",
"+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
],
- &results);
+ &results
+ );
// Second example were we temporarily move into the test data's parent directory and
// simulate a relative path, this requires registering an ObjectStore.
@@ -202,7 +203,8 @@ async fn query_parquet() -> Result<()> {
"| 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |",
"+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
],
- &results);
+ &results
+ );
// Reset the current directory
std::env::set_current_dir(cur_dir)?;
diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rs
index b4916e5bd0..fbb9e65248 100644
--- a/datafusion-examples/examples/udf/advanced_udaf.rs
+++ b/datafusion-examples/examples/udf/advanced_udaf.rs
@@ -28,13 +28,13 @@ use arrow::array::{
use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type};
use arrow::record_batch::RecordBatch;
use arrow_schema::FieldRef;
-use datafusion::common::{cast::as_float64_array, ScalarValue};
+use datafusion::common::{ScalarValue, cast::as_float64_array};
use datafusion::error::Result;
use datafusion::logical_expr::{
+ Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature,
expr::AggregateFunction,
function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},
simplify::SimplifyInfo,
- Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature,
};
use datafusion::prelude::*;
diff --git a/datafusion-examples/examples/udf/advanced_udf.rs b/datafusion-examples/examples/udf/advanced_udf.rs
index 7459501a1c..a00a7e7df4 100644
--- a/datafusion-examples/examples/udf/advanced_udf.rs
+++ b/datafusion-examples/examples/udf/advanced_udf.rs
@@ -21,15 +21,15 @@ use std::any::Any;
use std::sync::Arc;
use arrow::array::{
- new_null_array, Array, ArrayRef, AsArray, Float32Array, Float64Array,
+ Array, ArrayRef, AsArray, Float32Array, Float64Array, new_null_array,
};
use arrow::compute;
use arrow::datatypes::{DataType, Float64Type};
use arrow::record_batch::RecordBatch;
-use datafusion::common::{exec_err, internal_err, ScalarValue};
+use datafusion::common::{ScalarValue, exec_err, internal_err};
use datafusion::error::Result;
-use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion::logical_expr::Volatility;
+use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion::logical_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
};
diff --git a/datafusion-examples/examples/udf/async_udf.rs b/datafusion-examples/examples/udf/async_udf.rs
index 18630d33a3..c31e8290cc 100644
--- a/datafusion-examples/examples/udf/async_udf.rs
+++ b/datafusion-examples/examples/udf/async_udf.rs
@@ -93,19 +93,19 @@ pub async fn async_udf() -> Result<()> {
assert_batches_eq!(
[
- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
- "| plan_type | plan |",
- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
- "| logical_plan | SubqueryAlias: a |",
- "| | Filter: ask_llm(CAST(animal.name AS Utf8View), Utf8View(\"Is this animal furry?\")) |",
- "| | TableScan: animal projection=[id, name] |",
- "| physical_plan | FilterExec: __async_fn_0@2, projection=[id@0, name@1] |",
- "| | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |",
- "| | AsyncFuncExec: async_expr=[async_expr(name=__async_fn_0, expr=ask_llm(CAST(name@1 AS Utf8View), Is this animal furry?))] |",
- "| | CoalesceBatchesExec: target_batch_size=8192 |",
- "| | DataSourceExec: partitions=1, partition_sizes=[1] |",
- "| | |",
- "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
+ "| plan_type | plan |",
+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
+ "| logical_plan | SubqueryAlias: a |",
+ "| | Filter: ask_llm(CAST(animal.name AS Utf8View), Utf8View(\"Is this animal furry?\")) |",
+ "| | TableScan: animal projection=[id, name] |",
+ "| physical_plan | FilterExec: __async_fn_0@2, projection=[id@0, name@1] |",
+ "| | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |",
+ "| | AsyncFuncExec: async_expr=[async_expr(name=__async_fn_0, expr=ask_llm(CAST(name@1 AS Utf8View), Is this animal furry?))] |",
+ "| | CoalesceBatchesExec: target_batch_size=8192 |",
+ "| | DataSourceExec: partitions=1, partition_sizes=[1] |",
+ "| | |",
+ "+---------------+------------------------------------------------------------------------------------------------------------------------------+",
],
&results
);
diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rs
index 12ee74fc52..087b8ba73a 100644
--- a/datafusion-examples/examples/udf/simple_udtf.rs
+++ b/datafusion-examples/examples/udf/simple_udtf.rs
@@ -17,16 +17,16 @@
//! See `main.rs` for how to run it.
-use arrow::csv::reader::Format;
use arrow::csv::ReaderBuilder;
+use arrow::csv::reader::Format;
use async_trait::async_trait;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::catalog::Session;
use datafusion::catalog::TableFunctionImpl;
-use datafusion::common::{plan_err, ScalarValue};
-use datafusion::datasource::memory::MemorySourceConfig;
+use datafusion::common::{ScalarValue, plan_err};
use datafusion::datasource::TableProvider;
+use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::error::Result;
use datafusion::execution::context::ExecutionProps;
use datafusion::logical_expr::simplify::SimplifyContext;
@@ -134,8 +134,7 @@ struct LocalCsvTableFunc {}
impl TableFunctionImpl for LocalCsvTableFunc {
fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {
- let Some(Expr::Literal(ScalarValue::Utf8(Some(ref path)), _)) = exprs.first()
- else {
+ let Some(Expr::Literal(ScalarValue::Utf8(Some(path)), _)) = exprs.first() else {
return plan_err!("read_csv requires at least one string argument");
};
diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.toml
index 388cdac71f..be1374b371 100644
--- a/datafusion/catalog-listing/Cargo.toml
+++ b/datafusion/catalog-listing/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-catalog-listing"
description = "datafusion-catalog-listing"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/catalog/Cargo.toml b/datafusion/catalog/Cargo.toml
index 2d9f26ea50..1009e9aee4 100644
--- a/datafusion/catalog/Cargo.toml
+++ b/datafusion/catalog/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-catalog"
description = "datafusion-catalog"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/common-runtime/Cargo.toml b/datafusion/common-runtime/Cargo.toml
index 65e1544601..fd9a818bcb 100644
--- a/datafusion/common-runtime/Cargo.toml
+++ b/datafusion/common-runtime/Cargo.toml
@@ -21,7 +21,7 @@ description = "Common Runtime functionality for DataFusion query engine"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index dac06de659..5c36e33c72 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Common functionality for DataFusion query engine"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index b63a3f9ccf..d2ecd34886 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -22,7 +22,7 @@ keywords = ["arrow", "query", "sql"]
include = ["benches/*.rs", "src/**/*.rs", "Cargo.toml", "LICENSE.txt", "NOTICE.txt"]
readme = "../../README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/datasource-arrow/Cargo.toml b/datafusion/datasource-arrow/Cargo.toml
index 002408d643..fbadc8708c 100644
--- a/datafusion/datasource-arrow/Cargo.toml
+++ b/datafusion/datasource-arrow/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource-arrow"
description = "datafusion-datasource-arrow"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.toml
index fec17e133c..c9299aeb10 100644
--- a/datafusion/datasource-avro/Cargo.toml
+++ b/datafusion/datasource-avro/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource-avro"
description = "datafusion-datasource-avro"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/datasource-csv/Cargo.toml b/datafusion/datasource-csv/Cargo.toml
index 74e5ff3300..2950925127 100644
--- a/datafusion/datasource-csv/Cargo.toml
+++ b/datafusion/datasource-csv/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource-csv"
description = "datafusion-datasource-csv"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition = { workspace = true }
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.toml
index a8984849a2..37fa8d43a0 100644
--- a/datafusion/datasource-json/Cargo.toml
+++ b/datafusion/datasource-json/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource-json"
description = "datafusion-datasource-json"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml
index 19b814fd0b..a5f6f56ac6 100644
--- a/datafusion/datasource-parquet/Cargo.toml
+++ b/datafusion/datasource-parquet/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource-parquet"
description = "datafusion-datasource-parquet"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml
index a41c4d173c..48bf30f7a4 100644
--- a/datafusion/datasource/Cargo.toml
+++ b/datafusion/datasource/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-datasource"
description = "datafusion-datasource"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/doc/Cargo.toml b/datafusion/doc/Cargo.toml
index 94edf70bc7..c1368c1531 100644
--- a/datafusion/doc/Cargo.toml
+++ b/datafusion/doc/Cargo.toml
@@ -21,7 +21,7 @@ description = "Documentation module for DataFusion query engine"
keywords = ["datafusion", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.toml
index 444a060d8b..ca1fba07ca 100644
--- a/datafusion/execution/Cargo.toml
+++ b/datafusion/execution/Cargo.toml
@@ -21,7 +21,7 @@ description = "Execution configuration support for DataFusion query engine"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.toml
index 7b13334a6a..5ee46b454e 100644
--- a/datafusion/expr-common/Cargo.toml
+++ b/datafusion/expr-common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Logical plan and expression representation for DataFusion query e
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index 37de339605..75aa59595b 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -21,7 +21,7 @@ description = "Logical plan and expression representation for DataFusion query e
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/ffi/src/arrow_wrappers.rs b/datafusion/ffi/src/arrow_wrappers.rs
index df3f1e3b4c..4b91fb551b 100644
--- a/datafusion/ffi/src/arrow_wrappers.rs
+++ b/datafusion/ffi/src/arrow_wrappers.rs
@@ -19,10 +19,10 @@ use std::sync::Arc;
use abi_stable::StableAbi;
use arrow::{
- array::{make_array, ArrayRef},
+ array::{ArrayRef, make_array},
datatypes::{Schema, SchemaRef},
error::ArrowError,
- ffi::{from_ffi, to_ffi, FFI_ArrowArray, FFI_ArrowSchema},
+ ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi, to_ffi},
};
use datafusion_common::{DataFusionError, ScalarValue};
use log::error;
@@ -38,7 +38,9 @@ impl From<SchemaRef> for WrappedSchema {
let ffi_schema = match FFI_ArrowSchema::try_from(value.as_ref()) {
Ok(s) => s,
Err(e) => {
- error!("Unable to convert DataFusion Schema to FFI_ArrowSchema in FFI_PlanProperties. {e}");
+ error!(
+ "Unable to convert DataFusion Schema to FFI_ArrowSchema in FFI_PlanProperties. {e}"
+ );
FFI_ArrowSchema::empty()
}
};
@@ -52,7 +54,9 @@ impl From<SchemaRef> for WrappedSchema {
/// empty schema.
#[cfg(not(tarpaulin_include))]
fn catch_df_schema_error(e: &ArrowError) -> Schema {
- error!("Unable to convert from FFI_ArrowSchema to DataFusion Schema in FFI_PlanProperties. {e}");
+ error!(
+ "Unable to convert from FFI_ArrowSchema to DataFusion Schema in FFI_PlanProperties. {e}"
+ );
Schema::empty()
}
diff --git a/datafusion/ffi/src/catalog_provider.rs b/datafusion/ffi/src/catalog_provider.rs
index 8c3e4cbc15..fe3cce1065 100644
--- a/datafusion/ffi/src/catalog_provider.rs
+++ b/datafusion/ffi/src/catalog_provider.rs
@@ -18,8 +18,8 @@
use std::{any::Any, ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{ROption, RResult, RString, RVec},
StableAbi,
+ std_types::{ROption, RResult, RString, RVec},
};
use datafusion::catalog::{CatalogProvider, SchemaProvider};
use tokio::runtime::Handle;
@@ -88,31 +88,39 @@ struct ProviderPrivateData {
impl FFI_CatalogProvider {
unsafe fn inner(&self) -> &Arc<dyn CatalogProvider + Send> {
- let private_data = self.private_data as *const ProviderPrivateData;
- &(*private_data).provider
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ &(*private_data).provider
+ }
}
unsafe fn runtime(&self) -> Option<Handle> {
- let private_data = self.private_data as *const ProviderPrivateData;
- (*private_data).runtime.clone()
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ (*private_data).runtime.clone()
+ }
}
}
unsafe extern "C" fn schema_names_fn_wrapper(
provider: &FFI_CatalogProvider,
) -> RVec<RString> {
- let names = provider.inner().schema_names();
- names.into_iter().map(|s| s.into()).collect()
+ unsafe {
+ let names = provider.inner().schema_names();
+ names.into_iter().map(|s| s.into()).collect()
+ }
}
unsafe extern "C" fn schema_fn_wrapper(
provider: &FFI_CatalogProvider,
name: RString,
) -> ROption<FFI_SchemaProvider> {
- let maybe_schema = provider.inner().schema(name.as_str());
- maybe_schema
- .map(|schema| FFI_SchemaProvider::new(schema, provider.runtime()))
- .into()
+ unsafe {
+ let maybe_schema = provider.inner().schema(name.as_str());
+ maybe_schema
+ .map(|schema| FFI_SchemaProvider::new(schema, provider.runtime()))
+ .into()
+ }
}
unsafe extern "C" fn register_schema_fn_wrapper(
@@ -120,16 +128,18 @@ unsafe extern "C" fn register_schema_fn_wrapper(
name: RString,
schema: &FFI_SchemaProvider,
) -> FFIResult<ROption<FFI_SchemaProvider>> {
- let runtime = provider.runtime();
- let provider = provider.inner();
- let schema: Arc<dyn SchemaProvider + Send> = schema.into();
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
+ let schema: Arc<dyn SchemaProvider + Send> = schema.into();
- let returned_schema =
- rresult_return!(provider.register_schema(name.as_str(), schema))
- .map(|schema| FFI_SchemaProvider::new(schema, runtime))
- .into();
+ let returned_schema =
+ rresult_return!(provider.register_schema(name.as_str(), schema))
+ .map(|schema| FFI_SchemaProvider::new(schema, runtime))
+ .into();
- RResult::ROk(returned_schema)
+ RResult::ROk(returned_schema)
+ }
}
unsafe extern "C" fn deregister_schema_fn_wrapper(
@@ -137,47 +147,54 @@ unsafe extern "C" fn deregister_schema_fn_wrapper(
name: RString,
cascade: bool,
) -> FFIResult<ROption<FFI_SchemaProvider>> {
- let runtime = provider.runtime();
- let provider = provider.inner();
-
- let maybe_schema =
- rresult_return!(provider.deregister_schema(name.as_str(), cascade));
-
- RResult::ROk(
- maybe_schema
- .map(|schema| FFI_SchemaProvider::new(schema, runtime))
- .into(),
- )
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
+
+ let maybe_schema =
+ rresult_return!(provider.deregister_schema(name.as_str(), cascade));
+
+ RResult::ROk(
+ maybe_schema
+ .map(|schema| FFI_SchemaProvider::new(schema, runtime))
+ .into(),
+ )
+ }
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_CatalogProvider) {
- debug_assert!(!provider.private_data.is_null());
- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);
- drop(private_data);
- provider.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!provider.private_data.is_null());
+ let private_data =
+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);
+ drop(private_data);
+ provider.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
provider: &FFI_CatalogProvider,
) -> FFI_CatalogProvider {
- let old_private_data = provider.private_data as *const ProviderPrivateData;
- let runtime = (*old_private_data).runtime.clone();
-
- let private_data = Box::into_raw(Box::new(ProviderPrivateData {
- provider: Arc::clone(&(*old_private_data).provider),
- runtime,
- })) as *mut c_void;
-
- FFI_CatalogProvider {
- schema_names: schema_names_fn_wrapper,
- schema: schema_fn_wrapper,
- register_schema: register_schema_fn_wrapper,
- deregister_schema: deregister_schema_fn_wrapper,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- version: super::version,
- private_data,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ let old_private_data = provider.private_data as *const ProviderPrivateData;
+ let runtime = (*old_private_data).runtime.clone();
+
+ let private_data = Box::into_raw(Box::new(ProviderPrivateData {
+ provider: Arc::clone(&(*old_private_data).provider),
+ runtime,
+ })) as *mut c_void;
+
+ FFI_CatalogProvider {
+ schema_names: schema_names_fn_wrapper,
+ schema: schema_fn_wrapper,
+ register_schema: register_schema_fn_wrapper,
+ deregister_schema: deregister_schema_fn_wrapper,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ version: super::version,
+ private_data,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -307,11 +324,13 @@ mod tests {
let prior_schema = Arc::new(MemorySchemaProvider::new());
let catalog = Arc::new(MemoryCatalogProvider::new());
- assert!(catalog
- .as_ref()
- .register_schema("prior_schema", prior_schema)
- .unwrap()
- .is_none());
+ assert!(
+ catalog
+ .as_ref()
+ .register_schema("prior_schema", prior_schema)
+ .unwrap()
+ .is_none()
+ );
let mut ffi_catalog = FFI_CatalogProvider::new(catalog, None);
ffi_catalog.library_marker_id = crate::mock_foreign_marker_id;
@@ -360,17 +379,21 @@ mod tests {
// Verify local libraries can be downcast to their original
let foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();
- assert!(foreign_catalog
- .as_any()
- .downcast_ref::<MemoryCatalogProvider>()
- .is_some());
+ assert!(
+ foreign_catalog
+ .as_any()
+ .downcast_ref::<MemoryCatalogProvider>()
+ .is_some()
+ );
// Verify different library markers generate foreign providers
ffi_catalog.library_marker_id = crate::mock_foreign_marker_id;
let foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();
- assert!(foreign_catalog
- .as_any()
- .downcast_ref::<ForeignCatalogProvider>()
- .is_some());
+ assert!(
+ foreign_catalog
+ .as_any()
+ .downcast_ref::<ForeignCatalogProvider>()
+ .is_some()
+ );
}
}
diff --git a/datafusion/ffi/src/catalog_provider_list.rs b/datafusion/ffi/src/catalog_provider_list.rs
index 102e9e4ef5..a119dd9485 100644
--- a/datafusion/ffi/src/catalog_provider_list.rs
+++ b/datafusion/ffi/src/catalog_provider_list.rs
@@ -18,8 +18,8 @@
use std::{any::Any, ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{ROption, RString, RVec},
StableAbi,
+ std_types::{ROption, RString, RVec},
};
use datafusion::catalog::{CatalogProvider, CatalogProviderList};
use tokio::runtime::Handle;
@@ -75,21 +75,27 @@ struct ProviderPrivateData {
impl FFI_CatalogProviderList {
unsafe fn inner(&self) -> &Arc<dyn CatalogProviderList + Send> {
- let private_data = self.private_data as *const ProviderPrivateData;
- &(*private_data).provider
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ &(*private_data).provider
+ }
}
unsafe fn runtime(&self) -> Option<Handle> {
- let private_data = self.private_data as *const ProviderPrivateData;
- (*private_data).runtime.clone()
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ (*private_data).runtime.clone()
+ }
}
}
unsafe extern "C" fn catalog_names_fn_wrapper(
provider: &FFI_CatalogProviderList,
) -> RVec<RString> {
- let names = provider.inner().catalog_names();
- names.into_iter().map(|s| s.into()).collect()
+ unsafe {
+ let names = provider.inner().catalog_names();
+ names.into_iter().map(|s| s.into()).collect()
+ }
}
unsafe extern "C" fn register_catalog_fn_wrapper(
@@ -97,55 +103,64 @@ unsafe extern "C" fn register_catalog_fn_wrapper(
name: RString,
catalog: &FFI_CatalogProvider,
) -> ROption<FFI_CatalogProvider> {
- let runtime = provider.runtime();
- let provider = provider.inner();
- let catalog: Arc<dyn CatalogProvider + Send> = catalog.into();
-
- provider
- .register_catalog(name.into(), catalog)
- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))
- .into()
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
+ let catalog: Arc<dyn CatalogProvider + Send> = catalog.into();
+
+ provider
+ .register_catalog(name.into(), catalog)
+ .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))
+ .into()
+ }
}
unsafe extern "C" fn catalog_fn_wrapper(
provider: &FFI_CatalogProviderList,
name: RString,
) -> ROption<FFI_CatalogProvider> {
- let runtime = provider.runtime();
- let provider = provider.inner();
- provider
- .catalog(name.as_str())
- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))
- .into()
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
+ provider
+ .catalog(name.as_str())
+ .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))
+ .into()
+ }
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_CatalogProviderList) {
- debug_assert!(!provider.private_data.is_null());
- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);
- drop(private_data);
- provider.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!provider.private_data.is_null());
+ let private_data =
+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);
+ drop(private_data);
+ provider.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
provider: &FFI_CatalogProviderList,
) -> FFI_CatalogProviderList {
- let old_private_data = provider.private_data as *const ProviderPrivateData;
- let runtime = (*old_private_data).runtime.clone();
-
- let private_data = Box::into_raw(Box::new(ProviderPrivateData {
- provider: Arc::clone(&(*old_private_data).provider),
- runtime,
- })) as *mut c_void;
-
- FFI_CatalogProviderList {
- register_catalog: register_catalog_fn_wrapper,
- catalog_names: catalog_names_fn_wrapper,
- catalog: catalog_fn_wrapper,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- version: super::version,
- private_data,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ let old_private_data = provider.private_data as *const ProviderPrivateData;
+ let runtime = (*old_private_data).runtime.clone();
+
+ let private_data = Box::into_raw(Box::new(ProviderPrivateData {
+ provider: Arc::clone(&(*old_private_data).provider),
+ runtime,
+ })) as *mut c_void;
+
+ FFI_CatalogProviderList {
+ register_catalog: register_catalog_fn_wrapper,
+ catalog_names: catalog_names_fn_wrapper,
+ catalog: catalog_fn_wrapper,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ version: super::version,
+ private_data,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -257,10 +272,12 @@ mod tests {
let prior_catalog = Arc::new(MemoryCatalogProvider::new());
let catalog_list = Arc::new(MemoryCatalogProviderList::new());
- assert!(catalog_list
- .as_ref()
- .register_catalog("prior_catalog".to_owned(), prior_catalog)
- .is_none());
+ assert!(
+ catalog_list
+ .as_ref()
+ .register_catalog("prior_catalog".to_owned(), prior_catalog)
+ .is_none()
+ );
let mut ffi_catalog_list = FFI_CatalogProviderList::new(catalog_list, None);
ffi_catalog_list.library_marker_id = crate::mock_foreign_marker_id;
@@ -306,18 +323,22 @@ mod tests {
// Verify local libraries can be downcast to their original
let foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =
(&ffi_catalog_list).into();
- assert!(foreign_catalog_list
- .as_any()
- .downcast_ref::<MemoryCatalogProviderList>()
- .is_some());
+ assert!(
+ foreign_catalog_list
+ .as_any()
+ .downcast_ref::<MemoryCatalogProviderList>()
+ .is_some()
+ );
// Verify different library markers generate foreign providers
ffi_catalog_list.library_marker_id = crate::mock_foreign_marker_id;
let foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =
(&ffi_catalog_list).into();
- assert!(foreign_catalog_list
- .as_any()
- .downcast_ref::<ForeignCatalogProviderList>()
- .is_some());
+ assert!(
+ foreign_catalog_list
+ .as_any()
+ .downcast_ref::<ForeignCatalogProviderList>()
+ .is_some()
+ );
}
}
diff --git a/datafusion/ffi/src/execution/task_ctx.rs b/datafusion/ffi/src/execution/task_ctx.rs
index 602e8dec8e..999a33bc1a 100644
--- a/datafusion/ffi/src/execution/task_ctx.rs
+++ b/datafusion/ffi/src/execution/task_ctx.rs
@@ -18,12 +18,12 @@
use std::ffi::c_void;
use std::sync::Arc;
+use abi_stable::StableAbi;
use abi_stable::pmr::ROption;
use abi_stable::std_types::{RHashMap, RString};
-use abi_stable::StableAbi;
+use datafusion_execution::TaskContext;
use datafusion_execution::config::SessionConfig;
use datafusion_execution::runtime_env::RuntimeEnv;
-use datafusion_execution::TaskContext;
use datafusion_expr::{
AggregateUDF, AggregateUDFImpl, ScalarUDF, ScalarUDFImpl, WindowUDF, WindowUDFImpl,
};
@@ -76,66 +76,84 @@ struct TaskContextPrivateData {
impl FFI_TaskContext {
unsafe fn inner(&self) -> &Arc<TaskContext> {
- let private_data = self.private_data as *const TaskContextPrivateData;
- &(*private_data).ctx
+ unsafe {
+ let private_data = self.private_data as *const TaskContextPrivateData;
+ &(*private_data).ctx
+ }
}
}
unsafe extern "C" fn session_id_fn_wrapper(ctx: &FFI_TaskContext) -> RString {
- let ctx = ctx.inner();
- ctx.session_id().into()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.session_id().into()
+ }
}
unsafe extern "C" fn task_id_fn_wrapper(ctx: &FFI_TaskContext) -> ROption<RString> {
- let ctx = ctx.inner();
- ctx.task_id().map(|s| s.as_str().into()).into()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.task_id().map(|s| s.as_str().into()).into()
+ }
}
unsafe extern "C" fn session_config_fn_wrapper(
ctx: &FFI_TaskContext,
) -> FFI_SessionConfig {
- let ctx = ctx.inner();
- ctx.session_config().into()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.session_config().into()
+ }
}
unsafe extern "C" fn scalar_functions_fn_wrapper(
ctx: &FFI_TaskContext,
) -> RHashMap<RString, FFI_ScalarUDF> {
- let ctx = ctx.inner();
- ctx.scalar_functions()
- .iter()
- .map(|(name, udf)| (name.to_owned().into(), Arc::clone(udf).into()))
- .collect()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.scalar_functions()
+ .iter()
+ .map(|(name, udf)| (name.to_owned().into(), Arc::clone(udf).into()))
+ .collect()
+ }
}
unsafe extern "C" fn aggregate_functions_fn_wrapper(
ctx: &FFI_TaskContext,
) -> RHashMap<RString, FFI_AggregateUDF> {
- let ctx = ctx.inner();
- ctx.aggregate_functions()
- .iter()
- .map(|(name, udaf)| {
- (
- name.to_owned().into(),
- FFI_AggregateUDF::from(Arc::clone(udaf)),
- )
- })
- .collect()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.aggregate_functions()
+ .iter()
+ .map(|(name, udaf)| {
+ (
+ name.to_owned().into(),
+ FFI_AggregateUDF::from(Arc::clone(udaf)),
+ )
+ })
+ .collect()
+ }
}
unsafe extern "C" fn window_functions_fn_wrapper(
ctx: &FFI_TaskContext,
) -> RHashMap<RString, FFI_WindowUDF> {
- let ctx = ctx.inner();
- ctx.window_functions()
- .iter()
- .map(|(name, udf)| (name.to_owned().into(), FFI_WindowUDF::from(Arc::clone(udf))))
- .collect()
+ unsafe {
+ let ctx = ctx.inner();
+ ctx.window_functions()
+ .iter()
+ .map(|(name, udf)| {
+ (name.to_owned().into(), FFI_WindowUDF::from(Arc::clone(udf)))
+ })
+ .collect()
+ }
}
unsafe extern "C" fn release_fn_wrapper(ctx: &mut FFI_TaskContext) {
- let private_data = Box::from_raw(ctx.private_data as *mut TaskContextPrivateData);
- drop(private_data);
+ unsafe {
+ let private_data = Box::from_raw(ctx.private_data as *mut TaskContextPrivateData);
+ drop(private_data);
+ }
}
impl Drop for FFI_TaskContext {
diff --git a/datafusion/ffi/src/execution/task_ctx_provider.rs b/datafusion/ffi/src/execution/task_ctx_provider.rs
index b9861a68d0..e04080c0bc 100644
--- a/datafusion/ffi/src/execution/task_ctx_provider.rs
+++ b/datafusion/ffi/src/execution/task_ctx_provider.rs
@@ -19,7 +19,7 @@ use std::ffi::c_void;
use std::sync::{Arc, Weak};
use abi_stable::StableAbi;
-use datafusion_common::{ffi_datafusion_err, DataFusionError};
+use datafusion_common::{DataFusionError, ffi_datafusion_err};
use datafusion_execution::{TaskContext, TaskContextProvider};
use crate::execution::task_ctx::FFI_TaskContext;
@@ -67,44 +67,54 @@ struct TaskContextProviderPrivateData {
impl FFI_TaskContextProvider {
unsafe fn inner(&self) -> Option<Arc<TaskContext>> {
- let private_data = self.private_data as *const TaskContextProviderPrivateData;
- (*private_data).ctx.upgrade().map(|ctx| ctx.task_ctx())
+ unsafe {
+ let private_data = self.private_data as *const TaskContextProviderPrivateData;
+ (*private_data).ctx.upgrade().map(|ctx| ctx.task_ctx())
+ }
}
}
unsafe extern "C" fn task_ctx_fn_wrapper(
ctx_provider: &FFI_TaskContextProvider,
) -> FFIResult<FFI_TaskContext> {
- rresult!(ctx_provider
- .inner()
- .map(FFI_TaskContext::from)
- .ok_or_else(|| {
- ffi_datafusion_err!(
- "TaskContextProvider went out of scope over FFI boundary."
- )
- }))
+ unsafe {
+ rresult!(
+ ctx_provider
+ .inner()
+ .map(FFI_TaskContext::from)
+ .ok_or_else(|| {
+ ffi_datafusion_err!(
+ "TaskContextProvider went out of scope over FFI boundary."
+ )
+ })
+ )
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
provider: &FFI_TaskContextProvider,
) -> FFI_TaskContextProvider {
- let private_data = provider.private_data as *const TaskContextProviderPrivateData;
- let ctx = Weak::clone(&(*private_data).ctx);
+ unsafe {
+ let private_data = provider.private_data as *const TaskContextProviderPrivateData;
+ let ctx = Weak::clone(&(*private_data).ctx);
- let private_data = Box::new(TaskContextProviderPrivateData { ctx });
+ let private_data = Box::new(TaskContextProviderPrivateData { ctx });
- FFI_TaskContextProvider {
- task_ctx: task_ctx_fn_wrapper,
- release: release_fn_wrapper,
- clone: clone_fn_wrapper,
- private_data: Box::into_raw(private_data) as *mut c_void,
- library_marker_id: crate::get_library_marker_id,
+ FFI_TaskContextProvider {
+ task_ctx: task_ctx_fn_wrapper,
+ release: release_fn_wrapper,
+ clone: clone_fn_wrapper,
+ private_data: Box::into_raw(private_data) as *mut c_void,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
unsafe extern "C" fn release_fn_wrapper(ctx: &mut FFI_TaskContextProvider) {
- let private_data =
- Box::from_raw(ctx.private_data as *mut TaskContextProviderPrivateData);
- drop(private_data);
+ unsafe {
+ let private_data =
+ Box::from_raw(ctx.private_data as *mut TaskContextProviderPrivateData);
+ drop(private_data);
+ }
}
impl Drop for FFI_TaskContextProvider {
fn drop(&mut self) {
diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs
index 8ee67d7e18..d8ef1c9827 100644
--- a/datafusion/ffi/src/execution_plan.rs
+++ b/datafusion/ffi/src/execution_plan.rs
@@ -18,8 +18,8 @@
use std::{ffi::c_void, pin::Pin, sync::Arc};
use abi_stable::{
- std_types::{RString, RVec},
StableAbi,
+ std_types::{RString, RVec},
};
use datafusion::{
error::DataFusionError,
@@ -98,34 +98,43 @@ unsafe extern "C" fn properties_fn_wrapper(
unsafe extern "C" fn children_fn_wrapper(
plan: &FFI_ExecutionPlan,
) -> RVec<FFI_ExecutionPlan> {
- let private_data = plan.private_data as *const ExecutionPlanPrivateData;
- let plan = &(*private_data).plan;
- let ctx = &(*private_data).context;
- let runtime = &(*private_data).runtime;
-
- let children: Vec<_> = plan
- .children()
- .into_iter()
- .map(|child| {
- FFI_ExecutionPlan::new(Arc::clone(child), Arc::clone(ctx), runtime.clone())
- })
- .collect();
-
- children.into()
+ unsafe {
+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;
+ let plan = &(*private_data).plan;
+ let ctx = &(*private_data).context;
+ let runtime = &(*private_data).runtime;
+
+ let children: Vec<_> = plan
+ .children()
+ .into_iter()
+ .map(|child| {
+ FFI_ExecutionPlan::new(
+ Arc::clone(child),
+ Arc::clone(ctx),
+ runtime.clone(),
+ )
+ })
+ .collect();
+
+ children.into()
+ }
}
unsafe extern "C" fn execute_fn_wrapper(
plan: &FFI_ExecutionPlan,
partition: usize,
) -> FFIResult<FFI_RecordBatchStream> {
- let private_data = plan.private_data as *const ExecutionPlanPrivateData;
- let plan = &(*private_data).plan;
- let ctx = &(*private_data).context;
- let runtime = (*private_data).runtime.clone();
-
- rresult!(plan
- .execute(partition, Arc::clone(ctx))
- .map(|rbs| FFI_RecordBatchStream::new(rbs, runtime)))
+ unsafe {
+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;
+ let plan = &(*private_data).plan;
+ let ctx = &(*private_data).context;
+ let runtime = (*private_data).runtime.clone();
+
+ rresult!(
+ plan.execute(partition, Arc::clone(ctx))
+ .map(|rbs| FFI_RecordBatchStream::new(rbs, runtime))
+ )
+ }
}
unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> RString {
@@ -133,21 +142,26 @@ unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> RString {
}
unsafe extern "C" fn release_fn_wrapper(plan: &mut FFI_ExecutionPlan) {
- debug_assert!(!plan.private_data.is_null());
- let private_data = Box::from_raw(plan.private_data as *mut ExecutionPlanPrivateData);
- drop(private_data);
- plan.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!plan.private_data.is_null());
+ let private_data =
+ Box::from_raw(plan.private_data as *mut ExecutionPlanPrivateData);
+ drop(private_data);
+ plan.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(plan: &FFI_ExecutionPlan) -> FFI_ExecutionPlan {
- let private_data = plan.private_data as *const ExecutionPlanPrivateData;
- let plan_data = &(*private_data);
-
- FFI_ExecutionPlan::new(
- Arc::clone(&plan_data.plan),
- Arc::clone(&plan_data.context),
- plan_data.runtime.clone(),
- )
+ unsafe {
+ let private_data = plan.private_data as *const ExecutionPlanPrivateData;
+ let plan_data = &(*private_data);
+
+ FFI_ExecutionPlan::new(
+ Arc::clone(&plan_data.plan),
+ Arc::clone(&plan_data.context),
+ plan_data.runtime.clone(),
+ )
+ }
}
impl Clone for FFI_ExecutionPlan {
@@ -306,8 +320,8 @@ pub(crate) mod tests {
use arrow::datatypes::{DataType, Field, Schema};
use datafusion::{
physical_plan::{
- execution_plan::{Boundedness, EmissionType},
Partitioning,
+ execution_plan::{Boundedness, EmissionType},
},
prelude::SessionContext,
};
@@ -468,9 +482,11 @@ pub(crate) mod tests {
// Verify different library markers generate foreign providers
ffi_plan.library_marker_id = crate::mock_foreign_marker_id;
let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into().unwrap();
- assert!(foreign_plan
- .as_any()
- .downcast_ref::<ForeignExecutionPlan>()
- .is_some());
+ assert!(
+ foreign_plan
+ .as_any()
+ .downcast_ref::<ForeignExecutionPlan>()
+ .is_some()
+ );
}
}
diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rs
index 720d83ce9a..abd07d0a27 100644
--- a/datafusion/ffi/src/physical_expr/mod.rs
+++ b/datafusion/ffi/src/physical_expr/mod.rs
@@ -27,18 +27,18 @@ use std::{
};
use abi_stable::{
- std_types::{ROption, RResult, RString, RVec},
StableAbi,
+ std_types::{ROption, RResult, RString, RVec},
};
use arrow::{
array::{ArrayRef, BooleanArray, RecordBatch},
datatypes::SchemaRef,
};
-use arrow_schema::{ffi::FFI_ArrowSchema, DataType, Field, FieldRef, Schema};
-use datafusion_common::{ffi_datafusion_err, Result};
+use arrow_schema::{DataType, Field, FieldRef, Schema, ffi::FFI_ArrowSchema};
+use datafusion_common::{Result, ffi_datafusion_err};
use datafusion_expr::{
- interval_arithmetic::Interval, sort_properties::ExprProperties,
- statistics::Distribution, ColumnarValue,
+ ColumnarValue, interval_arithmetic::Interval, sort_properties::ExprProperties,
+ statistics::Distribution,
};
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
@@ -191,10 +191,11 @@ unsafe extern "C" fn evaluate_fn_wrapper(
batch: WrappedArray,
) -> FFIResult<FFI_ColumnarValue> {
let batch = rresult_return!(wrapped_array_to_record_batch(batch));
- rresult!(expr
- .inner()
- .evaluate(&batch)
- .and_then(FFI_ColumnarValue::try_from))
+ rresult!(
+ expr.inner()
+ .evaluate(&batch)
+ .and_then(FFI_ColumnarValue::try_from)
+ )
}
unsafe extern "C" fn return_field_fn_wrapper(
@@ -203,10 +204,11 @@ unsafe extern "C" fn return_field_fn_wrapper(
) -> FFIResult<WrappedSchema> {
let expr = expr.inner();
let schema: SchemaRef = input_schema.into();
- rresult!(expr
- .return_field(&schema)
- .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(Into::into))
- .map(WrappedSchema))
+ rresult!(
+ expr.return_field(&schema)
+ .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(Into::into))
+ .map(WrappedSchema)
+ )
}
unsafe extern "C" fn evaluate_selection_fn_wrapper(
@@ -216,14 +218,17 @@ unsafe extern "C" fn evaluate_selection_fn_wrapper(
) -> FFIResult<FFI_ColumnarValue> {
let batch = rresult_return!(wrapped_array_to_record_batch(batch));
let selection: ArrayRef = rresult_return!(selection.try_into());
- let selection = rresult_return!(selection
- .as_any()
- .downcast_ref::<BooleanArray>()
- .ok_or(ffi_datafusion_err!("Unexpected selection array type")));
- rresult!(expr
- .inner()
- .evaluate_selection(&batch, selection)
- .and_then(FFI_ColumnarValue::try_from))
+ let selection = rresult_return!(
+ selection
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .ok_or(ffi_datafusion_err!("Unexpected selection array type"))
+ );
+ rresult!(
+ expr.inner()
+ .evaluate_selection(&batch, selection)
+ .and_then(FFI_ColumnarValue::try_from)
+ )
}
unsafe extern "C" fn children_fn_wrapper(
@@ -251,15 +256,18 @@ unsafe extern "C" fn evaluate_bounds_fn_wrapper(
children: RVec<FFI_Interval>,
) -> FFIResult<FFI_Interval> {
let expr = expr.inner();
- let children = rresult_return!(children
- .into_iter()
- .map(Interval::try_from)
- .collect::<Result<Vec<_>>>());
+ let children = rresult_return!(
+ children
+ .into_iter()
+ .map(Interval::try_from)
+ .collect::<Result<Vec<_>>>()
+ );
let children_borrowed = children.iter().collect::<Vec<_>>();
- rresult!(expr
- .evaluate_bounds(&children_borrowed)
- .and_then(FFI_Interval::try_from))
+ rresult!(
+ expr.evaluate_bounds(&children_borrowed)
+ .and_then(FFI_Interval::try_from)
+ )
}
unsafe extern "C" fn propagate_constraints_fn_wrapper(
@@ -269,21 +277,25 @@ unsafe extern "C" fn propagate_constraints_fn_wrapper(
) -> FFIResult<ROption<RVec<FFI_Interval>>> {
let expr = expr.inner();
let interval = rresult_return!(Interval::try_from(interval));
- let children = rresult_return!(children
- .into_iter()
- .map(Interval::try_from)
- .collect::<Result<Vec<_>>>());
+ let children = rresult_return!(
+ children
+ .into_iter()
+ .map(Interval::try_from)
+ .collect::<Result<Vec<_>>>()
+ );
let children_borrowed = children.iter().collect::<Vec<_>>();
let result =
rresult_return!(expr.propagate_constraints(&interval, &children_borrowed));
- let result = rresult_return!(result
- .map(|intervals| intervals
- .into_iter()
- .map(FFI_Interval::try_from)
- .collect::<Result<RVec<_>>>())
- .transpose());
+ let result = rresult_return!(
+ result
+ .map(|intervals| intervals
+ .into_iter()
+ .map(FFI_Interval::try_from)
+ .collect::<Result<RVec<_>>>())
+ .transpose()
+ );
RResult::ROk(result.into())
}
@@ -293,14 +305,17 @@ unsafe extern "C" fn evaluate_statistics_fn_wrapper(
children: RVec<FFI_Distribution>,
) -> FFIResult<FFI_Distribution> {
let expr = expr.inner();
- let children = rresult_return!(children
- .into_iter()
- .map(Distribution::try_from)
- .collect::<Result<Vec<_>>>());
+ let children = rresult_return!(
+ children
+ .into_iter()
+ .map(Distribution::try_from)
+ .collect::<Result<Vec<_>>>()
+ );
let children_borrowed = children.iter().collect::<Vec<_>>();
- rresult!(expr
- .evaluate_statistics(&children_borrowed)
- .and_then(|dist| FFI_Distribution::try_from(&dist)))
+ rresult!(
+ expr.evaluate_statistics(&children_borrowed)
+ .and_then(|dist| FFI_Distribution::try_from(&dist))
+ )
}
unsafe extern "C" fn propagate_statistics_fn_wrapper(
@@ -310,19 +325,23 @@ unsafe extern "C" fn propagate_statistics_fn_wrapper(
) -> FFIResult<ROption<RVec<FFI_Distribution>>> {
let expr = expr.inner();
let parent = rresult_return!(Distribution::try_from(parent));
- let children = rresult_return!(children
- .into_iter()
- .map(Distribution::try_from)
- .collect::<Result<Vec<_>>>());
+ let children = rresult_return!(
+ children
+ .into_iter()
+ .map(Distribution::try_from)
+ .collect::<Result<Vec<_>>>()
+ );
let children_borrowed = children.iter().collect::<Vec<_>>();
let result = rresult_return!(expr.propagate_statistics(&parent, &children_borrowed));
- let result = rresult_return!(result
- .map(|dists| dists
- .iter()
- .map(FFI_Distribution::try_from)
- .collect::<Result<RVec<_>>>())
- .transpose());
+ let result = rresult_return!(
+ result
+ .map(|dists| dists
+ .iter()
+ .map(FFI_Distribution::try_from)
+ .collect::<Result<RVec<_>>>())
+ .transpose()
+ );
RResult::ROk(result.into())
}
@@ -332,13 +351,16 @@ unsafe extern "C" fn get_properties_fn_wrapper(
children: RVec<FFI_ExprProperties>,
) -> FFIResult<FFI_ExprProperties> {
let expr = expr.inner();
- let children = rresult_return!(children
- .into_iter()
- .map(ExprProperties::try_from)
- .collect::<Result<Vec<_>>>());
- rresult!(expr
- .get_properties(&children)
- .and_then(|p| FFI_ExprProperties::try_from(&p)))
+ let children = rresult_return!(
+ children
+ .into_iter()
+ .map(ExprProperties::try_from)
+ .collect::<Result<Vec<_>>>()
+ );
+ rresult!(
+ expr.get_properties(&children)
+ .and_then(|p| FFI_ExprProperties::try_from(&p))
+ )
}
unsafe extern "C" fn fmt_sql_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFIResult<RString> {
@@ -351,9 +373,10 @@ unsafe extern "C" fn snapshot_fn_wrapper(
expr: &FFI_PhysicalExpr,
) -> FFIResult<ROption<FFI_PhysicalExpr>> {
let expr = expr.inner();
- rresult!(expr
- .snapshot()
- .map(|snapshot| snapshot.map(FFI_PhysicalExpr::from).into()))
+ rresult!(
+ expr.snapshot()
+ .map(|snapshot| snapshot.map(FFI_PhysicalExpr::from).into())
+ )
}
unsafe extern "C" fn snapshot_generation_fn_wrapper(expr: &FFI_PhysicalExpr) -> u64 {
@@ -378,43 +401,48 @@ unsafe extern "C" fn hash_fn_wrapper(expr: &FFI_PhysicalExpr) -> u64 {
}
unsafe extern "C" fn release_fn_wrapper(expr: &mut FFI_PhysicalExpr) {
- debug_assert!(!expr.private_data.is_null());
- let private_data = Box::from_raw(expr.private_data as *mut PhysicalExprPrivateData);
- drop(private_data);
- expr.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!expr.private_data.is_null());
+ let private_data =
+ Box::from_raw(expr.private_data as *mut PhysicalExprPrivateData);
+ drop(private_data);
+ expr.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFI_PhysicalExpr {
- let old_private_data = expr.private_data as *const PhysicalExprPrivateData;
-
- let private_data = Box::into_raw(Box::new(PhysicalExprPrivateData {
- expr: Arc::clone(&(*old_private_data).expr),
- })) as *mut c_void;
-
- FFI_PhysicalExpr {
- data_type: data_type_fn_wrapper,
- nullable: nullable_fn_wrapper,
- evaluate: evaluate_fn_wrapper,
- return_field: return_field_fn_wrapper,
- evaluate_selection: evaluate_selection_fn_wrapper,
- children: children_fn_wrapper,
- new_with_children: new_with_children_fn_wrapper,
- evaluate_bounds: evaluate_bounds_fn_wrapper,
- propagate_constraints: propagate_constraints_fn_wrapper,
- evaluate_statistics: evaluate_statistics_fn_wrapper,
- propagate_statistics: propagate_statistics_fn_wrapper,
- get_properties: get_properties_fn_wrapper,
- fmt_sql: fmt_sql_fn_wrapper,
- snapshot: snapshot_fn_wrapper,
- snapshot_generation: snapshot_generation_fn_wrapper,
- is_volatile_node: is_volatile_node_fn_wrapper,
- display: display_fn_wrapper,
- hash: hash_fn_wrapper,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- version: super::version,
- private_data,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ let old_private_data = expr.private_data as *const PhysicalExprPrivateData;
+
+ let private_data = Box::into_raw(Box::new(PhysicalExprPrivateData {
+ expr: Arc::clone(&(*old_private_data).expr),
+ })) as *mut c_void;
+
+ FFI_PhysicalExpr {
+ data_type: data_type_fn_wrapper,
+ nullable: nullable_fn_wrapper,
+ evaluate: evaluate_fn_wrapper,
+ return_field: return_field_fn_wrapper,
+ evaluate_selection: evaluate_selection_fn_wrapper,
+ children: children_fn_wrapper,
+ new_with_children: new_with_children_fn_wrapper,
+ evaluate_bounds: evaluate_bounds_fn_wrapper,
+ propagate_constraints: propagate_constraints_fn_wrapper,
+ evaluate_statistics: evaluate_statistics_fn_wrapper,
+ propagate_statistics: propagate_statistics_fn_wrapper,
+ get_properties: get_properties_fn_wrapper,
+ fmt_sql: fmt_sql_fn_wrapper,
+ snapshot: snapshot_fn_wrapper,
+ snapshot_generation: snapshot_generation_fn_wrapper,
+ is_volatile_node: is_volatile_node_fn_wrapper,
+ display: display_fn_wrapper,
+ hash: hash_fn_wrapper,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ version: super::version,
+ private_data,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -559,8 +587,13 @@ impl PhysicalExpr for ForeignPhysicalExpr {
) -> Result<Arc<dyn PhysicalExpr>> {
unsafe {
let children = children.into_iter().map(FFI_PhysicalExpr::from).collect();
- df_result!((self.expr.new_with_children)(&self.expr, &children)
- .map(|expr| <Arc<dyn PhysicalExpr>>::from(&expr)))
+ df_result!(
+ (self.expr.new_with_children)(&self.expr, &children).map(|expr| <Arc<
+ dyn PhysicalExpr,
+ >>::from(
+ &expr
+ ))
+ )
}
}
@@ -709,11 +742,11 @@ mod tests {
sync::Arc,
};
- use arrow::array::{record_batch, BooleanArray, RecordBatch};
- use datafusion_common::{tree_node::DynTreeNode, DataFusionError, ScalarValue};
+ use arrow::array::{BooleanArray, RecordBatch, record_batch};
+ use datafusion_common::{DataFusionError, ScalarValue, tree_node::DynTreeNode};
use datafusion_expr::{interval_arithmetic::Interval, statistics::Distribution};
use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};
- use datafusion_physical_expr_common::physical_expr::{fmt_sql, PhysicalExpr};
+ use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};
use crate::physical_expr::FFI_PhysicalExpr;
diff --git a/datafusion/ffi/src/physical_expr/partitioning.rs b/datafusion/ffi/src/physical_expr/partitioning.rs
index 32ca1f4835..fb16705e68 100644
--- a/datafusion/ffi/src/physical_expr/partitioning.rs
+++ b/datafusion/ffi/src/physical_expr/partitioning.rs
@@ -17,7 +17,7 @@
use std::sync::Arc;
-use abi_stable::{std_types::RVec, StableAbi};
+use abi_stable::{StableAbi, std_types::RVec};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -70,7 +70,7 @@ impl From<&FFI_Partitioning> for Partitioning {
#[cfg(test)]
mod tests {
- use datafusion_physical_expr::{expressions::lit, Partitioning};
+ use datafusion_physical_expr::{Partitioning, expressions::lit};
use crate::physical_expr::partitioning::FFI_Partitioning;
diff --git a/datafusion/ffi/src/plan_properties.rs b/datafusion/ffi/src/plan_properties.rs
index 40ac82c39d..430b34c984 100644
--- a/datafusion/ffi/src/plan_properties.rs
+++ b/datafusion/ffi/src/plan_properties.rs
@@ -18,24 +18,24 @@
use std::{ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{RResult::ROk, RVec},
StableAbi,
+ std_types::{RResult::ROk, RVec},
};
use arrow::datatypes::SchemaRef;
use datafusion::{
error::{DataFusionError, Result},
physical_expr::EquivalenceProperties,
physical_plan::{
- execution_plan::{Boundedness, EmissionType},
PlanProperties,
+ execution_plan::{Boundedness, EmissionType},
},
prelude::SessionContext,
};
use datafusion_proto::{
physical_plan::{
+ DefaultPhysicalExtensionCodec,
from_proto::{parse_physical_sort_exprs, parse_protobuf_partitioning},
to_proto::{serialize_partitioning, serialize_physical_sort_exprs},
- DefaultPhysicalExtensionCodec,
},
protobuf::{Partitioning, PhysicalSortExprNodeCollection},
};
@@ -141,11 +141,13 @@ unsafe extern "C" fn schema_fn_wrapper(properties: &FFI_PlanProperties) -> Wrapp
}
unsafe extern "C" fn release_fn_wrapper(props: &mut FFI_PlanProperties) {
- debug_assert!(!props.private_data.is_null());
- let private_data =
- Box::from_raw(props.private_data as *mut PlanPropertiesPrivateData);
- drop(private_data);
- props.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!props.private_data.is_null());
+ let private_data =
+ Box::from_raw(props.private_data as *mut PlanPropertiesPrivateData);
+ drop(private_data);
+ props.private_data = std::ptr::null_mut();
+ }
}
impl Drop for FFI_PlanProperties {
diff --git a/datafusion/ffi/src/proto/logical_extension_codec.rs b/datafusion/ffi/src/proto/logical_extension_codec.rs
index 928a7c5bdb..3f79cfc732 100644
--- a/datafusion/ffi/src/proto/logical_extension_codec.rs
+++ b/datafusion/ffi/src/proto/logical_extension_codec.rs
@@ -18,12 +18,12 @@
use std::ffi::c_void;
use std::sync::Arc;
-use abi_stable::std_types::{RResult, RSlice, RStr, RVec};
use abi_stable::StableAbi;
+use abi_stable::std_types::{RResult, RSlice, RStr, RVec};
use arrow::datatypes::SchemaRef;
use datafusion_catalog::TableProvider;
use datafusion_common::error::Result;
-use datafusion_common::{not_impl_err, TableReference};
+use datafusion_common::{TableReference, not_impl_err};
use datafusion_datasource::file_format::FileFormatFactory;
use datafusion_execution::TaskContext;
use datafusion_expr::{
@@ -261,9 +261,11 @@ unsafe extern "C" fn try_encode_udwf_fn_wrapper(
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_LogicalExtensionCodec) {
- let private_data =
- Box::from_raw(provider.private_data as *mut LogicalExtensionCodecPrivateData);
- drop(private_data);
+ unsafe {
+ let private_data =
+ Box::from_raw(provider.private_data as *mut LogicalExtensionCodecPrivateData);
+ drop(private_data);
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
@@ -472,7 +474,7 @@ mod tests {
use arrow::array::record_batch;
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use datafusion_catalog::{MemTable, TableProvider};
- use datafusion_common::{exec_err, Result, TableReference};
+ use datafusion_common::{Result, TableReference, exec_err};
use datafusion_datasource::file_format::FileFormatFactory;
use datafusion_execution::TaskContext;
use datafusion_expr::ptr_eq::arc_ptr_eq;
diff --git a/datafusion/ffi/src/proto/physical_extension_codec.rs b/datafusion/ffi/src/proto/physical_extension_codec.rs
index 8f3667400e..89a9a2cead 100644
--- a/datafusion/ffi/src/proto/physical_extension_codec.rs
+++ b/datafusion/ffi/src/proto/physical_extension_codec.rs
@@ -18,8 +18,8 @@
use std::ffi::c_void;
use std::sync::Arc;
-use abi_stable::std_types::{RResult, RSlice, RStr, RVec};
use abi_stable::StableAbi;
+use abi_stable::std_types::{RResult, RSlice, RStr, RVec};
use datafusion_common::error::Result;
use datafusion_execution::TaskContext;
use datafusion_expr::{
@@ -242,9 +242,12 @@ unsafe extern "C" fn try_encode_udwf_fn_wrapper(
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_PhysicalExtensionCodec) {
- let private_data =
- Box::from_raw(provider.private_data as *mut PhysicalExtensionCodecPrivateData);
- drop(private_data);
+ unsafe {
+ let private_data = Box::from_raw(
+ provider.private_data as *mut PhysicalExtensionCodecPrivateData,
+ );
+ drop(private_data);
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
@@ -412,7 +415,7 @@ pub(crate) mod tests {
use std::sync::Arc;
use arrow_schema::{DataType, Field, Schema};
- use datafusion_common::{exec_err, Result};
+ use datafusion_common::{Result, exec_err};
use datafusion_execution::TaskContext;
use datafusion_expr::ptr_eq::arc_ptr_eq;
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF, WindowUDFImpl};
diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs
index b515125f9a..d84b710c6f 100644
--- a/datafusion/ffi/src/record_batch_stream.rs
+++ b/datafusion/ffi/src/record_batch_stream.rs
@@ -18,12 +18,12 @@
use std::{ffi::c_void, task::Poll};
use abi_stable::{
- std_types::{ROption, RResult},
StableAbi,
+ std_types::{ROption, RResult},
};
use arrow::array::{Array, RecordBatch};
use arrow::{
- array::{make_array, StructArray},
+ array::{StructArray, make_array},
ffi::{from_ffi, to_ffi},
};
use async_ffi::{ContextExt, FfiContext, FfiPoll};
@@ -95,18 +95,22 @@ impl FFI_RecordBatchStream {
unsafe impl Send for FFI_RecordBatchStream {}
unsafe extern "C" fn schema_fn_wrapper(stream: &FFI_RecordBatchStream) -> WrappedSchema {
- let private_data = stream.private_data as *const RecordBatchStreamPrivateData;
- let stream = &(*private_data).rbs;
+ unsafe {
+ let private_data = stream.private_data as *const RecordBatchStreamPrivateData;
+ let stream = &(*private_data).rbs;
- (*stream).schema().into()
+ (*stream).schema().into()
+ }
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_RecordBatchStream) {
- debug_assert!(!provider.private_data.is_null());
- let private_data =
- Box::from_raw(provider.private_data as *mut RecordBatchStreamPrivateData);
- drop(private_data);
- provider.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!provider.private_data.is_null());
+ let private_data =
+ Box::from_raw(provider.private_data as *mut RecordBatchStreamPrivateData);
+ drop(private_data);
+ provider.private_data = std::ptr::null_mut();
+ }
}
pub(crate) fn record_batch_to_wrapped_array(
@@ -138,18 +142,20 @@ unsafe extern "C" fn poll_next_fn_wrapper(
stream: &FFI_RecordBatchStream,
cx: &mut FfiContext,
) -> FfiPoll<ROption<FFIResult<WrappedArray>>> {
- let private_data = stream.private_data as *mut RecordBatchStreamPrivateData;
- let stream = &mut (*private_data).rbs;
+ unsafe {
+ let private_data = stream.private_data as *mut RecordBatchStreamPrivateData;
+ let stream = &mut (*private_data).rbs;
- let _guard = (*private_data).runtime.as_ref().map(|rt| rt.enter());
+ let _guard = (*private_data).runtime.as_ref().map(|rt| rt.enter());
- let poll_result = cx.with_context(|std_cx| {
- (*stream)
- .try_poll_next_unpin(std_cx)
- .map(maybe_record_batch_to_wrapped_stream)
- });
+ let poll_result = cx.with_context(|std_cx| {
+ (*stream)
+ .try_poll_next_unpin(std_cx)
+ .map(maybe_record_batch_to_wrapped_stream)
+ });
- poll_result.into()
+ poll_result.into()
+ }
}
impl RecordBatchStream for FFI_RecordBatchStream {
diff --git a/datafusion/ffi/src/schema_provider.rs b/datafusion/ffi/src/schema_provider.rs
index eafdb2e694..cea41ae30f 100644
--- a/datafusion/ffi/src/schema_provider.rs
+++ b/datafusion/ffi/src/schema_provider.rs
@@ -18,8 +18,8 @@
use std::{any::Any, ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{ROption, RResult, RString, RVec},
StableAbi,
+ std_types::{ROption, RResult, RString, RVec},
};
use async_ffi::{FfiFuture, FutureExt};
use async_trait::async_trait;
@@ -97,40 +97,48 @@ struct ProviderPrivateData {
impl FFI_SchemaProvider {
unsafe fn inner(&self) -> &Arc<dyn SchemaProvider + Send> {
- let private_data = self.private_data as *const ProviderPrivateData;
- &(*private_data).provider
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ &(*private_data).provider
+ }
}
unsafe fn runtime(&self) -> Option<Handle> {
- let private_data = self.private_data as *const ProviderPrivateData;
- (*private_data).runtime.clone()
+ unsafe {
+ let private_data = self.private_data as *const ProviderPrivateData;
+ (*private_data).runtime.clone()
+ }
}
}
unsafe extern "C" fn table_names_fn_wrapper(
provider: &FFI_SchemaProvider,
) -> RVec<RString> {
- let provider = provider.inner();
+ unsafe {
+ let provider = provider.inner();
- let table_names = provider.table_names();
- table_names.into_iter().map(|s| s.into()).collect()
+ let table_names = provider.table_names();
+ table_names.into_iter().map(|s| s.into()).collect()
+ }
}
unsafe extern "C" fn table_fn_wrapper(
provider: &FFI_SchemaProvider,
name: RString,
) -> FfiFuture<FFIResult<ROption<FFI_TableProvider>>> {
- let runtime = provider.runtime();
- let provider = Arc::clone(provider.inner());
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = Arc::clone(provider.inner());
- async move {
- let table = rresult_return!(provider.table(name.as_str()).await)
- .map(|t| FFI_TableProvider::new(t, true, runtime))
- .into();
+ async move {
+ let table = rresult_return!(provider.table(name.as_str()).await)
+ .map(|t| FFI_TableProvider::new(t, true, runtime))
+ .into();
- RResult::ROk(table)
+ RResult::ROk(table)
+ }
+ .into_ffi()
}
- .into_ffi()
}
unsafe extern "C" fn register_table_fn_wrapper(
@@ -138,67 +146,76 @@ unsafe extern "C" fn register_table_fn_wrapper(
name: RString,
table: FFI_TableProvider,
) -> FFIResult<ROption<FFI_TableProvider>> {
- let runtime = provider.runtime();
- let provider = provider.inner();
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
- let table = Arc::new(ForeignTableProvider(table));
+ let table = Arc::new(ForeignTableProvider(table));
- let returned_table = rresult_return!(provider.register_table(name.into(), table))
- .map(|t| FFI_TableProvider::new(t, true, runtime));
+ let returned_table = rresult_return!(provider.register_table(name.into(), table))
+ .map(|t| FFI_TableProvider::new(t, true, runtime));
- RResult::ROk(returned_table.into())
+ RResult::ROk(returned_table.into())
+ }
}
unsafe extern "C" fn deregister_table_fn_wrapper(
provider: &FFI_SchemaProvider,
name: RString,
) -> FFIResult<ROption<FFI_TableProvider>> {
- let runtime = provider.runtime();
- let provider = provider.inner();
+ unsafe {
+ let runtime = provider.runtime();
+ let provider = provider.inner();
- let returned_table = rresult_return!(provider.deregister_table(name.as_str()))
- .map(|t| FFI_TableProvider::new(t, true, runtime));
+ let returned_table = rresult_return!(provider.deregister_table(name.as_str()))
+ .map(|t| FFI_TableProvider::new(t, true, runtime));
- RResult::ROk(returned_table.into())
+ RResult::ROk(returned_table.into())
+ }
}
unsafe extern "C" fn table_exist_fn_wrapper(
provider: &FFI_SchemaProvider,
name: RString,
) -> bool {
- provider.inner().table_exist(name.as_str())
+ unsafe { provider.inner().table_exist(name.as_str()) }
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_SchemaProvider) {
- debug_assert!(!provider.private_data.is_null());
- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);
- drop(private_data);
- provider.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!provider.private_data.is_null());
+ let private_data =
+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);
+ drop(private_data);
+ provider.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(
provider: &FFI_SchemaProvider,
) -> FFI_SchemaProvider {
- let old_private_data = provider.private_data as *const ProviderPrivateData;
- let runtime = (*old_private_data).runtime.clone();
-
- let private_data = Box::into_raw(Box::new(ProviderPrivateData {
- provider: Arc::clone(&(*old_private_data).provider),
- runtime,
- })) as *mut c_void;
-
- FFI_SchemaProvider {
- owner_name: provider.owner_name.clone(),
- table_names: table_names_fn_wrapper,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- version: super::version,
- private_data,
- table: table_fn_wrapper,
- register_table: register_table_fn_wrapper,
- deregister_table: deregister_table_fn_wrapper,
- table_exist: table_exist_fn_wrapper,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ let old_private_data = provider.private_data as *const ProviderPrivateData;
+ let runtime = (*old_private_data).runtime.clone();
+
+ let private_data = Box::into_raw(Box::new(ProviderPrivateData {
+ provider: Arc::clone(&(*old_private_data).provider),
+ runtime,
+ })) as *mut c_void;
+
+ FFI_SchemaProvider {
+ owner_name: provider.owner_name.clone(),
+ table_names: table_names_fn_wrapper,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ version: super::version,
+ private_data,
+ table: table_fn_wrapper,
+ register_table: register_table_fn_wrapper,
+ deregister_table: deregister_table_fn_wrapper,
+ table_exist: table_exist_fn_wrapper,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -342,11 +359,13 @@ mod tests {
#[tokio::test]
async fn test_round_trip_ffi_schema_provider() {
let schema_provider = Arc::new(MemorySchemaProvider::new());
- assert!(schema_provider
- .as_ref()
- .register_table("prior_table".to_string(), empty_table())
- .unwrap()
- .is_none());
+ assert!(
+ schema_provider
+ .as_ref()
+ .register_table("prior_table".to_string(), empty_table())
+ .unwrap()
+ .is_none()
+ );
let mut ffi_schema_provider = FFI_SchemaProvider::new(schema_provider, None);
ffi_schema_provider.library_marker_id = crate::mock_foreign_marker_id;
@@ -403,17 +422,21 @@ mod tests {
// Verify local libraries can be downcast to their original
let foreign_schema: Arc<dyn SchemaProvider + Send> = (&ffi_schema).into();
- assert!(foreign_schema
- .as_any()
- .downcast_ref::<MemorySchemaProvider>()
- .is_some());
+ assert!(
+ foreign_schema
+ .as_any()
+ .downcast_ref::<MemorySchemaProvider>()
+ .is_some()
+ );
// Verify different library markers generate foreign providers
ffi_schema.library_marker_id = crate::mock_foreign_marker_id;
let foreign_schema: Arc<dyn SchemaProvider + Send> = (&ffi_schema).into();
- assert!(foreign_schema
- .as_any()
- .downcast_ref::<ForeignSchemaProvider>()
- .is_some());
+ assert!(
+ foreign_schema
+ .as_any()
+ .downcast_ref::<ForeignSchemaProvider>()
+ .is_some()
+ );
}
}
diff --git a/datafusion/ffi/src/session/config.rs b/datafusion/ffi/src/session/config.rs
index 5fbd6175e0..da026bbd8c 100644
--- a/datafusion/ffi/src/session/config.rs
+++ b/datafusion/ffi/src/session/config.rs
@@ -18,8 +18,8 @@
use std::collections::HashMap;
use std::ffi::c_void;
-use abi_stable::std_types::{RHashMap, RString};
use abi_stable::StableAbi;
+use abi_stable::std_types::{RHashMap, RString};
use datafusion_common::error::{DataFusionError, Result};
use datafusion_execution::config::SessionConfig;
@@ -84,25 +84,29 @@ unsafe extern "C" fn config_options_fn_wrapper(
}
unsafe extern "C" fn release_fn_wrapper(config: &mut FFI_SessionConfig) {
- debug_assert!(!config.private_data.is_null());
- let private_data =
- Box::from_raw(config.private_data as *mut SessionConfigPrivateData);
- drop(private_data);
- config.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!config.private_data.is_null());
+ let private_data =
+ Box::from_raw(config.private_data as *mut SessionConfigPrivateData);
+ drop(private_data);
+ config.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(config: &FFI_SessionConfig) -> FFI_SessionConfig {
- let old_private_data = config.private_data as *mut SessionConfigPrivateData;
- let old_config = (*old_private_data).config.clone();
+ unsafe {
+ let old_private_data = config.private_data as *mut SessionConfigPrivateData;
+ let old_config = (*old_private_data).config.clone();
- let private_data = Box::new(SessionConfigPrivateData { config: old_config });
+ let private_data = Box::new(SessionConfigPrivateData { config: old_config });
- FFI_SessionConfig {
- config_options: config_options_fn_wrapper,
- private_data: Box::into_raw(private_data) as *mut c_void,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- library_marker_id: crate::get_library_marker_id,
+ FFI_SessionConfig {
+ config_options: config_options_fn_wrapper,
+ private_data: Box::into_raw(private_data) as *mut c_void,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rs
index dea892d432..1694f46dc7 100644
--- a/datafusion/ffi/src/session/mod.rs
+++ b/datafusion/ffi/src/session/mod.rs
@@ -20,17 +20,17 @@ use std::collections::HashMap;
use std::ffi::c_void;
use std::sync::Arc;
-use abi_stable::std_types::{RHashMap, RResult, RStr, RString, RVec};
use abi_stable::StableAbi;
-use arrow_schema::ffi::FFI_ArrowSchema;
+use abi_stable::std_types::{RHashMap, RResult, RStr, RString, RVec};
use arrow_schema::SchemaRef;
+use arrow_schema::ffi::FFI_ArrowSchema;
use async_ffi::{FfiFuture, FutureExt};
use async_trait::async_trait;
use datafusion_common::config::{ConfigOptions, TableOptions};
use datafusion_common::{DFSchema, DataFusionError};
+use datafusion_execution::TaskContext;
use datafusion_execution::config::SessionConfig;
use datafusion_execution::runtime_env::RuntimeEnv;
-use datafusion_execution::TaskContext;
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::{
AggregateUDF, AggregateUDFImpl, Expr, LogicalPlan, ScalarUDF, ScalarUDFImpl,
@@ -39,9 +39,9 @@ use datafusion_expr::{
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_proto::bytes::{logical_plan_from_bytes, logical_plan_to_bytes};
+use datafusion_proto::logical_plan::LogicalExtensionCodec;
use datafusion_proto::logical_plan::from_proto::parse_expr;
use datafusion_proto::logical_plan::to_proto::serialize_expr;
-use datafusion_proto::logical_plan::LogicalExtensionCodec;
use datafusion_proto::protobuf::LogicalExprNode;
use datafusion_session::Session;
use prost::Message;
@@ -142,8 +142,10 @@ impl FFI_SessionRef {
}
unsafe fn runtime(&self) -> &Option<Handle> {
- let private_data = self.private_data as *const SessionPrivateData;
- &(*private_data).runtime
+ unsafe {
+ let private_data = self.private_data as *const SessionPrivateData;
+ &(*private_data).runtime
+ }
}
}
@@ -161,22 +163,26 @@ unsafe extern "C" fn create_physical_plan_fn_wrapper(
session: &FFI_SessionRef,
logical_plan_serialized: RVec<u8>,
) -> FfiFuture<FFIResult<FFI_ExecutionPlan>> {
- let runtime = session.runtime().clone();
- let session = session.clone();
- async move {
- let session = session.inner();
- let task_ctx = session.task_ctx();
-
- let logical_plan = rresult_return!(logical_plan_from_bytes(
- logical_plan_serialized.as_slice(),
- task_ctx.as_ref(),
- ));
-
- let physical_plan = session.create_physical_plan(&logical_plan).await;
-
- rresult!(physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, task_ctx, runtime)))
+ unsafe {
+ let runtime = session.runtime().clone();
+ let session = session.clone();
+ async move {
+ let session = session.inner();
+ let task_ctx = session.task_ctx();
+
+ let logical_plan = rresult_return!(logical_plan_from_bytes(
+ logical_plan_serialized.as_slice(),
+ task_ctx.as_ref(),
+ ));
+
+ let physical_plan = session.create_physical_plan(&logical_plan).await;
+
+ rresult!(
+ physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, task_ctx, runtime))
+ )
+ }
+ .into_ffi()
}
- .into_ffi()
}
unsafe extern "C" fn create_physical_expr_fn_wrapper(
@@ -267,36 +273,41 @@ unsafe extern "C" fn task_ctx_fn_wrapper(session: &FFI_SessionRef) -> FFI_TaskCo
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_SessionRef) {
- let private_data = Box::from_raw(provider.private_data as *mut SessionPrivateData);
- drop(private_data);
+ unsafe {
+ let private_data =
+ Box::from_raw(provider.private_data as *mut SessionPrivateData);
+ drop(private_data);
+ }
}
unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_SessionRef) -> FFI_SessionRef {
- let old_private_data = provider.private_data as *const SessionPrivateData;
-
- let private_data = Box::into_raw(Box::new(SessionPrivateData {
- session: (*old_private_data).session,
- runtime: (*old_private_data).runtime.clone(),
- })) as *mut c_void;
-
- FFI_SessionRef {
- session_id: session_id_fn_wrapper,
- config: config_fn_wrapper,
- create_physical_plan: create_physical_plan_fn_wrapper,
- create_physical_expr: create_physical_expr_fn_wrapper,
- scalar_functions: scalar_functions_fn_wrapper,
- aggregate_functions: aggregate_functions_fn_wrapper,
- window_functions: window_functions_fn_wrapper,
- table_options: table_options_fn_wrapper,
- default_table_options: default_table_options_fn_wrapper,
- task_ctx: task_ctx_fn_wrapper,
- logical_codec: provider.logical_codec.clone(),
-
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- version: super::version,
- private_data,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ let old_private_data = provider.private_data as *const SessionPrivateData;
+
+ let private_data = Box::into_raw(Box::new(SessionPrivateData {
+ session: (*old_private_data).session,
+ runtime: (*old_private_data).runtime.clone(),
+ })) as *mut c_void;
+
+ FFI_SessionRef {
+ session_id: session_id_fn_wrapper,
+ config: config_fn_wrapper,
+ create_physical_plan: create_physical_plan_fn_wrapper,
+ create_physical_expr: create_physical_expr_fn_wrapper,
+ scalar_functions: scalar_functions_fn_wrapper,
+ aggregate_functions: aggregate_functions_fn_wrapper,
+ window_functions: window_functions_fn_wrapper,
+ table_options: table_options_fn_wrapper,
+ default_table_options: default_table_options_fn_wrapper,
+ task_ctx: task_ctx_fn_wrapper,
+ logical_codec: provider.logical_codec.clone(),
+
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ version: super::version,
+ private_data,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -532,7 +543,9 @@ impl Session for ForeignSession {
}
fn table_options_mut(&mut self) -> &mut TableOptions {
- log::warn!("Mutating table options is not supported via FFI. Changes will not have an effect.");
+ log::warn!(
+ "Mutating table options is not supported via FFI. Changes will not have an effect."
+ );
&mut self.table_options
}
@@ -577,7 +590,10 @@ mod tests {
let logical_plan = LogicalPlan::default();
let physical_plan = foreign_session.create_physical_plan(&logical_plan).await?;
- assert_eq!(format!("{physical_plan:?}"), "EmptyExec { schema: Schema { fields: [], metadata: {} }, partitions: 1, cache: PlanProperties { eq_properties: EquivalenceProperties { eq_group: EquivalenceGroup { map: {}, classes: [] }, oeq_class: OrderingEquivalenceClass { orderings: [] }, oeq_cache: OrderingEquivalenceCache { normal_cls: OrderingEquivalenceClass { orderings: [] }, leading_map: {} }, constraints: Constraints { inner: [] }, schema: Schema { fields: [], metadata: {} } }, partitioning: UnknownPartitioning(1), emission_type: Incremental, boundedness: Bounded, evaluation_type: Lazy, scheduling_type: Cooperative, output_ordering: None } }");
+ assert_eq!(
+ format!("{physical_plan:?}"),
+ "EmptyExec { schema: Schema { fields: [], metadata: {} }, partitions: 1, cache: PlanProperties { eq_properties: EquivalenceProperties { eq_group: EquivalenceGroup { map: {}, classes: [] }, oeq_class: OrderingEquivalenceClass { orderings: [] }, oeq_cache: OrderingEquivalenceCache { normal_cls: OrderingEquivalenceClass { orderings: [] }, leading_map: {} }, constraints: Constraints { inner: [] }, schema: Schema { fields: [], metadata: {} } }, partitioning: UnknownPartitioning(1), emission_type: Incremental, boundedness: Bounded, evaluation_type: Lazy, scheduling_type: Cooperative, output_ordering: None } }"
+ );
assert_eq!(
format!("{:?}", foreign_session.default_table_options()),
diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rs
index 01672bb69e..8929d02b0e 100644
--- a/datafusion/ffi/src/table_provider.rs
+++ b/datafusion/ffi/src/table_provider.rs
@@ -18,8 +18,8 @@
use std::{any::Any, ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{ROption, RResult, RVec},
StableAbi,
+ std_types::{ROption, RResult, RVec},
};
use arrow::datatypes::SchemaRef;
use async_ffi::{FfiFuture, FutureExt};
@@ -28,14 +28,14 @@ use datafusion::{
catalog::{Session, TableProvider},
datasource::TableType,
error::DataFusionError,
- execution::{session_state::SessionStateBuilder, TaskContext},
- logical_expr::{logical_plan::dml::InsertOp, TableProviderFilterPushDown},
+ execution::{TaskContext, session_state::SessionStateBuilder},
+ logical_expr::{TableProviderFilterPushDown, logical_plan::dml::InsertOp},
physical_plan::ExecutionPlan,
prelude::{Expr, SessionContext},
};
use datafusion_proto::{
logical_plan::{
- from_proto::parse_exprs, to_proto::serialize_exprs, DefaultLogicalExtensionCodec,
+ DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,
},
protobuf::LogicalExprList,
};
@@ -319,10 +319,13 @@ unsafe extern "C" fn insert_into_fn_wrapper(
}
unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_TableProvider) {
- debug_assert!(!provider.private_data.is_null());
- let private_data = Box::from_raw(provider.private_data as *mut ProviderPrivateData);
- drop(private_data);
- provider.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!provider.private_data.is_null());
+ let private_data =
+ Box::from_raw(provider.private_data as *mut ProviderPrivateData);
+ drop(private_data);
+ provider.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_TableProvider) -> FFI_TableProvider {
@@ -469,7 +472,7 @@ impl TableProvider for ForeignTableProvider {
return Ok(vec![
TableProviderFilterPushDown::Unsupported;
filters.len()
- ])
+ ]);
}
};
@@ -647,18 +650,22 @@ mod tests {
// Verify local libraries can be downcast to their original
let foreign_table: Arc<dyn TableProvider> = (&ffi_table).into();
- assert!(foreign_table
- .as_any()
- .downcast_ref::<datafusion::datasource::MemTable>()
- .is_some());
+ assert!(
+ foreign_table
+ .as_any()
+ .downcast_ref::<datafusion::datasource::MemTable>()
+ .is_some()
+ );
// Verify different library markers generate foreign providers
ffi_table.library_marker_id = crate::mock_foreign_marker_id;
let foreign_table: Arc<dyn TableProvider> = (&ffi_table).into();
- assert!(foreign_table
- .as_any()
- .downcast_ref::<ForeignTableProvider>()
- .is_some());
+ assert!(
+ foreign_table
+ .as_any()
+ .downcast_ref::<ForeignTableProvider>()
+ .is_some()
+ );
Ok(())
}
diff --git a/datafusion/ffi/src/tests/catalog.rs b/datafusion/ffi/src/tests/catalog.rs
index b6efbdf726..de012659f2 100644
--- a/datafusion/ffi/src/tests/catalog.rs
+++ b/datafusion/ffi/src/tests/catalog.rs
@@ -163,7 +163,9 @@ impl CatalogProvider for FixedCatalogProvider {
schema: Arc<dyn SchemaProvider>,
) -> Result<Option<Arc<dyn SchemaProvider>>> {
if !["apple", "banana", "cherry", "date"].contains(&name) {
- return exec_err!("FixedCatalogProvider only provides four schemas: apple, banana, cherry, date");
+ return exec_err!(
+ "FixedCatalogProvider only provides four schemas: apple, banana, cherry, date"
+ );
}
self.inner.register_schema(name, schema)
@@ -222,7 +224,9 @@ impl CatalogProviderList for FixedCatalogProviderList {
catalog: Arc<dyn CatalogProvider>,
) -> Option<Arc<dyn CatalogProvider>> {
if !["blue", "red", "green", "yellow"].contains(&name.as_str()) {
- log::warn!("FixedCatalogProviderList only provides four catalogs: blue, red, green, yellow");
+ log::warn!(
+ "FixedCatalogProviderList only provides four catalogs: blue, red, green, yellow"
+ );
return None;
}
diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs
index d9b4a61579..dfb4756107 100644
--- a/datafusion/ffi/src/tests/mod.rs
+++ b/datafusion/ffi/src/tests/mod.rs
@@ -18,12 +18,11 @@
use std::sync::Arc;
use abi_stable::{
- declare_root_module_statics, export_root_module,
+ StableAbi, declare_root_module_statics, export_root_module,
library::{LibraryError, RootModule},
package_version_strings,
prefix_type::PrefixTypeTrait,
sabi_types::VersionStrings,
- StableAbi,
};
use catalog::create_catalog_provider;
diff --git a/datafusion/ffi/src/udaf/accumulator.rs b/datafusion/ffi/src/udaf/accumulator.rs
index c0ab0bdd5b..924371538c 100644
--- a/datafusion/ffi/src/udaf/accumulator.rs
+++ b/datafusion/ffi/src/udaf/accumulator.rs
@@ -16,8 +16,8 @@
// under the License.
use abi_stable::{
- std_types::{RResult, RVec},
StableAbi,
+ std_types::{RResult, RVec},
};
use arrow::{array::ArrayRef, error::ArrowError};
use datafusion::{
@@ -86,14 +86,18 @@ pub struct AccumulatorPrivateData {
impl FFI_Accumulator {
#[inline]
unsafe fn inner_mut(&mut self) -> &mut Box<dyn Accumulator> {
- let private_data = self.private_data as *mut AccumulatorPrivateData;
- &mut (*private_data).accumulator
+ unsafe {
+ let private_data = self.private_data as *mut AccumulatorPrivateData;
+ &mut (*private_data).accumulator
+ }
}
#[inline]
unsafe fn inner(&self) -> &dyn Accumulator {
- let private_data = self.private_data as *const AccumulatorPrivateData;
- (*private_data).accumulator.deref()
+ unsafe {
+ let private_data = self.private_data as *const AccumulatorPrivateData;
+ (*private_data).accumulator.deref()
+ }
}
}
@@ -101,87 +105,101 @@ unsafe extern "C" fn update_batch_fn_wrapper(
accumulator: &mut FFI_Accumulator,
values: RVec<WrappedArray>,
) -> FFIResult<()> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let values_arrays = values
- .into_iter()
- .map(|v| v.try_into().map_err(DataFusionError::from))
- .collect::<Result<Vec<ArrayRef>>>();
- let values_arrays = rresult_return!(values_arrays);
+ let values_arrays = values
+ .into_iter()
+ .map(|v| v.try_into().map_err(DataFusionError::from))
+ .collect::<Result<Vec<ArrayRef>>>();
+ let values_arrays = rresult_return!(values_arrays);
- rresult!(accumulator.update_batch(&values_arrays))
+ rresult!(accumulator.update_batch(&values_arrays))
+ }
}
unsafe extern "C" fn evaluate_fn_wrapper(
accumulator: &mut FFI_Accumulator,
) -> FFIResult<RVec<u8>> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let scalar_result = rresult_return!(accumulator.evaluate());
- let proto_result: datafusion_proto::protobuf::ScalarValue =
- rresult_return!((&scalar_result).try_into());
+ let scalar_result = rresult_return!(accumulator.evaluate());
+ let proto_result: datafusion_proto::protobuf::ScalarValue =
+ rresult_return!((&scalar_result).try_into());
- RResult::ROk(proto_result.encode_to_vec().into())
+ RResult::ROk(proto_result.encode_to_vec().into())
+ }
}
unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_Accumulator) -> usize {
- accumulator.inner().size()
+ unsafe { accumulator.inner().size() }
}
unsafe extern "C" fn state_fn_wrapper(
accumulator: &mut FFI_Accumulator,
) -> FFIResult<RVec<RVec<u8>>> {
- let accumulator = accumulator.inner_mut();
-
- let state = rresult_return!(accumulator.state());
- let state = state
- .into_iter()
- .map(|state_val| {
- datafusion_proto::protobuf::ScalarValue::try_from(&state_val)
- .map_err(DataFusionError::from)
- .map(|v| RVec::from(v.encode_to_vec()))
- })
- .collect::<Result<Vec<_>>>()
- .map(|state_vec| state_vec.into());
-
- rresult!(state)
+ unsafe {
+ let accumulator = accumulator.inner_mut();
+
+ let state = rresult_return!(accumulator.state());
+ let state = state
+ .into_iter()
+ .map(|state_val| {
+ datafusion_proto::protobuf::ScalarValue::try_from(&state_val)
+ .map_err(DataFusionError::from)
+ .map(|v| RVec::from(v.encode_to_vec()))
+ })
+ .collect::<Result<Vec<_>>>()
+ .map(|state_vec| state_vec.into());
+
+ rresult!(state)
+ }
}
unsafe extern "C" fn merge_batch_fn_wrapper(
accumulator: &mut FFI_Accumulator,
states: RVec<WrappedArray>,
) -> FFIResult<()> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let states = rresult_return!(states
- .into_iter()
- .map(|state| ArrayRef::try_from(state).map_err(DataFusionError::from))
- .collect::<Result<Vec<_>>>());
+ let states = rresult_return!(
+ states
+ .into_iter()
+ .map(|state| ArrayRef::try_from(state).map_err(DataFusionError::from))
+ .collect::<Result<Vec<_>>>()
+ );
- rresult!(accumulator.merge_batch(&states))
+ rresult!(accumulator.merge_batch(&states))
+ }
}
unsafe extern "C" fn retract_batch_fn_wrapper(
accumulator: &mut FFI_Accumulator,
values: RVec<WrappedArray>,
) -> FFIResult<()> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let values_arrays = values
- .into_iter()
- .map(|v| v.try_into().map_err(DataFusionError::from))
- .collect::<Result<Vec<ArrayRef>>>();
- let values_arrays = rresult_return!(values_arrays);
+ let values_arrays = values
+ .into_iter()
+ .map(|v| v.try_into().map_err(DataFusionError::from))
+ .collect::<Result<Vec<ArrayRef>>>();
+ let values_arrays = rresult_return!(values_arrays);
- rresult!(accumulator.retract_batch(&values_arrays))
+ rresult!(accumulator.retract_batch(&values_arrays))
+ }
}
unsafe extern "C" fn release_fn_wrapper(accumulator: &mut FFI_Accumulator) {
- if !accumulator.private_data.is_null() {
- let private_data =
- Box::from_raw(accumulator.private_data as *mut AccumulatorPrivateData);
- drop(private_data);
- accumulator.private_data = null_mut();
+ unsafe {
+ if !accumulator.private_data.is_null() {
+ let private_data =
+ Box::from_raw(accumulator.private_data as *mut AccumulatorPrivateData);
+ drop(private_data);
+ accumulator.private_data = null_mut();
+ }
}
}
@@ -323,7 +341,7 @@ impl Accumulator for ForeignAccumulator {
#[cfg(test)]
mod tests {
use super::{FFI_Accumulator, ForeignAccumulator};
- use arrow::array::{make_array, Array};
+ use arrow::array::{Array, make_array};
use datafusion::{
common::create_array, error::Result,
functions_aggregate::average::AvgAccumulator, logical_expr::Accumulator,
diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rs
index d79ecba14c..a5cc370436 100644
--- a/datafusion/ffi/src/udaf/accumulator_args.rs
+++ b/datafusion/ffi/src/udaf/accumulator_args.rs
@@ -19,8 +19,8 @@ use std::sync::Arc;
use crate::arrow_wrappers::WrappedSchema;
use abi_stable::{
- std_types::{RString, RVec},
StableAbi,
+ std_types::{RString, RVec},
};
use arrow::{datatypes::Schema, ffi::FFI_ArrowSchema};
use arrow_schema::FieldRef;
@@ -33,9 +33,9 @@ use datafusion::{
use datafusion_common::ffi_datafusion_err;
use datafusion_proto::{
physical_plan::{
+ DefaultPhysicalExtensionCodec,
from_proto::{parse_physical_exprs, parse_physical_sort_exprs},
to_proto::{serialize_physical_exprs, serialize_physical_sort_exprs},
- DefaultPhysicalExtensionCodec,
},
protobuf::PhysicalAggregateExprNode,
};
diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rs
index 69113d8128..403c09fb51 100644
--- a/datafusion/ffi/src/udaf/groups_accumulator.rs
+++ b/datafusion/ffi/src/udaf/groups_accumulator.rs
@@ -21,8 +21,8 @@ use crate::{
df_result, rresult, rresult_return,
};
use abi_stable::{
- std_types::{ROption, RVec},
StableAbi,
+ std_types::{ROption, RVec},
};
use arrow::{
array::{Array, ArrayRef, BooleanArray},
@@ -100,14 +100,18 @@ pub struct GroupsAccumulatorPrivateData {
impl FFI_GroupsAccumulator {
#[inline]
unsafe fn inner_mut(&mut self) -> &mut Box<dyn GroupsAccumulator> {
- let private_data = self.private_data as *mut GroupsAccumulatorPrivateData;
- &mut (*private_data).accumulator
+ unsafe {
+ let private_data = self.private_data as *mut GroupsAccumulatorPrivateData;
+ &mut (*private_data).accumulator
+ }
}
#[inline]
unsafe fn inner(&self) -> &dyn GroupsAccumulator {
- let private_data = self.private_data as *const GroupsAccumulatorPrivateData;
- (*private_data).accumulator.deref()
+ unsafe {
+ let private_data = self.private_data as *const GroupsAccumulatorPrivateData;
+ (*private_data).accumulator.deref()
+ }
}
}
@@ -137,46 +141,56 @@ unsafe extern "C" fn update_batch_fn_wrapper(
opt_filter: ROption<WrappedArray>,
total_num_groups: usize,
) -> FFIResult<()> {
- let accumulator = accumulator.inner_mut();
- let values = rresult_return!(process_values(values));
- let group_indices: Vec<usize> = group_indices.into_iter().collect();
- let opt_filter = rresult_return!(process_opt_filter(opt_filter));
-
- rresult!(accumulator.update_batch(
- &values,
- &group_indices,
- opt_filter.as_ref(),
- total_num_groups
- ))
+ unsafe {
+ let accumulator = accumulator.inner_mut();
+ let values = rresult_return!(process_values(values));
+ let group_indices: Vec<usize> = group_indices.into_iter().collect();
+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));
+
+ rresult!(accumulator.update_batch(
+ &values,
+ &group_indices,
+ opt_filter.as_ref(),
+ total_num_groups
+ ))
+ }
}
unsafe extern "C" fn evaluate_fn_wrapper(
accumulator: &mut FFI_GroupsAccumulator,
emit_to: FFI_EmitTo,
) -> FFIResult<WrappedArray> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let result = rresult_return!(accumulator.evaluate(emit_to.into()));
+ let result = rresult_return!(accumulator.evaluate(emit_to.into()));
- rresult!(WrappedArray::try_from(&result))
+ rresult!(WrappedArray::try_from(&result))
+ }
}
unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_GroupsAccumulator) -> usize {
- let accumulator = accumulator.inner();
- accumulator.size()
+ unsafe {
+ let accumulator = accumulator.inner();
+ accumulator.size()
+ }
}
unsafe extern "C" fn state_fn_wrapper(
accumulator: &mut FFI_GroupsAccumulator,
emit_to: FFI_EmitTo,
) -> FFIResult<RVec<WrappedArray>> {
- let accumulator = accumulator.inner_mut();
+ unsafe {
+ let accumulator = accumulator.inner_mut();
- let state = rresult_return!(accumulator.state(emit_to.into()));
- rresult!(state
- .into_iter()
- .map(|arr| WrappedArray::try_from(&arr).map_err(DataFusionError::from))
- .collect::<Result<RVec<_>>>())
+ let state = rresult_return!(accumulator.state(emit_to.into()));
+ rresult!(
+ state
+ .into_iter()
+ .map(|arr| WrappedArray::try_from(&arr).map_err(DataFusionError::from))
+ .collect::<Result<RVec<_>>>()
+ )
+ }
}
unsafe extern "C" fn merge_batch_fn_wrapper(
@@ -186,17 +200,19 @@ unsafe extern "C" fn merge_batch_fn_wrapper(
opt_filter: ROption<WrappedArray>,
total_num_groups: usize,
) -> FFIResult<()> {
- let accumulator = accumulator.inner_mut();
- let values = rresult_return!(process_values(values));
- let group_indices: Vec<usize> = group_indices.into_iter().collect();
- let opt_filter = rresult_return!(process_opt_filter(opt_filter));
-
- rresult!(accumulator.merge_batch(
- &values,
- &group_indices,
- opt_filter.as_ref(),
- total_num_groups
- ))
+ unsafe {
+ let accumulator = accumulator.inner_mut();
+ let values = rresult_return!(process_values(values));
+ let group_indices: Vec<usize> = group_indices.into_iter().collect();
+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));
+
+ rresult!(accumulator.merge_batch(
+ &values,
+ &group_indices,
+ opt_filter.as_ref(),
+ total_num_groups
+ ))
+ }
}
unsafe extern "C" fn convert_to_state_fn_wrapper(
@@ -204,24 +220,31 @@ unsafe extern "C" fn convert_to_state_fn_wrapper(
values: RVec<WrappedArray>,
opt_filter: ROption<WrappedArray>,
) -> FFIResult<RVec<WrappedArray>> {
- let accumulator = accumulator.inner();
- let values = rresult_return!(process_values(values));
- let opt_filter = rresult_return!(process_opt_filter(opt_filter));
- let state =
- rresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref()));
-
- rresult!(state
- .iter()
- .map(|arr| WrappedArray::try_from(arr).map_err(DataFusionError::from))
- .collect::<Result<RVec<_>>>())
+ unsafe {
+ let accumulator = accumulator.inner();
+ let values = rresult_return!(process_values(values));
+ let opt_filter = rresult_return!(process_opt_filter(opt_filter));
+ let state =
+ rresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref()));
+
+ rresult!(
+ state
+ .iter()
+ .map(|arr| WrappedArray::try_from(arr).map_err(DataFusionError::from))
+ .collect::<Result<RVec<_>>>()
+ )
+ }
}
unsafe extern "C" fn release_fn_wrapper(accumulator: &mut FFI_GroupsAccumulator) {
- if !accumulator.private_data.is_null() {
- let private_data =
- Box::from_raw(accumulator.private_data as *mut GroupsAccumulatorPrivateData);
- drop(private_data);
- accumulator.private_data = null_mut();
+ unsafe {
+ if !accumulator.private_data.is_null() {
+ let private_data = Box::from_raw(
+ accumulator.private_data as *mut GroupsAccumulatorPrivateData,
+ );
+ drop(private_data);
+ accumulator.private_data = null_mut();
+ }
}
}
@@ -446,7 +469,7 @@ impl From<FFI_EmitTo> for EmitTo {
#[cfg(test)]
mod tests {
use super::{FFI_EmitTo, FFI_GroupsAccumulator, ForeignGroupsAccumulator};
- use arrow::array::{make_array, Array, BooleanArray};
+ use arrow::array::{Array, BooleanArray, make_array};
use datafusion::functions_aggregate::stddev::StddevGroupsAccumulator;
use datafusion::{
common::create_array,
diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs
index 838c3f909b..352b804338 100644
--- a/datafusion/ffi/src/udaf/mod.rs
+++ b/datafusion/ffi/src/udaf/mod.rs
@@ -16,8 +16,8 @@
// under the License.
use abi_stable::{
- std_types::{ROption, RResult, RStr, RString, RVec},
StableAbi,
+ std_types::{ROption, RResult, RStr, RString, RVec},
};
use accumulator::FFI_Accumulator;
use accumulator_args::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};
@@ -27,10 +27,10 @@ use arrow_schema::FieldRef;
use datafusion::{
error::DataFusionError,
logical_expr::{
+ Accumulator, GroupsAccumulator,
function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},
type_coercion::functions::fields_with_aggregate_udf,
utils::AggregateOrderSensitivity,
- Accumulator, GroupsAccumulator,
},
};
use datafusion::{
@@ -44,7 +44,7 @@ use std::hash::{Hash, Hasher};
use std::{ffi::c_void, sync::Arc};
use crate::util::{
- rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped, FFIResult,
+ FFIResult, rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped,
};
use crate::{
arrow_wrappers::WrappedSchema,
@@ -163,8 +163,10 @@ pub struct AggregateUDFPrivateData {
impl FFI_AggregateUDF {
unsafe fn inner(&self) -> &Arc<AggregateUDF> {
- let private_data = self.private_data as *const AggregateUDFPrivateData;
- &(*private_data).udaf
+ unsafe {
+ let private_data = self.private_data as *const AggregateUDFPrivateData;
+ &(*private_data).udaf
+ }
}
}
@@ -172,87 +174,104 @@ unsafe extern "C" fn return_field_fn_wrapper(
udaf: &FFI_AggregateUDF,
arg_fields: RVec<WrappedSchema>,
) -> FFIResult<WrappedSchema> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let arg_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields));
+ let arg_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields));
- let return_field = udaf
- .return_field(&arg_fields)
- .and_then(|v| {
- FFI_ArrowSchema::try_from(v.as_ref()).map_err(DataFusionError::from)
- })
- .map(WrappedSchema);
+ let return_field = udaf
+ .return_field(&arg_fields)
+ .and_then(|v| {
+ FFI_ArrowSchema::try_from(v.as_ref()).map_err(DataFusionError::from)
+ })
+ .map(WrappedSchema);
- rresult!(return_field)
+ rresult!(return_field)
+ }
}
unsafe extern "C" fn accumulator_fn_wrapper(
udaf: &FFI_AggregateUDF,
args: FFI_AccumulatorArgs,
) -> FFIResult<FFI_Accumulator> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
- rresult!(udaf
- .accumulator(accumulator_args.into())
- .map(FFI_Accumulator::from))
+ rresult!(
+ udaf.accumulator(accumulator_args.into())
+ .map(FFI_Accumulator::from)
+ )
+ }
}
unsafe extern "C" fn create_sliding_accumulator_fn_wrapper(
udaf: &FFI_AggregateUDF,
args: FFI_AccumulatorArgs,
) -> FFIResult<FFI_Accumulator> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
- rresult!(udaf
- .create_sliding_accumulator(accumulator_args.into())
- .map(FFI_Accumulator::from))
+ rresult!(
+ udaf.create_sliding_accumulator(accumulator_args.into())
+ .map(FFI_Accumulator::from)
+ )
+ }
}
unsafe extern "C" fn create_groups_accumulator_fn_wrapper(
udaf: &FFI_AggregateUDF,
args: FFI_AccumulatorArgs,
) -> FFIResult<FFI_GroupsAccumulator> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
+ let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args));
- rresult!(udaf
- .create_groups_accumulator(accumulator_args.into())
- .map(FFI_GroupsAccumulator::from))
+ rresult!(
+ udaf.create_groups_accumulator(accumulator_args.into())
+ .map(FFI_GroupsAccumulator::from)
+ )
+ }
}
unsafe extern "C" fn groups_accumulator_supported_fn_wrapper(
udaf: &FFI_AggregateUDF,
args: FFI_AccumulatorArgs,
) -> bool {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- ForeignAccumulatorArgs::try_from(args)
- .map(|a| udaf.groups_accumulator_supported((&a).into()))
- .unwrap_or_else(|e| {
- log::warn!("Unable to parse accumulator args. {e}");
- false
- })
+ ForeignAccumulatorArgs::try_from(args)
+ .map(|a| udaf.groups_accumulator_supported((&a).into()))
+ .unwrap_or_else(|e| {
+ log::warn!("Unable to parse accumulator args. {e}");
+ false
+ })
+ }
}
unsafe extern "C" fn with_beneficial_ordering_fn_wrapper(
udaf: &FFI_AggregateUDF,
beneficial_ordering: bool,
) -> FFIResult<ROption<FFI_AggregateUDF>> {
- let udaf = udaf.inner().as_ref().clone();
+ unsafe {
+ let udaf = udaf.inner().as_ref().clone();
- let result = rresult_return!(udaf.with_beneficial_ordering(beneficial_ordering));
- let result = rresult_return!(result
- .map(|func| func.with_beneficial_ordering(beneficial_ordering))
- .transpose())
- .flatten()
- .map(|func| FFI_AggregateUDF::from(Arc::new(func)));
+ let result = rresult_return!(udaf.with_beneficial_ordering(beneficial_ordering));
+ let result = rresult_return!(
+ result
+ .map(|func| func.with_beneficial_ordering(beneficial_ordering))
+ .transpose()
+ )
+ .flatten()
+ .map(|func| FFI_AggregateUDF::from(Arc::new(func)));
- RResult::ROk(result.into())
+ RResult::ROk(result.into())
+ }
}
unsafe extern "C" fn state_fields_fn_wrapper(
@@ -263,79 +282,93 @@ unsafe extern "C" fn state_fields_fn_wrapper(
ordering_fields: RVec<RVec<u8>>,
is_distinct: bool,
) -> FFIResult<RVec<RVec<u8>>> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let input_fields = &rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));
- let return_field = rresult_return!(Field::try_from(&return_field.0)).into();
+ let input_fields = &rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));
+ let return_field = rresult_return!(Field::try_from(&return_field.0)).into();
- let ordering_fields = &rresult_return!(ordering_fields
- .into_iter()
- .map(|field_bytes| datafusion_proto_common::Field::decode(field_bytes.as_ref()))
- .collect::<std::result::Result<Vec<_>, DecodeError>>());
+ let ordering_fields = &rresult_return!(
+ ordering_fields
+ .into_iter()
+ .map(|field_bytes| datafusion_proto_common::Field::decode(
+ field_bytes.as_ref()
+ ))
+ .collect::<std::result::Result<Vec<_>, DecodeError>>()
+ );
+
+ let ordering_fields =
+ &rresult_return!(parse_proto_fields_to_fields(ordering_fields))
+ .into_iter()
+ .map(Arc::new)
+ .collect::<Vec<_>>();
+
+ let args = StateFieldsArgs {
+ name: name.as_str(),
+ input_fields,
+ return_field,
+ ordering_fields,
+ is_distinct,
+ };
- let ordering_fields = &rresult_return!(parse_proto_fields_to_fields(ordering_fields))
+ let state_fields = rresult_return!(udaf.state_fields(args));
+ let state_fields = rresult_return!(
+ state_fields
+ .iter()
+ .map(|f| f.as_ref())
+ .map(datafusion_proto::protobuf::Field::try_from)
+ .map(|v| v.map_err(DataFusionError::from))
+ .collect::<Result<Vec<_>>>()
+ )
.into_iter()
- .map(Arc::new)
- .collect::<Vec<_>>();
-
- let args = StateFieldsArgs {
- name: name.as_str(),
- input_fields,
- return_field,
- ordering_fields,
- is_distinct,
- };
+ .map(|field| field.encode_to_vec().into())
+ .collect();
- let state_fields = rresult_return!(udaf.state_fields(args));
- let state_fields = rresult_return!(state_fields
- .iter()
- .map(|f| f.as_ref())
- .map(datafusion_proto::protobuf::Field::try_from)
- .map(|v| v.map_err(DataFusionError::from))
- .collect::<Result<Vec<_>>>())
- .into_iter()
- .map(|field| field.encode_to_vec().into())
- .collect();
-
- RResult::ROk(state_fields)
+ RResult::ROk(state_fields)
+ }
}
unsafe extern "C" fn order_sensitivity_fn_wrapper(
udaf: &FFI_AggregateUDF,
) -> FFI_AggregateOrderSensitivity {
- udaf.inner().order_sensitivity().into()
+ unsafe { udaf.inner().order_sensitivity().into() }
}
unsafe extern "C" fn coerce_types_fn_wrapper(
udaf: &FFI_AggregateUDF,
arg_types: RVec<WrappedSchema>,
) -> FFIResult<RVec<WrappedSchema>> {
- let udaf = udaf.inner();
+ unsafe {
+ let udaf = udaf.inner();
- let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types));
+ let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types));
- let arg_fields = arg_types
- .iter()
- .map(|dt| Field::new("f", dt.clone(), true))
- .map(Arc::new)
- .collect::<Vec<_>>();
- let return_types = rresult_return!(fields_with_aggregate_udf(&arg_fields, udaf))
- .into_iter()
- .map(|f| f.data_type().to_owned())
- .collect::<Vec<_>>();
+ let arg_fields = arg_types
+ .iter()
+ .map(|dt| Field::new("f", dt.clone(), true))
+ .map(Arc::new)
+ .collect::<Vec<_>>();
+ let return_types = rresult_return!(fields_with_aggregate_udf(&arg_fields, udaf))
+ .into_iter()
+ .map(|f| f.data_type().to_owned())
+ .collect::<Vec<_>>();
- rresult!(vec_datatype_to_rvec_wrapped(&return_types))
+ rresult!(vec_datatype_to_rvec_wrapped(&return_types))
+ }
}
unsafe extern "C" fn release_fn_wrapper(udaf: &mut FFI_AggregateUDF) {
- debug_assert!(!udaf.private_data.is_null());
- let private_data = Box::from_raw(udaf.private_data as *mut AggregateUDFPrivateData);
- drop(private_data);
- udaf.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!udaf.private_data.is_null());
+ let private_data =
+ Box::from_raw(udaf.private_data as *mut AggregateUDFPrivateData);
+ drop(private_data);
+ udaf.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(udaf: &FFI_AggregateUDF) -> FFI_AggregateUDF {
- Arc::clone(udaf.inner()).into()
+ unsafe { Arc::clone(udaf.inner()).into() }
}
impl Clone for FFI_AggregateUDF {
@@ -835,10 +868,12 @@ mod tests {
// Verify different library markers generate foreign providers
ffi_udaf.library_marker_id = crate::mock_foreign_marker_id;
let foreign_udaf: Arc<dyn AggregateUDFImpl> = (&ffi_udaf).into();
- assert!(foreign_udaf
- .as_any()
- .downcast_ref::<ForeignAggregateUDF>()
- .is_some());
+ assert!(
+ foreign_udaf
+ .as_any()
+ .downcast_ref::<ForeignAggregateUDF>()
+ .is_some()
+ );
Ok(())
}
diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rs
index 7ed3105e63..6df0e0bca5 100644
--- a/datafusion/ffi/src/udf/mod.rs
+++ b/datafusion/ffi/src/udf/mod.rs
@@ -23,14 +23,14 @@ use crate::{
volatility::FFI_Volatility,
};
use abi_stable::{
- std_types::{RResult, RString, RVec},
StableAbi,
+ std_types::{RResult, RString, RVec},
};
use arrow::datatypes::{DataType, Field};
use arrow::{
array::ArrayRef,
error::ArrowError,
- ffi::{from_ffi, to_ffi, FFI_ArrowSchema},
+ ffi::{FFI_ArrowSchema, from_ffi, to_ffi},
};
use arrow_schema::FieldRef;
use datafusion::config::ConfigOptions;
@@ -163,61 +163,68 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper(
number_rows: usize,
return_field: WrappedSchema,
) -> FFIResult<WrappedArray> {
- let args = args
- .into_iter()
- .map(|arr| {
- from_ffi(arr.array, &arr.schema.0)
- .map(|v| ColumnarValue::Array(arrow::array::make_array(v)))
- })
- .collect::<std::result::Result<_, _>>();
+ unsafe {
+ let args = args
+ .into_iter()
+ .map(|arr| {
+ from_ffi(arr.array, &arr.schema.0)
+ .map(|v| ColumnarValue::Array(arrow::array::make_array(v)))
+ })
+ .collect::<std::result::Result<_, _>>();
- let args = rresult_return!(args);
- let return_field = rresult_return!(Field::try_from(&return_field.0)).into();
+ let args = rresult_return!(args);
+ let return_field = rresult_return!(Field::try_from(&return_field.0)).into();
- let arg_fields = arg_fields
- .into_iter()
- .map(|wrapped_field| {
- Field::try_from(&wrapped_field.0)
- .map(Arc::new)
- .map_err(DataFusionError::from)
- })
- .collect::<Result<Vec<FieldRef>>>();
- let arg_fields = rresult_return!(arg_fields);
-
- let args = ScalarFunctionArgs {
- args,
- arg_fields,
- number_rows,
- return_field,
- // TODO: pass config options: https://github.com/apache/datafusion/issues/17035
- config_options: Arc::new(ConfigOptions::default()),
- };
-
- let result = rresult_return!(udf
- .inner()
- .invoke_with_args(args)
- .and_then(|r| r.to_array(number_rows)));
+ let arg_fields = arg_fields
+ .into_iter()
+ .map(|wrapped_field| {
+ Field::try_from(&wrapped_field.0)
+ .map(Arc::new)
+ .map_err(DataFusionError::from)
+ })
+ .collect::<Result<Vec<FieldRef>>>();
+ let arg_fields = rresult_return!(arg_fields);
- let (result_array, result_schema) = rresult_return!(to_ffi(&result.to_data()));
+ let args = ScalarFunctionArgs {
+ args,
+ arg_fields,
+ number_rows,
+ return_field,
+ // TODO: pass config options: https://github.com/apache/datafusion/issues/17035
+ config_options: Arc::new(ConfigOptions::default()),
+ };
- RResult::ROk(WrappedArray {
- array: result_array,
- schema: WrappedSchema(result_schema),
- })
+ let result = rresult_return!(
+ udf.inner()
+ .invoke_with_args(args)
+ .and_then(|r| r.to_array(number_rows))
+ );
+
+ let (result_array, result_schema) = rresult_return!(to_ffi(&result.to_data()));
+
+ RResult::ROk(WrappedArray {
+ array: result_array,
+ schema: WrappedSchema(result_schema),
+ })
+ }
}
unsafe extern "C" fn release_fn_wrapper(udf: &mut FFI_ScalarUDF) {
- debug_assert!(!udf.private_data.is_null());
- let private_data = Box::from_raw(udf.private_data as *mut ScalarUDFPrivateData);
- drop(private_data);
- udf.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!udf.private_data.is_null());
+ let private_data = Box::from_raw(udf.private_data as *mut ScalarUDFPrivateData);
+ drop(private_data);
+ udf.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(udf: &FFI_ScalarUDF) -> FFI_ScalarUDF {
- let private_data = udf.private_data as *const ScalarUDFPrivateData;
- let udf_data = &(*private_data);
+ unsafe {
+ let private_data = udf.private_data as *const ScalarUDFPrivateData;
+ let udf_data = &(*private_data);
- Arc::clone(&udf_data.udf).into()
+ Arc::clone(&udf_data.udf).into()
+ }
}
impl Clone for FFI_ScalarUDF {
@@ -460,10 +467,12 @@ mod tests {
// Verify different library markers generate foreign providers
ffi_udf.library_marker_id = crate::mock_foreign_marker_id;
let foreign_udf: Arc<dyn ScalarUDFImpl> = (&ffi_udf).into();
- assert!(foreign_udf
- .as_any()
- .downcast_ref::<ForeignScalarUDF>()
- .is_some());
+ assert!(
+ foreign_udf
+ .as_any()
+ .downcast_ref::<ForeignScalarUDF>()
+ .is_some()
+ );
Ok(())
}
diff --git a/datafusion/ffi/src/udf/return_type_args.rs b/datafusion/ffi/src/udf/return_type_args.rs
index c9396b7971..2803558e56 100644
--- a/datafusion/ffi/src/udf/return_type_args.rs
+++ b/datafusion/ffi/src/udf/return_type_args.rs
@@ -16,8 +16,8 @@
// under the License.
use abi_stable::{
- std_types::{ROption, RVec},
StableAbi,
+ std_types::{ROption, RVec},
};
use arrow_schema::FieldRef;
use datafusion::{
diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rs
index f154295958..ccecee2217 100644
--- a/datafusion/ffi/src/udtf.rs
+++ b/datafusion/ffi/src/udtf.rs
@@ -18,8 +18,8 @@
use std::{ffi::c_void, sync::Arc};
use abi_stable::{
- std_types::{RResult, RVec},
StableAbi,
+ std_types::{RResult, RVec},
};
use datafusion::error::Result;
@@ -29,7 +29,7 @@ use datafusion::{
};
use datafusion_proto::{
logical_plan::{
- from_proto::parse_exprs, to_proto::serialize_exprs, DefaultLogicalExtensionCodec,
+ DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,
},
protobuf::LogicalExprList,
};
@@ -106,10 +106,13 @@ unsafe extern "C" fn call_fn_wrapper(
}
unsafe extern "C" fn release_fn_wrapper(udtf: &mut FFI_TableFunction) {
- debug_assert!(!udtf.private_data.is_null());
- let private_data = Box::from_raw(udtf.private_data as *mut TableFunctionPrivateData);
- drop(private_data);
- udtf.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!udtf.private_data.is_null());
+ let private_data =
+ Box::from_raw(udtf.private_data as *mut TableFunctionPrivateData);
+ drop(private_data);
+ udtf.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(udtf: &FFI_TableFunction) -> FFI_TableFunction {
@@ -206,7 +209,7 @@ mod tests {
use super::*;
use arrow::{
array::{
- record_batch, ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array,
+ ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array, record_batch,
},
datatypes::{DataType, Field, Schema},
};
diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rs
index 7a65a4784f..f0e871fb42 100644
--- a/datafusion/ffi/src/udwf/mod.rs
+++ b/datafusion/ffi/src/udwf/mod.rs
@@ -19,8 +19,8 @@ use std::ffi::c_void;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
-use abi_stable::std_types::{ROption, RResult, RString, RVec};
use abi_stable::StableAbi;
+use abi_stable::std_types::{ROption, RResult, RString, RVec};
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Schema, SchemaRef};
use arrow_schema::{Field, FieldRef};
@@ -43,8 +43,8 @@ mod range;
use crate::arrow_wrappers::WrappedSchema;
use crate::util::{
- rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref,
- vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped, FFIResult,
+ FFIResult, rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref,
+ vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped,
};
use crate::volatility::FFI_Volatility;
use crate::{df_result, rresult, rresult_return};
@@ -112,8 +112,10 @@ pub struct WindowUDFPrivateData {
impl FFI_WindowUDF {
unsafe fn inner(&self) -> &Arc<WindowUDF> {
- let private_data = self.private_data as *const WindowUDFPrivateData;
- &(*private_data).udf
+ unsafe {
+ let private_data = self.private_data as *const WindowUDFPrivateData;
+ &(*private_data).udf
+ }
}
}
@@ -121,13 +123,16 @@ unsafe extern "C" fn partition_evaluator_fn_wrapper(
udwf: &FFI_WindowUDF,
args: FFI_PartitionEvaluatorArgs,
) -> FFIResult<FFI_PartitionEvaluator> {
- let inner = udwf.inner();
+ unsafe {
+ let inner = udwf.inner();
- let args = rresult_return!(ForeignPartitionEvaluatorArgs::try_from(args));
+ let args = rresult_return!(ForeignPartitionEvaluatorArgs::try_from(args));
- let evaluator = rresult_return!(inner.partition_evaluator_factory((&args).into()));
+ let evaluator =
+ rresult_return!(inner.partition_evaluator_factory((&args).into()));
- RResult::ROk(evaluator.into())
+ RResult::ROk(evaluator.into())
+ }
}
unsafe extern "C" fn field_fn_wrapper(
@@ -135,71 +140,79 @@ unsafe extern "C" fn field_fn_wrapper(
input_fields: RVec<WrappedSchema>,
display_name: RString,
) -> FFIResult<WrappedSchema> {
- let inner = udwf.inner();
+ unsafe {
+ let inner = udwf.inner();
- let input_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));
+ let input_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields));
- let field = rresult_return!(inner.field(WindowUDFFieldArgs::new(
- &input_fields,
- display_name.as_str()
- )));
+ let field = rresult_return!(inner.field(WindowUDFFieldArgs::new(
+ &input_fields,
+ display_name.as_str()
+ )));
- let schema = Arc::new(Schema::new(vec![field]));
+ let schema = Arc::new(Schema::new(vec![field]));
- RResult::ROk(WrappedSchema::from(schema))
+ RResult::ROk(WrappedSchema::from(schema))
+ }
}
unsafe extern "C" fn coerce_types_fn_wrapper(
udwf: &FFI_WindowUDF,
arg_types: RVec<WrappedSchema>,
) -> FFIResult<RVec<WrappedSchema>> {
- let inner = udwf.inner();
-
- let arg_fields = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types))
- .into_iter()
- .map(|dt| Field::new("f", dt, false))
- .map(Arc::new)
- .collect::<Vec<_>>();
-
- let return_fields = rresult_return!(fields_with_window_udf(&arg_fields, inner));
- let return_types = return_fields
- .into_iter()
- .map(|f| f.data_type().to_owned())
- .collect::<Vec<_>>();
-
- rresult!(vec_datatype_to_rvec_wrapped(&return_types))
+ unsafe {
+ let inner = udwf.inner();
+
+ let arg_fields = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types))
+ .into_iter()
+ .map(|dt| Field::new("f", dt, false))
+ .map(Arc::new)
+ .collect::<Vec<_>>();
+
+ let return_fields = rresult_return!(fields_with_window_udf(&arg_fields, inner));
+ let return_types = return_fields
+ .into_iter()
+ .map(|f| f.data_type().to_owned())
+ .collect::<Vec<_>>();
+
+ rresult!(vec_datatype_to_rvec_wrapped(&return_types))
+ }
}
unsafe extern "C" fn release_fn_wrapper(udwf: &mut FFI_WindowUDF) {
- debug_assert!(!udwf.private_data.is_null());
- let private_data = Box::from_raw(udwf.private_data as *mut WindowUDFPrivateData);
- drop(private_data);
- udwf.private_data = std::ptr::null_mut();
+ unsafe {
+ debug_assert!(!udwf.private_data.is_null());
+ let private_data = Box::from_raw(udwf.private_data as *mut WindowUDFPrivateData);
+ drop(private_data);
+ udwf.private_data = std::ptr::null_mut();
+ }
}
unsafe extern "C" fn clone_fn_wrapper(udwf: &FFI_WindowUDF) -> FFI_WindowUDF {
- // let private_data = udf.private_data as *const WindowUDFPrivateData;
- // let udf_data = &(*private_data);
-
- // let private_data = Box::new(WindowUDFPrivateData {
- // udf: Arc::clone(&udf_data.udf),
- // });
- let private_data = Box::new(WindowUDFPrivateData {
- udf: Arc::clone(udwf.inner()),
- });
-
- FFI_WindowUDF {
- name: udwf.name.clone(),
- aliases: udwf.aliases.clone(),
- volatility: udwf.volatility.clone(),
- partition_evaluator: partition_evaluator_fn_wrapper,
- sort_options: udwf.sort_options.clone(),
- coerce_types: coerce_types_fn_wrapper,
- field: field_fn_wrapper,
- clone: clone_fn_wrapper,
- release: release_fn_wrapper,
- private_data: Box::into_raw(private_data) as *mut c_void,
- library_marker_id: crate::get_library_marker_id,
+ unsafe {
+ // let private_data = udf.private_data as *const WindowUDFPrivateData;
+ // let udf_data = &(*private_data);
+
+ // let private_data = Box::new(WindowUDFPrivateData {
+ // udf: Arc::clone(&udf_data.udf),
+ // });
+ let private_data = Box::new(WindowUDFPrivateData {
+ udf: Arc::clone(udwf.inner()),
+ });
+
+ FFI_WindowUDF {
+ name: udwf.name.clone(),
+ aliases: udwf.aliases.clone(),
+ volatility: udwf.volatility.clone(),
+ partition_evaluator: partition_evaluator_fn_wrapper,
+ sort_options: udwf.sort_options.clone(),
+ coerce_types: coerce_types_fn_wrapper,
+ field: field_fn_wrapper,
+ clone: clone_fn_wrapper,
+ release: release_fn_wrapper,
+ private_data: Box::into_raw(private_data) as *mut c_void,
+ library_marker_id: crate::get_library_marker_id,
+ }
}
}
@@ -387,10 +400,10 @@ impl From<&FFI_SortOptions> for SortOptions {
mod tests {
use std::sync::Arc;
- use arrow::array::{create_array, ArrayRef};
- use datafusion::functions_window::lead_lag::{lag_udwf, WindowShift};
+ use arrow::array::{ArrayRef, create_array};
+ use datafusion::functions_window::lead_lag::{WindowShift, lag_udwf};
use datafusion::logical_expr::expr::Sort;
- use datafusion::logical_expr::{col, ExprFunctionExt, WindowUDF, WindowUDFImpl};
+ use datafusion::logical_expr::{ExprFunctionExt, WindowUDF, WindowUDFImpl, col};
use datafusion::prelude::SessionContext;
use crate::tests::create_record_batch;
@@ -462,18 +475,22 @@ mod tests {
// Verify local libraries can be downcast to their original
let foreign_udwf: Arc<dyn WindowUDFImpl> = (&ffi_udwf).into();
- assert!(foreign_udwf
- .as_any()
- .downcast_ref::<WindowShift>()
- .is_some());
+ assert!(
+ foreign_udwf
+ .as_any()
+ .downcast_ref::<WindowShift>()
+ .is_some()
+ );
// Verify different library markers generate foreign providers
ffi_udwf.library_marker_id = crate::mock_foreign_marker_id;
let foreign_udwf: Arc<dyn WindowUDFImpl> = (&ffi_udwf).into();
- assert!(foreign_udwf
- .as_any()
- .downcast_ref::<ForeignWindowUDF>()
- .is_some());
+ assert!(
+ foreign_udwf
+ .as_any()
+ .downcast_ref::<ForeignWindowUDF>()
+ .is_some()
+ );
Ok(())
}
diff --git a/datafusion/ffi/src/udwf/partition_evaluator.rs b/datafusion/ffi/src/udwf/partition_evaluator.rs
index 57cc8f73aa..b3fa9e1ef5 100644
--- a/datafusion/ffi/src/udwf/partition_evaluator.rs
+++ b/datafusion/ffi/src/udwf/partition_evaluator.rs
@@ -21,11 +21,11 @@ use super::range::FFI_Range;
use crate::util::FFIResult;
use crate::{arrow_wrappers::WrappedArray, df_result, rresult, rresult_return};
use abi_stable::std_types::RResult;
-use abi_stable::{std_types::RVec, StableAbi};
+use abi_stable::{StableAbi, std_types::RVec};
use arrow::{array::ArrayRef, error::ArrowError};
use datafusion::{
error::{DataFusionError, Result},
- logical_expr::{window_state::WindowAggState, PartitionEvaluator},
+ logical_expr::{PartitionEvaluator, window_state::WindowAggState},
scalar::ScalarValue,
};
use prost::Message;
@@ -89,13 +89,17 @@ pub struct PartitionEvaluatorPrivateData {
impl FFI_PartitionEvaluator {
unsafe fn inner_mut(&mut self) -> &mut Box<dyn PartitionEvaluator + 'static> {
- let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;
- &mut (*private_data).evaluator
+ unsafe {
+ let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;
+ &mut (*private_data).evaluator
+ }
}
unsafe fn inner(&self) -> &(dyn PartitionEvaluator + 'static) {
- let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;
- (*private_data).evaluator.as_ref()
+ unsafe {
+ let private_data = self.private_data as *mut PartitionEvaluatorPrivateData;
+ (*private_data).evaluator.as_ref()
+ }
}
}
@@ -104,19 +108,24 @@ unsafe extern "C" fn evaluate_all_fn_wrapper(
values: RVec<WrappedArray>,
num_rows: usize,
) -> FFIResult<WrappedArray> {
- let inner = evaluator.inner_mut();
-
- let values_arrays = values
- .into_iter()
- .map(|v| v.try_into().map_err(DataFusionError::from))
- .collect::<Result<Vec<ArrayRef>>>();
- let values_arrays = rresult_return!(values_arrays);
-
- let return_array = inner
- .evaluate_all(&values_arrays, num_rows)
- .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));
-
- rresult!(return_array)
+ unsafe {
+ let inner = evaluator.inner_mut();
+
+ let values_arrays = values
+ .into_iter()
+ .map(|v| v.try_into().map_err(DataFusionError::from))
+ .collect::<Result<Vec<ArrayRef>>>();
+ let values_arrays = rresult_return!(values_arrays);
+
+ let return_array =
+ inner
+ .evaluate_all(&values_arrays, num_rows)
+ .and_then(|array| {
+ WrappedArray::try_from(&array).map_err(DataFusionError::from)
+ });
+
+ rresult!(return_array)
+ }
}
unsafe extern "C" fn evaluate_fn_wrapper(
@@ -124,21 +133,24 @@ unsafe extern "C" fn evaluate_fn_wrapper(
values: RVec<WrappedArray>,
range: FFI_Range,
) -> FFIResult<RVec<u8>> {
- let inner = evaluator.inner_mut();
-
- let values_arrays = values
- .into_iter()
- .map(|v| v.try_into().map_err(DataFusionError::from))
- .collect::<Result<Vec<ArrayRef>>>();
- let values_arrays = rresult_return!(values_arrays);
-
- // let return_array = (inner.evaluate(&values_arrays, &range.into()));
- // .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));
- let scalar_result = rresult_return!(inner.evaluate(&values_arrays, &range.into()));
- let proto_result: datafusion_proto::protobuf::ScalarValue =
- rresult_return!((&scalar_result).try_into());
-
- RResult::ROk(proto_result.encode_to_vec().into())
+ unsafe {
+ let inner = evaluator.inner_mut();
+
+ let values_arrays = values
+ .into_iter()
+ .map(|v| v.try_into().map_err(DataFusionError::from))
+ .collect::<Result<Vec<ArrayRef>>>();
+ let values_arrays = rresult_return!(values_arrays);
+
+ // let return_array = (inner.evaluate(&values_arrays, &range.into()));
+ // .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));
+ let scalar_result =
+ rresult_return!(inner.evaluate(&values_arrays, &range.into()));
+ let proto_result: datafusion_proto::protobuf::ScalarValue =
+ rresult_return!((&scalar_result).try_into());
+
+ RResult::ROk(proto_result.encode_to_vec().into())
+ }
}
unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper(
@@ -146,18 +158,22 @@ unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper(
num_rows: usize,
ranks_in_partition: RVec<FFI_Range>,
) -> FFIResult<WrappedArray> {
- let inner = evaluator.inner();
+ unsafe {
+ let inner = evaluator.inner();
- let ranks_in_partition = ranks_in_partition
- .into_iter()
- .map(Range::from)
- .collect::<Vec<_>>();
+ let ranks_in_partition = ranks_in_partition
+ .into_iter()
+ .map(Range::from)
+ .collect::<Vec<_>>();
- let return_array = inner
- .evaluate_all_with_rank(num_rows, &ranks_in_partition)
- .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from));
+ let return_array = inner
+ .evaluate_all_with_rank(num_rows, &ranks_in_partition)
+ .and_then(|array| {
+ WrappedArray::try_from(&array).map_err(DataFusionError::from)
+ });
- rresult!(return_array)
+ rresult!(return_array)
+ }
}
unsafe extern "C" fn get_range_fn_wrapper(
@@ -165,18 +181,23 @@ unsafe extern "C" fn get_range_fn_wrapper(
idx: usize,
n_rows: usize,
) -> FFIResult<FFI_Range> {
- let inner = evaluator.inner();
- let range = inner.get_range(idx, n_rows).map(FFI_Range::from);
+ unsafe {
+ let inner = evaluator.inner();
+ let range = inner.get_range(idx, n_rows).map(FFI_Range::from);
- rresult!(range)
+ rresult!(range)
+ }
}
unsafe extern "C" fn release_fn_wrapper(evaluator: &mut FFI_PartitionEvaluator) {
- if !evaluator.private_data.is_null() {
- let private_data =
- Box::from_raw(evaluator.private_data as *mut PartitionEvaluatorPrivateData);
- drop(private_data);
- evaluator.private_data = std::ptr::null_mut();
+ unsafe {
+ if !evaluator.private_data.is_null() {
+ let private_data = Box::from_raw(
+ evaluator.private_data as *mut PartitionEvaluatorPrivateData,
+ );
+ drop(private_data);
+ evaluator.private_data = std::ptr::null_mut();
+ }
}
}
diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rs
index 93090af6f7..e272b14166 100644
--- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs
+++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs
@@ -18,7 +18,7 @@
use std::{collections::HashMap, sync::Arc};
use crate::arrow_wrappers::WrappedSchema;
-use abi_stable::{std_types::RVec, StableAbi};
+use abi_stable::{StableAbi, std_types::RVec};
use arrow::{
datatypes::{DataType, Field, Schema, SchemaRef},
error::ArrowError,
@@ -28,14 +28,14 @@ use arrow_schema::FieldRef;
use datafusion::{
error::{DataFusionError, Result},
logical_expr::function::PartitionEvaluatorArgs,
- physical_plan::{expressions::Column, PhysicalExpr},
+ physical_plan::{PhysicalExpr, expressions::Column},
prelude::SessionContext,
};
use datafusion_common::ffi_datafusion_err;
use datafusion_proto::{
physical_plan::{
- from_proto::parse_physical_expr, to_proto::serialize_physical_exprs,
- DefaultPhysicalExtensionCodec,
+ DefaultPhysicalExtensionCodec, from_proto::parse_physical_expr,
+ to_proto::serialize_physical_exprs,
},
protobuf::PhysicalExprNode,
};
diff --git a/datafusion/ffi/src/util.rs b/datafusion/ffi/src/util.rs
index 330286e3d5..db6eb0552d 100644
--- a/datafusion/ffi/src/util.rs
+++ b/datafusion/ffi/src/util.rs
@@ -180,8 +180,10 @@ pub(crate) mod tests {
let returned_err_r_result = wrap_result(err_result);
assert!(returned_err_r_result.is_err());
- assert!(returned_err_r_result
- .unwrap_err()
- .starts_with(format!("FFI error: {ERROR_VALUE}").as_str()));
+ assert!(
+ returned_err_r_result
+ .unwrap_err()
+ .starts_with(format!("FFI error: {ERROR_VALUE}").as_str())
+ );
}
}
diff --git a/datafusion/ffi/tests/ffi_udaf.rs b/datafusion/ffi/tests/ffi_udaf.rs
index 23ca913fca..6aac88d494 100644
--- a/datafusion/ffi/tests/ffi_udaf.rs
+++ b/datafusion/ffi/tests/ffi_udaf.rs
@@ -23,7 +23,7 @@ mod tests {
use datafusion::common::record_batch;
use datafusion::error::{DataFusionError, Result};
use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};
- use datafusion::prelude::{col, SessionContext};
+ use datafusion::prelude::{SessionContext, col};
use std::sync::Arc;
use datafusion_ffi::tests::utils::get_module;
diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rs
index 5f96308355..8f63265fd7 100644
--- a/datafusion/ffi/tests/ffi_udf.rs
+++ b/datafusion/ffi/tests/ffi_udf.rs
@@ -23,7 +23,7 @@ mod tests {
use datafusion::common::record_batch;
use datafusion::error::{DataFusionError, Result};
use datafusion::logical_expr::{ScalarUDF, ScalarUDFImpl};
- use datafusion::prelude::{col, SessionContext};
+ use datafusion::prelude::{SessionContext, col};
use std::sync::Arc;
use datafusion_ffi::tests::create_record_batch;
diff --git a/datafusion/ffi/tests/ffi_udtf.rs b/datafusion/ffi/tests/ffi_udtf.rs
index 8339b350cd..a0135a903f 100644
--- a/datafusion/ffi/tests/ffi_udtf.rs
+++ b/datafusion/ffi/tests/ffi_udtf.rs
@@ -22,7 +22,7 @@ mod tests {
use std::sync::Arc;
- use arrow::array::{create_array, ArrayRef};
+ use arrow::array::{ArrayRef, create_array};
use datafusion::catalog::TableFunctionImpl;
use datafusion::error::{DataFusionError, Result};
use datafusion::prelude::SessionContext;
diff --git a/datafusion/ffi/tests/ffi_udwf.rs b/datafusion/ffi/tests/ffi_udwf.rs
index e2a823cb16..01ffe59973 100644
--- a/datafusion/ffi/tests/ffi_udwf.rs
+++ b/datafusion/ffi/tests/ffi_udwf.rs
@@ -19,10 +19,10 @@
/// when the feature integration-tests is built
#[cfg(feature = "integration-tests")]
mod tests {
- use arrow::array::{create_array, ArrayRef};
+ use arrow::array::{ArrayRef, create_array};
use datafusion::error::{DataFusionError, Result};
use datafusion::logical_expr::expr::Sort;
- use datafusion::logical_expr::{col, ExprFunctionExt, WindowUDF, WindowUDFImpl};
+ use datafusion::logical_expr::{ExprFunctionExt, WindowUDF, WindowUDFImpl, col};
use datafusion::prelude::SessionContext;
use datafusion_ffi::tests::create_record_batch;
use datafusion_ffi::tests::utils::get_module;
diff --git a/datafusion/functions-aggregate-common/Cargo.toml b/datafusion/functions-aggregate-common/Cargo.toml
index dd7a97d9c7..1d4fb29d9c 100644
--- a/datafusion/functions-aggregate-common/Cargo.toml
+++ b/datafusion/functions-aggregate-common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Utility functions for implementing aggregate functions for the Da
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml
index 1d07c1f8a0..8f8697fef0 100644
--- a/datafusion/functions-aggregate/Cargo.toml
+++ b/datafusion/functions-aggregate/Cargo.toml
@@ -21,7 +21,7 @@ description = "Traits and types for logical plans and expressions for DataFusion
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index 52cddadd0a..6b0241a10a 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -21,7 +21,7 @@ description = "Nested Type Function packages for the DataFusion query engine"
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/functions-table/Cargo.toml b/datafusion/functions-table/Cargo.toml
index 8894ef8c4a..aa401fbd7d 100644
--- a/datafusion/functions-table/Cargo.toml
+++ b/datafusion/functions-table/Cargo.toml
@@ -21,7 +21,7 @@ description = "Traits and types for logical plans and expressions for DataFusion
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/functions-window-common/Cargo.toml b/datafusion/functions-window-common/Cargo.toml
index d15542ffae..6af668c145 100644
--- a/datafusion/functions-window-common/Cargo.toml
+++ b/datafusion/functions-window-common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Common functions for implementing user-defined window functions f
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
authors = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
license = { workspace = true }
repository = { workspace = true }
diff --git a/datafusion/functions-window/Cargo.toml b/datafusion/functions-window/Cargo.toml
index fbe6a34f29..42690907ae 100644
--- a/datafusion/functions-window/Cargo.toml
+++ b/datafusion/functions-window/Cargo.toml
@@ -21,7 +21,7 @@ description = "Window function packages for the DataFusion query engine"
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml
index 443dca898b..3e832691f9 100644
--- a/datafusion/functions/Cargo.toml
+++ b/datafusion/functions/Cargo.toml
@@ -21,7 +21,7 @@ description = "Function packages for the DataFusion query engine"
keywords = ["datafusion", "logical", "plan", "expressions"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/macros/Cargo.toml b/datafusion/macros/Cargo.toml
index 8e33833250..53691edf5a 100644
--- a/datafusion/macros/Cargo.toml
+++ b/datafusion/macros/Cargo.toml
@@ -21,7 +21,7 @@ description = "Procedural macros for DataFusion query engine"
keywords = ["datafusion", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index 0fb08684cd..15d3261ca5 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -21,7 +21,7 @@ description = "DataFusion Query Optimizer"
keywords = ["datafusion", "query", "optimizer"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/physical-expr-adapter/Cargo.toml b/datafusion/physical-expr-adapter/Cargo.toml
index 2436c2dbb3..03e1b1f065 100644
--- a/datafusion/physical-expr-adapter/Cargo.toml
+++ b/datafusion/physical-expr-adapter/Cargo.toml
@@ -4,7 +4,7 @@ description = "Physical expression schema adaptation utilities for DataFusion"
keywords = ["datafusion", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.toml
index df46be6f51..4602e59c42 100644
--- a/datafusion/physical-expr-common/Cargo.toml
+++ b/datafusion/physical-expr-common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Common functionality of physical expression for DataFusion query
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index 70cf88b5af..772a1f8f4a 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -21,7 +21,7 @@ description = "Physical expression implementation for DataFusion query engine"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/physical-optimizer/Cargo.toml b/datafusion/physical-optimizer/Cargo.toml
index 1c94b8cd49..395da10d62 100644
--- a/datafusion/physical-optimizer/Cargo.toml
+++ b/datafusion/physical-optimizer/Cargo.toml
@@ -21,7 +21,7 @@ description = "DataFusion Physical Optimizer"
keywords = ["datafusion", "query", "optimizer"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml
index 398dc2e793..68e67fa018 100644
--- a/datafusion/physical-plan/Cargo.toml
+++ b/datafusion/physical-plan/Cargo.toml
@@ -21,7 +21,7 @@ description = "Physical (ExecutionPlan) implementations for DataFusion query eng
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/proto-common/Cargo.toml b/datafusion/proto-common/Cargo.toml
index f92ee554c4..c67c8892a3 100644
--- a/datafusion/proto-common/Cargo.toml
+++ b/datafusion/proto-common/Cargo.toml
@@ -21,7 +21,7 @@ description = "Protobuf serialization of DataFusion common types"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index df46fb2a9f..b00bd0dcc6 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -21,7 +21,7 @@ description = "Protobuf serialization of DataFusion logical plan expressions"
keywords = ["arrow", "query", "sql"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/pruning/Cargo.toml b/datafusion/pruning/Cargo.toml
index 97d9035d1e..bd898cba20 100644
--- a/datafusion/pruning/Cargo.toml
+++ b/datafusion/pruning/Cargo.toml
@@ -3,7 +3,7 @@ name = "datafusion-pruning"
description = "DataFusion Pruning Logic"
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/session/Cargo.toml b/datafusion/session/Cargo.toml
index 3bed851425..230e26d1fc 100644
--- a/datafusion/session/Cargo.toml
+++ b/datafusion/session/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-session"
description = "datafusion-session"
readme = "README.md"
authors.workspace = true
-edition = "2024"
+edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 02d747df6d..09959db41f 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -24,7 +24,7 @@ repository = { workspace = true }
authors = { workspace = true }
readme = "README.md"
license = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
[package.metadata.docs.rs]
all-features = true
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index f7afbe9c31..a814292a3d 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -21,7 +21,7 @@ description = "DataFusion SQL Query Planner"
keywords = ["datafusion", "sql", "parser", "planner"]
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/sqllogictest/bin/postgres_container.rs b/datafusion/sqllogictest/bin/postgres_container.rs
index 411562a7cc..e5addf5408 100644
--- a/datafusion/sqllogictest/bin/postgres_container.rs
+++ b/datafusion/sqllogictest/bin/postgres_container.rs
@@ -16,19 +16,19 @@
// under the License.
use crate::Options;
+use ContainerCommands::{FetchHost, FetchPort};
use datafusion::common::Result;
use log::info;
use std::env::set_var;
use std::future::Future;
use std::sync::LazyLock;
use std::{env, thread};
+use testcontainers::ImageExt;
use testcontainers::core::IntoContainerPort;
use testcontainers::runners::AsyncRunner;
-use testcontainers::ImageExt;
use testcontainers_modules::postgres;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
-use tokio::sync::{mpsc, Mutex};
-use ContainerCommands::{FetchHost, FetchPort};
+use tokio::sync::{Mutex, mpsc};
#[derive(Debug)]
pub enum ContainerCommands {
@@ -86,7 +86,9 @@ pub async fn initialize_postgres_container(options: &Options) -> Result<()> {
let pg_uri = format!("postgresql://postgres:postgres@{db_host}:{db_port}/test");
info!("Postgres uri is {pg_uri}");
- set_var("PG_URI", pg_uri);
+ unsafe {
+ set_var("PG_URI", pg_uri);
+ }
} else {
// close receiver
POSTGRES_IN.rx.lock().await.close();
diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs
index 0f9f6c83dc..8037532c09 100644
--- a/datafusion/sqllogictest/bin/sqllogictests.rs
+++ b/datafusion/sqllogictest/bin/sqllogictests.rs
@@ -18,11 +18,11 @@
use clap::Parser;
use datafusion::common::instant::Instant;
use datafusion::common::utils::get_available_parallelism;
-use datafusion::common::{exec_datafusion_err, exec_err, DataFusionError, Result};
+use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};
use datafusion_sqllogictest::{
- df_value_validator, read_dir_recursive, setup_scratch_dir, should_skip_file,
- should_skip_record, value_normalizer, CurrentlyExecutingSqlTracker, DataFusion,
- DataFusionSubstraitRoundTrip, Filter, TestContext,
+ CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,
+ TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,
+ should_skip_file, should_skip_record, value_normalizer,
};
use futures::stream::StreamExt;
use indicatif::{
@@ -32,8 +32,8 @@ use itertools::Itertools;
use log::Level::Info;
use log::{info, log_enabled};
use sqllogictest::{
- parse_file, strict_column_validator, AsyncDB, Condition, MakeConnection, Normalizer,
- Record, Validator,
+ AsyncDB, Condition, MakeConnection, Normalizer, Record, Validator, parse_file,
+ strict_column_validator,
};
#[cfg(feature = "postgres")]
@@ -135,7 +135,9 @@ async fn run_tests() -> Result<()> {
eprintln!(" {error}");
}
- eprintln!("\nTemporary file check failed. Please ensure that within each test file, any scratch file created is placed under a folder with the same name as the test file (without extension).\nExample: inside `join.slt`, temporary files must be created under `.../scratch/join/`\n");
+ eprintln!(
+ "\nTemporary file check failed. Please ensure that within each test file, any scratch file created is placed under a folder with the same name as the test file (without extension).\nExample: inside `join.slt`, temporary files must be created under `.../scratch/join/`\n"
+ );
return exec_err!("sqllogictests scratch file check failed");
}
@@ -857,18 +859,18 @@ fn scratch_file_check(test_files: &[TestFile]) -> Result<Vec<String>> {
let lines: Vec<&str> = content.lines().collect();
for (line_num, line) in lines.iter().enumerate() {
- if let Some(captures) = scratch_pattern.captures(line) {
- if let Some(found_target) = captures.get(1) {
- let found_target = found_target.as_str();
- if found_target != expected_target {
- errors.push(format!(
- "File {}:{}: scratch target '{}' does not match file name '{}'",
- test_file.path.display(),
- line_num + 1,
- found_target,
- expected_target
- ));
- }
+ if let Some(captures) = scratch_pattern.captures(line)
+ && let Some(found_target) = captures.get(1)
+ {
+ let found_target = found_target.as_str();
+ if found_target != expected_target {
+ errors.push(format!(
+ "File {}:{}: scratch target '{}' does not match file name '{}'",
+ test_file.path.display(),
+ line_num + 1,
+ found_target,
+ expected_target
+ ));
}
}
}
diff --git a/datafusion/sqllogictest/src/engines/conversion.rs b/datafusion/sqllogictest/src/engines/conversion.rs
index 984845b584..633029a2de 100644
--- a/datafusion/sqllogictest/src/engines/conversion.rs
+++ b/datafusion/sqllogictest/src/engines/conversion.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::datatypes::{i256, Decimal128Type, Decimal256Type, DecimalType};
+use arrow::datatypes::{Decimal128Type, Decimal256Type, DecimalType, i256};
use bigdecimal::BigDecimal;
use half::f16;
use rust_decimal::prelude::*;
@@ -133,7 +133,7 @@ pub(crate) fn big_decimal_to_str(value: BigDecimal, round_digits: Option<i64>) -
#[cfg(test)]
mod tests {
use super::big_decimal_to_str;
- use bigdecimal::{num_bigint::BigInt, BigDecimal};
+ use bigdecimal::{BigDecimal, num_bigint::BigInt};
macro_rules! assert_decimal_str_eq {
($integer:expr, $scale:expr, $round_digits:expr, $expected:expr) => {
diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
index 13c107408c..df43a9a34c 100644
--- a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
+++ b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
@@ -18,7 +18,7 @@
use std::sync::Arc;
use std::{path::PathBuf, time::Duration};
-use super::{error::Result, normalize, DFSqlLogicTestError};
+use super::{DFSqlLogicTestError, error::Result, normalize};
use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;
use crate::engines::output::{DFColumnType, DFOutput};
use crate::is_spark_path;
diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
index 086e3af6f0..d4b4377e30 100644
--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs
@@ -21,7 +21,7 @@ use std::{path::PathBuf, time::Duration};
use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;
use crate::engines::datafusion_engine::Result;
use crate::engines::output::{DFColumnType, DFOutput};
-use crate::{convert_batches, convert_schema_to_types, DFSqlLogicTestError};
+use crate::{DFSqlLogicTestError, convert_batches, convert_schema_to_types};
use arrow::record_batch::RecordBatch;
use async_trait::async_trait;
use datafusion::logical_expr::LogicalPlan;
diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rs
index 8a6f44de43..ee2987db07 100644
--- a/datafusion/sqllogictest/src/engines/mod.rs
+++ b/datafusion/sqllogictest/src/engines/mod.rs
@@ -22,10 +22,10 @@ mod datafusion_engine;
mod datafusion_substrait_roundtrip_engine;
mod output;
-pub use datafusion_engine::convert_batches;
-pub use datafusion_engine::convert_schema_to_types;
pub use datafusion_engine::DFSqlLogicTestError;
pub use datafusion_engine::DataFusion;
+pub use datafusion_engine::convert_batches;
+pub use datafusion_engine::convert_schema_to_types;
pub use datafusion_substrait_roundtrip_engine::DataFusionSubstraitRoundTrip;
pub use output::DFColumnType;
pub use output::DFOutput;
diff --git a/datafusion/sqllogictest/src/filters.rs b/datafusion/sqllogictest/src/filters.rs
index 44482236f7..568fa3f666 100644
--- a/datafusion/sqllogictest/src/filters.rs
+++ b/datafusion/sqllogictest/src/filters.rs
@@ -120,10 +120,10 @@ pub fn should_skip_record<D: AsyncDB>(
if !loc.file().contains(&filter.file_substring) {
continue;
}
- if let Some(line_num) = filter.line_number {
- if loc.line() != line_num {
- continue;
- }
+ if let Some(line_num) = filter.line_number
+ && loc.line() != line_num
+ {
+ continue;
}
// This filter matches both file name substring and the exact
@@ -142,12 +142,11 @@ fn statement_is_skippable(statement: &Statement) -> bool {
// Cannot skip SELECT INTO statements, as they can also create tables
// that further test cases will use.
- if let SqlStatement::Query(v) = sql_stmt.as_ref() {
- if let SetExpr::Select(v) = v.body.as_ref() {
- if v.into.is_some() {
- return false;
- }
- }
+ if let SqlStatement::Query(v) = sql_stmt.as_ref()
+ && let SetExpr::Select(v) = v.body.as_ref()
+ && v.into.is_some()
+ {
+ return false;
}
// Only SELECT and EXPLAIN statements can be skipped, as any other
diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rs
index d9319dbbbd..f228ee89ab 100644
--- a/datafusion/sqllogictest/src/lib.rs
+++ b/datafusion/sqllogictest/src/lib.rs
@@ -28,14 +28,14 @@
mod engines;
-pub use engines::convert_batches;
-pub use engines::convert_schema_to_types;
pub use engines::CurrentlyExecutingSqlTracker;
pub use engines::DFColumnType;
pub use engines::DFOutput;
pub use engines::DFSqlLogicTestError;
pub use engines::DataFusion;
pub use engines::DataFusionSubstraitRoundTrip;
+pub use engines::convert_batches;
+pub use engines::convert_schema_to_types;
#[cfg(feature = "postgres")]
pub use engines::Postgres;
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 110108a554..dc6dc29fdc 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -32,12 +32,12 @@ use arrow::record_batch::RecordBatch;
use datafusion::catalog::{
CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, Session,
};
-use datafusion::common::{exec_err, not_impl_err, DataFusionError, Result, ScalarValue};
+use datafusion::common::{DataFusionError, Result, ScalarValue, exec_err, not_impl_err};
use datafusion::functions::math::abs;
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
use datafusion::logical_expr::{
- create_udf, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
- Signature, Volatility,
+ ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
+ Volatility, create_udf,
};
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::*;
@@ -49,8 +49,8 @@ use datafusion::{
use crate::is_spark_path;
use async_trait::async_trait;
use datafusion::common::cast::as_float64_array;
-use datafusion::execution::runtime_env::RuntimeEnv;
use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnv;
use log::info;
use tempfile::TempDir;
@@ -440,7 +440,9 @@ impl ScalarUDFImpl for GetMetadataUdf {
let key = match &args.args[1] {
ColumnarValue::Scalar(ScalarValue::Utf8(Some(k))) => k.clone(),
_ => {
- return exec_err!("get_metadata second argument must be a string literal")
+ return exec_err!(
+ "get_metadata second argument must be a string literal"
+ );
}
};
diff --git a/datafusion/sqllogictest/src/util.rs b/datafusion/sqllogictest/src/util.rs
index 487bafc4c9..6a3d3944e4 100644
--- a/datafusion/sqllogictest/src/util.rs
+++ b/datafusion/sqllogictest/src/util.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use datafusion::common::{exec_datafusion_err, Result};
+use datafusion::common::{Result, exec_datafusion_err};
use itertools::Itertools;
use log::Level::Warn;
use log::{info, log_enabled, warn};
diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml
index 67097ff522..8bfec86497 100644
--- a/datafusion/substrait/Cargo.toml
+++ b/datafusion/substrait/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-substrait"
description = "DataFusion Substrait Producer and Consumer"
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/datafusion/wasmtest/Cargo.toml b/datafusion/wasmtest/Cargo.toml
index dfe6d84dd6..16fa9790f6 100644
--- a/datafusion/wasmtest/Cargo.toml
+++ b/datafusion/wasmtest/Cargo.toml
@@ -20,7 +20,7 @@ name = "datafusion-wasmtest"
description = "Test library to compile datafusion crates to wasm"
readme = "README.md"
version = { workspace = true }
-edition = "2024"
+edition = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
diff --git a/rustfmt.toml b/rustfmt.toml
index 4522e520a4..c680d9d068 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-edition = "2021"
+edition = "2024"
max_width = 90
# ignore generated files
diff --git a/test-utils/src/array_gen/binary.rs b/test-utils/src/array_gen/binary.rs
index 9740eeae5e..ab0530a9ab 100644
--- a/test-utils/src/array_gen/binary.rs
+++ b/test-utils/src/array_gen/binary.rs
@@ -19,8 +19,8 @@ use arrow::array::{
ArrayRef, BinaryViewArray, GenericBinaryArray, OffsetSizeTrait, UInt32Array,
};
use arrow::compute;
-use rand::rngs::StdRng;
use rand::Rng;
+use rand::rngs::StdRng;
/// Randomly generate binary arrays
pub struct BinaryArrayGenerator {
diff --git a/test-utils/src/array_gen/boolean.rs b/test-utils/src/array_gen/boolean.rs
index 004d615b4c..c9104c0b8e 100644
--- a/test-utils/src/array_gen/boolean.rs
+++ b/test-utils/src/array_gen/boolean.rs
@@ -17,8 +17,8 @@
use arrow::array::{ArrayRef, BooleanArray, BooleanBuilder, UInt32Array};
use arrow::compute::take;
-use rand::rngs::StdRng;
use rand::Rng;
+use rand::rngs::StdRng;
/// Randomly generate boolean arrays
pub struct BooleanArrayGenerator {
diff --git a/test-utils/src/array_gen/decimal.rs b/test-utils/src/array_gen/decimal.rs
index c5ec8ac5e8..54fa2269d6 100644
--- a/test-utils/src/array_gen/decimal.rs
+++ b/test-utils/src/array_gen/decimal.rs
@@ -17,8 +17,8 @@
use arrow::array::{ArrayRef, PrimitiveArray, PrimitiveBuilder, UInt32Array};
use arrow::datatypes::DecimalType;
-use rand::rngs::StdRng;
use rand::Rng;
+use rand::rngs::StdRng;
use super::random_data::RandomNativeData;
diff --git a/test-utils/src/array_gen/primitive.rs b/test-utils/src/array_gen/primitive.rs
index 62a38a1b4c..5944879600 100644
--- a/test-utils/src/array_gen/primitive.rs
+++ b/test-utils/src/array_gen/primitive.rs
@@ -17,9 +17,9 @@
use arrow::array::{ArrayRef, ArrowPrimitiveType, PrimitiveArray, UInt32Array};
use arrow::datatypes::DataType;
-use chrono_tz::{Tz, TZ_VARIANTS};
+use chrono_tz::{TZ_VARIANTS, Tz};
use rand::prelude::IndexedRandom;
-use rand::{rng, rngs::StdRng, Rng};
+use rand::{Rng, rng, rngs::StdRng};
use std::sync::Arc;
use super::random_data::RandomNativeData;
diff --git a/test-utils/src/array_gen/random_data.rs b/test-utils/src/array_gen/random_data.rs
index ea2b872f7d..f341d23417 100644
--- a/test-utils/src/array_gen/random_data.rs
+++ b/test-utils/src/array_gen/random_data.rs
@@ -17,19 +17,19 @@
use arrow::array::ArrowPrimitiveType;
use arrow::datatypes::{
- i256, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Decimal32Type,
- Decimal64Type, DurationMicrosecondType, DurationMillisecondType,
- DurationNanosecondType, DurationSecondType, Float32Type, Float64Type, Int16Type,
- Int32Type, Int64Type, Int8Type, IntervalDayTime, IntervalDayTimeType,
- IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalYearMonthType,
- Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
+ Date32Type, Date64Type, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,
+ DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
+ DurationSecondType, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,
+ Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,
+ IntervalMonthDayNanoType, IntervalYearMonthType, Time32MillisecondType,
+ Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
- TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, i256,
};
+use rand::Rng;
use rand::distr::StandardUniform;
use rand::prelude::Distribution;
use rand::rngs::StdRng;
-use rand::Rng;
/// Generate corresponding NativeType value randomly according to
/// ArrowPrimitiveType.
diff --git a/test-utils/src/array_gen/string.rs b/test-utils/src/array_gen/string.rs
index 546485fd8d..896182290c 100644
--- a/test-utils/src/array_gen/string.rs
+++ b/test-utils/src/array_gen/string.rs
@@ -18,9 +18,9 @@
use arrow::array::{
ArrayRef, GenericStringArray, OffsetSizeTrait, StringViewArray, UInt32Array,
};
+use rand::Rng;
use rand::distr::StandardUniform;
use rand::rngs::StdRng;
-use rand::Rng;
/// Randomly generate string arrays
pub struct StringArrayGenerator {
diff --git a/test-utils/src/string_gen.rs b/test-utils/src/string_gen.rs
index 75ed03898a..21eecc05b8 100644
--- a/test-utils/src/string_gen.rs
+++ b/test-utils/src/string_gen.rs
@@ -19,7 +19,7 @@ use crate::array_gen::StringArrayGenerator;
use crate::stagger_batch;
use arrow::record_batch::RecordBatch;
use rand::rngs::StdRng;
-use rand::{rng, Rng, SeedableRng};
+use rand::{Rng, SeedableRng, rng};
/// Randomly generate strings
pub struct StringBatchGenerator(StringArrayGenerator);