Apache DataFusion SQL Query Engine
Remove core dependency from ffi (#19422)
Tim Saucer1e591640dde4
diff --git a/Cargo.lock b/Cargo.lockindex aee84eafab..22ec582536 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2302,6 +2302,7 @@ dependencies = ["datafusion-functions","datafusion-functions-aggregate","datafusion-functions-aggregate-common",+ "datafusion-functions-table","datafusion-functions-window","datafusion-physical-expr","datafusion-physical-expr-common",diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.tomlindex 58e04b246c..28e1b2ee56 100644--- a/datafusion/ffi/Cargo.toml+++ b/datafusion/ffi/Cargo.toml@@ -40,19 +40,25 @@ workspace = truename = "datafusion_ffi"crate-type = ["cdylib", "rlib"]+# Note to developers: do *not* add `datafusion` as a dependency in this crate.+# It increases build times and library binary size for users.+[dependencies]abi_stable = "0.11.3"arrow = { workspace = true, features = ["ffi"] }arrow-schema = { workspace = true }async-ffi = { version = "0.5.0", features = ["abi_stable"] }async-trait = { workspace = true }-datafusion = { workspace = true, default-features = false }datafusion-catalog = { workspace = true }datafusion-common = { workspace = true }datafusion-datasource = { workspace = true }datafusion-execution = { workspace = true }datafusion-expr = { workspace = true }+datafusion-functions = { workspace = true, optional = true }+datafusion-functions-aggregate = { workspace = true, optional = true }datafusion-functions-aggregate-common = { workspace = true }+datafusion-functions-table = { workspace = true, optional = true }+datafusion-functions-window = { workspace = true, optional = true }datafusion-physical-expr = { workspace = true }datafusion-physical-expr-common = { workspace = true }datafusion-physical-plan = { workspace = true }@@ -74,5 +80,10 @@ datafusion-functions-window = { workspace = true }doc-comment = { workspace = true }[features]-integration-tests = []+integration-tests = [+ "datafusion-functions",+ "datafusion-functions-aggregate",+ "datafusion-functions-table",+ "datafusion-functions-window",+]tarpaulin_include = [] # Exists only to prevent warnings on stable and still have accurate coveragediff --git a/datafusion/ffi/src/arrow_wrappers.rs b/datafusion/ffi/src/arrow_wrappers.rsindex 4b91fb551b..c83e412310 100644--- a/datafusion/ffi/src/arrow_wrappers.rs+++ b/datafusion/ffi/src/arrow_wrappers.rs@@ -18,12 +18,10 @@use std::sync::Arc;use abi_stable::StableAbi;-use arrow::{- array::{ArrayRef, make_array},- datatypes::{Schema, SchemaRef},- error::ArrowError,- ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi, to_ffi},-};+use arrow::array::{ArrayRef, make_array};+use arrow::datatypes::{Schema, SchemaRef};+use arrow::error::ArrowError;+use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi, to_ffi};use datafusion_common::{DataFusionError, ScalarValue};use log::error;diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex fdd3d5649e..b65355b39b 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -15,26 +15,24 @@// specific language governing permissions and limitations// under the License.-use std::{ffi::c_void, pin::Pin, sync::Arc};--use abi_stable::{- StableAbi,- std_types::{RString, RVec},-};-use datafusion::{- error::DataFusionError,- execution::{SendableRecordBatchStream, TaskContext},- physical_plan::{DisplayAs, ExecutionPlan, PlanProperties},+use std::ffi::c_void;+use std::pin::Pin;+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{RString, RVec};+use datafusion_common::{DataFusionError, Result};+use datafusion_execution::{SendableRecordBatchStream, TaskContext};+use datafusion_physical_plan::{+ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,};-use datafusion::{error::Result, physical_plan::DisplayFormatType};use tokio::runtime::Handle;use crate::execution::FFI_TaskContext;+use crate::plan_properties::FFI_PlanProperties;+use crate::record_batch_stream::FFI_RecordBatchStream;use crate::util::FFIResult;-use crate::{- df_result, plan_properties::FFI_PlanProperties,- record_batch_stream::FFI_RecordBatchStream, rresult,-};+use crate::{df_result, rresult};/// A stable struct for sharing a [`ExecutionPlan`] across FFI boundaries.#[repr(C)]@@ -300,12 +298,11 @@ impl ExecutionPlan for ForeignExecutionPlan {#[cfg(test)]pub(crate) mod tests {- use super::*;use arrow::datatypes::{DataType, Field, Schema};- use datafusion::physical_plan::{- Partitioning,- execution_plan::{Boundedness, EmissionType},- };+ use datafusion::physical_plan::Partitioning;+ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};++ use super::*;#[derive(Debug)]pub struct EmptyExec {diff --git a/datafusion/ffi/src/expr/distribution.rs b/datafusion/ffi/src/expr/distribution.rsindex c35bb4c0df..1a68fff152 100644--- a/datafusion/ffi/src/expr/distribution.rs+++ b/datafusion/ffi/src/expr/distribution.rs@@ -15,8 +15,6 @@// specific language governing permissions and limitations// under the License.-use crate::arrow_wrappers::WrappedArray;-use crate::expr::interval::FFI_Interval;use abi_stable::StableAbi;use datafusion_common::DataFusionError;use datafusion_expr::statistics::{@@ -24,6 +22,9 @@ use datafusion_expr::statistics::{GenericDistribution, UniformDistribution,};+use crate::arrow_wrappers::WrappedArray;+use crate::expr::interval::FFI_Interval;+/// A stable struct for sharing [`Distribution`] across FFI boundaries./// See ['Distribution'] for the meaning of each variant.#[repr(C)]diff --git a/datafusion/ffi/src/expr/interval.rs b/datafusion/ffi/src/expr/interval.rsindex d636ded6eb..8996b449b4 100644--- a/datafusion/ffi/src/expr/interval.rs+++ b/datafusion/ffi/src/expr/interval.rs@@ -15,11 +15,12 @@// specific language governing permissions and limitations// under the License.-use crate::arrow_wrappers::WrappedArray;use abi_stable::StableAbi;use datafusion_common::DataFusionError;use datafusion_expr::interval_arithmetic::Interval;+use crate::arrow_wrappers::WrappedArray;+/// A stable struct for sharing [`Interval`] across FFI boundaries./// See [`Interval`] for the meaning of each field. Scalar values/// are passed as Arrow arrays of length 1.diff --git a/datafusion/ffi/src/insert_op.rs b/datafusion/ffi/src/insert_op.rsindex 8e8693076c..acba1aa2ae 100644--- a/datafusion/ffi/src/insert_op.rs+++ b/datafusion/ffi/src/insert_op.rs@@ -16,7 +16,7 @@// under the License.use abi_stable::StableAbi;-use datafusion::logical_expr::logical_plan::dml::InsertOp;+use datafusion_expr::logical_plan::dml::InsertOp;/// FFI safe version of [`InsertOp`].#[repr(C)]diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rsindex abd07d0a27..c8d3e3e9a5 100644--- a/datafusion/ffi/src/physical_expr/mod.rs+++ b/datafusion/ffi/src/physical_expr/mod.rs@@ -18,42 +18,36 @@pub(crate) mod partitioning;pub(crate) mod sort;-use std::{- any::Any,- ffi::c_void,- fmt::{Display, Formatter},- hash::{DefaultHasher, Hash, Hasher},- sync::Arc,-};--use abi_stable::{- StableAbi,- std_types::{ROption, RResult, RString, RVec},-};-use arrow::{- array::{ArrayRef, BooleanArray, RecordBatch},- datatypes::SchemaRef,-};-use arrow_schema::{DataType, Field, FieldRef, Schema, ffi::FFI_ArrowSchema};+use std::any::Any;+use std::ffi::c_void;+use std::fmt::{Display, Formatter};+use std::hash::{DefaultHasher, Hash, Hasher};+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RString, RVec};+use arrow::array::{ArrayRef, BooleanArray, RecordBatch};+use arrow::datatypes::SchemaRef;+use arrow_schema::ffi::FFI_ArrowSchema;+use arrow_schema::{DataType, Field, FieldRef, Schema};use datafusion_common::{Result, ffi_datafusion_err};-use datafusion_expr::{- ColumnarValue, interval_arithmetic::Interval, sort_properties::ExprProperties,- statistics::Distribution,-};+use datafusion_expr::ColumnarValue;+use datafusion_expr::interval_arithmetic::Interval;+use datafusion_expr::sort_properties::ExprProperties;+use datafusion_expr::statistics::Distribution;use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;-use crate::{- arrow_wrappers::{WrappedArray, WrappedSchema},- df_result,- expr::{- columnar_value::FFI_ColumnarValue, distribution::FFI_Distribution,- expr_properties::FFI_ExprProperties, interval::FFI_Interval,- },- record_batch_stream::{record_batch_to_wrapped_array, wrapped_array_to_record_batch},- rresult, rresult_return,- util::FFIResult,+use crate::arrow_wrappers::{WrappedArray, WrappedSchema};+use crate::expr::columnar_value::FFI_ColumnarValue;+use crate::expr::distribution::FFI_Distribution;+use crate::expr::expr_properties::FFI_ExprProperties;+use crate::expr::interval::FFI_Interval;+use crate::record_batch_stream::{+ record_batch_to_wrapped_array, wrapped_array_to_record_batch,};+use crate::util::FFIResult;+use crate::{df_result, rresult, rresult_return};#[repr(C)]#[derive(Debug, StableAbi)]@@ -737,14 +731,14 @@ impl Display for ForeignPhysicalExpr {#[cfg(test)]mod tests {- use std::{- hash::{DefaultHasher, Hash, Hasher},- sync::Arc,- };+ use std::hash::{DefaultHasher, Hash, Hasher};+ use std::sync::Arc;use arrow::array::{BooleanArray, RecordBatch, record_batch};- use datafusion_common::{DataFusionError, ScalarValue, tree_node::DynTreeNode};- use datafusion_expr::{interval_arithmetic::Interval, statistics::Distribution};+ use datafusion_common::tree_node::DynTreeNode;+ use datafusion_common::{DataFusionError, ScalarValue};+ use datafusion_expr::interval_arithmetic::Interval;+ use datafusion_expr::statistics::Distribution;use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};diff --git a/datafusion/ffi/src/physical_expr/partitioning.rs b/datafusion/ffi/src/physical_expr/partitioning.rsindex fb16705e68..ed755d7ebe 100644--- a/datafusion/ffi/src/physical_expr/partitioning.rs+++ b/datafusion/ffi/src/physical_expr/partitioning.rs@@ -17,7 +17,8 @@use std::sync::Arc;-use abi_stable::{StableAbi, std_types::RVec};+use abi_stable::StableAbi;+use abi_stable::std_types::RVec;use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -70,7 +71,8 @@ impl From<&FFI_Partitioning> for Partitioning {#[cfg(test)]mod tests {- use datafusion_physical_expr::{Partitioning, expressions::lit};+ use datafusion_physical_expr::Partitioning;+ use datafusion_physical_expr::expressions::lit;use crate::physical_expr::partitioning::FFI_Partitioning;diff --git a/datafusion/ffi/src/physical_expr/sort.rs b/datafusion/ffi/src/physical_expr/sort.rsindex 8c5df9f4ae..58a06e7fcd 100644--- a/datafusion/ffi/src/physical_expr/sort.rs+++ b/datafusion/ffi/src/physical_expr/sort.rs@@ -55,12 +55,14 @@ impl From<&FFI_PhysicalSortExpr> for PhysicalSortExpr {#[cfg(test)]mod tests {- use crate::physical_expr::sort::FFI_PhysicalSortExpr;+ use std::sync::Arc;+use arrow_schema::SortOptions;use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;- use std::sync::Arc;++ use crate::physical_expr::sort::FFI_PhysicalSortExpr;#[test]fn ffi_sort_expr_round_trip() {diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rsindex d84b710c6f..7063d7e8c3 100644--- a/datafusion/ffi/src/record_batch_stream.rs+++ b/datafusion/ffi/src/record_batch_stream.rs@@ -15,32 +15,22 @@// specific language governing permissions and limitations// under the License.-use std::{ffi::c_void, task::Poll};--use abi_stable::{- StableAbi,- std_types::{ROption, RResult},-};-use arrow::array::{Array, RecordBatch};-use arrow::{- array::{StructArray, make_array},- ffi::{from_ffi, to_ffi},-};+use std::ffi::c_void;+use std::task::Poll;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult};+use arrow::array::{Array, RecordBatch, StructArray, make_array};+use arrow::ffi::{from_ffi, to_ffi};use async_ffi::{ContextExt, FfiContext, FfiPoll};-use datafusion::error::Result;-use datafusion::{- error::DataFusionError,- execution::{RecordBatchStream, SendableRecordBatchStream},-};-use datafusion_common::{ffi_datafusion_err, ffi_err};+use datafusion_common::{DataFusionError, Result, ffi_datafusion_err, ffi_err};+use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};use futures::{Stream, TryStreamExt};use tokio::runtime::Handle;+use crate::arrow_wrappers::{WrappedArray, WrappedSchema};+use crate::rresult;use crate::util::FFIResult;-use crate::{- arrow_wrappers::{WrappedArray, WrappedSchema},- rresult,-};/// A stable struct for sharing [`RecordBatchStream`] across FFI boundaries./// We use the async-ffi crate for handling async calls across libraries.@@ -224,13 +214,13 @@ mod tests {use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema};- use datafusion::{- common::record_batch, error::Result, execution::SendableRecordBatchStream,- test_util::bounded_stream,- };+ use datafusion::common::record_batch;+ use datafusion::error::Result;+ use datafusion::execution::SendableRecordBatchStream;+ use datafusion::test_util::bounded_stream;+ use futures::StreamExt;use super::FFI_RecordBatchStream;- use futures::StreamExt;#[tokio::test]async fn test_round_trip_record_batch_stream() -> Result<()> {diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rsindex 4e1ef96d30..4cb33f7d44 100644--- a/datafusion/ffi/src/session/mod.rs+++ b/datafusion/ffi/src/session/mod.rs@@ -556,13 +556,14 @@ impl Session for ForeignSession {mod tests {use std::sync::Arc;- use super::*;use arrow_schema::{DataType, Field, Schema};use datafusion_common::DataFusionError;use datafusion_expr::col;use datafusion_expr::registry::FunctionRegistry;use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec;+ use super::*;+#[tokio::test]async fn test_ffi_session() -> Result<(), DataFusionError> {let (ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();diff --git a/datafusion/ffi/src/table_source.rs b/datafusion/ffi/src/table_source.rsindex 418fdf16a5..6cd8f1efa7 100644--- a/datafusion/ffi/src/table_source.rs+++ b/datafusion/ffi/src/table_source.rs@@ -16,7 +16,7 @@// under the License.use abi_stable::StableAbi;-use datafusion::{datasource::TableType, logical_expr::TableProviderFilterPushDown};+use datafusion_expr::{TableProviderFilterPushDown, TableType};/// FFI safe version of [`TableProviderFilterPushDown`].#[repr(C)]@@ -88,9 +88,10 @@ impl From<TableType> for FFI_TableType {#[cfg(test)]mod tests {- use super::*;use datafusion::error::Result;+ use super::*;+fn round_trip_filter_pushdown(pushdown: TableProviderFilterPushDown) -> Result<()> {let ffi_pushdown: FFI_TableProviderFilterPushDown = (&pushdown).into();let round_trip: TableProviderFilterPushDown = (&ffi_pushdown).into();diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex 5e43a3e8ad..2d098f8adf 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -25,28 +25,27 @@//! access the runtime, then you will get a panic when trying to do operations//! such as spawning a tokio task.-use std::{any::Any, fmt::Debug, sync::Arc};+use std::any::Any;+use std::fmt::Debug;+use std::sync::Arc;-use super::create_record_batch;-use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;-use crate::table_provider::FFI_TableProvider;use arrow::array::RecordBatch;use arrow::datatypes::Schema;use async_trait::async_trait;-use datafusion::{- catalog::{Session, TableProvider},- error::Result,- execution::RecordBatchStream,- physical_expr::EquivalenceProperties,- physical_plan::{ExecutionPlan, Partitioning},- prelude::Expr,-};-use datafusion_common::exec_err;+use datafusion_catalog::TableProvider;+use datafusion_common::{Result, exec_err};+use datafusion_execution::RecordBatchStream;+use datafusion_expr::Expr;+use datafusion_physical_expr::{EquivalenceProperties, Partitioning};+use datafusion_physical_plan::ExecutionPlan;+use datafusion_session::Session;use futures::Stream;-use tokio::{- runtime::Handle,- sync::{broadcast, mpsc},-};+use tokio::runtime::Handle;+use tokio::sync::{broadcast, mpsc};++use super::create_record_batch;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::table_provider::FFI_TableProvider;#[derive(Debug)]pub struct AsyncTableProvider {@@ -135,8 +134,8 @@ impl TableProvider for AsyncTableProvider {super::create_test_schema()}- fn table_type(&self) -> datafusion::logical_expr::TableType {- datafusion::logical_expr::TableType::Base+ fn table_type(&self) -> datafusion_expr::TableType {+ datafusion_expr::TableType::Base}async fn scan(@@ -163,7 +162,7 @@ impl Drop for AsyncTableProvider {#[derive(Debug)]struct AsyncTestExecutionPlan {- properties: datafusion::physical_plan::PlanProperties,+ properties: datafusion_physical_plan::PlanProperties,batch_request: mpsc::Sender<bool>,batch_receiver: broadcast::Receiver<Option<RecordBatch>>,}@@ -174,11 +173,11 @@ impl AsyncTestExecutionPlan {batch_receiver: broadcast::Receiver<Option<RecordBatch>>,) -> Self {Self {- properties: datafusion::physical_plan::PlanProperties::new(+ properties: datafusion_physical_plan::PlanProperties::new(EquivalenceProperties::new(super::create_test_schema()),Partitioning::UnknownPartitioning(3),- datafusion::physical_plan::execution_plan::EmissionType::Incremental,- datafusion::physical_plan::execution_plan::Boundedness::Bounded,+ datafusion_physical_plan::execution_plan::EmissionType::Incremental,+ datafusion_physical_plan::execution_plan::Boundedness::Bounded,),batch_request,batch_receiver,@@ -195,7 +194,7 @@ impl ExecutionPlan for AsyncTestExecutionPlan {self}- fn properties(&self) -> &datafusion::physical_plan::PlanProperties {+ fn properties(&self) -> &datafusion_physical_plan::PlanProperties {&self.properties}@@ -213,8 +212,8 @@ impl ExecutionPlan for AsyncTestExecutionPlan {fn execute(&self,_partition: usize,- _context: Arc<datafusion::execution::TaskContext>,- ) -> Result<datafusion::execution::SendableRecordBatchStream> {+ _context: Arc<datafusion_execution::TaskContext>,+ ) -> Result<datafusion_execution::SendableRecordBatchStream> {Ok(Box::pin(AsyncTestRecordBatchStream {batch_request: self.batch_request.clone(),batch_receiver: self.batch_receiver.resubscribe(),@@ -222,10 +221,10 @@ impl ExecutionPlan for AsyncTestExecutionPlan {}}-impl datafusion::physical_plan::DisplayAs for AsyncTestExecutionPlan {+impl datafusion_physical_plan::DisplayAs for AsyncTestExecutionPlan {fn fmt_as(&self,- _t: datafusion::physical_plan::DisplayFormatType,+ _t: datafusion_physical_plan::DisplayFormatType,_f: &mut std::fmt::Formatter,) -> std::fmt::Result {// Do nothing, just a testdiff --git a/datafusion/ffi/src/tests/catalog.rs b/datafusion/ffi/src/tests/catalog.rsindex 122971eb0c..76d60ee379 100644--- a/datafusion/ffi/src/tests/catalog.rs+++ b/datafusion/ffi/src/tests/catalog.rs@@ -25,22 +25,21 @@//! access the runtime, then you will get a panic when trying to do operations//! such as spawning a tokio task.-use std::{any::Any, fmt::Debug, sync::Arc};+use std::any::Any;+use std::fmt::Debug;+use std::sync::Arc;-use crate::catalog_provider::FFI_CatalogProvider;-use crate::catalog_provider_list::FFI_CatalogProviderList;-use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use arrow::datatypes::Schema;use async_trait::async_trait;-use datafusion::{- catalog::{- CatalogProvider, CatalogProviderList, MemoryCatalogProvider,- MemoryCatalogProviderList, MemorySchemaProvider, SchemaProvider, TableProvider,- },- common::exec_err,- datasource::MemTable,- error::{DataFusionError, Result},+use datafusion_catalog::{+ CatalogProvider, CatalogProviderList, MemTable, MemoryCatalogProvider,+ MemoryCatalogProviderList, MemorySchemaProvider, SchemaProvider, TableProvider,};+use datafusion_common::{Result, exec_err};++use crate::catalog_provider::FFI_CatalogProvider;+use crate::catalog_provider_list::FFI_CatalogProviderList;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;/// This schema provider is intended only for unit tests. It prepopulates with one/// table and only allows for tables named sales and purchases.@@ -51,7 +50,7 @@ pub struct FixedSchemaProvider {pub fn fruit_table() -> Arc<dyn TableProvider + 'static> {use arrow::datatypes::{DataType, Field};- use datafusion::common::record_batch;+ use datafusion_common::record_batch;let schema = Arc::new(Schema::new(vec![Field::new("units", DataType::Int32, true),@@ -98,10 +97,7 @@ impl SchemaProvider for FixedSchemaProvider {self.inner.table_names()}- async fn table(- &self,- name: &str,- ) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {+ async fn table(&self, name: &str) -> Result<Option<Arc<dyn TableProvider>>> {self.inner.table(name).await}diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex e87d465db0..9bcd7e0031 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -17,37 +17,33 @@use std::sync::Arc;+use abi_stable::library::{LibraryError, RootModule};+use abi_stable::prefix_type::PrefixTypeTrait;+use abi_stable::sabi_types::VersionStrings;use abi_stable::{- StableAbi, declare_root_module_statics, export_root_module,- library::{LibraryError, RootModule},- package_version_strings,- prefix_type::PrefixTypeTrait,- sabi_types::VersionStrings,+ StableAbi, declare_root_module_statics, export_root_module, package_version_strings,};-use catalog::create_catalog_provider;--use crate::{catalog_provider::FFI_CatalogProvider, udtf::FFI_TableFunction};--use crate::udaf::FFI_AggregateUDF;--use crate::udwf::FFI_WindowUDF;--use super::{table_provider::FFI_TableProvider, udf::FFI_ScalarUDF};-use crate::catalog_provider_list::FFI_CatalogProviderList;-use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;-use crate::tests::catalog::create_catalog_provider_list;use arrow::array::RecordBatch;+use arrow_schema::{DataType, Field, Schema};use async_provider::create_async_table_provider;-use datafusion::{- arrow::datatypes::{DataType, Field, Schema},- common::record_batch,-};+use catalog::create_catalog_provider;+use datafusion_common::record_batch;use sync_provider::create_sync_table_provider;use udf_udaf_udwf::{create_ffi_abs_func, create_ffi_random_func, create_ffi_rank_func,create_ffi_stddev_func, create_ffi_sum_func, create_ffi_table_func,};+use super::table_provider::FFI_TableProvider;+use super::udf::FFI_ScalarUDF;+use crate::catalog_provider::FFI_CatalogProvider;+use crate::catalog_provider_list::FFI_CatalogProviderList;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::tests::catalog::create_catalog_provider_list;+use crate::udaf::FFI_AggregateUDF;+use crate::udtf::FFI_TableFunction;+use crate::udwf::FFI_WindowUDF;+mod async_provider;pub mod catalog;mod sync_provider;diff --git a/datafusion/ffi/src/tests/sync_provider.rs b/datafusion/ffi/src/tests/sync_provider.rsindex cbad8343dd..e3cb54fff9 100644--- a/datafusion/ffi/src/tests/sync_provider.rs+++ b/datafusion/ffi/src/tests/sync_provider.rs@@ -17,10 +17,11 @@use std::sync::Arc;+use datafusion_catalog::MemTable;+use super::{create_record_batch, create_test_schema};use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use crate::table_provider::FFI_TableProvider;-use datafusion::datasource::MemTable;pub(crate) fn create_sync_table_provider(codec: FFI_LogicalExtensionCodec,diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rsindex 5eee18f00a..d88ddfb28d 100644--- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs+++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs@@ -15,25 +15,27 @@// specific language governing permissions and limitations// under the License.-use crate::{- udaf::FFI_AggregateUDF, udf::FFI_ScalarUDF, udtf::FFI_TableFunction,- udwf::FFI_WindowUDF,-};-use datafusion::{- catalog::TableFunctionImpl,- functions::math::{abs::AbsFunc, random::RandomFunc},- functions_aggregate::{stddev::Stddev, sum::Sum},- functions_table::generate_series::RangeFunc,- functions_window::rank::Rank,- logical_expr::{AggregateUDF, ScalarUDF, WindowUDF},-};use std::any::Any;+use std::sync::Arc;-use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use arrow_schema::DataType;-use datafusion::logical_expr::{ColumnarValue, Signature};-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};-use std::sync::Arc;+use datafusion_catalog::TableFunctionImpl;+use datafusion_expr::{+ AggregateUDF, ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,+ WindowUDF,+};+use datafusion_functions::math::abs::AbsFunc;+use datafusion_functions::math::random::RandomFunc;+use datafusion_functions_aggregate::stddev::Stddev;+use datafusion_functions_aggregate::sum::Sum;+use datafusion_functions_table::generate_series::RangeFunc;+use datafusion_functions_window::rank::Rank;++use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::udaf::FFI_AggregateUDF;+use crate::udf::FFI_ScalarUDF;+use crate::udtf::FFI_TableFunction;+use crate::udwf::FFI_WindowUDF;pub(crate) extern "C" fn create_ffi_abs_func() -> FFI_ScalarUDF {let inner = WrappedAbs(Arc::new(AbsFunc::new().into()));@@ -100,7 +102,7 @@ pub(crate) extern "C" fn create_ffi_rank_func() -> FFI_WindowUDF {let udwf: Arc<WindowUDF> = Arc::new(Rank::new("rank_demo".to_string(),- datafusion::functions_window::rank::RankType::Basic,+ datafusion_functions_window::rank::RankType::Basic,).into(),);diff --git a/datafusion/ffi/src/tests/utils.rs b/datafusion/ffi/src/tests/utils.rsindex 6465b17d9b..9659a51f04 100644--- a/datafusion/ffi/src/tests/utils.rs+++ b/datafusion/ffi/src/tests/utils.rs@@ -15,11 +15,13 @@// specific language governing permissions and limitations// under the License.-use crate::tests::ForeignLibraryModuleRef;-use abi_stable::library::RootModule;-use datafusion::error::{DataFusionError, Result};use std::path::Path;+use abi_stable::library::RootModule;+use datafusion_common::{DataFusionError, Result};++use crate::tests::ForeignLibraryModuleRef;+/// Compute the path to the library. It would be preferable to simply use/// abi_stable::library::development_utils::compute_library_path however/// our current CI pipeline has a `ci` profile that we need to use todiff --git a/datafusion/ffi/src/udaf/accumulator.rs b/datafusion/ffi/src/udaf/accumulator.rsindex 924371538c..44bfc5bdeb 100644--- a/datafusion/ffi/src/udaf/accumulator.rs+++ b/datafusion/ffi/src/udaf/accumulator.rs@@ -15,22 +15,22 @@// specific language governing permissions and limitations// under the License.-use abi_stable::{- StableAbi,- std_types::{RResult, RVec},-};-use arrow::{array::ArrayRef, error::ArrowError};-use datafusion::{- error::{DataFusionError, Result},- logical_expr::Accumulator,- scalar::ScalarValue,-};-use prost::Message;+use std::ffi::c_void;+use std::ops::Deref;use std::ptr::null_mut;-use std::{ffi::c_void, ops::Deref};+use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RVec};+use arrow::array::ArrayRef;+use arrow::error::ArrowError;+use datafusion_common::error::{DataFusionError, Result};+use datafusion_common::scalar::ScalarValue;+use datafusion_expr::Accumulator;+use prost::Message;++use crate::arrow_wrappers::WrappedArray;use crate::util::FFIResult;-use crate::{arrow_wrappers::WrappedArray, df_result, rresult, rresult_return};+use crate::{df_result, rresult, rresult_return};/// A stable struct for sharing [`Accumulator`] across FFI boundaries./// For an explanation of each field, see the corresponding function@@ -340,13 +340,14 @@ impl Accumulator for ForeignAccumulator {#[cfg(test)]mod tests {- use super::{FFI_Accumulator, ForeignAccumulator};use arrow::array::{Array, make_array};- use datafusion::{- common::create_array, error::Result,- functions_aggregate::average::AvgAccumulator, logical_expr::Accumulator,- scalar::ScalarValue,- };+ use datafusion::common::create_array;+ use datafusion::error::Result;+ use datafusion::functions_aggregate::average::AvgAccumulator;+ use datafusion::logical_expr::Accumulator;+ use datafusion::scalar::ScalarValue;++ use super::{FFI_Accumulator, ForeignAccumulator};#[test]fn test_foreign_avg_accumulator() -> Result<()> {diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rsindex c5b7b7110e..5bb0f2103f 100644--- a/datafusion/ffi/src/udaf/accumulator_args.rs+++ b/datafusion/ffi/src/udaf/accumulator_args.rs@@ -17,21 +17,19 @@use std::sync::Arc;-use abi_stable::{- StableAbi,- std_types::{RString, RVec},-};-use arrow::{datatypes::Schema, ffi::FFI_ArrowSchema};+use abi_stable::StableAbi;+use abi_stable::std_types::{RString, RVec};+use arrow::datatypes::Schema;+use arrow::ffi::FFI_ArrowSchema;use arrow_schema::FieldRef;use datafusion_common::error::DataFusionError;use datafusion_expr::function::AccumulatorArgs;use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};-use crate::{- arrow_wrappers::WrappedSchema,- physical_expr::{FFI_PhysicalExpr, sort::FFI_PhysicalSortExpr},- util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped},-};+use crate::arrow_wrappers::WrappedSchema;+use crate::physical_expr::FFI_PhysicalExpr;+use crate::physical_expr::sort::FFI_PhysicalSortExpr;+use crate::util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped};/// A stable struct for sharing [`AccumulatorArgs`] across FFI boundaries./// For an explanation of each field, see the corresponding field@@ -153,10 +151,10 @@ impl<'a> From<&'a ForeignAccumulatorArgs> for AccumulatorArgs<'a> {#[cfg(test)]mod tests {use arrow::datatypes::{DataType, Field, Schema};- use datafusion::{- error::Result, logical_expr::function::AccumulatorArgs,- physical_expr::PhysicalSortExpr, physical_plan::expressions::col,- };+ use datafusion::error::Result;+ use datafusion::logical_expr::function::AccumulatorArgs;+ use datafusion::physical_expr::PhysicalSortExpr;+ use datafusion::physical_plan::expressions::col;use super::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rsindex 403c09fb51..73a6befd81 100644--- a/datafusion/ffi/src/udaf/groups_accumulator.rs+++ b/datafusion/ffi/src/udaf/groups_accumulator.rs@@ -15,26 +15,22 @@// specific language governing permissions and limitations// under the License.-use crate::util::FFIResult;-use crate::{- arrow_wrappers::{WrappedArray, WrappedSchema},- df_result, rresult, rresult_return,-};-use abi_stable::{- StableAbi,- std_types::{ROption, RVec},-};-use arrow::{- array::{Array, ArrayRef, BooleanArray},- error::ArrowError,- ffi::to_ffi,-};-use datafusion::{- error::{DataFusionError, Result},- logical_expr::{EmitTo, GroupsAccumulator},-};+use std::ffi::c_void;+use std::ops::Deref;use std::ptr::null_mut;-use std::{ffi::c_void, ops::Deref, sync::Arc};+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RVec};+use arrow::array::{Array, ArrayRef, BooleanArray};+use arrow::error::ArrowError;+use arrow::ffi::to_ffi;+use datafusion_common::error::{DataFusionError, Result};+use datafusion_expr::{EmitTo, GroupsAccumulator};++use crate::arrow_wrappers::{WrappedArray, WrappedSchema};+use crate::util::FFIResult;+use crate::{df_result, rresult, rresult_return};/// A stable struct for sharing [`GroupsAccumulator`] across FFI boundaries./// For an explanation of each field, see the corresponding function@@ -468,17 +464,16 @@ impl From<FFI_EmitTo> for EmitTo {#[cfg(test)]mod tests {- use super::{FFI_EmitTo, FFI_GroupsAccumulator, ForeignGroupsAccumulator};use arrow::array::{Array, BooleanArray, make_array};+ use datafusion::common::create_array;+ use datafusion::error::Result;use datafusion::functions_aggregate::stddev::StddevGroupsAccumulator;- use datafusion::{- common::create_array,- error::Result,- logical_expr::{EmitTo, GroupsAccumulator},- };+ use datafusion::logical_expr::{EmitTo, GroupsAccumulator};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::bool_op::BooleanGroupsAccumulator;use datafusion_functions_aggregate_common::stats::StatsType;+ use super::{FFI_EmitTo, FFI_GroupsAccumulator, ForeignGroupsAccumulator};+#[test]fn test_foreign_avg_accumulator() -> Result<()> {let boxed_accum: Box<dyn GroupsAccumulator> =diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rsindex 352b804338..9711dd7fd2 100644--- a/datafusion/ffi/src/udaf/mod.rs+++ b/datafusion/ffi/src/udaf/mod.rs@@ -15,44 +15,38 @@// specific language governing permissions and limitations// under the License.-use abi_stable::{- StableAbi,- std_types::{ROption, RResult, RStr, RString, RVec},-};+use std::ffi::c_void;+use std::hash::{Hash, Hasher};+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RStr, RString, RVec};use accumulator::FFI_Accumulator;use accumulator_args::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};use arrow::datatypes::{DataType, Field};use arrow::ffi::FFI_ArrowSchema;use arrow_schema::FieldRef;-use datafusion::{- error::DataFusionError,- logical_expr::{- Accumulator, GroupsAccumulator,- function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},- type_coercion::functions::fields_with_aggregate_udf,- utils::AggregateOrderSensitivity,- },+use datafusion_common::{DataFusionError, Result, ffi_datafusion_err};+use datafusion_expr::function::AggregateFunctionSimplification;+use datafusion_expr::type_coercion::functions::fields_with_aggregate_udf;+use datafusion_expr::{+ Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature,};-use datafusion::{- error::Result,- logical_expr::{AggregateUDF, AggregateUDFImpl, Signature},+use datafusion_functions_aggregate_common::accumulator::{+ AccumulatorArgs, StateFieldsArgs,};-use datafusion_common::ffi_datafusion_err;+use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity;use datafusion_proto_common::from_proto::parse_proto_fields_to_fields;use groups_accumulator::FFI_GroupsAccumulator;-use std::hash::{Hash, Hasher};-use std::{ffi::c_void, sync::Arc};+use prost::{DecodeError, Message};+use crate::arrow_wrappers::WrappedSchema;use crate::util::{- FFIResult, rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped,-};-use crate::{- arrow_wrappers::WrappedSchema,- df_result, rresult, rresult_return,- util::{rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped},- volatility::FFI_Volatility,+ FFIResult, rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref,+ vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped,};-use prost::{DecodeError, Message};+use crate::volatility::FFI_Volatility;+use crate::{df_result, rresult, rresult_return};mod accumulator;mod accumulator_args;@@ -650,16 +644,18 @@ impl From<AggregateOrderSensitivity> for FFI_AggregateOrderSensitivity {#[cfg(test)]mod tests {- use super::*;- use arrow::datatypes::Schema;- use datafusion::{- common::create_array, functions_aggregate::sum::Sum,- physical_expr::PhysicalSortExpr, physical_plan::expressions::col,- scalar::ScalarValue,- };use std::any::Any;use std::collections::HashMap;+ use arrow::datatypes::Schema;+ use datafusion::common::create_array;+ use datafusion::functions_aggregate::sum::Sum;+ use datafusion::physical_expr::PhysicalSortExpr;+ use datafusion::physical_plan::expressions::col;+ use datafusion::scalar::ScalarValue;++ use super::*;+#[derive(Default, Debug, Hash, Eq, PartialEq)]struct SumWithCopiedMetadata {inner: Sum,diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rsindex 6df0e0bca5..0b2efd45c1 100644--- a/datafusion/ffi/src/udf/mod.rs+++ b/datafusion/ffi/src/udf/mod.rs@@ -15,42 +15,34 @@// specific language governing permissions and limitations// under the License.-use crate::util::FFIResult;-use crate::{- arrow_wrappers::{WrappedArray, WrappedSchema},- df_result, rresult, rresult_return,- util::{rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped},- volatility::FFI_Volatility,-};-use abi_stable::{- StableAbi,- std_types::{RResult, RString, RVec},-};+use std::ffi::c_void;+use std::hash::{Hash, Hasher};+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RString, RVec};+use arrow::array::ArrayRef;use arrow::datatypes::{DataType, Field};-use arrow::{- array::ArrayRef,- error::ArrowError,- ffi::{FFI_ArrowSchema, from_ffi, to_ffi},-};+use arrow::error::ArrowError;+use arrow::ffi::{FFI_ArrowSchema, from_ffi, to_ffi};use arrow_schema::FieldRef;-use datafusion::config::ConfigOptions;-use datafusion::logical_expr::ReturnFieldArgs;-use datafusion::{- error::DataFusionError,- logical_expr::type_coercion::functions::data_types_with_scalar_udf,+use datafusion_common::config::ConfigOptions;+use datafusion_common::{DataFusionError, Result, internal_err};+use datafusion_expr::type_coercion::functions::data_types_with_scalar_udf;+use datafusion_expr::{+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,+ Signature,};-use datafusion::{- error::Result,- logical_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,- },-};-use datafusion_common::internal_err;use return_type_args::{FFI_ReturnFieldArgs, ForeignReturnFieldArgs, ForeignReturnFieldArgsOwned,};-use std::hash::{Hash, Hasher};-use std::{ffi::c_void, sync::Arc};++use crate::arrow_wrappers::{WrappedArray, WrappedSchema};+use crate::util::{+ FFIResult, rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped,+};+use crate::volatility::FFI_Volatility;+use crate::{df_result, rresult, rresult_return};pub mod return_type_args;diff --git a/datafusion/ffi/src/udf/return_type_args.rs b/datafusion/ffi/src/udf/return_type_args.rsindex 2803558e56..554b0a1eee 100644--- a/datafusion/ffi/src/udf/return_type_args.rs+++ b/datafusion/ffi/src/udf/return_type_args.rs@@ -15,19 +15,16 @@// specific language governing permissions and limitations// under the License.-use abi_stable::{- StableAbi,- std_types::{ROption, RVec},-};+use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RVec};use arrow_schema::FieldRef;-use datafusion::{- common::ffi_datafusion_err, error::DataFusionError, logical_expr::ReturnFieldArgs,- scalar::ScalarValue,-};+use datafusion_common::scalar::ScalarValue;+use datafusion_common::{DataFusionError, ffi_datafusion_err};+use datafusion_expr::ReturnFieldArgs;+use prost::Message;use crate::arrow_wrappers::WrappedSchema;use crate::util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped};-use prost::Message;/// A stable struct for sharing a [`ReturnFieldArgs`] across FFI boundaries.#[repr(C)]diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rsindex f0e871fb42..e4ac97f5d4 100644--- a/datafusion/ffi/src/udwf/mod.rs+++ b/datafusion/ffi/src/udwf/mod.rs@@ -24,14 +24,13 @@ use abi_stable::std_types::{ROption, RResult, RString, RVec};use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Schema, SchemaRef};use arrow_schema::{Field, FieldRef};-use datafusion::error::Result;-use datafusion::logical_expr::function::WindowUDFFieldArgs;-use datafusion::logical_expr::type_coercion::functions::fields_with_window_udf;-use datafusion::logical_expr::{+use datafusion_common::{Result, ffi_err};+use datafusion_expr::function::WindowUDFFieldArgs;+use datafusion_expr::type_coercion::functions::fields_with_window_udf;+use datafusion_expr::{LimitEffect, PartitionEvaluator, Signature, WindowUDF, WindowUDFImpl,};-use datafusion::physical_expr::PhysicalExpr;-use datafusion_common::ffi_err;+use datafusion_physical_expr::PhysicalExpr;use partition_evaluator::FFI_PartitionEvaluator;use partition_evaluator_args::{FFI_PartitionEvaluatorArgs, ForeignPartitionEvaluatorArgs,@@ -330,7 +329,7 @@ impl WindowUDFImpl for ForeignWindowUDF {fn partition_evaluator(&self,- args: datafusion::logical_expr::function::PartitionEvaluatorArgs,+ args: datafusion_expr::function::PartitionEvaluatorArgs,) -> Result<Box<dyn PartitionEvaluator>> {let evaluator = unsafe {let args = FFI_PartitionEvaluatorArgs::try_from(args)?;diff --git a/datafusion/ffi/src/udwf/partition_evaluator.rs b/datafusion/ffi/src/udwf/partition_evaluator.rsindex b3fa9e1ef5..70149efeb1 100644--- a/datafusion/ffi/src/udwf/partition_evaluator.rs+++ b/datafusion/ffi/src/udwf/partition_evaluator.rs@@ -15,20 +15,23 @@// specific language governing permissions and limitations// under the License.-use std::{ffi::c_void, ops::Range};+use std::ffi::c_void;+use std::ops::Range;++use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RVec};+use arrow::array::ArrayRef;+use arrow::error::ArrowError;+use datafusion_common::scalar::ScalarValue;+use datafusion_common::{DataFusionError, Result};+use datafusion_expr::PartitionEvaluator;+use datafusion_expr::window_state::WindowAggState;+use prost::Message;use super::range::FFI_Range;+use crate::arrow_wrappers::WrappedArray;use crate::util::FFIResult;-use crate::{arrow_wrappers::WrappedArray, df_result, rresult, rresult_return};-use abi_stable::std_types::RResult;-use abi_stable::{StableAbi, std_types::RVec};-use arrow::{array::ArrayRef, error::ArrowError};-use datafusion::{- error::{DataFusionError, Result},- logical_expr::{PartitionEvaluator, window_state::WindowAggState},- scalar::ScalarValue,-};-use prost::Message;+use crate::{df_result, rresult, rresult_return};/// A stable struct for sharing [`PartitionEvaluator`] across FFI boundaries./// For an explanation of each field, see the corresponding function@@ -353,11 +356,12 @@ impl PartitionEvaluator for ForeignPartitionEvaluator {#[cfg(test)]mod tests {+ use arrow::array::ArrayRef;+ use datafusion::logical_expr::PartitionEvaluator;+use crate::udwf::partition_evaluator::{FFI_PartitionEvaluator, ForeignPartitionEvaluator,};- use arrow::array::ArrayRef;- use datafusion::logical_expr::PartitionEvaluator;#[derive(Debug)]struct TestPartitionEvaluator {}diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rsindex 20238fdb8b..dca53a7a34 100644--- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs+++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs@@ -17,17 +17,18 @@use std::sync::Arc;-use abi_stable::{StableAbi, std_types::RVec};-use arrow::{error::ArrowError, ffi::FFI_ArrowSchema};+use abi_stable::StableAbi;+use abi_stable::std_types::RVec;+use arrow::error::ArrowError;+use arrow::ffi::FFI_ArrowSchema;use arrow_schema::FieldRef;use datafusion_common::{DataFusionError, Result};use datafusion_expr::function::PartitionEvaluatorArgs;use datafusion_physical_plan::PhysicalExpr;-use crate::{- arrow_wrappers::WrappedSchema, physical_expr::FFI_PhysicalExpr,- util::rvec_wrapped_to_vec_fieldref,-};+use crate::arrow_wrappers::WrappedSchema;+use crate::physical_expr::FFI_PhysicalExpr;+use crate::util::rvec_wrapped_to_vec_fieldref;/// A stable struct for sharing [`PartitionEvaluatorArgs`] across FFI boundaries./// For an explanation of each field, see the corresponding functiondiff --git a/datafusion/ffi/src/volatility.rs b/datafusion/ffi/src/volatility.rsindex f1705da294..a3faf8098b 100644--- a/datafusion/ffi/src/volatility.rs+++ b/datafusion/ffi/src/volatility.rs@@ -16,7 +16,7 @@// under the License.use abi_stable::StableAbi;-use datafusion::logical_expr::Volatility;+use datafusion_expr::Volatility;#[repr(C)]#[derive(Debug, StableAbi, Clone)]diff --git a/datafusion/ffi/tests/ffi_catalog.rs b/datafusion/ffi/tests/ffi_catalog.rsindex a464d3df5a..28bb5f406f 100644--- a/datafusion/ffi/tests/ffi_catalog.rs+++ b/datafusion/ffi/tests/ffi_catalog.rs@@ -21,10 +21,11 @@ mod utils;/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {+ use std::sync::Arc;+use datafusion::catalog::{CatalogProvider, CatalogProviderList};use datafusion_common::DataFusionError;use datafusion_ffi::tests::utils::get_module;- use std::sync::Arc;#[tokio::test]async fn test_catalog() -> datafusion_common::Result<()> {diff --git a/datafusion/ffi/tests/ffi_integration.rs b/datafusion/ffi/tests/ffi_integration.rsindex 78650abf2f..2d18679cb0 100644--- a/datafusion/ffi/tests/ffi_integration.rs+++ b/datafusion/ffi/tests/ffi_integration.rs@@ -21,11 +21,12 @@ mod utils;/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {+ use std::sync::Arc;+use datafusion::catalog::TableProvider;use datafusion::error::{DataFusionError, Result};use datafusion_ffi::tests::create_record_batch;use datafusion_ffi::tests::utils::get_module;- use std::sync::Arc;/// It is important that this test is in the `tests` directory and not in the/// library directory so we can verify we are building a dynamic library anddiff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rsindex 8f63265fd7..c659e27f02 100644--- a/datafusion/ffi/tests/ffi_udf.rs+++ b/datafusion/ffi/tests/ffi_udf.rs@@ -19,13 +19,13 @@/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {+ use std::sync::Arc;+use arrow::datatypes::DataType;use datafusion::common::record_batch;use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::{ScalarUDF, ScalarUDFImpl};use datafusion::prelude::{SessionContext, col};- use std::sync::Arc;-use datafusion_ffi::tests::create_record_batch;use datafusion_ffi::tests::utils::get_module;diff --git a/datafusion/ffi/tests/ffi_udtf.rs b/datafusion/ffi/tests/ffi_udtf.rsindex 097ed75f35..ab78189329 100644--- a/datafusion/ffi/tests/ffi_udtf.rs+++ b/datafusion/ffi/tests/ffi_udtf.rs@@ -27,7 +27,6 @@ mod tests {use arrow::array::{ArrayRef, create_array};use datafusion::catalog::TableFunctionImpl;use datafusion::error::{DataFusionError, Result};-use datafusion_ffi::tests::utils::get_module;/// This test validates that we can load an external module and use a scalardiff --git a/datafusion/ffi/tests/ffi_udwf.rs b/datafusion/ffi/tests/ffi_udwf.rsindex 01ffe59973..c4e889b796 100644--- a/datafusion/ffi/tests/ffi_udwf.rs+++ b/datafusion/ffi/tests/ffi_udwf.rs@@ -19,6 +19,8 @@/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {+ use std::sync::Arc;+use arrow::array::{ArrayRef, create_array};use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::expr::Sort;@@ -26,7 +28,6 @@ mod tests {use datafusion::prelude::SessionContext;use datafusion_ffi::tests::create_record_batch;use datafusion_ffi::tests::utils::get_module;- use std::sync::Arc;#[tokio::test]async fn test_rank_udwf() -> Result<()> {diff --git a/datafusion/ffi/tests/utils/mod.rs b/datafusion/ffi/tests/utils/mod.rsindex 25d1464811..a83512f663 100644--- a/datafusion/ffi/tests/utils/mod.rs+++ b/datafusion/ffi/tests/utils/mod.rs@@ -15,12 +15,13 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;+use datafusion::prelude::SessionContext;use datafusion_execution::TaskContextProvider;use datafusion_ffi::execution::FFI_TaskContextProvider;use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec;-use std::sync::Arc;pub fn ctx_and_codec() -> (Arc<SessionContext>, FFI_LogicalExtensionCodec) {let ctx = Arc::new(SessionContext::default());