Apache DataFusion SQL Query Engine
feat: update FFI TableProvider and ExecutionPlan to use FFI Session and TaskContext (#19281)
Tim Saucer32e6fe887a23
diff --git a/Cargo.lock b/Cargo.lockindex 594e63cd68..866c4a34c0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3054,6 +3054,7 @@ version = "0.1.0"dependencies = ["abi_stable","datafusion",+ "datafusion-ffi","ffi_module_interface","tokio",]diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rsindex a83f15926f..eb217ef9e4 100644--- a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs+++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs@@ -21,6 +21,7 @@ use abi_stable::{export_root_module, prefix_type::PrefixTypeTrait};use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema};use datafusion::{common::record_batch, datasource::MemTable};+use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use datafusion_ffi::table_provider::FFI_TableProvider;use ffi_module_interface::{TableProviderModule, TableProviderModuleRef};@@ -34,7 +35,9 @@ fn create_record_batch(start_value: i32, num_values: usize) -> RecordBatch {/// Here we only wish to create a simple table provider as an example./// We create an in-memory table and convert it to it's FFI counterpart.-extern "C" fn construct_simple_table_provider() -> FFI_TableProvider {+extern "C" fn construct_simple_table_provider(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_TableProvider {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true),Field::new("b", DataType::Float64, true),@@ -50,7 +53,7 @@ extern "C" fn construct_simple_table_provider() -> FFI_TableProvider {let table_provider = MemTable::try_new(schema, vec![batches]).unwrap();- FFI_TableProvider::new(Arc::new(table_provider), true, None)+ FFI_TableProvider::new_with_ffi_codec(Arc::new(table_provider), true, None, codec)}#[export_root_module]diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rsindex 1166eeb707..3b2b9e1871 100644--- a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs+++ b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs@@ -21,6 +21,7 @@ use abi_stable::{package_version_strings,sabi_types::VersionStrings,};+use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use datafusion_ffi::table_provider::FFI_TableProvider;#[repr(C)]@@ -33,7 +34,8 @@ use datafusion_ffi::table_provider::FFI_TableProvider;/// how a user may wish to separate these concerns.pub struct TableProviderModule {/// Constructs the table provider- pub create_table: extern "C" fn() -> FFI_TableProvider,+ pub create_table:+ extern "C" fn(codec: FFI_LogicalExtensionCodec) -> FFI_TableProvider,}impl RootModule for TableProviderModuleRef {diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.tomlindex c60cd7c029..823c9afdde 100644--- a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml+++ b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml@@ -24,5 +24,6 @@ publish = false[dependencies]abi_stable = "0.11.3"datafusion = { workspace = true }+datafusion-ffi = { workspace = true }ffi_module_interface = { path = "../ffi_module_interface" }tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rsindex d76af96058..8ce5b156df 100644--- a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs+++ b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs@@ -24,6 +24,8 @@ use datafusion::{use abi_stable::library::{RootModule, development_utils::compute_library_path};use datafusion::datasource::TableProvider;+use datafusion::execution::TaskContextProvider;+use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use ffi_module_interface::TableProviderModuleRef;#[tokio::main]@@ -39,6 +41,11 @@ async fn main() -> Result<()> {TableProviderModuleRef::load_from_directory(&library_path).map_err(|e| DataFusionError::External(Box::new(e)))?;+ let ctx = Arc::new(SessionContext::new());+ let codec = FFI_LogicalExtensionCodec::new_default(+ &(Arc::clone(&ctx) as Arc<dyn TaskContextProvider>),+ );+// By calling the code below, the table provided will be created within// the module's code.let ffi_table_provider =@@ -46,14 +53,12 @@ async fn main() -> Result<()> {.create_table().ok_or(DataFusionError::NotImplemented("External table provider failed to implement create_table".to_string(),- ))?();+ ))?(codec);// In order to access the table provider within this executable, we need to// turn it into a `TableProvider`.let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_table_provider).into();- let ctx = SessionContext::new();-// Display the data to show the full cycle works.ctx.register_table("external_table", foreign_table_provider)?;let df = ctx.table("external_table").await?;diff --git a/datafusion/ffi/src/catalog_provider.rs b/datafusion/ffi/src/catalog_provider.rsindex fe3cce1065..25e398c4dd 100644--- a/datafusion/ffi/src/catalog_provider.rs+++ b/datafusion/ffi/src/catalog_provider.rs@@ -15,22 +15,24 @@// specific language governing permissions and limitations// under the License.-use std::{any::Any, ffi::c_void, sync::Arc};--use abi_stable::{- StableAbi,- std_types::{ROption, RResult, RString, RVec},+use std::any::Any;+use std::ffi::c_void;+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RString, RVec};+use datafusion_catalog::{CatalogProvider, SchemaProvider};+use datafusion_common::error::Result;+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,};-use datafusion::catalog::{CatalogProvider, SchemaProvider};use tokio::runtime::Handle;-use crate::{- df_result, rresult_return,- schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider},-};-+use crate::execution::FFI_TaskContextProvider;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider};use crate::util::FFIResult;-use datafusion::error::Result;+use crate::{df_result, rresult_return};/// A stable struct for sharing [`CatalogProvider`] across FFI boundaries.#[repr(C)]@@ -58,6 +60,8 @@ pub struct FFI_CatalogProvider {)-> FFIResult<ROption<FFI_SchemaProvider>>,+ pub logical_codec: FFI_LogicalExtensionCodec,+/// Used to create a clone on the provider of the execution plan. This should/// only need to be called by the receiver of the plan.pub clone: unsafe extern "C" fn(plan: &Self) -> Self,@@ -118,7 +122,13 @@ unsafe extern "C" fn schema_fn_wrapper(unsafe {let maybe_schema = provider.inner().schema(name.as_str());maybe_schema- .map(|schema| FFI_SchemaProvider::new(schema, provider.runtime()))+ .map(|schema| {+ FFI_SchemaProvider::new_with_ffi_codec(+ schema,+ provider.runtime(),+ provider.logical_codec.clone(),+ )+ }).into()}}@@ -130,12 +140,18 @@ unsafe extern "C" fn register_schema_fn_wrapper() -> FFIResult<ROption<FFI_SchemaProvider>> {unsafe {let runtime = provider.runtime();- let provider = provider.inner();+ let inner_provider = provider.inner();let schema: Arc<dyn SchemaProvider + Send> = schema.into();let returned_schema =- rresult_return!(provider.register_schema(name.as_str(), schema))- .map(|schema| FFI_SchemaProvider::new(schema, runtime))+ rresult_return!(inner_provider.register_schema(name.as_str(), schema))+ .map(|schema| {+ FFI_SchemaProvider::new_with_ffi_codec(+ schema,+ runtime,+ provider.logical_codec.clone(),+ )+ }).into();RResult::ROk(returned_schema)@@ -149,14 +165,20 @@ unsafe extern "C" fn deregister_schema_fn_wrapper() -> FFIResult<ROption<FFI_SchemaProvider>> {unsafe {let runtime = provider.runtime();- let provider = provider.inner();+ let inner_provider = provider.inner();let maybe_schema =- rresult_return!(provider.deregister_schema(name.as_str(), cascade));+ rresult_return!(inner_provider.deregister_schema(name.as_str(), cascade));RResult::ROk(maybe_schema- .map(|schema| FFI_SchemaProvider::new(schema, runtime))+ .map(|schema| {+ FFI_SchemaProvider::new_with_ffi_codec(+ schema,+ runtime,+ provider.logical_codec.clone(),+ )+ }).into(),)}@@ -189,6 +211,7 @@ unsafe extern "C" fn clone_fn_wrapper(schema: schema_fn_wrapper,register_schema: register_schema_fn_wrapper,deregister_schema: deregister_schema_fn_wrapper,+ logical_codec: provider.logical_codec.clone(),clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -209,6 +232,24 @@ impl FFI_CatalogProvider {pub fn new(provider: Arc<dyn CatalogProvider + Send>,runtime: Option<Handle>,+ task_ctx_provider: impl Into<FFI_TaskContextProvider>,+ logical_codec: Option<Arc<dyn LogicalExtensionCodec>>,+ ) -> Self {+ let task_ctx_provider = task_ctx_provider.into();+ let logical_codec =+ logical_codec.unwrap_or_else(|| Arc::new(DefaultLogicalExtensionCodec {}));+ let logical_codec = FFI_LogicalExtensionCodec::new(+ logical_codec,+ runtime.clone(),+ task_ctx_provider.clone(),+ );+ Self::new_with_ffi_codec(provider, runtime, logical_codec)+ }++ pub fn new_with_ffi_codec(+ provider: Arc<dyn CatalogProvider + Send>,+ runtime: Option<Handle>,+ logical_codec: FFI_LogicalExtensionCodec,) -> Self {let private_data = Box::new(ProviderPrivateData { provider, runtime });@@ -217,6 +258,7 @@ impl FFI_CatalogProvider {schema: schema_fn_wrapper,register_schema: register_schema_fn_wrapper,deregister_schema: deregister_schema_fn_wrapper,+ logical_codec,clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -286,7 +328,11 @@ impl CatalogProvider for ForeignCatalogProvider {unsafe {let schema = match schema.as_any().downcast_ref::<ForeignSchemaProvider>() {Some(s) => &s.0,- None => &FFI_SchemaProvider::new(schema, None),+ None => &FFI_SchemaProvider::new_with_ffi_codec(+ schema,+ None,+ self.0.logical_codec.clone(),+ ),};let returned_schema: Option<FFI_SchemaProvider> =df_result!((self.0.register_schema)(&self.0, name.into(), schema))?@@ -331,8 +377,10 @@ mod tests {.unwrap().is_none());+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();- let mut ffi_catalog = FFI_CatalogProvider::new(catalog, None);+ let mut ffi_catalog =+ FFI_CatalogProvider::new(catalog, None, task_ctx_provider, None);ffi_catalog.library_marker_id = crate::mock_foreign_marker_id;let foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();@@ -375,7 +423,9 @@ mod tests {fn test_ffi_catalog_provider_local_bypass() {let catalog = Arc::new(MemoryCatalogProvider::new());- let mut ffi_catalog = FFI_CatalogProvider::new(catalog, None);+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();+ let mut ffi_catalog =+ FFI_CatalogProvider::new(catalog, None, task_ctx_provider, None);// Verify local libraries can be downcast to their originallet foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();diff --git a/datafusion/ffi/src/catalog_provider_list.rs b/datafusion/ffi/src/catalog_provider_list.rsindex a119dd9485..c6ff5f7da7 100644--- a/datafusion/ffi/src/catalog_provider_list.rs+++ b/datafusion/ffi/src/catalog_provider_list.rs@@ -15,16 +15,21 @@// specific language governing permissions and limitations// under the License.-use std::{any::Any, ffi::c_void, sync::Arc};--use abi_stable::{- StableAbi,- std_types::{ROption, RString, RVec},+use std::any::Any;+use std::ffi::c_void;+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RString, RVec};+use datafusion_catalog::{CatalogProvider, CatalogProviderList};+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,};-use datafusion::catalog::{CatalogProvider, CatalogProviderList};use tokio::runtime::Handle;use crate::catalog_provider::{FFI_CatalogProvider, ForeignCatalogProvider};+use crate::execution::FFI_TaskContextProvider;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;/// A stable struct for sharing [`CatalogProviderList`] across FFI boundaries.#[repr(C)]@@ -45,8 +50,10 @@ pub struct FFI_CatalogProviderList {pub catalog:unsafe extern "C" fn(&Self, name: RString) -> ROption<FFI_CatalogProvider>,- /// Used to create a clone on the provider. This should only need to be called- /// by the receiver of the plan.+ pub logical_codec: FFI_LogicalExtensionCodec,++ /// Used to create a clone on the provider of the execution plan. This should+ /// only need to be called by the receiver of the plan.pub clone: unsafe extern "C" fn(plan: &Self) -> Self,/// Release the memory of the private data when it is no longer being used.@@ -105,12 +112,18 @@ unsafe extern "C" fn register_catalog_fn_wrapper() -> ROption<FFI_CatalogProvider> {unsafe {let runtime = provider.runtime();- let provider = provider.inner();+ let inner_provider = provider.inner();let catalog: Arc<dyn CatalogProvider + Send> = catalog.into();- provider+ inner_provider.register_catalog(name.into(), catalog)- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))+ .map(|catalog| {+ FFI_CatalogProvider::new_with_ffi_codec(+ catalog,+ runtime,+ provider.logical_codec.clone(),+ )+ }).into()}}@@ -121,10 +134,16 @@ unsafe extern "C" fn catalog_fn_wrapper() -> ROption<FFI_CatalogProvider> {unsafe {let runtime = provider.runtime();- let provider = provider.inner();- provider+ let inner_provider = provider.inner();+ inner_provider.catalog(name.as_str())- .map(|catalog| FFI_CatalogProvider::new(catalog, runtime))+ .map(|catalog| {+ FFI_CatalogProvider::new_with_ffi_codec(+ catalog,+ runtime,+ provider.logical_codec.clone(),+ )+ }).into()}}@@ -155,6 +174,7 @@ unsafe extern "C" fn clone_fn_wrapper(register_catalog: register_catalog_fn_wrapper,catalog_names: catalog_names_fn_wrapper,catalog: catalog_fn_wrapper,+ logical_codec: provider.logical_codec.clone(),clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -175,6 +195,23 @@ impl FFI_CatalogProviderList {pub fn new(provider: Arc<dyn CatalogProviderList + Send>,runtime: Option<Handle>,+ task_ctx_provider: impl Into<FFI_TaskContextProvider>,+ logical_codec: Option<Arc<dyn LogicalExtensionCodec>>,+ ) -> Self {+ let task_ctx_provider = task_ctx_provider.into();+ let logical_codec =+ logical_codec.unwrap_or_else(|| Arc::new(DefaultLogicalExtensionCodec {}));+ let logical_codec = FFI_LogicalExtensionCodec::new(+ logical_codec,+ runtime.clone(),+ task_ctx_provider.clone(),+ );+ Self::new_with_ffi_codec(provider, runtime, logical_codec)+ }+ pub fn new_with_ffi_codec(+ provider: Arc<dyn CatalogProviderList + Send>,+ runtime: Option<Handle>,+ logical_codec: FFI_LogicalExtensionCodec,) -> Self {let private_data = Box::new(ProviderPrivateData { provider, runtime });@@ -182,6 +219,7 @@ impl FFI_CatalogProviderList {register_catalog: register_catalog_fn_wrapper,catalog_names: catalog_names_fn_wrapper,catalog: catalog_fn_wrapper,+ logical_codec,clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -232,7 +270,11 @@ impl CatalogProviderList for ForeignCatalogProviderList {let catalog = match catalog.as_any().downcast_ref::<ForeignCatalogProvider>(){Some(s) => &s.0,- None => &FFI_CatalogProvider::new(catalog, None),+ None => &FFI_CatalogProvider::new_with_ffi_codec(+ catalog,+ None,+ self.0.logical_codec.clone(),+ ),};(self.0.register_catalog)(&self.0, name.into(), catalog)@@ -279,7 +321,9 @@ mod tests {.is_none());- let mut ffi_catalog_list = FFI_CatalogProviderList::new(catalog_list, None);+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();+ let mut ffi_catalog_list =+ FFI_CatalogProviderList::new(catalog_list, None, task_ctx_provider, None);ffi_catalog_list.library_marker_id = crate::mock_foreign_marker_id;let foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =@@ -318,7 +362,9 @@ mod tests {fn test_ffi_catalog_provider_list_local_bypass() {let catalog_list = Arc::new(MemoryCatalogProviderList::new());- let mut ffi_catalog_list = FFI_CatalogProviderList::new(catalog_list, None);+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();+ let mut ffi_catalog_list =+ FFI_CatalogProviderList::new(catalog_list, None, task_ctx_provider, None);// Verify local libraries can be downcast to their originallet foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex d8ef1c9827..fdd3d5649e 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -29,6 +29,7 @@ use datafusion::{use datafusion::{error::Result, physical_plan::DisplayFormatType};use tokio::runtime::Handle;+use crate::execution::FFI_TaskContext;use crate::util::FFIResult;use crate::{df_result, plan_properties::FFI_PlanProperties,@@ -54,6 +55,7 @@ pub struct FFI_ExecutionPlan {pub execute: unsafe extern "C" fn(plan: &Self,partition: usize,+ context: FFI_TaskContext,) -> FFIResult<FFI_RecordBatchStream>,/// Used to create a clone on the provider of the execution plan. This should@@ -78,7 +80,6 @@ unsafe impl Sync for FFI_ExecutionPlan {}pub struct ExecutionPlanPrivateData {pub plan: Arc<dyn ExecutionPlan>,- pub context: Arc<TaskContext>,pub runtime: Option<Handle>,}@@ -101,19 +102,12 @@ unsafe extern "C" fn children_fn_wrapper(unsafe {let private_data = plan.private_data as *const ExecutionPlanPrivateData;let plan = &(*private_data).plan;- let ctx = &(*private_data).context;let runtime = &(*private_data).runtime;let children: Vec<_> = plan.children().into_iter()- .map(|child| {- FFI_ExecutionPlan::new(- Arc::clone(child),- Arc::clone(ctx),- runtime.clone(),- )- })+ .map(|child| FFI_ExecutionPlan::new(Arc::clone(child), runtime.clone())).collect();children.into()@@ -123,15 +117,16 @@ unsafe extern "C" fn children_fn_wrapper(unsafe extern "C" fn execute_fn_wrapper(plan: &FFI_ExecutionPlan,partition: usize,+ context: FFI_TaskContext,) -> FFIResult<FFI_RecordBatchStream> {unsafe {+ let ctx = context.into();let private_data = plan.private_data as *const ExecutionPlanPrivateData;let plan = &(*private_data).plan;- let ctx = &(*private_data).context;let runtime = (*private_data).runtime.clone();rresult!(- plan.execute(partition, Arc::clone(ctx))+ plan.execute(partition, ctx).map(|rbs| FFI_RecordBatchStream::new(rbs, runtime)))}@@ -156,11 +151,7 @@ unsafe extern "C" fn clone_fn_wrapper(plan: &FFI_ExecutionPlan) -> FFI_Executionlet private_data = plan.private_data as *const ExecutionPlanPrivateData;let plan_data = &(*private_data);- FFI_ExecutionPlan::new(- Arc::clone(&plan_data.plan),- Arc::clone(&plan_data.context),- plan_data.runtime.clone(),- )+ FFI_ExecutionPlan::new(Arc::clone(&plan_data.plan), plan_data.runtime.clone())}}@@ -172,16 +163,8 @@ impl Clone for FFI_ExecutionPlan {impl FFI_ExecutionPlan {/// This function is called on the provider's side.- pub fn new(- plan: Arc<dyn ExecutionPlan>,- context: Arc<TaskContext>,- runtime: Option<Handle>,- ) -> Self {- let private_data = Box::new(ExecutionPlanPrivateData {- plan,- context,- runtime,- });+ pub fn new(plan: Arc<dyn ExecutionPlan>, runtime: Option<Handle>) -> Self {+ let private_data = Box::new(ExecutionPlanPrivateData { plan, runtime });Self {properties: properties_fn_wrapper,@@ -305,10 +288,11 @@ impl ExecutionPlan for ForeignExecutionPlan {fn execute(&self,partition: usize,- _context: Arc<TaskContext>,+ context: Arc<TaskContext>,) -> Result<SendableRecordBatchStream> {+ let context = FFI_TaskContext::from(context);unsafe {- df_result!((self.plan.execute)(&self.plan, partition))+ df_result!((self.plan.execute)(&self.plan, partition, context)).map(|stream| Pin::new(Box::new(stream)) as SendableRecordBatchStream)}}@@ -318,12 +302,9 @@ impl ExecutionPlan for ForeignExecutionPlan {pub(crate) mod tests {use super::*;use arrow::datatypes::{DataType, Field, Schema};- use datafusion::{- physical_plan::{- Partitioning,- execution_plan::{Boundedness, EmissionType},- },- prelude::SessionContext,+ use datafusion::physical_plan::{+ Partitioning,+ execution_plan::{Boundedness, EmissionType},};#[derive(Debug)]@@ -400,17 +381,16 @@ pub(crate) mod tests {fn test_round_trip_ffi_execution_plan() -> Result<()> {let schema =Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));- let ctx = SessionContext::new();let original_plan = Arc::new(EmptyExec::new(schema));let original_name = original_plan.name().to_string();- let mut local_plan = FFI_ExecutionPlan::new(original_plan, ctx.task_ctx(), None);+ let mut local_plan = FFI_ExecutionPlan::new(original_plan, None);local_plan.library_marker_id = crate::mock_foreign_marker_id;let foreign_plan: Arc<dyn ExecutionPlan> = (&local_plan).try_into()?;- assert!(original_name == foreign_plan.name());+ assert_eq!(original_name, foreign_plan.name());let display = datafusion::physical_plan::display::DisplayableExecutionPlan::new(foreign_plan.as_ref(),@@ -429,16 +409,15 @@ pub(crate) mod tests {fn test_ffi_execution_plan_children() -> Result<()> {let schema =Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));- let ctx = SessionContext::new();// Version 1: Adding child to the foreign planlet child_plan = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let mut child_local = FFI_ExecutionPlan::new(child_plan, ctx.task_ctx(), None);+ let mut child_local = FFI_ExecutionPlan::new(child_plan, None);child_local.library_marker_id = crate::mock_foreign_marker_id;let child_foreign = <Arc<dyn ExecutionPlan>>::try_from(&child_local)?;let parent_plan = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let mut parent_local = FFI_ExecutionPlan::new(parent_plan, ctx.task_ctx(), None);+ let mut parent_local = FFI_ExecutionPlan::new(parent_plan, None);parent_local.library_marker_id = crate::mock_foreign_marker_id;let parent_foreign = <Arc<dyn ExecutionPlan>>::try_from(&parent_local)?;@@ -450,13 +429,13 @@ pub(crate) mod tests {// Version 2: Adding child to the local planlet child_plan = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let mut child_local = FFI_ExecutionPlan::new(child_plan, ctx.task_ctx(), None);+ let mut child_local = FFI_ExecutionPlan::new(child_plan, None);child_local.library_marker_id = crate::mock_foreign_marker_id;let child_foreign = <Arc<dyn ExecutionPlan>>::try_from(&child_local)?;let parent_plan = Arc::new(EmptyExec::new(Arc::clone(&schema)));let parent_plan = parent_plan.with_new_children(vec![child_foreign])?;- let mut parent_local = FFI_ExecutionPlan::new(parent_plan, ctx.task_ctx(), None);+ let mut parent_local = FFI_ExecutionPlan::new(parent_plan, None);parent_local.library_marker_id = crate::mock_foreign_marker_id;let parent_foreign = <Arc<dyn ExecutionPlan>>::try_from(&parent_local)?;@@ -469,11 +448,10 @@ pub(crate) mod tests {fn test_ffi_execution_plan_local_bypass() {let schema =Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));- let ctx = SessionContext::new();let plan = Arc::new(EmptyExec::new(schema));- let mut ffi_plan = FFI_ExecutionPlan::new(plan, ctx.task_ctx(), None);+ let mut ffi_plan = FFI_ExecutionPlan::new(plan, None);// Verify local libraries can be downcast to their originallet foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into().unwrap();diff --git a/datafusion/ffi/src/plan_properties.rs b/datafusion/ffi/src/plan_properties.rsindex 430b34c984..ac388a3653 100644--- a/datafusion/ffi/src/plan_properties.rs+++ b/datafusion/ffi/src/plan_properties.rs@@ -15,43 +15,29 @@// specific language governing permissions and limitations// under the License.-use std::{ffi::c_void, sync::Arc};+use std::ffi::c_void;+use std::sync::Arc;-use abi_stable::{- StableAbi,- std_types::{RResult::ROk, RVec},-};+use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RVec};use arrow::datatypes::SchemaRef;-use datafusion::{- error::{DataFusionError, Result},- physical_expr::EquivalenceProperties,- physical_plan::{- PlanProperties,- execution_plan::{Boundedness, EmissionType},- },- prelude::SessionContext,-};-use datafusion_proto::{- physical_plan::{- DefaultPhysicalExtensionCodec,- from_proto::{parse_physical_sort_exprs, parse_protobuf_partitioning},- to_proto::{serialize_partitioning, serialize_physical_sort_exprs},- },- protobuf::{Partitioning, PhysicalSortExprNodeCollection},-};-use prost::Message;--use crate::util::FFIResult;-use crate::{arrow_wrappers::WrappedSchema, df_result, rresult_return};+use datafusion_common::error::{DataFusionError, Result};+use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_plan::PlanProperties;+use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};++use crate::arrow_wrappers::WrappedSchema;+use crate::physical_expr::partitioning::FFI_Partitioning;+use crate::physical_expr::sort::FFI_PhysicalSortExpr;/// A stable struct for sharing [`PlanProperties`] across FFI boundaries.#[repr(C)]#[derive(Debug, StableAbi)]#[allow(non_camel_case_types)]pub struct FFI_PlanProperties {- /// The output partitioning is a [`Partitioning`] protobuf message serialized- /// into bytes to pass across the FFI boundary.- pub output_partitioning: unsafe extern "C" fn(plan: &Self) -> FFIResult<RVec<u8>>,+ /// The output partitioning of the plan.+ pub output_partitioning: unsafe extern "C" fn(plan: &Self) -> FFI_Partitioning,/// Return the emission type of the plan.pub emission_type: unsafe extern "C" fn(plan: &Self) -> FFI_EmissionType,@@ -59,9 +45,9 @@ pub struct FFI_PlanProperties {/// Indicate boundedness of the plan and its memory requirements.pub boundedness: unsafe extern "C" fn(plan: &Self) -> FFI_Boundedness,- /// The output ordering is a [`PhysicalSortExprNodeCollection`] protobuf message- /// serialized into bytes to pass across the FFI boundary.- pub output_ordering: unsafe extern "C" fn(plan: &Self) -> FFIResult<RVec<u8>>,+ /// The output ordering of the plan.+ pub output_ordering:+ unsafe extern "C" fn(plan: &Self) -> ROption<RVec<FFI_PhysicalSortExpr>>,/// Return the schema of the plan.pub schema: unsafe extern "C" fn(plan: &Self) -> WrappedSchema,@@ -92,15 +78,8 @@ impl FFI_PlanProperties {unsafe extern "C" fn output_partitioning_fn_wrapper(properties: &FFI_PlanProperties,-) -> FFIResult<RVec<u8>> {- let codec = DefaultPhysicalExtensionCodec {};- let partitioning_data = rresult_return!(serialize_partitioning(- properties.inner().output_partitioning(),- &codec- ));- let output_partitioning = partitioning_data.encode_to_vec();-- ROk(output_partitioning.into())+) -> FFI_Partitioning {+ properties.inner().output_partitioning().into()}unsafe extern "C" fn emission_type_fn_wrapper(@@ -117,22 +96,17 @@ unsafe extern "C" fn boundedness_fn_wrapper(unsafe extern "C" fn output_ordering_fn_wrapper(properties: &FFI_PlanProperties,-) -> FFIResult<RVec<u8>> {- let codec = DefaultPhysicalExtensionCodec {};- let output_ordering = match properties.inner().output_ordering() {- Some(ordering) => {- let physical_sort_expr_nodes = rresult_return!(- serialize_physical_sort_exprs(ordering.to_owned(), &codec)- );- let ordering_data = PhysicalSortExprNodeCollection {- physical_sort_expr_nodes,- };-- ordering_data.encode_to_vec()- }- None => Vec::default(),- };- ROk(output_ordering.into())+) -> ROption<RVec<FFI_PhysicalSortExpr>> {+ let ordering: Option<RVec<FFI_PhysicalSortExpr>> =+ properties.inner().output_ordering().map(|lex_ordering| {+ let vec_ordering: Vec<PhysicalSortExpr> = lex_ordering.clone().into();+ vec_ordering+ .iter()+ .map(FFI_PhysicalSortExpr::from)+ .collect()+ });++ ordering.into()}unsafe extern "C" fn schema_fn_wrapper(properties: &FFI_PlanProperties) -> WrappedSchema {@@ -186,38 +160,18 @@ impl TryFrom<FFI_PlanProperties> for PlanProperties {let ffi_schema = unsafe { (ffi_props.schema)(&ffi_props) };let schema = (&ffi_schema.0).try_into()?;- // TODO Extend FFI to get the registry and codex- let default_ctx = SessionContext::new();- let task_context = default_ctx.task_ctx();- let codex = DefaultPhysicalExtensionCodec {};-- let ffi_orderings = unsafe { (ffi_props.output_ordering)(&ffi_props) };-- let proto_output_ordering =- PhysicalSortExprNodeCollection::decode(df_result!(ffi_orderings)?.as_ref())- .map_err(|e| DataFusionError::External(Box::new(e)))?;- let sort_exprs = parse_physical_sort_exprs(- &proto_output_ordering.physical_sort_expr_nodes,- &task_context,- &schema,- &codex,- )?;-- let partitioning_vec =- unsafe { df_result!((ffi_props.output_partitioning)(&ffi_props))? };- let proto_output_partitioning =- Partitioning::decode(partitioning_vec.as_ref())- .map_err(|e| DataFusionError::External(Box::new(e)))?;- let partitioning = parse_protobuf_partitioning(- Some(&proto_output_partitioning),- &task_context,- &schema,- &codex,- )?- .ok_or(DataFusionError::Plan(- "Unable to deserialize partitioning protobuf in FFI_PlanProperties"- .to_string(),- ))?;+ let ffi_orderings: Option<RVec<FFI_PhysicalSortExpr>> =+ unsafe { (ffi_props.output_ordering)(&ffi_props) }.into();+ let sort_exprs = ffi_orderings+ .map(|ordering_vec| {+ ordering_vec+ .iter()+ .map(PhysicalSortExpr::from)+ .collect::<Vec<_>>()+ })+ .unwrap_or_default();++ let partitioning = unsafe { (ffi_props.output_partitioning)(&ffi_props) };let eq_properties = if sort_exprs.is_empty() {EquivalenceProperties::new(Arc::new(schema))@@ -233,7 +187,7 @@ impl TryFrom<FFI_PlanProperties> for PlanProperties {Ok(PlanProperties::new(eq_properties,- partitioning,+ (&partitioning).into(),emission_type,boundedness,))@@ -307,7 +261,8 @@ impl From<FFI_EmissionType> for EmissionType {#[cfg(test)]mod tests {- use datafusion::{physical_expr::PhysicalSortExpr, physical_plan::Partitioning};+ use datafusion::physical_expr::PhysicalSortExpr;+ use datafusion::physical_plan::Partitioning;use super::*;@@ -331,6 +286,7 @@ mod tests {#[test]fn test_round_trip_ffi_plan_properties() -> Result<()> {let original_props = create_test_props()?;+let mut local_props_ptr = FFI_PlanProperties::from(&original_props);local_props_ptr.library_marker_id = crate::mock_foreign_marker_id;@@ -342,7 +298,7 @@ mod tests {}#[test]- fn test_ffi_execution_plan_local_bypass() -> Result<()> {+ fn test_ffi_plan_properties_local_bypass() -> Result<()> {let props = create_test_props()?;let ffi_plan = FFI_PlanProperties::from(&props);diff --git a/datafusion/ffi/src/proto/logical_extension_codec.rs b/datafusion/ffi/src/proto/logical_extension_codec.rsindex 3f79cfc732..3cb62f4a53 100644--- a/datafusion/ffi/src/proto/logical_extension_codec.rs+++ b/datafusion/ffi/src/proto/logical_extension_codec.rs@@ -25,12 +25,14 @@ use datafusion_catalog::TableProvider;use datafusion_common::error::Result;use datafusion_common::{TableReference, not_impl_err};use datafusion_datasource::file_format::FileFormatFactory;-use datafusion_execution::TaskContext;+use datafusion_execution::{TaskContext, TaskContextProvider};use datafusion_expr::{AggregateUDF, AggregateUDFImpl, Extension, LogicalPlan, ScalarUDF, ScalarUDFImpl,WindowUDF, WindowUDFImpl,};-use datafusion_proto::logical_plan::LogicalExtensionCodec;+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,+};use tokio::runtime::Handle;use crate::arrow_wrappers::WrappedSchema;@@ -95,7 +97,7 @@ pub struct FFI_LogicalExtensionCodec {try_encode_udwf:unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFIResult<RVec<u8>>,- task_ctx_provider: FFI_TaskContextProvider,+ pub task_ctx_provider: FFI_TaskContextProvider,/// Used to create a clone on the provider of the execution plan. This should/// only need to be called by the receiver of the plan.@@ -120,14 +122,14 @@ unsafe impl Send for FFI_LogicalExtensionCodec {}unsafe impl Sync for FFI_LogicalExtensionCodec {}struct LogicalExtensionCodecPrivateData {- provider: Arc<dyn LogicalExtensionCodec>,+ codec: Arc<dyn LogicalExtensionCodec>,runtime: Option<Handle>,}impl FFI_LogicalExtensionCodec {fn inner(&self) -> &Arc<dyn LogicalExtensionCodec> {let private_data = self.private_data as *const LogicalExtensionCodecPrivateData;- unsafe { &(*private_data).provider }+ unsafe { &(*private_data).codec }}fn runtime(&self) -> &Option<Handle> {@@ -148,18 +150,23 @@ unsafe extern "C" fn try_decode_table_provider_fn_wrapper() -> FFIResult<FFI_TableProvider> {let ctx = rresult_return!(codec.task_ctx());let runtime = codec.runtime().clone();- let codec = codec.inner();+ let codec_inner = codec.inner();let table_ref = TableReference::from(table_ref.as_str());let schema: SchemaRef = schema.into();- let table_provider = rresult_return!(codec.try_decode_table_provider(+ let table_provider = rresult_return!(codec_inner.try_decode_table_provider(buf.as_ref(),&table_ref,schema,ctx.as_ref()));- RResult::ROk(FFI_TableProvider::new(table_provider, true, runtime))+ RResult::ROk(FFI_TableProvider::new_with_ffi_codec(+ table_provider,+ true,+ runtime,+ codec.clone(),+ ))}unsafe extern "C" fn try_encode_table_provider_fn_wrapper(@@ -286,13 +293,12 @@ impl Drop for FFI_LogicalExtensionCodec {impl FFI_LogicalExtensionCodec {/// Creates a new [`FFI_LogicalExtensionCodec`].pub fn new(- provider: Arc<dyn LogicalExtensionCodec + Send>,+ codec: Arc<dyn LogicalExtensionCodec + Send>,runtime: Option<Handle>,task_ctx_provider: impl Into<FFI_TaskContextProvider>,) -> Self {let task_ctx_provider = task_ctx_provider.into();- let private_data =- Box::new(LogicalExtensionCodecPrivateData { provider, runtime });+ let private_data = Box::new(LogicalExtensionCodecPrivateData { codec, runtime });Self {try_decode_table_provider: try_decode_table_provider_fn_wrapper,@@ -312,6 +318,13 @@ impl FFI_LogicalExtensionCodec {library_marker_id: crate::get_library_marker_id,}}++ pub fn new_default(task_ctx_provider: &Arc<dyn TaskContextProvider>) -> Self {+ let task_ctx_provider = FFI_TaskContextProvider::from(task_ctx_provider);+ let codec = Arc::new(DefaultLogicalExtensionCodec {});++ Self::new(codec, None, task_ctx_provider)+ }}/// This wrapper struct exists on the receiver side of the FFI interface, so it has@@ -383,7 +396,8 @@ impl LogicalExtensionCodec for ForeignLogicalExtensionCodec {buf: &mut Vec<u8>,) -> Result<()> {let table_ref = table_ref.to_string();- let node = FFI_TableProvider::new(node, true, None);+ let node =+ FFI_TableProvider::new_with_ffi_codec(node, true, None, self.0.clone());let bytes = df_result!(unsafe {(self.0.try_encode_table_provider)(&self.0, table_ref.as_str().into(), node)diff --git a/datafusion/ffi/src/proto/physical_extension_codec.rs b/datafusion/ffi/src/proto/physical_extension_codec.rsindex 89a9a2cead..5025c962d3 100644--- a/datafusion/ffi/src/proto/physical_extension_codec.rs+++ b/datafusion/ffi/src/proto/physical_extension_codec.rs@@ -145,7 +145,7 @@ unsafe extern "C" fn try_decode_fn_wrapper(let plan =rresult_return!(codec.try_decode(buf.as_ref(), &inputs, task_ctx.as_ref()));- RResult::ROk(FFI_ExecutionPlan::new(plan, task_ctx, None))+ RResult::ROk(FFI_ExecutionPlan::new(plan, None))}unsafe extern "C" fn try_encode_fn_wrapper(@@ -329,12 +329,9 @@ impl PhysicalExtensionCodec for ForeignPhysicalExtensionCodec {inputs: &[Arc<dyn ExecutionPlan>],_ctx: &TaskContext,) -> Result<Arc<dyn ExecutionPlan>> {- let task_ctx = (&self.0.task_ctx_provider).try_into()?;let inputs = inputs.iter()- .map(|plan| {- FFI_ExecutionPlan::new(Arc::clone(plan), Arc::clone(&task_ctx), None)- })+ .map(|plan| FFI_ExecutionPlan::new(Arc::clone(plan), None)).collect();let plan =@@ -345,8 +342,7 @@ impl PhysicalExtensionCodec for ForeignPhysicalExtensionCodec {}fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {- let task_ctx = (&self.0.task_ctx_provider).try_into()?;- let plan = FFI_ExecutionPlan::new(node, task_ctx, None);+ let plan = FFI_ExecutionPlan::new(node, None);let bytes = df_result!(unsafe { (self.0.try_encode)(&self.0, plan) })?;buf.extend(bytes);diff --git a/datafusion/ffi/src/schema_provider.rs b/datafusion/ffi/src/schema_provider.rsindex cea41ae30f..84cff6e8b3 100644--- a/datafusion/ffi/src/schema_provider.rs+++ b/datafusion/ffi/src/schema_provider.rs@@ -15,27 +15,26 @@// specific language governing permissions and limitations// under the License.-use std::{any::Any, ffi::c_void, sync::Arc};+use std::any::Any;+use std::ffi::c_void;+use std::sync::Arc;-use abi_stable::{- StableAbi,- std_types::{ROption, RResult, RString, RVec},-};+use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RString, RVec};use async_ffi::{FfiFuture, FutureExt};use async_trait::async_trait;-use datafusion::{- catalog::{SchemaProvider, TableProvider},- error::DataFusionError,+use datafusion_catalog::{SchemaProvider, TableProvider};+use datafusion_common::error::{DataFusionError, Result};+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,};use tokio::runtime::Handle;-use crate::{- df_result, rresult_return,- table_provider::{FFI_TableProvider, ForeignTableProvider},-};-+use crate::execution::FFI_TaskContextProvider;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::table_provider::{FFI_TableProvider, ForeignTableProvider};use crate::util::FFIResult;-use datafusion::error::Result;+use crate::{df_result, rresult_return};/// A stable struct for sharing [`SchemaProvider`] across FFI boundaries.#[repr(C)]@@ -67,6 +66,8 @@ pub struct FFI_SchemaProvider {pub table_exist: unsafe extern "C" fn(provider: &Self, name: RString) -> bool,+ pub logical_codec: FFI_LogicalExtensionCodec,+/// Used to create a clone on the provider of the execution plan. This should/// only need to be called by the receiver of the plan.pub clone: unsafe extern "C" fn(plan: &Self) -> Self,@@ -128,11 +129,14 @@ unsafe extern "C" fn table_fn_wrapper() -> FfiFuture<FFIResult<ROption<FFI_TableProvider>>> {unsafe {let runtime = provider.runtime();+ let logical_codec = provider.logical_codec.clone();let provider = Arc::clone(provider.inner());async move {let table = rresult_return!(provider.table(name.as_str()).await)- .map(|t| FFI_TableProvider::new(t, true, runtime))+ .map(|t| {+ FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec)+ }).into();RResult::ROk(table)@@ -148,12 +152,15 @@ unsafe extern "C" fn register_table_fn_wrapper() -> FFIResult<ROption<FFI_TableProvider>> {unsafe {let runtime = provider.runtime();+ let logical_codec = provider.logical_codec.clone();let provider = provider.inner();let table = Arc::new(ForeignTableProvider(table));let returned_table = rresult_return!(provider.register_table(name.into(), table))- .map(|t| FFI_TableProvider::new(t, true, runtime));+ .map(|t| {+ FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec)+ });RResult::ROk(returned_table.into())}@@ -165,10 +172,13 @@ unsafe extern "C" fn deregister_table_fn_wrapper() -> FFIResult<ROption<FFI_TableProvider>> {unsafe {let runtime = provider.runtime();+ let logical_codec = provider.logical_codec.clone();let provider = provider.inner();let returned_table = rresult_return!(provider.deregister_table(name.as_str()))- .map(|t| FFI_TableProvider::new(t, true, runtime));+ .map(|t| {+ FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec)+ });RResult::ROk(returned_table.into())}@@ -206,14 +216,15 @@ unsafe extern "C" fn clone_fn_wrapper(FFI_SchemaProvider {owner_name: provider.owner_name.clone(),table_names: table_names_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data,table: table_fn_wrapper,register_table: register_table_fn_wrapper,deregister_table: deregister_table_fn_wrapper,table_exist: table_exist_fn_wrapper,+ logical_codec: provider.logical_codec.clone(),+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data,library_marker_id: crate::get_library_marker_id,}}@@ -230,6 +241,24 @@ impl FFI_SchemaProvider {pub fn new(provider: Arc<dyn SchemaProvider + Send>,runtime: Option<Handle>,+ task_ctx_provider: impl Into<FFI_TaskContextProvider>,+ logical_codec: Option<Arc<dyn LogicalExtensionCodec>>,+ ) -> Self {+ let task_ctx_provider = task_ctx_provider.into();+ let logical_codec =+ logical_codec.unwrap_or_else(|| Arc::new(DefaultLogicalExtensionCodec {}));+ let logical_codec = FFI_LogicalExtensionCodec::new(+ logical_codec,+ runtime.clone(),+ task_ctx_provider.clone(),+ );+ Self::new_with_ffi_codec(provider, runtime, logical_codec)+ }++ pub fn new_with_ffi_codec(+ provider: Arc<dyn SchemaProvider + Send>,+ runtime: Option<Handle>,+ logical_codec: FFI_LogicalExtensionCodec,) -> Self {let owner_name = provider.owner_name().map(|s| s.into()).into();let private_data = Box::new(ProviderPrivateData { provider, runtime });@@ -237,14 +266,15 @@ impl FFI_SchemaProvider {Self {owner_name,table_names: table_names_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- version: super::version,- private_data: Box::into_raw(private_data) as *mut c_void,table: table_fn_wrapper,register_table: register_table_fn_wrapper,deregister_table: deregister_table_fn_wrapper,table_exist: table_exist_fn_wrapper,+ logical_codec,+ clone: clone_fn_wrapper,+ release: release_fn_wrapper,+ version: super::version,+ private_data: Box::into_raw(private_data) as *mut c_void,library_marker_id: crate::get_library_marker_id,}}@@ -319,7 +349,12 @@ impl SchemaProvider for ForeignSchemaProvider {unsafe {let ffi_table = match table.as_any().downcast_ref::<ForeignTableProvider>() {Some(t) => t.0.clone(),- None => FFI_TableProvider::new(table, true, None),+ None => FFI_TableProvider::new_with_ffi_codec(+ table,+ true,+ None,+ self.0.logical_codec.clone(),+ ),};let returned_provider: Option<FFI_TableProvider> =@@ -348,9 +383,11 @@ impl SchemaProvider for ForeignSchemaProvider {#[cfg(test)]mod tests {- use super::*;use arrow::datatypes::Schema;- use datafusion::{catalog::MemorySchemaProvider, datasource::empty::EmptyTable};+ use datafusion::catalog::MemorySchemaProvider;+ use datafusion::datasource::empty::EmptyTable;++ use super::*;fn empty_table() -> Arc<dyn TableProvider> {Arc::new(EmptyTable::new(Arc::new(Schema::empty())))@@ -367,7 +404,10 @@ mod tests {.is_none());- let mut ffi_schema_provider = FFI_SchemaProvider::new(schema_provider, None);+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();++ let mut ffi_schema_provider =+ FFI_SchemaProvider::new(schema_provider, None, task_ctx_provider, None);ffi_schema_provider.library_marker_id = crate::mock_foreign_marker_id;let foreign_schema_provider: Arc<dyn SchemaProvider + Send> =@@ -418,7 +458,9 @@ mod tests {fn test_ffi_schema_provider_local_bypass() {let schema_provider = Arc::new(MemorySchemaProvider::new());- let mut ffi_schema = FFI_SchemaProvider::new(schema_provider, None);+ let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();+ let mut ffi_schema =+ FFI_SchemaProvider::new(schema_provider, None, task_ctx_provider, None);// Verify local libraries can be downcast to their originallet foreign_schema: Arc<dyn SchemaProvider + Send> = (&ffi_schema).into();diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rsindex 1694f46dc7..4e1ef96d30 100644--- a/datafusion/ffi/src/session/mod.rs+++ b/datafusion/ffi/src/session/mod.rs@@ -76,7 +76,7 @@ pub mod config;#[repr(C)]#[derive(Debug, StableAbi)]#[allow(non_camel_case_types)]-pub struct FFI_SessionRef {+pub(crate) struct FFI_SessionRef {session_id: unsafe extern "C" fn(&Self) -> RStr,config: unsafe extern "C" fn(&Self) -> FFI_SessionConfig,@@ -177,9 +177,7 @@ unsafe extern "C" fn create_physical_plan_fn_wrapper(let physical_plan = session.create_physical_plan(&logical_plan).await;- rresult!(- physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, task_ctx, runtime))- )+ rresult!(physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, runtime)))}.into_ffi()}diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rsindex 8929d02b0e..a5f9940128 100644--- a/datafusion/ffi/src/table_provider.rs+++ b/datafusion/ffi/src/table_provider.rs@@ -15,46 +15,39 @@// specific language governing permissions and limitations// under the License.-use std::{any::Any, ffi::c_void, sync::Arc};+use std::any::Any;+use std::ffi::c_void;+use std::sync::Arc;-use abi_stable::{- StableAbi,- std_types::{ROption, RResult, RVec},-};+use abi_stable::StableAbi;+use abi_stable::std_types::{ROption, RResult, RVec};use arrow::datatypes::SchemaRef;use async_ffi::{FfiFuture, FutureExt};use async_trait::async_trait;-use datafusion::{- catalog::{Session, TableProvider},- datasource::TableType,- error::DataFusionError,- execution::{TaskContext, session_state::SessionStateBuilder},- logical_expr::{TableProviderFilterPushDown, logical_plan::dml::InsertOp},- physical_plan::ExecutionPlan,- prelude::{Expr, SessionContext},-};-use datafusion_proto::{- logical_plan::{- DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,- },- protobuf::LogicalExprList,+use datafusion_catalog::{Session, TableProvider};+use datafusion_common::error::{DataFusionError, Result};+use datafusion_execution::TaskContext;+use datafusion_expr::dml::InsertOp;+use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};+use datafusion_physical_plan::ExecutionPlan;+use datafusion_proto::logical_plan::from_proto::parse_exprs;+use datafusion_proto::logical_plan::to_proto::serialize_exprs;+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,};+use datafusion_proto::protobuf::LogicalExprList;use prost::Message;use tokio::runtime::Handle;-use crate::{- arrow_wrappers::WrappedSchema,- df_result, rresult_return,- table_source::{FFI_TableProviderFilterPushDown, FFI_TableType},-};--use super::{- execution_plan::FFI_ExecutionPlan, insert_op::FFI_InsertOp,- session::config::FFI_SessionConfig,-};+use super::execution_plan::FFI_ExecutionPlan;+use super::insert_op::FFI_InsertOp;+use crate::arrow_wrappers::WrappedSchema;+use crate::execution::FFI_TaskContextProvider;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::session::{FFI_SessionRef, ForeignSession};+use crate::table_source::{FFI_TableProviderFilterPushDown, FFI_TableType};use crate::util::FFIResult;-use datafusion::error::Result;-use datafusion_execution::config::SessionConfig;+use crate::{df_result, rresult_return};/// A stable struct for sharing [`TableProvider`] across FFI boundaries.///@@ -100,60 +93,62 @@ use datafusion_execution::config::SessionConfig;#[allow(non_camel_case_types)]pub struct FFI_TableProvider {/// Return the table schema- pub schema: unsafe extern "C" fn(provider: &Self) -> WrappedSchema,+ schema: unsafe extern "C" fn(provider: &Self) -> WrappedSchema,/// Perform a scan on the table. See [`TableProvider`] for detailed usage information.////// # Arguments////// * `provider` - the table provider- /// * `session_config` - session configuration+ /// * `session` - session/// * `projections` - if specified, only a subset of the columns are returned/// * `filters_serialized` - filters to apply to the scan, which are a/// [`LogicalExprList`] protobuf message serialized into bytes to pass/// across the FFI boundary./// * `limit` - if specified, limit the number of rows returned- pub scan: unsafe extern "C" fn(+ scan: unsafe extern "C" fn(provider: &Self,- session_config: &FFI_SessionConfig,+ session: FFI_SessionRef,projections: RVec<usize>,filters_serialized: RVec<u8>,limit: ROption<usize>,) -> FfiFuture<FFIResult<FFI_ExecutionPlan>>,/// Return the type of table. See [`TableType`] for options.- pub table_type: unsafe extern "C" fn(provider: &Self) -> FFI_TableType,+ table_type: unsafe extern "C" fn(provider: &Self) -> FFI_TableType,/// Based upon the input filters, identify which are supported. The filters/// are a [`LogicalExprList`] protobuf message serialized into bytes to pass/// across the FFI boundary.- pub supports_filters_pushdown: Option<+ supports_filters_pushdown: Option<unsafe extern "C" fn(provider: &FFI_TableProvider,filters_serialized: RVec<u8>,) -> FFIResult<RVec<FFI_TableProviderFilterPushDown>>,>,- pub insert_into: unsafe extern "C" fn(+ insert_into: unsafe extern "C" fn(provider: &Self,- session_config: &FFI_SessionConfig,+ session: FFI_SessionRef,input: &FFI_ExecutionPlan,insert_op: FFI_InsertOp,) -> FfiFuture<FFIResult<FFI_ExecutionPlan>>,+ pub logical_codec: FFI_LogicalExtensionCodec,+/// Used to create a clone on the provider of the execution plan. This should/// only need to be called by the receiver of the plan.- pub clone: unsafe extern "C" fn(plan: &Self) -> Self,+ clone: unsafe extern "C" fn(plan: &Self) -> Self,/// Release the memory of the private data when it is no longer being used.- pub release: unsafe extern "C" fn(arg: &mut Self),+ release: unsafe extern "C" fn(arg: &mut Self),/// Return the major DataFusion version number of this provider.pub version: unsafe extern "C" fn() -> u64,/// Internal data. This is only to be accessed by the provider of the plan./// A [`ForeignTableProvider`] should never attempt to access this data.- pub private_data: *mut c_void,+ private_data: *mut c_void,/// Utility to identify when FFI objects are accessed locally through/// the foreign interface. See [`crate::get_library_marker_id`] and@@ -194,17 +189,16 @@ unsafe extern "C" fn table_type_fn_wrapper(fn supports_filters_pushdown_internal(provider: &Arc<dyn TableProvider + Send>,filters_serialized: &[u8],+ task_ctx: &Arc<TaskContext>,+ codec: &dyn LogicalExtensionCodec,) -> Result<RVec<FFI_TableProviderFilterPushDown>> {- let default_ctx = SessionContext::new();- let codec = DefaultLogicalExtensionCodec {};-let filters = match filters_serialized.is_empty() {true => vec![],false => {let proto_filters = LogicalExprList::decode(filters_serialized).map_err(|e| DataFusionError::Plan(e.to_string()))?;- parse_exprs(proto_filters.expr.iter(), &default_ctx, &codec)?+ parse_exprs(proto_filters.expr.iter(), task_ctx.as_ref(), codec)?}};let filters_borrowed: Vec<&Expr> = filters.iter().collect();@@ -222,43 +216,56 @@ unsafe extern "C" fn supports_filters_pushdown_fn_wrapper(provider: &FFI_TableProvider,filters_serialized: RVec<u8>,) -> FFIResult<RVec<FFI_TableProviderFilterPushDown>> {- supports_filters_pushdown_internal(provider.inner(), &filters_serialized)- .map_err(|e| e.to_string().into())- .into()+ let logical_codec: Arc<dyn LogicalExtensionCodec> = (&provider.logical_codec).into();+ let task_ctx = rresult_return!(<Arc<TaskContext>>::try_from(+ &provider.logical_codec.task_ctx_provider+ ));+ supports_filters_pushdown_internal(+ provider.inner(),+ &filters_serialized,+ &task_ctx,+ logical_codec.as_ref(),+ )+ .map_err(|e| e.to_string().into())+ .into()}unsafe extern "C" fn scan_fn_wrapper(provider: &FFI_TableProvider,- session_config: &FFI_SessionConfig,+ session: FFI_SessionRef,projections: RVec<usize>,filters_serialized: RVec<u8>,limit: ROption<usize>,) -> FfiFuture<FFIResult<FFI_ExecutionPlan>> {+ let task_ctx: Result<Arc<TaskContext>, DataFusionError> =+ (&provider.logical_codec.task_ctx_provider).try_into();let runtime = provider.runtime().clone();+ let logical_codec: Arc<dyn LogicalExtensionCodec> = (&provider.logical_codec).into();let internal_provider = Arc::clone(provider.inner());- let session_config = session_config.clone();async move {- let config = rresult_return!(SessionConfig::try_from(&session_config));- let session = SessionStateBuilder::new()- .with_default_features()- .with_config(config)- .build();- let ctx = SessionContext::new_with_state(session);+ let mut foreign_session = None;+ let session = rresult_return!(+ session+ .as_local()+ .map(Ok::<&(dyn Session + Send + Sync), DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );+ let task_ctx = rresult_return!(task_ctx);let filters = match filters_serialized.is_empty() {true => vec![],false => {- let default_ctx = SessionContext::new();- let codec = DefaultLogicalExtensionCodec {};-let proto_filters =rresult_return!(LogicalExprList::decode(filters_serialized.as_ref()));rresult_return!(parse_exprs(proto_filters.expr.iter(),- &default_ctx,- &codec+ task_ctx.as_ref(),+ logical_codec.as_ref(),))}};@@ -267,37 +274,36 @@ unsafe extern "C" fn scan_fn_wrapper(let plan = rresult_return!(internal_provider- .scan(&ctx.state(), Some(&projections), &filters, limit.into())+ .scan(session, Some(&projections), &filters, limit.into()).await);- RResult::ROk(FFI_ExecutionPlan::new(- plan,- ctx.task_ctx(),- runtime.clone(),- ))+ RResult::ROk(FFI_ExecutionPlan::new(plan, runtime.clone()))}.into_ffi()}unsafe extern "C" fn insert_into_fn_wrapper(provider: &FFI_TableProvider,- session_config: &FFI_SessionConfig,+ session: FFI_SessionRef,input: &FFI_ExecutionPlan,insert_op: FFI_InsertOp,) -> FfiFuture<FFIResult<FFI_ExecutionPlan>> {let runtime = provider.runtime().clone();let internal_provider = Arc::clone(provider.inner());- let session_config = session_config.clone();let input = input.clone();async move {- let config = rresult_return!(SessionConfig::try_from(&session_config));- let session = SessionStateBuilder::new()- .with_default_features()- .with_config(config)- .build();- let ctx = SessionContext::new_with_state(session);+ let mut foreign_session = None;+ let session = rresult_return!(+ session+ .as_local()+ .map(Ok::<&(dyn Session + Send + Sync), DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );let input = rresult_return!(<Arc<dyn ExecutionPlan>>::try_from(&input));@@ -305,15 +311,11 @@ unsafe extern "C" fn insert_into_fn_wrapper(let plan = rresult_return!(internal_provider- .insert_into(&ctx.state(), input, insert_op)+ .insert_into(session, input, insert_op).await);- RResult::ROk(FFI_ExecutionPlan::new(- plan,- ctx.task_ctx(),- runtime.clone(),- ))+ RResult::ROk(FFI_ExecutionPlan::new(plan, runtime.clone()))}.into_ffi()}@@ -343,6 +345,7 @@ unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_TableProvider) -> FFI_Tabletable_type: table_type_fn_wrapper,supports_filters_pushdown: provider.supports_filters_pushdown,insert_into: provider.insert_into,+ logical_codec: provider.logical_codec.clone(),clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -363,6 +366,30 @@ impl FFI_TableProvider {provider: Arc<dyn TableProvider + Send>,can_support_pushdown_filters: bool,runtime: Option<Handle>,+ task_ctx_provider: impl Into<FFI_TaskContextProvider>,+ logical_codec: Option<Arc<dyn LogicalExtensionCodec>>,+ ) -> Self {+ let task_ctx_provider = task_ctx_provider.into();+ let logical_codec =+ logical_codec.unwrap_or_else(|| Arc::new(DefaultLogicalExtensionCodec {}));+ let logical_codec = FFI_LogicalExtensionCodec::new(+ logical_codec,+ runtime.clone(),+ task_ctx_provider.clone(),+ );+ Self::new_with_ffi_codec(+ provider,+ can_support_pushdown_filters,+ runtime,+ logical_codec,+ )+ }++ pub fn new_with_ffi_codec(+ provider: Arc<dyn TableProvider + Send>,+ can_support_pushdown_filters: bool,+ runtime: Option<Handle>,+ logical_codec: FFI_LogicalExtensionCodec,) -> Self {let private_data = Box::new(ProviderPrivateData { provider, runtime });@@ -375,6 +402,7 @@ impl FFI_TableProvider {false => None,},insert_into: insert_into_fn_wrapper,+ logical_codec,clone: clone_fn_wrapper,release: release_fn_wrapper,version: super::version,@@ -432,21 +460,21 @@ impl TableProvider for ForeignTableProvider {filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {- let session_config: FFI_SessionConfig = session.config().into();+ let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone());let projections: Option<RVec<usize>> =projection.map(|p| p.iter().map(|v| v.to_owned()).collect());- let codec = DefaultLogicalExtensionCodec {};+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();let filter_list = LogicalExprList {- expr: serialize_exprs(filters, &codec)?,+ expr: serialize_exprs(filters, codec.as_ref())?,};let filters_serialized = filter_list.encode_to_vec().into();let plan = unsafe {let maybe_plan = (self.0.scan)(&self.0,- &session_config,+ session,projections.unwrap_or_default(),filters_serialized,limit.into(),@@ -476,10 +504,13 @@ impl TableProvider for ForeignTableProvider {}};- let codec = DefaultLogicalExtensionCodec {};+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();let expr_list = LogicalExprList {- expr: serialize_exprs(filters.iter().map(|f| f.to_owned()), &codec)?,+ expr: serialize_exprs(+ filters.iter().map(|f| f.to_owned()),+ codec.as_ref(),+ )?,};let serialized_filters = expr_list.encode_to_vec();@@ -495,16 +526,15 @@ impl TableProvider for ForeignTableProvider {input: Arc<dyn ExecutionPlan>,insert_op: InsertOp,) -> Result<Arc<dyn ExecutionPlan>> {- let session_config: FFI_SessionConfig = session.config().into();+ let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone());let rc = Handle::try_current().ok();- let input =- FFI_ExecutionPlan::new(input, Arc::new(TaskContext::from(session)), rc);+ let input = FFI_ExecutionPlan::new(input, rc);let insert_op: FFI_InsertOp = insert_op.into();let plan = unsafe {let maybe_plan =- (self.0.insert_into)(&self.0, &session_config, &input, insert_op).await;+ (self.0.insert_into)(&self.0, session, &input, insert_op).await;<Arc<dyn ExecutionPlan>>::try_from(&df_result!(maybe_plan)?)?};@@ -515,15 +545,17 @@ impl TableProvider for ForeignTableProvider {#[cfg(test)]mod tests {- use super::*;use arrow::datatypes::Schema;- use datafusion::prelude::{col, lit};+ use datafusion::prelude::{SessionContext, col, lit};+ use datafusion_execution::TaskContextProvider;++ use super::*;fn create_test_table_provider() -> Result<Arc<dyn TableProvider>> {use arrow::datatypes::Field;- use datafusion::arrow::{- array::Float32Array, datatypes::DataType, record_batch::RecordBatch,- };+ use datafusion::arrow::array::Float32Array;+ use datafusion::arrow::datatypes::DataType;+ use datafusion::arrow::record_batch::RecordBatch;use datafusion::datasource::MemTable;let schema =@@ -548,9 +580,12 @@ mod tests {#[tokio::test]async fn test_round_trip_ffi_table_provider_scan() -> Result<()> {let provider = create_test_table_provider()?;- let ctx = SessionContext::new();+ let ctx = Arc::new(SessionContext::new());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);- let mut ffi_provider = FFI_TableProvider::new(provider, true, None);+ let mut ffi_provider =+ FFI_TableProvider::new(provider, true, None, task_ctx_provider, None);ffi_provider.library_marker_id = crate::mock_foreign_marker_id;let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_provider).into();@@ -570,9 +605,12 @@ mod tests {#[tokio::test]async fn test_round_trip_ffi_table_provider_insert_into() -> Result<()> {let provider = create_test_table_provider()?;- let ctx = SessionContext::new();+ let ctx = Arc::new(SessionContext::new());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);- let mut ffi_provider = FFI_TableProvider::new(provider, true, None);+ let mut ffi_provider =+ FFI_TableProvider::new(provider, true, None, task_ctx_provider, None);ffi_provider.library_marker_id = crate::mock_foreign_marker_id;let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_provider).into();@@ -600,9 +638,9 @@ mod tests {#[tokio::test]async fn test_aggregation() -> Result<()> {use arrow::datatypes::Field;- use datafusion::arrow::{- array::Float32Array, datatypes::DataType, record_batch::RecordBatch,- };+ use datafusion::arrow::array::Float32Array;+ use datafusion::arrow::datatypes::DataType;+ use datafusion::arrow::record_batch::RecordBatch;use datafusion::common::assert_batches_eq;use datafusion::datasource::MemTable;@@ -615,11 +653,14 @@ mod tests {vec![Arc::new(Float32Array::from(vec![2.0, 4.0, 8.0]))],)?;- let ctx = SessionContext::new();+ let ctx = Arc::new(SessionContext::new());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);let provider = Arc::new(MemTable::try_new(schema, vec![vec![batch1]])?);- let ffi_provider = FFI_TableProvider::new(provider, true, None);+ let ffi_provider =+ FFI_TableProvider::new(provider, true, None, task_ctx_provider, None);let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_provider).into();@@ -646,7 +687,10 @@ mod tests {fn test_ffi_table_provider_local_bypass() -> Result<()> {let table_provider = create_test_table_provider()?;- let mut ffi_table = FFI_TableProvider::new(table_provider, false, None);+ let ctx = Arc::new(SessionContext::new()) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&ctx);+ let mut ffi_table =+ FFI_TableProvider::new(table_provider, false, None, task_ctx_provider, None);// Verify local libraries can be downcast to their originallet foreign_table: Arc<dyn TableProvider> = (&ffi_table).into();diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex 67421f5880..5e43a3e8ad 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -27,6 +27,8 @@use std::{any::Any, fmt::Debug, sync::Arc};+use super::create_record_batch;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use crate::table_provider::FFI_TableProvider;use arrow::array::RecordBatch;use arrow::datatypes::Schema;@@ -46,8 +48,6 @@ use tokio::{sync::{broadcast, mpsc},};-use super::create_record_batch;-#[derive(Debug)]pub struct AsyncTableProvider {batch_request: mpsc::Sender<bool>,@@ -277,7 +277,14 @@ impl Stream for AsyncTestRecordBatchStream {}}-pub(crate) fn create_async_table_provider() -> FFI_TableProvider {+pub(crate) fn create_async_table_provider(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_TableProvider {let (table_provider, tokio_rt) = start_async_provider();- FFI_TableProvider::new(Arc::new(table_provider), true, Some(tokio_rt))+ FFI_TableProvider::new_with_ffi_codec(+ Arc::new(table_provider),+ true,+ Some(tokio_rt),+ codec,+ )}diff --git a/datafusion/ffi/src/tests/catalog.rs b/datafusion/ffi/src/tests/catalog.rsindex de012659f2..122971eb0c 100644--- a/datafusion/ffi/src/tests/catalog.rs+++ b/datafusion/ffi/src/tests/catalog.rs@@ -29,6 +29,7 @@ use std::{any::Any, fmt::Debug, sync::Arc};use crate::catalog_provider::FFI_CatalogProvider;use crate::catalog_provider_list::FFI_CatalogProviderList;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use arrow::datatypes::Schema;use async_trait::async_trait;use datafusion::{@@ -180,9 +181,11 @@ impl CatalogProvider for FixedCatalogProvider {}}-pub(crate) extern "C" fn create_catalog_provider() -> FFI_CatalogProvider {+pub(crate) extern "C" fn create_catalog_provider(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_CatalogProvider {let catalog_provider = Arc::new(FixedCatalogProvider::default());- FFI_CatalogProvider::new(catalog_provider, None)+ FFI_CatalogProvider::new_with_ffi_codec(catalog_provider, None, codec)}/// This catalog provider list is intended only for unit tests. It prepopulates with one@@ -234,7 +237,9 @@ impl CatalogProviderList for FixedCatalogProviderList {}}-pub(crate) extern "C" fn create_catalog_provider_list() -> FFI_CatalogProviderList {+pub(crate) extern "C" fn create_catalog_provider_list(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_CatalogProviderList {let catalog_provider_list = Arc::new(FixedCatalogProviderList::default());- FFI_CatalogProviderList::new(catalog_provider_list, None)+ FFI_CatalogProviderList::new_with_ffi_codec(catalog_provider_list, None, codec)}diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex dfb4756107..e87d465db0 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -34,6 +34,7 @@ use crate::udwf::FFI_WindowUDF;use super::{table_provider::FFI_TableProvider, udf::FFI_ScalarUDF};use crate::catalog_provider_list::FFI_CatalogProviderList;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use crate::tests::catalog::create_catalog_provider_list;use arrow::array::RecordBatch;use async_provider::create_async_table_provider;@@ -61,20 +62,26 @@ pub mod utils;/// module.pub struct ForeignLibraryModule {/// Construct an opinionated catalog provider- pub create_catalog: extern "C" fn() -> FFI_CatalogProvider,+ pub create_catalog:+ extern "C" fn(codec: FFI_LogicalExtensionCodec) -> FFI_CatalogProvider,/// Construct an opinionated catalog provider list- pub create_catalog_list: extern "C" fn() -> FFI_CatalogProviderList,+ pub create_catalog_list:+ extern "C" fn(codec: FFI_LogicalExtensionCodec) -> FFI_CatalogProviderList,/// Constructs the table provider- pub create_table: extern "C" fn(synchronous: bool) -> FFI_TableProvider,+ pub create_table: extern "C" fn(+ synchronous: bool,+ codec: FFI_LogicalExtensionCodec,+ ) -> FFI_TableProvider,/// Create a scalar UDFpub create_scalar_udf: extern "C" fn() -> FFI_ScalarUDF,pub create_nullary_udf: extern "C" fn() -> FFI_ScalarUDF,- pub create_table_function: extern "C" fn() -> FFI_TableFunction,+ pub create_table_function:+ extern "C" fn(FFI_LogicalExtensionCodec) -> FFI_TableFunction,/// Create an aggregate UDAF using sumpub create_sum_udaf: extern "C" fn() -> FFI_AggregateUDF,@@ -115,10 +122,13 @@ pub fn create_record_batch(start_value: i32, num_values: usize) -> RecordBatch {/// Here we only wish to create a simple table provider as an example./// We create an in-memory table and convert it to it's FFI counterpart.-extern "C" fn construct_table_provider(synchronous: bool) -> FFI_TableProvider {+extern "C" fn construct_table_provider(+ synchronous: bool,+ codec: FFI_LogicalExtensionCodec,+) -> FFI_TableProvider {match synchronous {- true => create_sync_table_provider(),- false => create_async_table_provider(),+ true => create_sync_table_provider(codec),+ false => create_async_table_provider(codec),}}diff --git a/datafusion/ffi/src/tests/sync_provider.rs b/datafusion/ffi/src/tests/sync_provider.rsindex ff85e0b15b..cbad8343dd 100644--- a/datafusion/ffi/src/tests/sync_provider.rs+++ b/datafusion/ffi/src/tests/sync_provider.rs@@ -17,12 +17,14 @@use std::sync::Arc;+use super::{create_record_batch, create_test_schema};+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;use crate::table_provider::FFI_TableProvider;use datafusion::datasource::MemTable;-use super::{create_record_batch, create_test_schema};--pub(crate) fn create_sync_table_provider() -> FFI_TableProvider {+pub(crate) fn create_sync_table_provider(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_TableProvider {let schema = create_test_schema();// It is useful to create these as multiple record batches@@ -35,5 +37,5 @@ pub(crate) fn create_sync_table_provider() -> FFI_TableProvider {let table_provider = MemTable::try_new(schema, vec![batches]).unwrap();- FFI_TableProvider::new(Arc::new(table_provider), true, None)+ FFI_TableProvider::new_with_ffi_codec(Arc::new(table_provider), true, None, codec)}diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rsindex 55e31ef3ab..5eee18f00a 100644--- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs+++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs@@ -27,25 +27,61 @@ use datafusion::{functions_window::rank::Rank,logical_expr::{AggregateUDF, ScalarUDF, WindowUDF},};+use std::any::Any;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use arrow_schema::DataType;+use datafusion::logical_expr::{ColumnarValue, Signature};+use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};use std::sync::Arc;pub(crate) extern "C" fn create_ffi_abs_func() -> FFI_ScalarUDF {- let udf: Arc<ScalarUDF> = Arc::new(AbsFunc::new().into());+ let inner = WrappedAbs(Arc::new(AbsFunc::new().into()));+ let udf: Arc<ScalarUDF> = Arc::new(inner.into());udf.into()}+#[derive(Debug, Hash, Eq, PartialEq)]+struct WrappedAbs(Arc<ScalarUDF>);++impl ScalarUDFImpl for WrappedAbs {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "ffi_abs"+ }++ fn signature(&self) -> &Signature {+ self.0.signature()+ }++ fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result<DataType> {+ self.0.return_type(arg_types)+ }++ fn invoke_with_args(+ &self,+ args: ScalarFunctionArgs,+ ) -> datafusion_common::Result<ColumnarValue> {+ self.0.invoke_with_args(args)+ }+}+pub(crate) extern "C" fn create_ffi_random_func() -> FFI_ScalarUDF {let udf: Arc<ScalarUDF> = Arc::new(RandomFunc::new().into());udf.into()}-pub(crate) extern "C" fn create_ffi_table_func() -> FFI_TableFunction {+pub(crate) extern "C" fn create_ffi_table_func(+ codec: FFI_LogicalExtensionCodec,+) -> FFI_TableFunction {let udtf: Arc<dyn TableFunctionImpl> = Arc::new(RangeFunc {});- FFI_TableFunction::new(udtf, None)+ FFI_TableFunction::new_with_ffi_codec(udtf, None, codec)}pub(crate) extern "C" fn create_ffi_sum_func() -> FFI_AggregateUDF {diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rsindex a5cc370436..c5b7b7110e 100644--- a/datafusion/ffi/src/udaf/accumulator_args.rs+++ b/datafusion/ffi/src/udaf/accumulator_args.rs@@ -17,29 +17,21 @@use std::sync::Arc;-use crate::arrow_wrappers::WrappedSchema;use abi_stable::{StableAbi,std_types::{RString, RVec},};use arrow::{datatypes::Schema, ffi::FFI_ArrowSchema};use arrow_schema::FieldRef;-use datafusion::{- error::DataFusionError,- logical_expr::function::AccumulatorArgs,- physical_expr::{PhysicalExpr, PhysicalSortExpr},- prelude::SessionContext,+use datafusion_common::error::DataFusionError;+use datafusion_expr::function::AccumulatorArgs;+use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};++use crate::{+ arrow_wrappers::WrappedSchema,+ physical_expr::{FFI_PhysicalExpr, sort::FFI_PhysicalSortExpr},+ util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped},};-use datafusion_common::ffi_datafusion_err;-use datafusion_proto::{- physical_plan::{- DefaultPhysicalExtensionCodec,- from_proto::{parse_physical_exprs, parse_physical_sort_exprs},- to_proto::{serialize_physical_exprs, serialize_physical_sort_exprs},- },- protobuf::PhysicalAggregateExprNode,-};-use prost::Message;/// A stable struct for sharing [`AccumulatorArgs`] across FFI boundaries./// For an explanation of each field, see the corresponding field@@ -50,42 +42,47 @@ use prost::Message;pub struct FFI_AccumulatorArgs {return_field: WrappedSchema,schema: WrappedSchema,+ ignore_nulls: bool,+ order_bys: RVec<FFI_PhysicalSortExpr>,is_reversed: bool,name: RString,- physical_expr_def: RVec<u8>,+ is_distinct: bool,+ exprs: RVec<FFI_PhysicalExpr>,+ expr_fields: RVec<WrappedSchema>,}impl TryFrom<AccumulatorArgs<'_>> for FFI_AccumulatorArgs {type Error = DataFusionError;-- fn try_from(args: AccumulatorArgs) -> Result<Self, Self::Error> {+ fn try_from(args: AccumulatorArgs) -> Result<Self, DataFusionError> {let return_field =WrappedSchema(FFI_ArrowSchema::try_from(args.return_field.as_ref())?);let schema = WrappedSchema(FFI_ArrowSchema::try_from(args.schema)?);- let codec = DefaultPhysicalExtensionCodec {};- let ordering_req =- serialize_physical_sort_exprs(args.order_bys.to_owned(), &codec)?;+ let order_bys: RVec<_> = args+ .order_bys+ .iter()+ .map(FFI_PhysicalSortExpr::from)+ .collect();- let expr = serialize_physical_exprs(args.exprs, &codec)?;+ let exprs = args+ .exprs+ .iter()+ .map(Arc::clone)+ .map(FFI_PhysicalExpr::from)+ .collect();- let physical_expr_def = PhysicalAggregateExprNode {- expr,- ordering_req,- distinct: args.is_distinct,- ignore_nulls: args.ignore_nulls,- fun_definition: None,- aggregate_function: None,- human_display: args.name.to_string(),- };- let physical_expr_def = physical_expr_def.encode_to_vec().into();+ let expr_fields = vec_fieldref_to_rvec_wrapped(args.expr_fields)?;Ok(Self {return_field,schema,+ ignore_nulls: args.ignore_nulls,+ order_bys,is_reversed: args.is_reversed,name: args.name.into(),- physical_expr_def,+ is_distinct: args.is_distinct,+ exprs,+ expr_fields,})}}@@ -110,43 +107,28 @@ impl TryFrom<FFI_AccumulatorArgs> for ForeignAccumulatorArgs {type Error = DataFusionError;fn try_from(value: FFI_AccumulatorArgs) -> Result<Self, Self::Error> {- let proto_def = PhysicalAggregateExprNode::decode(- value.physical_expr_def.as_ref(),- )- .map_err(|e| {- ffi_datafusion_err!("Failed to decode PhysicalAggregateExprNode: {e}")- })?;-let return_field = Arc::new((&value.return_field.0).try_into()?);let schema = Schema::try_from(&value.schema.0)?;- let default_ctx = SessionContext::new();- let task_ctx = default_ctx.task_ctx();- let codex = DefaultPhysicalExtensionCodec {};-- let order_bys = parse_physical_sort_exprs(- &proto_def.ordering_req,- &task_ctx,- &schema,- &codex,- )?;-- let exprs = parse_physical_exprs(&proto_def.expr, &task_ctx, &schema, &codex)?;+ let order_bys = value.order_bys.iter().map(PhysicalSortExpr::from).collect();- let expr_fields = exprs+ let exprs = value+ .exprs.iter()- .map(|e| e.return_field(&schema))- .collect::<Result<Vec<_>, _>>()?;+ .map(<Arc<dyn PhysicalExpr>>::from)+ .collect();++ let expr_fields = rvec_wrapped_to_vec_fieldref(&value.expr_fields)?;Ok(Self {return_field,schema,expr_fields,- ignore_nulls: proto_def.ignore_nulls,+ ignore_nulls: value.ignore_nulls,order_bys,is_reversed: value.is_reversed,name: value.name.to_string(),- is_distinct: proto_def.distinct,+ is_distinct: value.is_distinct,exprs,})}@@ -170,13 +152,14 @@ impl<'a> From<&'a ForeignAccumulatorArgs> for AccumulatorArgs<'a> {#[cfg(test)]mod tests {- use super::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};use arrow::datatypes::{DataType, Field, Schema};use datafusion::{error::Result, logical_expr::function::AccumulatorArgs,physical_expr::PhysicalSortExpr, physical_plan::expressions::col,};+ use super::{FFI_AccumulatorArgs, ForeignAccumulatorArgs};+#[test]fn test_round_trip_accumulator_args() -> Result<()> {let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);@@ -193,7 +176,7 @@ mod tests {};let orig_str = format!("{orig_args:?}");- let ffi_args: FFI_AccumulatorArgs = orig_args.try_into()?;+ let ffi_args = FFI_AccumulatorArgs::try_from(orig_args)?;let foreign_args: ForeignAccumulatorArgs = ffi_args.try_into()?;let round_trip_args: AccumulatorArgs = (&foreign_args).into();diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rsindex ccecee2217..bd35839d96 100644--- a/datafusion/ffi/src/udtf.rs+++ b/datafusion/ffi/src/udtf.rs@@ -15,29 +15,29 @@// specific language governing permissions and limitations// under the License.-use std::{ffi::c_void, sync::Arc};--use abi_stable::{- StableAbi,- std_types::{RResult, RVec},-};--use datafusion::error::Result;-use datafusion::{- catalog::{TableFunctionImpl, TableProvider},- prelude::{Expr, SessionContext},-};-use datafusion_proto::{- logical_plan::{- DefaultLogicalExtensionCodec, from_proto::parse_exprs, to_proto::serialize_exprs,- },- protobuf::LogicalExprList,+use std::ffi::c_void;+use std::sync::Arc;++use abi_stable::StableAbi;+use abi_stable::std_types::{RResult, RVec};+use datafusion_catalog::{TableFunctionImpl, TableProvider};+use datafusion_common::error::Result;+use datafusion_execution::TaskContext;+use datafusion_expr::Expr;+use datafusion_proto::logical_plan::from_proto::parse_exprs;+use datafusion_proto::logical_plan::to_proto::serialize_exprs;+use datafusion_proto::logical_plan::{+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,};+use datafusion_proto::protobuf::LogicalExprList;use prost::Message;use tokio::runtime::Handle;+use crate::execution::FFI_TaskContextProvider;+use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::table_provider::FFI_TableProvider;use crate::util::FFIResult;-use crate::{df_result, rresult_return, table_provider::FFI_TableProvider};+use crate::{df_result, rresult_return};/// A stable struct for sharing a [`TableFunctionImpl`] across FFI boundaries.#[repr(C)]@@ -49,6 +49,8 @@ pub struct FFI_TableFunction {pub call:unsafe extern "C" fn(udtf: &Self, args: RVec<u8>) -> FFIResult<FFI_TableProvider>,+ pub logical_codec: FFI_LogicalExtensionCodec,+/// Used to create a clone on the provider of the udtf. This should/// only need to be called by the receiver of the udtf.pub clone: unsafe extern "C" fn(udtf: &Self) -> Self,@@ -91,18 +93,27 @@ unsafe extern "C" fn call_fn_wrapper(args: RVec<u8>,) -> FFIResult<FFI_TableProvider> {let runtime = udtf.runtime();- let udtf = udtf.inner();+ let udtf_inner = udtf.inner();- let default_ctx = SessionContext::new();- let codec = DefaultLogicalExtensionCodec {};+ let ctx: Arc<TaskContext> =+ rresult_return!((&udtf.logical_codec.task_ctx_provider).try_into());+ let codec: Arc<dyn LogicalExtensionCodec> = (&udtf.logical_codec).into();let proto_filters = rresult_return!(LogicalExprList::decode(args.as_ref()));- let args =- rresult_return!(parse_exprs(proto_filters.expr.iter(), &default_ctx, &codec));-- let table_provider = rresult_return!(udtf.call(&args));- RResult::ROk(FFI_TableProvider::new(table_provider, false, runtime))+ let args = rresult_return!(parse_exprs(+ proto_filters.expr.iter(),+ ctx.as_ref(),+ codec.as_ref()+ ));++ let table_provider = rresult_return!(udtf_inner.call(&args));+ RResult::ROk(FFI_TableProvider::new_with_ffi_codec(+ table_provider,+ false,+ runtime,+ udtf.logical_codec.clone(),+ ))}unsafe extern "C" fn release_fn_wrapper(udtf: &mut FFI_TableFunction) {@@ -117,9 +128,13 @@ unsafe extern "C" fn release_fn_wrapper(udtf: &mut FFI_TableFunction) {unsafe extern "C" fn clone_fn_wrapper(udtf: &FFI_TableFunction) -> FFI_TableFunction {let runtime = udtf.runtime();- let udtf = udtf.inner();+ let udtf_inner = udtf.inner();- FFI_TableFunction::new(Arc::clone(udtf), runtime)+ FFI_TableFunction::new_with_ffi_codec(+ Arc::clone(udtf_inner),+ runtime,+ udtf.logical_codec.clone(),+ )}impl Clone for FFI_TableFunction {@@ -129,28 +144,34 @@ impl Clone for FFI_TableFunction {}impl FFI_TableFunction {- pub fn new(udtf: Arc<dyn TableFunctionImpl>, runtime: Option<Handle>) -> Self {- let private_data = Box::new(TableFunctionPrivateData { udtf, runtime });-- Self {- call: call_fn_wrapper,- clone: clone_fn_wrapper,- release: release_fn_wrapper,- private_data: Box::into_raw(private_data) as *mut c_void,- library_marker_id: crate::get_library_marker_id,- }+ pub fn new(+ udtf: Arc<dyn TableFunctionImpl>,+ runtime: Option<Handle>,+ task_ctx_provider: impl Into<FFI_TaskContextProvider>,+ logical_codec: Option<Arc<dyn LogicalExtensionCodec>>,+ ) -> Self {+ let task_ctx_provider = task_ctx_provider.into();+ let logical_codec =+ logical_codec.unwrap_or_else(|| Arc::new(DefaultLogicalExtensionCodec {}));+ let logical_codec = FFI_LogicalExtensionCodec::new(+ logical_codec,+ runtime.clone(),+ task_ctx_provider.clone(),+ );++ Self::new_with_ffi_codec(udtf, runtime, logical_codec)}-}-impl From<Arc<dyn TableFunctionImpl>> for FFI_TableFunction {- fn from(udtf: Arc<dyn TableFunctionImpl>) -> Self {- let private_data = Box::new(TableFunctionPrivateData {- udtf,- runtime: None,- });+ pub fn new_with_ffi_codec(+ udtf: Arc<dyn TableFunctionImpl>,+ runtime: Option<Handle>,+ logical_codec: FFI_LogicalExtensionCodec,+ ) -> Self {+ let private_data = Box::new(TableFunctionPrivateData { udtf, runtime });Self {call: call_fn_wrapper,+ logical_codec,clone: clone_fn_wrapper,release: release_fn_wrapper,private_data: Box::into_raw(private_data) as *mut c_void,@@ -189,9 +210,9 @@ impl From<FFI_TableFunction> for Arc<dyn TableFunctionImpl> {impl TableFunctionImpl for ForeignTableFunction {fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {- let codec = DefaultLogicalExtensionCodec {};+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();let expr_list = LogicalExprList {- expr: serialize_exprs(args, &codec)?,+ expr: serialize_exprs(args, codec.as_ref())?,};let filters_serialized = expr_list.encode_to_vec().into();@@ -206,17 +227,18 @@ impl TableFunctionImpl for ForeignTableFunction {#[cfg(test)]mod tests {- use super::*;- use arrow::{- array::{- ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array, record_batch,- },- datatypes::{DataType, Field, Schema},+ use arrow::array::{+ ArrayRef, Float64Array, RecordBatch, StringArray, UInt64Array, record_batch,};+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion::catalog::MemTable;+ use datafusion::common::exec_err;use datafusion::logical_expr::ptr_eq::arc_ptr_eq;- use datafusion::{- catalog::MemTable, common::exec_err, prelude::lit, scalar::ScalarValue,- };+ use datafusion::prelude::{SessionContext, lit};+ use datafusion::scalar::ScalarValue;+ use datafusion_execution::TaskContextProvider;++ use super::*;#[derive(Debug)]struct TestUDTF {}@@ -299,16 +321,22 @@ mod tests {#[tokio::test]async fn test_round_trip_udtf() -> Result<()> {let original_udtf = Arc::new(TestUDTF {}) as Arc<dyn TableFunctionImpl>;-- let mut local_udtf: FFI_TableFunction =- FFI_TableFunction::new(Arc::clone(&original_udtf), None);+ let ctx = Arc::new(SessionContext::default());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);++ let mut local_udtf: FFI_TableFunction = FFI_TableFunction::new(+ Arc::clone(&original_udtf),+ None,+ task_ctx_provider,+ None,+ );local_udtf.library_marker_id = crate::mock_foreign_marker_id;let foreign_udf: Arc<dyn TableFunctionImpl> = local_udtf.into();let table = foreign_udf.call(&[lit(6_u64), lit("one"), lit(2.0), lit(3_u64)])?;- let ctx = SessionContext::default();let _ = ctx.register_table("test-table", table)?;let returned_batches = ctx.table("test-table").await?.collect().await?;@@ -335,7 +363,14 @@ mod tests {fn test_ffi_udtf_local_bypass() -> Result<()> {let original_udtf = Arc::new(TestUDTF {}) as Arc<dyn TableFunctionImpl>;- let mut ffi_udtf = FFI_TableFunction::from(Arc::clone(&original_udtf));+ let ctx = Arc::new(SessionContext::default()) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&ctx);+ let mut ffi_udtf = FFI_TableFunction::new(+ Arc::clone(&original_udtf),+ None,+ task_ctx_provider,+ None,+ );// Verify local libraries can be downcast to their originallet foreign_udtf: Arc<dyn TableFunctionImpl> = ffi_udtf.clone().into();diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rsindex e272b14166..20238fdb8b 100644--- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs+++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs@@ -15,31 +15,19 @@// specific language governing permissions and limitations// under the License.-use std::{collections::HashMap, sync::Arc};+use std::sync::Arc;-use crate::arrow_wrappers::WrappedSchema;use abi_stable::{StableAbi, std_types::RVec};-use arrow::{- datatypes::{DataType, Field, Schema, SchemaRef},- error::ArrowError,- ffi::FFI_ArrowSchema,-};+use arrow::{error::ArrowError, ffi::FFI_ArrowSchema};use arrow_schema::FieldRef;-use datafusion::{- error::{DataFusionError, Result},- logical_expr::function::PartitionEvaluatorArgs,- physical_plan::{PhysicalExpr, expressions::Column},- prelude::SessionContext,-};-use datafusion_common::ffi_datafusion_err;-use datafusion_proto::{- physical_plan::{- DefaultPhysicalExtensionCodec, from_proto::parse_physical_expr,- to_proto::serialize_physical_exprs,- },- protobuf::PhysicalExprNode,+use datafusion_common::{DataFusionError, Result};+use datafusion_expr::function::PartitionEvaluatorArgs;+use datafusion_physical_plan::PhysicalExpr;++use crate::{+ arrow_wrappers::WrappedSchema, physical_expr::FFI_PhysicalExpr,+ util::rvec_wrapped_to_vec_fieldref,};-use prost::Message;/// A stable struct for sharing [`PartitionEvaluatorArgs`] across FFI boundaries./// For an explanation of each field, see the corresponding function@@ -48,58 +36,21 @@ use prost::Message;#[derive(Debug, StableAbi)]#[allow(non_camel_case_types)]pub struct FFI_PartitionEvaluatorArgs {- input_exprs: RVec<RVec<u8>>,+ input_exprs: RVec<FFI_PhysicalExpr>,input_fields: RVec<WrappedSchema>,is_reversed: bool,ignore_nulls: bool,- schema: WrappedSchema,}impl TryFrom<PartitionEvaluatorArgs<'_>> for FFI_PartitionEvaluatorArgs {type Error = DataFusionError;+fn try_from(args: PartitionEvaluatorArgs) -> Result<Self, DataFusionError> {- // This is a bit of a hack. Since PartitionEvaluatorArgs does not carry a schema- // around, and instead passes the data types directly we are unable to decode the- // protobuf PhysicalExpr correctly. In evaluating the code the only place these- // appear to be really used are the Column data types. So here we will find all- // of the required columns and create a schema that has empty fields except for- // the ones we require. Ideally we would enhance PartitionEvaluatorArgs to just- // pass along the schema, but that is a larger breaking change.- let required_columns: HashMap<usize, (&str, &DataType)> = args+ let input_exprs = args.input_exprs().iter()- .zip(args.input_fields())- .filter_map(|(expr, field)| {- expr.as_any()- .downcast_ref::<Column>()- .map(|column| (column.index(), (column.name(), field.data_type())))- })- .collect();-- let max_column = required_columns.keys().max();- let fields: Vec<_> = max_column- .map(|max_column| {- (0..(max_column + 1))- .map(|idx| match required_columns.get(&idx) {- Some((name, data_type)) => {- Field::new(*name, (*data_type).clone(), true)- }- None => Field::new(- format!("ffi_partition_evaluator_col_{idx}"),- DataType::Null,- true,- ),- })- .collect()- })- .unwrap_or_default();-- let schema = Arc::new(Schema::new(fields));-- let codec = DefaultPhysicalExtensionCodec {};- let input_exprs = serialize_physical_exprs(args.input_exprs(), &codec)?- .into_iter()- .map(|expr_node| expr_node.encode_to_vec().into())+ .map(Arc::clone)+ .map(FFI_PhysicalExpr::from).collect();let input_fields = args@@ -109,12 +60,9 @@ impl TryFrom<PartitionEvaluatorArgs<'_>> for FFI_PartitionEvaluatorArgs {.collect::<Result<Vec<_>, ArrowError>>()?.into();- let schema: WrappedSchema = schema.into();-Ok(Self {input_exprs,input_fields,- schema,is_reversed: args.is_reversed(),ignore_nulls: args.ignore_nulls(),})@@ -136,27 +84,9 @@ impl TryFrom<FFI_PartitionEvaluatorArgs> for ForeignPartitionEvaluatorArgs {type Error = DataFusionError;fn try_from(value: FFI_PartitionEvaluatorArgs) -> Result<Self> {- let default_ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};-- let schema: SchemaRef = value.schema.into();+ let input_exprs = value.input_exprs.iter().map(Into::into).collect();- let input_exprs = value- .input_exprs- .into_iter()- .map(|input_expr_bytes| PhysicalExprNode::decode(input_expr_bytes.as_ref()))- .collect::<std::result::Result<Vec<_>, prost::DecodeError>>()- .map_err(|e| ffi_datafusion_err!("Failed to decode PhysicalExprNode: {e}"))?- .iter()- .map(|expr_node| {- parse_physical_expr(expr_node, &default_ctx.task_ctx(), &schema, &codec)- })- .collect::<Result<Vec<_>>>()?;-- let input_fields = input_exprs- .iter()- .map(|expr| expr.return_field(&schema))- .collect::<Result<Vec<_>>>()?;+ let input_fields = rvec_wrapped_to_vec_fieldref(&value.input_fields)?;Ok(Self {input_exprs,diff --git a/datafusion/ffi/tests/ffi_catalog.rs b/datafusion/ffi/tests/ffi_catalog.rsindex c45a62ee50..a464d3df5a 100644--- a/datafusion/ffi/tests/ffi_catalog.rs+++ b/datafusion/ffi/tests/ffi_catalog.rs@@ -15,12 +15,13 @@// specific language governing permissions and limitations// under the License.+mod utils;+/// Add an additional module here for convenience to scope this to only/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {use datafusion::catalog::{CatalogProvider, CatalogProviderList};- use datafusion::prelude::SessionContext;use datafusion_common::DataFusionError;use datafusion_ffi::tests::utils::get_module;use std::sync::Arc;@@ -28,6 +29,7 @@ mod tests {#[tokio::test]async fn test_catalog() -> datafusion_common::Result<()> {let module = get_module()?;+ let (ctx, codec) = super::utils::ctx_and_codec();let ffi_catalog =module@@ -35,10 +37,9 @@ mod tests {.ok_or(DataFusionError::NotImplemented("External catalog provider failed to implement create_catalog".to_string(),- ))?();+ ))?(codec);let foreign_catalog: Arc<dyn CatalogProvider + Send> = (&ffi_catalog).into();- let ctx = SessionContext::default();let _ = ctx.register_catalog("fruit", foreign_catalog);let df = ctx.table("fruit.apple.purchases").await?;@@ -55,6 +56,7 @@ mod tests {#[tokio::test]async fn test_catalog_list() -> datafusion_common::Result<()> {let module = get_module()?;+ let (ctx, codec) = super::utils::ctx_and_codec();let ffi_catalog_list =module@@ -62,11 +64,10 @@ mod tests {.ok_or(DataFusionError::NotImplemented("External catalog provider failed to implement create_catalog_list".to_string(),- ))?();+ ))?(codec);let foreign_catalog_list: Arc<dyn CatalogProviderList + Send> =(&ffi_catalog_list).into();- let ctx = SessionContext::default();ctx.register_catalog_list(foreign_catalog_list);let df = ctx.table("blue.apple.purchases").await?;diff --git a/datafusion/ffi/tests/ffi_integration.rs b/datafusion/ffi/tests/ffi_integration.rsindex 216d6576a8..78650abf2f 100644--- a/datafusion/ffi/tests/ffi_integration.rs+++ b/datafusion/ffi/tests/ffi_integration.rs@@ -15,13 +15,14 @@// specific language governing permissions and limitations// under the License.+mod utils;+/// Add an additional module here for convenience to scope this to only/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {use datafusion::catalog::TableProvider;use datafusion::error::{DataFusionError, Result};- use datafusion::prelude::SessionContext;use datafusion_ffi::tests::create_record_batch;use datafusion_ffi::tests::utils::get_module;use std::sync::Arc;@@ -31,6 +32,7 @@ mod tests {/// testing it via a different executable.async fn test_table_provider(synchronous: bool) -> Result<()> {let table_provider_module = get_module()?;+ let (ctx, codec) = super::utils::ctx_and_codec();// By calling the code below, the table provided will be created within// the module's code.@@ -38,14 +40,12 @@ mod tests {DataFusionError::NotImplemented("External table provider failed to implement create_table".to_string(),),- )?(synchronous);+ )?(synchronous, codec);// In order to access the table provider within this executable, we need to// turn it into a `TableProvider`.let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_table_provider).into();- let ctx = SessionContext::new();-// Display the data to show the full cycle works.ctx.register_table("external_table", foreign_table_provider)?;let df = ctx.table("external_table").await?;diff --git a/datafusion/ffi/tests/ffi_udaf.rs b/datafusion/ffi/tests/ffi_udaf.rsindex 6aac88d494..f219979a85 100644--- a/datafusion/ffi/tests/ffi_udaf.rs+++ b/datafusion/ffi/tests/ffi_udaf.rs@@ -19,13 +19,15 @@/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]mod tests {+ use std::sync::Arc;+use arrow::array::Float64Array;use datafusion::common::record_batch;use datafusion::error::{DataFusionError, Result};use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};use datafusion::prelude::{SessionContext, col};- use std::sync::Arc;-+ use datafusion_catalog::MemTable;+ use datafusion_expr::{ScalarUDF, ScalarUDFImpl};use datafusion_ffi::tests::utils::get_module;#[tokio::test]@@ -126,4 +128,69 @@ mod tests {Ok(())}++ /// This test FFI UDFs can be used as inputs to FFI Aggregate UDFs.+ /// Really this is a test of the Protobuf serialization and deserialization+ /// using the TaskContextProvider. It can be demonstrated through the+ /// UDAF accumulator arguments as an end-to-end test.+ #[tokio::test]+ async fn udf_as_input_to_udf() -> Result<()> {+ let module = get_module()?;++ let ffi_abs_func =+ module+ .create_scalar_udf()+ .ok_or(DataFusionError::NotImplemented(+ "External table provider failed to implement create_scalar_udf"+ .to_string(),+ ))?();+ let foreign_abs_func: Arc<dyn ScalarUDFImpl> = (&ffi_abs_func).into();+ let abs_udf = ScalarUDF::new_from_shared_impl(foreign_abs_func);++ let ctx = SessionContext::new();+ ctx.deregister_udf("abs");++ let ffi_sum_func =+ module+ .create_sum_udaf()+ .ok_or(DataFusionError::NotImplemented(+ "External table provider failed to implement create_udaf".to_string(),+ ))?();+ let foreign_sum_func: Arc<dyn AggregateUDFImpl> = (&ffi_sum_func).into();++ let udaf = AggregateUDF::new_from_shared_impl(foreign_sum_func);++ // We need at least 2 record batches so we get an accumulator+ let ctx = SessionContext::default();+ let rb1 = record_batch!(+ ("a", Int32, vec![1, 2, 2, 4, 4, 4, 4]),+ ("b", Float64, vec![-1.0, 2.0, -2.0, 4.0, -4.0, -4.0, -4.0])+ )+ .unwrap();+ let rb2 = rb1.clone();++ let table = Arc::new(MemTable::try_new(rb1.schema(), vec![vec![rb1, rb2]])?);++ let df = ctx.read_table(table)?;++ let df = df+ .aggregate(+ vec![col("a")],+ vec![udaf.call(vec![abs_udf.call(vec![col("b")])]).alias("sum_b")],+ )?+ .sort_by(vec![col("a")])?;++ df.clone().show().await?;++ let result = df.collect().await?;++ let expected = record_batch!(+ ("a", Int32, vec![1, 2, 4]),+ ("sum_b", Float64, vec![2.0, 8.0, 32.0])+ )?;++ assert_eq!(result[0], expected);++ Ok(())+ }}diff --git a/datafusion/ffi/tests/ffi_udtf.rs b/datafusion/ffi/tests/ffi_udtf.rsindex a0135a903f..097ed75f35 100644--- a/datafusion/ffi/tests/ffi_udtf.rs+++ b/datafusion/ffi/tests/ffi_udtf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+mod utils;+/// Add an additional module here for convenience to scope this to only/// when the feature integration-tests is built#[cfg(feature = "integration-tests")]@@ -25,7 +27,6 @@ mod tests {use arrow::array::{ArrayRef, create_array};use datafusion::catalog::TableFunctionImpl;use datafusion::error::{DataFusionError, Result};- use datafusion::prelude::SessionContext;use datafusion_ffi::tests::utils::get_module;@@ -35,16 +36,16 @@ mod tests {#[tokio::test]async fn test_user_defined_table_function() -> Result<()> {let module = get_module()?;+ let (ctx, codec) = super::utils::ctx_and_codec();let ffi_table_func = module.create_table_function().ok_or(DataFusionError::NotImplemented("External table function provider failed to implement create_table_function".to_string(),- ))?();+ ))?(codec);let foreign_table_func: Arc<dyn TableFunctionImpl> = ffi_table_func.into();- let ctx = SessionContext::default();ctx.register_udtf("my_range", foreign_table_func);let result = ctxdiff --git a/datafusion/ffi/tests/utils/mod.rs b/datafusion/ffi/tests/utils/mod.rsnew file mode 100644index 0000000000..25d1464811--- /dev/null+++ b/datafusion/ffi/tests/utils/mod.rs@@ -0,0 +1,36 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use datafusion::prelude::SessionContext;+use datafusion_execution::TaskContextProvider;+use datafusion_ffi::execution::FFI_TaskContextProvider;+use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec;+use std::sync::Arc;++pub fn ctx_and_codec() -> (Arc<SessionContext>, FFI_LogicalExtensionCodec) {+ let ctx = Arc::new(SessionContext::default());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);+ let codec = FFI_LogicalExtensionCodec::new(+ Arc::new(DefaultLogicalExtensionCodec {}),+ None,+ task_ctx_provider,+ );++ (ctx, codec)+}diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex c12768c95a..39d52bd590 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -396,6 +396,47 @@ Instead this should now be:let foreign_udf = ScalarUDF::new_from_shared_impl(foreign_udf);```+When creating any of the following structs, we now require the user to+provide a `TaskContextProvider` and optionally a `LogicalExtensionCodec`:++- `FFI_CatalogListProvider`+- `FFI_CatalogProvider`+- `FFI_SchemaProvider`+- `FFI_TableProvider`+- `FFI_TableFunction`++Each of these structs has a `new()` and a `new_with_ffi_codec()` method for+instantiation. For example, when you previously would write++```rust,ignore+ let table = Arc::new(MyTableProvider::new());+ let ffi_table = FFI_TableProvider::new(table, None);+```++Now you will need to provide a `TaskContextProvider`. The most common+implementation of this trait is `SessionContext`.++```rust,ignore+ let ctx = Arc::new(SessionContext::default());+ let table = Arc::new(MyTableProvider::new());+ let ffi_table = FFI_TableProvider::new(table, None, ctx, None);+```++The alternative function to create these structures may be more convenient+if you are doing many of these operations. A `FFI_LogicalExtensionCodec` will+store the `TaskContextProvider` as well.++```rust,ignore+ let codec = Arc::new(DefaultLogicalExtensionCodec {});+ let ctx = Arc::new(SessionContext::default());+ let ffi_codec = FFI_LogicalExtensionCodec::new(codec, None, ctx);+ let table = Arc::new(MyTableProvider::new());+ let ffi_table = FFI_TableProvider::new_with_ffi_codec(table, None, ffi_codec);+```++Additional information about the usage of the `TaskContextProvider` can be+found in the crate README.+Additionally, the FFI structure for Scalar UDF's no longer contains a`return_type` call. This code was not used since the `ForeignScalarUDF`struct implements the `return_field_from_args` instead.