Apache DataFusion SQL Query Engine
fix(ffi): preserve TableProvider DML overrides (#23906)
Amogh Rameshab12f5e4be59
diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rsindex ee9377bff0..f1a79added 100644--- a/datafusion/ffi/src/table_provider.rs+++ b/datafusion/ffi/src/table_provider.rs@@ -28,14 +28,15 @@ use datafusion_execution::TaskContext;use datafusion_expr::dml::InsertOp;use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};use datafusion_physical_plan::ExecutionPlan;-use datafusion_proto::logical_plan::from_proto::parse_exprs;-use datafusion_proto::logical_plan::to_proto::serialize_exprs;+use datafusion_proto::bytes::{+ logical_exprs_from_bytes_with_extension_codec,+ logical_exprs_to_bytes_with_extension_codec,+};use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec,};-use datafusion_proto::protobuf::LogicalExprList;-use prost::Message;+use stabby::string::String as SString;use stabby::vec::Vec as SVec;use tokio::runtime::Handle;@@ -103,7 +104,7 @@ pub struct FFI_TableProvider {/// * `session` - session/// * `projections` - if specified, only a subset of the columns are returned/// * `filters_serialized` - filters to apply to the scan, which are a- /// [`LogicalExprList`] protobuf message serialized into bytes to pass+ /// [`LogicalExprList`][datafusion_proto::protobuf::LogicalExprList] protobuf message serialized into bytes to pass/// across the FFI boundary./// * `limit` - if specified, limit the number of rows returnedscan: unsafe extern "C" fn(@@ -118,7 +119,7 @@ pub struct FFI_TableProvider {table_type: unsafe extern "C" fn(provider: &Self) -> FFI_TableType,/// Based upon the input filters, identify which are supported. The filters- /// are a [`LogicalExprList`] protobuf message serialized into bytes to pass+ /// are a [`LogicalExprList`][datafusion_proto::protobuf::LogicalExprList] protobuf message serialized into bytes to pass/// across the FFI boundary.supports_filters_pushdown: Option<unsafe extern "C" fn(@@ -160,11 +161,36 @@ pub struct FFI_TableProvider {/// the foreign interface. See [`crate::get_library_marker_id`] and/// the crate's `README.md` for more information.pub library_marker_id: extern "C" fn() -> usize,++ delete_from: unsafe extern "C" fn(+ provider: &Self,+ session: FFI_SessionRef,+ filters_serialized: SVec<u8>,+ ) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>>,++ update: unsafe extern "C" fn(+ provider: &Self,+ session: FFI_SessionRef,+ assignments: SVec<FFI_TableProviderUpdateAssignment>,+ filters_serialized: SVec<u8>,+ ) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>>,++ truncate: unsafe extern "C" fn(+ provider: &Self,+ session: FFI_SessionRef,+ ) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>>,}unsafe impl Send for FFI_TableProvider {}unsafe impl Sync for FFI_TableProvider {}+#[repr(C)]+#[derive(Debug)]+struct FFI_TableProviderUpdateAssignment {+ column: SString,+ expr_serialized: SVec<u8>,+}+struct ProviderPrivateData {provider: Arc<dyn TableProvider>,runtime: Option<Handle>,@@ -202,21 +228,33 @@ unsafe extern "C" fn table_type_fn_wrapper(provider.inner().table_type().into()}+fn parse_serialized_exprs(+ exprs_serialized: &[u8],+ task_ctx: &Arc<TaskContext>,+ codec: &dyn LogicalExtensionCodec,+) -> Result<Vec<Expr>> {+ logical_exprs_from_bytes_with_extension_codec(+ exprs_serialized,+ task_ctx.as_ref(),+ codec,+ )+}++fn serialize_expr_list<'a>(+ exprs: impl IntoIterator<Item = &'a Expr>,+ codec: &dyn LogicalExtensionCodec,+) -> Result<SVec<u8>> {+ let bytes = logical_exprs_to_bytes_with_extension_codec(exprs, codec)?;+ Ok(SVec::from(bytes.as_ref()))+}+fn supports_filters_pushdown_internal(provider: &Arc<dyn TableProvider>,filters_serialized: &[u8],task_ctx: &Arc<TaskContext>,codec: &dyn LogicalExtensionCodec,) -> Result<SVec<FFI_TableProviderFilterPushDown>> {- let filters = match filters_serialized.is_empty() {- true => vec![],- false => {- let proto_filters = LogicalExprList::decode(filters_serialized)- .map_err(|e| DataFusionError::Plan(e.to_string()))?;-- parse_exprs(proto_filters.expr.iter(), task_ctx.as_ref(), codec)?- }- };+ let filters = parse_serialized_exprs(filters_serialized, task_ctx, codec)?;let filters_borrowed: Vec<&Expr> = filters.iter().collect();let results: SVec<_> = provider@@ -271,19 +309,11 @@ unsafe extern "C" fn scan_fn_wrapper();let task_ctx = sresult_return!(task_ctx);- let filters = match filters_serialized.is_empty() {- true => vec![],- false => {- let proto_filters =- sresult_return!(LogicalExprList::decode(filters_serialized.as_ref()));-- sresult_return!(parse_exprs(- proto_filters.expr.iter(),- task_ctx.as_ref(),- logical_codec.as_ref(),- ))- }- };+ let filters = sresult_return!(parse_serialized_exprs(+ &filters_serialized,+ &task_ctx,+ logical_codec.as_ref(),+ ));let projections: Option<Vec<usize>> =projections.into_option().map(|p| p.into_iter().collect());@@ -336,6 +366,135 @@ unsafe extern "C" fn insert_into_fn_wrapper(.into_ffi()}+unsafe extern "C" fn delete_from_fn_wrapper(+ provider: &FFI_TableProvider,+ session: FFI_SessionRef,+ filters_serialized: SVec<u8>,+) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>> {+ let task_ctx: Result<Arc<TaskContext>, DataFusionError> =+ (&provider.logical_codec.task_ctx_provider).try_into();+ let runtime = provider.runtime().clone();+ let logical_codec: Arc<dyn LogicalExtensionCodec> = (&provider.logical_codec).into();+ let internal_provider = Arc::clone(provider.inner());++ async move {+ let mut foreign_session = None;+ let session = sresult_return!(+ session+ .as_local()+ .map(Ok::<&dyn Session, DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );++ let task_ctx = sresult_return!(task_ctx);+ let filters = sresult_return!(parse_serialized_exprs(+ &filters_serialized,+ &task_ctx,+ logical_codec.as_ref(),+ ));++ let plan = sresult_return!(internal_provider.delete_from(session, filters).await);++ FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime))+ }+ .into_ffi()+}++unsafe extern "C" fn update_fn_wrapper(+ provider: &FFI_TableProvider,+ session: FFI_SessionRef,+ assignments: SVec<FFI_TableProviderUpdateAssignment>,+ filters_serialized: SVec<u8>,+) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>> {+ let task_ctx: Result<Arc<TaskContext>, DataFusionError> =+ (&provider.logical_codec.task_ctx_provider).try_into();+ let runtime = provider.runtime().clone();+ let logical_codec: Arc<dyn LogicalExtensionCodec> = (&provider.logical_codec).into();+ let internal_provider = Arc::clone(provider.inner());++ async move {+ let mut foreign_session = None;+ let session = sresult_return!(+ session+ .as_local()+ .map(Ok::<&dyn Session, DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );++ let task_ctx = sresult_return!(task_ctx);+ let assignments = sresult_return!(+ assignments+ .into_iter()+ .map(|assignment| {+ let column = assignment.column.to_string();+ let mut exprs = parse_serialized_exprs(+ &assignment.expr_serialized,+ &task_ctx,+ logical_codec.as_ref(),+ )?;+ let expression_count = exprs.len();+ let expr = match expression_count {+ 1 => exprs.remove(0),+ _ => {+ return Err(DataFusionError::Plan(format!(+ "Expected exactly one expression for update assignment to column \+ '{column}', got {expression_count}"+ )));+ }+ };+ Ok((column, expr))+ })+ .collect::<Result<Vec<_>>>()+ );+ let filters = sresult_return!(parse_serialized_exprs(+ &filters_serialized,+ &task_ctx,+ logical_codec.as_ref(),+ ));++ let plan = sresult_return!(+ internal_provider+ .update(session, assignments, filters)+ .await+ );++ FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime))+ }+ .into_ffi()+}++unsafe extern "C" fn truncate_fn_wrapper(+ provider: &FFI_TableProvider,+ session: FFI_SessionRef,+) -> FfiFuture<FFI_Result<FFI_ExecutionPlan>> {+ let runtime = provider.runtime().clone();+ let internal_provider = Arc::clone(provider.inner());++ async move {+ let mut foreign_session = None;+ let session = sresult_return!(+ session+ .as_local()+ .map(Ok::<&dyn Session, DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );++ let plan = sresult_return!(internal_provider.truncate(session).await);++ FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime))+ }+ .into_ffi()+}+unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_TableProvider) {unsafe {debug_assert!(!provider.private_data.is_null());@@ -368,6 +527,9 @@ unsafe extern "C" fn clone_fn_wrapper(provider: &FFI_TableProvider) -> FFI_Tableversion: super::version,private_data,library_marker_id: crate::get_library_marker_id,+ delete_from: provider.delete_from,+ update: provider.update,+ truncate: provider.truncate,}}@@ -429,6 +591,9 @@ impl FFI_TableProvider {version: super::version,private_data: Box::into_raw(private_data) as *mut c_void,library_marker_id: crate::get_library_marker_id,+ delete_from: delete_from_fn_wrapper,+ update: update_fn_wrapper,+ truncate: truncate_fn_wrapper,}}}@@ -499,10 +664,7 @@ impl TableProvider for ForeignTableProvider {.into();let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();- let filter_list = LogicalExprList {- expr: serialize_exprs(filters, codec.as_ref())?,- };- let filters_serialized = filter_list.encode_to_vec().into_iter().collect();+ let filters_serialized = serialize_expr_list(filters.iter(), codec.as_ref())?;let plan = unsafe {let maybe_plan = (self.0.scan)(@@ -539,13 +701,8 @@ impl TableProvider for ForeignTableProvider {let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();- let expr_list = LogicalExprList {- expr: serialize_exprs(- filters.iter().map(|f| f.to_owned()),- codec.as_ref(),- )?,- };- let serialized_filters = expr_list.encode_to_vec();+ let serialized_filters =+ serialize_expr_list(filters.iter().copied(), codec.as_ref())?;let pushdowns = df_result!(pushdown_fn(&self.0,@@ -577,11 +734,79 @@ impl TableProvider for ForeignTableProvider {Ok(plan)}++ async fn delete_from(+ &self,+ session: &dyn Session,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone());+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();+ let filters_serialized = serialize_expr_list(filters.iter(), codec.as_ref())?;++ let plan = unsafe {+ let maybe_plan =+ (self.0.delete_from)(&self.0, session, filters_serialized).await;++ <Arc<dyn ExecutionPlan>>::try_from(&df_result!(maybe_plan)?)?+ };++ Ok(plan)+ }++ async fn update(+ &self,+ session: &dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone());+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();++ let assignments: SVec<_> = assignments+ .iter()+ .map(|(column, expr)| {+ Ok(FFI_TableProviderUpdateAssignment {+ column: SString::from(column.as_str()),+ expr_serialized: serialize_expr_list(+ std::iter::once(expr),+ codec.as_ref(),+ )?,+ })+ })+ .collect::<Result<Vec<_>>>()?+ .into_iter()+ .collect();+ let filters_serialized = serialize_expr_list(filters.iter(), codec.as_ref())?;++ let plan = unsafe {+ let maybe_plan =+ (self.0.update)(&self.0, session, assignments, filters_serialized).await;++ <Arc<dyn ExecutionPlan>>::try_from(&df_result!(maybe_plan)?)?+ };++ Ok(plan)+ }++ async fn truncate(&self, session: &dyn Session) -> Result<Arc<dyn ExecutionPlan>> {+ let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone());++ let plan = unsafe {+ let maybe_plan = (self.0.truncate)(&self.0, session).await;++ <Arc<dyn ExecutionPlan>>::try_from(&df_result!(maybe_plan)?)?+ };++ Ok(plan)+ }}#[cfg(test)]mod tests {- use arrow::datatypes::Schema;+ use std::sync::atomic::{AtomicUsize, Ordering};++ use arrow::datatypes::{DataType, Field, Schema};use datafusion::prelude::{SessionContext, col, lit};use datafusion_execution::TaskContextProvider;@@ -613,6 +838,105 @@ mod tests {)?))}+ #[derive(Debug)]+ struct TableWithStats {+ inner: Arc<dyn TableProvider>,+ stats: Option<Statistics>,+ delete_calls: AtomicUsize,+ update_calls: AtomicUsize,+ }++ impl TableWithStats {+ fn new(inner: Arc<dyn TableProvider>, stats: Option<Statistics>) -> Self {+ Self {+ inner,+ stats,+ delete_calls: AtomicUsize::new(0),+ update_calls: AtomicUsize::new(0),+ }+ }+ }++ #[async_trait]+ impl TableProvider for TableWithStats {+ fn schema(&self) -> SchemaRef {+ self.inner.schema()+ }++ fn table_type(&self) -> TableType {+ self.inner.table_type()+ }++ fn statistics(&self) -> Option<Statistics> {+ self.stats.clone()+ }++ async fn scan(+ &self,+ session: &dyn Session,+ projection: Option<&Vec<usize>>,+ filters: &[Expr],+ limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.inner.scan(session, projection, filters, limit).await+ }++ async fn delete_from(+ &self,+ _state: &dyn Session,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let call = self.delete_calls.fetch_add(1, Ordering::Relaxed);+ let valid = match call {+ 0 => filters == vec![col("a").gt(lit(10_i64)), col("b").lt(lit(2.5_f64))],+ 1 => filters.is_empty(),+ _ => false,+ };++ if !valid {+ return Err(DataFusionError::Internal(format!(+ "Unexpected DELETE filters for call {call}"+ )));+ }+ Ok(dml_count_plan())+ }++ async fn update(+ &self,+ _state: &dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if assignments+ != vec![+ ("b".to_string(), lit(42_f64)),+ ("a".to_string(), lit(7_i64)),+ ]+ {+ return Err(DataFusionError::Internal(+ "Unexpected UPDATE assignments".to_string(),+ ));+ }+ let call = self.update_calls.fetch_add(1, Ordering::Relaxed);+ let valid = match call {+ 0 => filters == vec![col("a").eq(lit(7_i64)), col("b").gt(lit(1.5_f64))],+ 1 => filters.is_empty(),+ _ => false,+ };++ if !valid {+ return Err(DataFusionError::Internal(format!(+ "Unexpected UPDATE filters for call {call}"+ )));+ }+ Ok(dml_count_plan())+ }++ async fn truncate(&self, _state: &dyn Session) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(dml_count_plan())+ }+ }+#[tokio::test]async fn test_round_trip_ffi_table_provider_scan() -> Result<()> {let provider = create_test_table_provider()?;@@ -671,11 +995,91 @@ mod tests {Ok(())}+ fn dml_count_plan() -> Arc<dyn ExecutionPlan> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "count",+ DataType::UInt64,+ false,+ )]));+ Arc::new(crate::execution_plan::tests::EmptyExec::new(schema))+ }++ #[tokio::test]+ async fn test_round_trip_ffi_table_provider_dml() -> Result<()> {+ use datafusion::datasource::MemTable;++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Float64, true),+ ]));+ let inner = Arc::new(MemTable::try_new(schema, vec![vec![]])?);+ let provider = Arc::new(TableWithStats::new(inner, None));+ let ctx = Arc::new(SessionContext::new());+ let task_ctx_provider = Arc::clone(&ctx) as Arc<dyn TaskContextProvider>;+ let task_ctx_provider = FFI_TaskContextProvider::from(&task_ctx_provider);++ let mut ffi_provider =+ FFI_TableProvider::new(provider, true, None, task_ctx_provider, None);+ ffi_provider.library_marker_id = crate::mock_foreign_marker_id;++ let foreign_table_provider: Arc<dyn TableProvider> = (&ffi_provider).into();+ assert!(+ foreign_table_provider+ .downcast_ref::<ForeignTableProvider>()+ .is_some()+ );++ let state = ctx.state();+ let delete_filters = vec![col("a").gt(lit(10_i64)), col("b").lt(lit(2.5_f64))];++ let delete_plan = foreign_table_provider+ .delete_from(&state, delete_filters)+ .await?;+ assert_eq!(delete_plan.schema().field(0).name(), "count");++ let delete_all_plan = foreign_table_provider.delete_from(&state, vec![]).await?;+ assert_eq!(delete_all_plan.schema().field(0).name(), "count");++ let update_assignments = vec![+ ("b".to_string(), lit(42_f64)),+ ("a".to_string(), lit(7_i64)),+ ];+ let update_filters = vec![col("a").eq(lit(7_i64)), col("b").gt(lit(1.5_f64))];++ let update_plan = foreign_table_provider+ .update(&state, update_assignments.clone(), update_filters)+ .await?;+ assert_eq!(update_plan.schema().field(0).name(), "count");++ let update_all_plan = foreign_table_provider+ .update(&state, update_assignments, vec![])+ .await?;+ assert_eq!(update_all_plan.schema().field(0).name(), "count");++ let truncate_plan = foreign_table_provider.truncate(&state).await?;+ assert_eq!(truncate_plan.schema().field(0).name(), "count");++ let session =+ FFI_SessionRef::new(&state, None, ffi_provider.logical_codec.clone());+ let assignments = [FFI_TableProviderUpdateAssignment {+ column: SString::from("b"),+ expr_serialized: SVec::new(),+ }]+ .into_iter()+ .collect();+ let result = unsafe {+ (ffi_provider.update)(&ffi_provider, session, assignments, SVec::new()).await+ };+ assert!(df_result!(result).unwrap_err().to_string().contains(+ "Expected exactly one expression for update assignment to column 'b', got 0",+ ));++ Ok(())+ }+#[tokio::test]async fn test_aggregation() -> Result<()> {- use arrow::datatypes::Field;use datafusion::arrow::array::Float32Array;- use datafusion::arrow::datatypes::DataType;use datafusion::arrow::record_batch::RecordBatch;use datafusion::common::assert_batches_eq;use datafusion::datasource::MemTable;@@ -816,35 +1220,6 @@ mod tests {use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, ScalarValue};- // A thin wrapper that lets us inject statistics onto any TableProvider.- #[derive(Debug)]- struct TableWithStats {- inner: Arc<dyn TableProvider>,- stats: Option<Statistics>,- }-- #[async_trait]- impl TableProvider for TableWithStats {- fn schema(&self) -> SchemaRef {- self.inner.schema()- }- fn table_type(&self) -> TableType {- self.inner.table_type()- }- fn statistics(&self) -> Option<Statistics> {- self.stats.clone()- }- async fn scan(- &self,- session: &dyn Session,- projection: Option<&Vec<usize>>,- filters: &[Expr],- limit: Option<usize>,- ) -> Result<Arc<dyn ExecutionPlan>> {- self.inner.scan(session, projection, filters, limit).await- }- }-let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));let batch = RecordBatch::try_new(@@ -861,10 +1236,7 @@ mod tests {Arc::clone(&schema),vec![vec![batch.clone()]],)?);- let no_stats_provider = Arc::new(TableWithStats {- inner: no_stats_inner,- stats: None,- });+ let no_stats_provider = Arc::new(TableWithStats::new(no_stats_inner, None));let mut ffi_provider = FFI_TableProvider::new(no_stats_provider,true,@@ -891,10 +1263,10 @@ mod tests {};let stats_inner =Arc::new(MemTable::try_new(Arc::clone(&schema), vec![vec![batch]])?);- let stats_provider = Arc::new(TableWithStats {- inner: stats_inner,- stats: Some(original_stats.clone()),- });+ let stats_provider = Arc::new(TableWithStats::new(+ stats_inner,+ Some(original_stats.clone()),+ ));let mut ffi_provider =FFI_TableProvider::new(stats_provider, true, None, task_ctx_provider, None);ffi_provider.library_marker_id = crate::mock_foreign_marker_id;diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex 74310d9c28..ce922de664 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -16,6 +16,7 @@// under the License.use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};use arrow::array::{RecordBatch, record_batch};use arrow_schema::{DataType, Field, Schema};@@ -26,8 +27,8 @@ use datafusion_catalog::MemTable;use datafusion_catalog::{Session, TableProvider};use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Statistics};-use datafusion_common::{Result, ScalarValue};-use datafusion_expr::{Expr, TableType};+use datafusion_common::{Result, ScalarValue, exec_err};+use datafusion_expr::{Expr, TableType, col, lit};use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_plan::ExecutionPlan;use sync_provider::create_sync_table_provider;@@ -237,6 +238,8 @@ pub(crate) extern "C" fn create_exec_with_statistics() -> FFI_ExecutionPlan {struct TableWithStats {inner: Arc<dyn TableProvider>,stats: Statistics,+ delete_calls: AtomicUsize,+ update_calls: AtomicUsize,}#[async_trait]@@ -262,6 +265,56 @@ impl TableProvider for TableWithStats {) -> Result<Arc<dyn ExecutionPlan>> {self.inner.scan(session, projection, filters, limit).await}++ async fn delete_from(+ &self,+ _state: &dyn Session,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let call = self.delete_calls.fetch_add(1, Ordering::Relaxed);+ let valid = match call {+ 0 => filters == vec![col("a").gt(lit(10_i32)), col("b").lt(lit(2.5_f64))],+ 1 => filters.is_empty(),+ _ => false,+ };+ if !valid {+ return exec_err!("Unexpected DELETE filters for call {call}");+ }+ Ok(dml_count_plan())+ }++ async fn update(+ &self,+ _state: &dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if assignments+ != vec![+ ("b".to_string(), lit(42_f64)),+ ("a".to_string(), lit(7_i32)),+ ]+ {+ return exec_err!("Unexpected UPDATE assignments");+ }++ let call = self.update_calls.fetch_add(1, Ordering::Relaxed);+ let valid = match call {+ 0 => filters == vec![col("a").eq(lit(7_i32)), col("b").gt(lit(1.5_f64))],+ 1 => filters.is_empty(),+ _ => false,+ };++ if !valid {+ return exec_err!("Unexpected UPDATE filters for call {call}");+ }++ Ok(dml_count_plan())+ }++ async fn truncate(&self, _state: &dyn Session) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(dml_count_plan())+ }}pub(crate) extern "C" fn create_table_with_statistics(@@ -273,10 +326,21 @@ pub(crate) extern "C" fn create_table_with_statistics(let provider = Arc::new(TableWithStats {inner,stats: make_test_statistics(),+ delete_calls: AtomicUsize::new(0),+ update_calls: AtomicUsize::new(0),});FFI_TableProvider::new_with_ffi_codec(provider, true, None, codec)}+fn dml_count_plan() -> Arc<dyn ExecutionPlan> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "count",+ DataType::UInt64,+ false,+ )]));+ Arc::new(EmptyExec::new(schema))+}+/// This defines the entry point for using the module.#[unsafe(no_mangle)]pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rsindex fa28519d58..eb1744b7b1 100644--- a/datafusion/ffi/src/udtf.rs+++ b/datafusion/ffi/src/udtf.rs@@ -23,14 +23,14 @@ use datafusion_catalog::{TableFunctionArgs, TableFunctionImpl, TableProvider};use datafusion_common::DataFusionError;use datafusion_common::error::Result;use datafusion_execution::TaskContext;-use datafusion_proto::logical_plan::from_proto::parse_exprs;-use datafusion_proto::logical_plan::to_proto::serialize_exprs;+use datafusion_proto::bytes::{+ logical_exprs_from_bytes_with_extension_codec,+ logical_exprs_to_bytes_with_extension_codec,+};use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec,};-use datafusion_proto::protobuf::LogicalExprList;use datafusion_session::Session;-use prost::Message;use stabby::vec::Vec as SVec;use tokio::runtime::Handle;@@ -113,12 +113,10 @@ unsafe extern "C" fn call_fn_wrapper(sresult_return!((&udtf.logical_codec.task_ctx_provider).try_into());let codec: Arc<dyn LogicalExtensionCodec> = (&udtf.logical_codec).into();- let proto_filters = sresult_return!(LogicalExprList::decode(args.as_ref()));-- let args = sresult_return!(parse_exprs(- proto_filters.expr.iter(),+ let args = sresult_return!(logical_exprs_from_bytes_with_extension_codec(+ args.as_ref(),ctx.as_ref(),- codec.as_ref()+ codec.as_ref(),));#[expect(deprecated)]@@ -143,12 +141,10 @@ unsafe extern "C" fn call_with_args_wrapper(sresult_return!((&udtf.logical_codec.task_ctx_provider).try_into());let codec: Arc<dyn LogicalExtensionCodec> = (&udtf.logical_codec).into();- let proto_filters = sresult_return!(LogicalExprList::decode(args.as_ref()));-- let args = sresult_return!(parse_exprs(- proto_filters.expr.iter(),+ let args = sresult_return!(logical_exprs_from_bytes_with_extension_codec(+ args.as_ref(),ctx.as_ref(),- codec.as_ref()+ codec.as_ref(),));let mut foreign_session = None;@@ -280,13 +276,12 @@ impl TableFunctionImpl for ForeignTableFunction {self.0.logical_codec.clone(),);let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();- let expr_list = LogicalExprList {- expr: serialize_exprs(args.exprs(), codec.as_ref())?,- };- let filters_serialized = expr_list.encode_to_vec().into_iter().collect();+ let bytes =+ logical_exprs_to_bytes_with_extension_codec(args.exprs(), codec.as_ref())?;+ let args_serialized = SVec::from(bytes.as_ref());let table_provider =- unsafe { (self.0.call_with_args)(&self.0, filters_serialized, session) };+ unsafe { (self.0.call_with_args)(&self.0, args_serialized, session) };let table_provider = df_result!(table_provider)?;let table_provider: Arc<dyn TableProvider> = (&table_provider).into();@@ -296,13 +291,11 @@ impl TableFunctionImpl for ForeignTableFunction {fn call(&self, args: &[datafusion_expr::Expr]) -> Result<Arc<dyn TableProvider>> {let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();- let expr_list = LogicalExprList {- expr: serialize_exprs(args, codec.as_ref())?,- };- let filters_serialized = expr_list.encode_to_vec().into_iter().collect();+ let bytes = logical_exprs_to_bytes_with_extension_codec(args, codec.as_ref())?;+ let args_serialized = SVec::from(bytes.as_ref());#[expect(deprecated)]- let table_provider = unsafe { (self.0.call)(&self.0, filters_serialized) };+ let table_provider = unsafe { (self.0.call)(&self.0, args_serialized) };let table_provider = df_result!(table_provider)?;let table_provider: Arc<dyn TableProvider> = (&table_provider).into();diff --git a/datafusion/ffi/tests/ffi_integration.rs b/datafusion/ffi/tests/ffi_integration.rsindex 86f953e262..cba334e8fa 100644--- a/datafusion/ffi/tests/ffi_integration.rs+++ b/datafusion/ffi/tests/ffi_integration.rs@@ -27,6 +27,7 @@ mod tests {use arrow::datatypes::Schema;use datafusion::catalog::{TableProvider, TableProviderFactory};use datafusion::error::Result;+ use datafusion::prelude::{col, lit};use datafusion_common::TableReference;use datafusion_common::ToDFSchema;use datafusion_expr::CreateExternalTable;@@ -95,6 +96,47 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_ffi_table_provider_dml_cross_library() -> Result<()> {+ let module = get_module()?;+ let (ctx, codec) = super::utils::ctx_and_codec();++ let ffi_provider = (module.create_table_with_statistics)(codec);+ let foreign: Arc<dyn TableProvider> = (&ffi_provider).into();+ let state = ctx.state();++ let delete_plan = foreign+ .delete_from(+ &state,+ vec![col("a").gt(lit(10_i32)), col("b").lt(lit(2.5_f64))],+ )+ .await?;+ assert_eq!(delete_plan.schema().field(0).name(), "count");+ let delete_all_plan = foreign.delete_from(&state, vec![]).await?;+ assert_eq!(delete_all_plan.schema().field(0).name(), "count");++ let update_assignments = vec![+ ("b".to_string(), lit(42_f64)),+ ("a".to_string(), lit(7_i32)),+ ];+ let update_plan = foreign+ .update(+ &state,+ update_assignments.clone(),+ vec![col("a").eq(lit(7_i32)), col("b").gt(lit(1.5_f64))],+ )+ .await?;+ assert_eq!(update_plan.schema().field(0).name(), "count");++ let update_all_plan = foreign.update(&state, update_assignments, vec![]).await?;+ assert_eq!(update_all_plan.schema().field(0).name(), "count");++ let truncate_plan = foreign.truncate(&state).await?;+ assert_eq!(truncate_plan.schema().field(0).name(), "count");++ Ok(())+ }+#[tokio::test]async fn test_table_provider_factory() -> Result<()> {let table_provider_module = get_module()?;diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rsindex 388e373c3f..fd745041eb 100644--- a/datafusion/proto/src/bytes/mod.rs+++ b/datafusion/proto/src/bytes/mod.rs@@ -16,7 +16,7 @@// under the License.//! Serialization / Deserialization to Bytes-use crate::logical_plan::to_proto::serialize_expr;+use crate::logical_plan::to_proto::{serialize_expr, serialize_exprs};use crate::logical_plan::{self, AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,};@@ -107,6 +107,37 @@ impl Serializeable for Expr {}}+/// Serialize logical expressions as protobuf bytes using the provided+/// extension codec.+pub fn logical_exprs_to_bytes_with_extension_codec<'a>(+ exprs: impl IntoIterator<Item = &'a Expr>,+ extension_codec: &dyn LogicalExtensionCodec,+) -> Result<Bytes> {+ Ok(protobuf::LogicalExprList {+ expr: serialize_exprs(exprs, extension_codec)?,+ }+ .encode_to_vec()+ .into())+}++/// Deserialize logical expressions from protobuf bytes using the provided+/// extension codec.+pub fn logical_exprs_from_bytes_with_extension_codec(+ bytes: &[u8],+ ctx: &TaskContext,+ extension_codec: &dyn LogicalExtensionCodec,+) -> Result<Vec<Expr>> {+ let protobuf = protobuf::LogicalExprList::decode(bytes).map_err(|e| {+ plan_datafusion_err!("Error decoding expressions as protobuf: {e}")+ })?;++ Ok(logical_plan::from_proto::parse_exprs(+ protobuf.expr.iter(),+ ctx,+ extension_codec,+ )?)+}+/// Serialize a LogicalPlan as bytespub fn logical_plan_to_bytes(plan: &LogicalPlan) -> Result<Bytes> {let extension_codec = DefaultLogicalExtensionCodec {};diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rsindex a0a917e323..46ab8511f7 100644--- a/datafusion/proto/tests/cases/serialize.rs+++ b/datafusion/proto/tests/cases/serialize.rs@@ -25,14 +25,18 @@ use datafusion::prelude::SessionContext;use datafusion_common::ScalarValue;use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable, Placeholder};use datafusion_expr::{ColumnarValue, HigherOrderUDF, col, create_udf, lambda, lit};-use datafusion_expr::{Expr, Volatility};+use datafusion_expr::{Expr, ScalarUDF, Volatility};use datafusion_functions::string;-use datafusion_proto::bytes::Serializeable;+use datafusion_proto::bytes::{+ Serializeable, logical_exprs_from_bytes_with_extension_codec,+ logical_exprs_to_bytes_with_extension_codec,+};use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec;use datafusion_proto::logical_plan::from_proto::parse_expr;use datafusion_proto::logical_plan::to_proto::serialize_expr;-use crate::cases::MyHigherOrderUDF;+use crate::cases::roundtrip_logical_plan::UDFExtensionCodec;+use crate::cases::{MyHigherOrderUDF, MyRegexUdf};#[test]#[should_panic(@@ -42,6 +46,57 @@ fn bad_decode() {Expr::from_bytes(b"Leet").unwrap();}+#[test]+fn logical_exprs_roundtrip() {+ let ctx = SessionContext::new();+ let codec = DefaultLogicalExtensionCodec {};+ let exprs = vec![col("a").gt(lit(10_i32)), col("b").lt(lit(2.5_f64))];++ let bytes = logical_exprs_to_bytes_with_extension_codec(&exprs, &codec).unwrap();++ let decoded = logical_exprs_from_bytes_with_extension_codec(+ &bytes,+ ctx.task_ctx().as_ref(),+ &codec,+ )+ .unwrap();++ assert_eq!(decoded, exprs);++ let bytes =+ logical_exprs_to_bytes_with_extension_codec(std::iter::empty::<&Expr>(), &codec)+ .unwrap();++ assert!(bytes.is_empty());++ let decoded = logical_exprs_from_bytes_with_extension_codec(+ &bytes,+ ctx.task_ctx().as_ref(),+ &codec,+ )+ .unwrap();++ assert!(decoded.is_empty());+}++#[test]+fn logical_exprs_roundtrip_with_extension_codec() {+ let ctx = SessionContext::new();+ let codec = UDFExtensionCodec;+ let udf = ScalarUDF::from(MyRegexUdf::new(".*".to_owned()));+ let exprs = vec![udf.call(vec![lit("foo")])];++ let bytes = logical_exprs_to_bytes_with_extension_codec(&exprs, &codec).unwrap();++ let decoded = logical_exprs_from_bytes_with_extension_codec(+ &bytes,+ ctx.task_ctx().as_ref(),+ &codec,+ )+ .unwrap();+ assert_eq!(format!("{exprs:?}"), format!("{decoded:?}"));+}+#[test]#[cfg(feature = "json")]fn plan_to_json() {