Skip to content

Apache DataFusion SQL Query Engine

fix(ffi): don't discard downcast-delegating wrappers of foreign plans (#25749)

Tim Saucerc1786f7e021f
diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs
index de175e7686..1617e38817 100644
--- a/datafusion/ffi/src/execution_plan.rs
+++ b/datafusion/ffi/src/execution_plan.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::any::Any;
use std::ffi::c_void;
use std::pin::Pin;
use std::sync::Arc;
@@ -288,6 +289,9 @@ fn pass_runtime_to_children(
runtime: &Handle,
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
let mut updated_children = false;
+ // This intentionally uses the delegate-aware `is`: a transparent wrapper
+ // around a `ForeignExecutionPlan` typically forwards `replace_children` to
+ // it, so its children will still cross the FFI boundary and need the runtime.
let plan_is_foreign = plan.is::<ForeignExecutionPlan>();
let children = plan
@@ -306,8 +310,12 @@ fn pass_runtime_to_children(
// we called `children()` above we will get something other than a
// `ForeignExecutionPlan`. In this case wrap the plan in a `ForeignExecutionPlan`
// because when we call `replace_children` below it will extract the
- // FFI plan that does contain the runtime.
- if plan_is_foreign && !child.is::<ForeignExecutionPlan>() {
+ // FFI plan that does contain the runtime. Check the concrete type rather
+ // than the delegate-aware `is` so that a local wrapper which delegates to
+ // a `ForeignExecutionPlan` still receives the runtime.
+ if plan_is_foreign
+ && !(child.as_ref() as &dyn Any).is::<ForeignExecutionPlan>()
+ {
updated_children = true;
let ffi_child = FFI_ExecutionPlan::new(child, Some(runtime.clone()));
let foreign_child = ForeignExecutionPlan::try_from(ffi_child);
@@ -334,7 +342,13 @@ impl FFI_ExecutionPlan {
pub fn new(mut plan: Arc<dyn ExecutionPlan>, runtime: Option<Handle>) -> Self {
// Note to developers: `pass_runtime_to_children` relies on the logic here to
// get the underlying FFI plan during calls to `new_with_children`.
- if let Some(plan) = plan.downcast_ref::<ForeignExecutionPlan>() {
+ //
+ // Check the concrete type rather than using the delegate-aware
+ // `downcast_ref`, otherwise a wrapper whose `downcast_delegate` is a
+ // `ForeignExecutionPlan` would be discarded here.
+ if let Some(plan) =
+ (plan.as_ref() as &dyn Any).downcast_ref::<ForeignExecutionPlan>()
+ {
return plan.plan.clone();
}
@@ -571,6 +585,7 @@ pub mod tests {
dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,
metrics: Option<MetricsSet>,
statistics: Option<Statistics>,
+ downcast_delegate: bool,
}
impl EmptyExec {
@@ -587,6 +602,7 @@ pub mod tests {
dynamic_expressions: Vec::default(),
metrics: None,
statistics: None,
+ downcast_delegate: false,
}
}
@@ -600,6 +616,13 @@ pub mod tests {
self
}
+ /// Act as a transparent wrapper around `child` via `downcast_delegate`.
+ pub fn with_downcast_delegate(mut self, child: Arc<dyn ExecutionPlan>) -> Self {
+ self.children = vec![child];
+ self.downcast_delegate = true;
+ self
+ }
+
pub fn with_partitioning(mut self, partitioning: Partitioning) -> Self {
self.props =
Arc::new(self.props.as_ref().clone().with_partitioning(partitioning));
@@ -658,6 +681,7 @@ pub mod tests {
dynamic_expressions: self.dynamic_expressions.clone(),
metrics: self.metrics.clone(),
statistics: self.statistics.clone(),
+ downcast_delegate: self.downcast_delegate,
}))
}
@@ -679,6 +703,12 @@ pub mod tests {
unimplemented!()
}
+ fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {
+ self.downcast_delegate
+ .then(|| self.children.first().map(|c| c.as_ref()))
+ .flatten()
+ }
+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
self.dynamic_expressions.iter().map(Arc::clone).collect()
}
@@ -1007,4 +1037,36 @@ pub mod tests {
let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into().unwrap();
assert!(foreign_plan.is::<ForeignExecutionPlan>());
}
+
+ /// A wrapper whose `downcast_delegate` is a `ForeignExecutionPlan` must not be
+ /// mistaken for one: it must survive `FFI_ExecutionPlan::new` and still receive
+ /// the runtime when it is the local child of a foreign parent.
+ #[tokio::test]
+ async fn test_ffi_execution_plan_delegating_wrapper() -> Result<()> {
+ let schema = Arc::new(arrow::datatypes::Schema::empty());
+
+ let mut ffi_inner =
+ FFI_ExecutionPlan::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), None);
+ ffi_inner.library_marker_id = crate::mock_foreign_marker_id;
+ let foreign_inner: Arc<dyn ExecutionPlan> =
+ Arc::new(ForeignExecutionPlan::try_from(ffi_inner)?);
+ let wrapper: Arc<dyn ExecutionPlan> = Arc::new(
+ EmptyExec::new(Arc::clone(&schema)).with_downcast_delegate(foreign_inner),
+ );
+ assert!(wrapper.is::<ForeignExecutionPlan>());
+
+ let ffi_wrapper = FFI_ExecutionPlan::new(Arc::clone(&wrapper), None);
+ assert!(ffi_wrapper.inner().downcast_delegate().is_some());
+
+ let parent = Arc::new(EmptyExec::new(schema)).replace_children(
+ vec![wrapper],
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )?;
+ let mut ffi_parent = FFI_ExecutionPlan::new(parent, None);
+ ffi_parent.library_marker_id = crate::mock_foreign_marker_id;
+ let foreign_parent: Arc<dyn ExecutionPlan> = (&ffi_parent).try_into()?;
+ assert!(pass_runtime_to_children(&foreign_parent, &Handle::current())?.is_some());
+
+ Ok(())
+ }
}