Apache DataFusion SQL Query Engine
[Proto]: Serialization support for `AsyncFuncExec` (#19118)
David Stancuca67edcf9957
diff --git a/Cargo.lock b/Cargo.lockindex b449a5ba9b..7638249d98 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2591,6 +2591,7 @@ name = "datafusion-proto"version = "51.0.0"dependencies = ["arrow",+ "async-trait","chrono","datafusion","datafusion-catalog",diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rsindex d442307e94..57b124a618 100644--- a/datafusion/physical-plan/src/async_func.rs+++ b/datafusion/physical-plan/src/async_func.rs@@ -100,6 +100,14 @@ impl AsyncFuncExec {input.boundedness(),))}++ pub fn async_exprs(&self) -> &[Arc<AsyncFuncExpr>] {+ &self.async_exprs+ }++ pub fn input(&self) -> &Arc<dyn ExecutionPlan> {+ &self.input+ }}impl DisplayAs for AsyncFuncExec {diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 920e277b8c..b00bd0dcc6 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -73,6 +73,7 @@ serde = { version = "1.0", optional = true }serde_json = { workspace = true, optional = true }[dev-dependencies]+async-trait = { workspace = true }datafusion = { workspace = true, default-features = false, features = ["sql","datetime_expressions",diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 789176862b..cb2eb515ca 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -748,6 +748,7 @@ message PhysicalPlanNode {GenerateSeriesNode generate_series = 33;SortMergeJoinExecNode sort_merge_join = 34;MemoryScanExecNode memory_scan = 35;+ AsyncFuncExecNode async_func = 36;}}@@ -1393,3 +1394,9 @@ message SortMergeJoinExecNode {repeated SortExprNode sort_options = 6;datafusion_common.NullEquality null_equality = 7;}++message AsyncFuncExecNode {+ PhysicalPlanNode input = 1;+ repeated PhysicalExprNode async_exprs = 2;+ repeated string async_expr_names = 3;+}diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 230bfa495a..9b25b7b803 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -1260,6 +1260,133 @@ impl<'de> serde::Deserialize<'de> for AnalyzedLogicalPlanType {deserializer.deserialize_struct("datafusion.AnalyzedLogicalPlanType", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for AsyncFuncExecNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.input.is_some() {+ len += 1;+ }+ if !self.async_exprs.is_empty() {+ len += 1;+ }+ if !self.async_expr_names.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.AsyncFuncExecNode", len)?;+ if let Some(v) = self.input.as_ref() {+ struct_ser.serialize_field("input", v)?;+ }+ if !self.async_exprs.is_empty() {+ struct_ser.serialize_field("asyncExprs", &self.async_exprs)?;+ }+ if !self.async_expr_names.is_empty() {+ struct_ser.serialize_field("asyncExprNames", &self.async_expr_names)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for AsyncFuncExecNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "input",+ "async_exprs",+ "asyncExprs",+ "async_expr_names",+ "asyncExprNames",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Input,+ AsyncExprs,+ AsyncExprNames,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "input" => Ok(GeneratedField::Input),+ "asyncExprs" | "async_exprs" => Ok(GeneratedField::AsyncExprs),+ "asyncExprNames" | "async_expr_names" => Ok(GeneratedField::AsyncExprNames),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = AsyncFuncExecNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.AsyncFuncExecNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<AsyncFuncExecNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut input__ = None;+ let mut async_exprs__ = None;+ let mut async_expr_names__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Input => {+ if input__.is_some() {+ return Err(serde::de::Error::duplicate_field("input"));+ }+ input__ = map_.next_value()?;+ }+ GeneratedField::AsyncExprs => {+ if async_exprs__.is_some() {+ return Err(serde::de::Error::duplicate_field("asyncExprs"));+ }+ async_exprs__ = Some(map_.next_value()?);+ }+ GeneratedField::AsyncExprNames => {+ if async_expr_names__.is_some() {+ return Err(serde::de::Error::duplicate_field("asyncExprNames"));+ }+ async_expr_names__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(AsyncFuncExecNode {+ input: input__,+ async_exprs: async_exprs__.unwrap_or_default(),+ async_expr_names: async_expr_names__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.AsyncFuncExecNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for AvroScanExecNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -17136,6 +17263,9 @@ impl serde::Serialize for PhysicalPlanNode {physical_plan_node::PhysicalPlanType::MemoryScan(v) => {struct_ser.serialize_field("memoryScan", v)?;}+ physical_plan_node::PhysicalPlanType::AsyncFunc(v) => {+ struct_ser.serialize_field("asyncFunc", v)?;+ }}}struct_ser.end()@@ -17201,6 +17331,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {"sortMergeJoin","memory_scan","memoryScan",+ "async_func",+ "asyncFunc",];#[allow(clippy::enum_variant_names)]@@ -17239,6 +17371,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {GenerateSeries,SortMergeJoin,MemoryScan,+ AsyncFunc,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -17294,6 +17427,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {"generateSeries" | "generate_series" => Ok(GeneratedField::GenerateSeries),"sortMergeJoin" | "sort_merge_join" => Ok(GeneratedField::SortMergeJoin),"memoryScan" | "memory_scan" => Ok(GeneratedField::MemoryScan),+ "asyncFunc" | "async_func" => Ok(GeneratedField::AsyncFunc),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -17552,6 +17686,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {return Err(serde::de::Error::duplicate_field("memoryScan"));}physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::MemoryScan)+;+ }+ GeneratedField::AsyncFunc => {+ if physical_plan_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("asyncFunc"));+ }+ physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::AsyncFunc);}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex b2d0bc7751..4d16f50e14 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1076,7 +1076,7 @@ pub mod table_reference {pub struct PhysicalPlanNode {#[prost(oneof = "physical_plan_node::PhysicalPlanType",- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35"+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36")]pub physical_plan_type: ::core::option::Option<physical_plan_node::PhysicalPlanType>,}@@ -1154,6 +1154,8 @@ pub mod physical_plan_node {SortMergeJoin(::prost::alloc::boxed::Box<super::SortMergeJoinExecNode>),#[prost(message, tag = "35")]MemoryScan(super::MemoryScanExecNode),+ #[prost(message, tag = "36")]+ AsyncFunc(::prost::alloc::boxed::Box<super::AsyncFuncExecNode>),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -2104,6 +2106,15 @@ pub struct SortMergeJoinExecNode {#[prost(enumeration = "super::datafusion_common::NullEquality", tag = "7")]pub null_equality: i32,}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct AsyncFuncExecNode {+ #[prost(message, optional, boxed, tag = "1")]+ pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,+ #[prost(message, repeated, tag = "2")]+ pub async_exprs: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ #[prost(string, repeated, tag = "3")]+ pub async_expr_names: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,+}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]pub enum WindowFrameUnits {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 89dd0b5065..6f275d2a22 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -100,6 +100,8 @@ use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};+use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;+use datafusion_physical_plan::async_func::AsyncFuncExec;use prost::bytes::BufMut;use prost::Message;@@ -251,6 +253,9 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {PhysicalPlanType::SortMergeJoin(sort_join) => {self.try_into_sort_join(sort_join, ctx, extension_codec)}+ PhysicalPlanType::AsyncFunc(async_func) => {+ self.try_into_async_func_physical_plan(async_func, ctx, extension_codec)+ }}}@@ -462,6 +467,13 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {}}+ if let Some(exec) = plan.downcast_ref::<AsyncFuncExec>() {+ return protobuf::PhysicalPlanNode::try_from_async_func_exec(+ exec,+ extension_codec,+ );+ }+let mut buf: Vec<u8> = vec![];match extension_codec.try_encode(Arc::clone(&plan_clone), &mut buf) {Ok(_) => {@@ -1972,6 +1984,44 @@ impl protobuf::PhysicalPlanNode {Ok(Arc::new(CooperativeExec::new(input)))}+ fn try_into_async_func_physical_plan(+ &self,+ async_func: &protobuf::AsyncFuncExecNode,+ ctx: &TaskContext,+ extension_codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let input: Arc<dyn ExecutionPlan> =+ into_physical_plan(&async_func.input, ctx, extension_codec)?;++ if async_func.async_exprs.len() != async_func.async_expr_names.len() {+ return internal_err!(+ "AsyncFuncExecNode async_exprs length does not match async_expr_names"+ );+ }++ let async_exprs = async_func+ .async_exprs+ .iter()+ .zip(async_func.async_expr_names.iter())+ .map(|(expr, name)| {+ let physical_expr = parse_physical_expr(+ expr,+ ctx,+ input.schema().as_ref(),+ extension_codec,+ )?;++ Ok(Arc::new(AsyncFuncExpr::try_new(+ name.clone(),+ physical_expr,+ input.schema().as_ref(),+ )?))+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?))+ }+fn try_from_explain_exec(exec: &ExplainExec,_extension_codec: &dyn PhysicalExtensionCodec,@@ -3222,6 +3272,34 @@ impl protobuf::PhysicalPlanNode {Ok(None)}++ fn try_from_async_func_exec(+ exec: &AsyncFuncExec,+ extension_codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Self> {+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ Arc::clone(exec.input()),+ extension_codec,+ )?;++ let mut async_exprs = vec![];+ let mut async_expr_names = vec![];++ for async_expr in exec.async_exprs() {+ async_exprs.push(serialize_physical_expr(&async_expr.func, extension_codec)?);+ async_expr_names.push(async_expr.name.clone())+ }++ Ok(protobuf::PhysicalPlanNode {+ physical_plan_type: Some(PhysicalPlanType::AsyncFunc(Box::new(+ protobuf::AsyncFuncExecNode {+ input: Some(Box::new(input)),+ async_exprs,+ async_expr_names,+ },+ ))),+ })+ }}pub trait AsExecutionPlan: Debug + Send + Sync + Clone {@@ -3439,7 +3517,6 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {fn into_physical_plan(node: &Option<Box<protobuf::PhysicalPlanNode>>,ctx: &TaskContext,-extension_codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn ExecutionPlan>> {if let Some(field) = node {diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 0bcdd610c2..438e65f60b 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -107,9 +107,11 @@ use datafusion_common::{internal_datafusion_err, internal_err, not_impl_err, DataFusionError, NullEquality,Result, UnnestOptions,};+use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};use datafusion_expr::{- Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, ScalarUDF,- Signature, SimpleAggregateUDF, WindowFrame, WindowFrameBound, WindowUDF,+ Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue,+ ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,+ WindowFrame, WindowFrameBound, WindowUDF,};use datafusion_functions_aggregate::average::avg_udaf;use datafusion_functions_aggregate::nth_value::nth_value_udaf;@@ -2263,3 +2265,65 @@ async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {roundtrip_test(plan)}++#[tokio::test]+async fn roundtrip_async_func_exec() -> Result<()> {+ #[derive(Debug, PartialEq, Eq, Hash)]+ struct TestAsyncUDF {+ signature: Signature,+ }++ impl TestAsyncUDF {+ fn new() -> Self {+ Self {+ signature: Signature::exact(vec![DataType::Int64], Volatility::Volatile),+ }+ }+ }++ impl ScalarUDFImpl for TestAsyncUDF {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "test_async_udf"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Int64)+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ not_impl_err!("Must call from `invoke_async_with_args`")+ }+ }++ #[async_trait::async_trait]+ impl AsyncScalarUDFImpl for TestAsyncUDF {+ async fn invoke_async_with_args(+ &self,+ args: ScalarFunctionArgs,+ ) -> Result<ColumnarValue> {+ Ok(args.args[0].clone())+ }+ }++ let ctx = SessionContext::new();+ let async_udf = AsyncScalarUDF::new(Arc::new(TestAsyncUDF::new()));+ ctx.register_udf(async_udf.into_scalar_udf());++ let physical_plan = ctx+ .sql("select test_async_udf(1)")+ .await?+ .create_physical_plan()+ .await?;++ roundtrip_test_with_context(physical_plan, &ctx)?;++ Ok(())+}