Apache DataFusion SQL Query Engine
Add protobuf support for lambdas (#22362)
gstvg7ac784f9ea0b
diff --git a/datafusion/physical-expr/src/expressions/lambda.rs b/datafusion/physical-expr/src/expressions/lambda.rsindex 9275821ae9..cab2eea64f 100644--- a/datafusion/physical-expr/src/expressions/lambda.rs+++ b/datafusion/physical-expr/src/expressions/lambda.rs@@ -142,6 +142,27 @@ impl LambdaExpr {&self.body}+ #[cfg(feature = "proto")]+ /// Reconstruct a [`LambdaExpr`] from a proto node.+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalExprNode,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_physical_expr_common::expect_expr_variant;+ use datafusion_proto_models::protobuf;++ let lambda = expect_expr_variant!(+ node,+ protobuf::physical_expr_node::ExprType::Lambda,+ "LambdaExpr",+ );++ Ok(Arc::new(LambdaExpr::try_new(+ lambda.params.clone(),+ ctx.decode_required_expression(lambda.body.as_deref(), "LambdaExpr", "body")?,+ )?))+ }+pub(crate) fn projection(&self) -> &[usize] {&self.projection}@@ -193,6 +214,24 @@ impl PhysicalExpr for LambdaExpr {fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {write!(f, "({}) -> {}", self.params.join(", "), self.body)}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;++ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(+ protobuf::PhysicalLambdaExprNode {+ params: self.params().to_vec(),+ body: Some(Box::new(ctx.encode_child(self.body())?)),+ },+ ))),+ }))+ }}/// Create a lambda expressiondiff --git a/datafusion/physical-expr/src/expressions/lambda_variable.rs b/datafusion/physical-expr/src/expressions/lambda_variable.rsindex 1c130ab12e..f7e6910020 100644--- a/datafusion/physical-expr/src/expressions/lambda_variable.rs+++ b/datafusion/physical-expr/src/expressions/lambda_variable.rs@@ -72,6 +72,32 @@ impl LambdaVariable {pub fn field(&self) -> &FieldRef {&self.field}++ #[cfg(feature = "proto")]+ /// Reconstruct a [`LambdaVariable`] from a proto node.+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalExprNode,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_physical_expr_common::{+ expect_expr_variant, physical_expr::proto_decode::require_proto_field,+ };+ use datafusion_proto_models::protobuf;++ let var = expect_expr_variant!(+ node,+ protobuf::physical_expr_node::ExprType::LambdaVariable,+ "LambdaVariable",+ );++ Ok(Arc::new(LambdaVariable::new(+ var.index as usize,+ Arc::new(+ require_proto_field(var.field.as_ref(), "LambdaVariable", "field")?+ .try_into()?,+ ),+ )))+ }}impl std::fmt::Display for LambdaVariable {@@ -135,6 +161,24 @@ impl PhysicalExpr for LambdaVariable {fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {write!(f, "{}@{}", self.name(), self.index)}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;++ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::LambdaVariable(+ protobuf::PhysicalLambdaVariableExprNode {+ index: self.index() as u32,+ field: Some(self.field().as_ref().try_into()?),+ },+ )),+ }))+ }}/// Create a lambda variable expressiondiff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex d68973c44e..b22fad9c0e 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -501,6 +501,10 @@ message LogicalExprNode {// Subquery expressionsScalarSubqueryExprNode scalar_subquery_expr = 36;++ HigherOrderUDFExprNode higher_order_udf_expr = 37;+ Lambda lambda = 38;+ LambdaVariable lambda_variable = 39;}}@@ -638,6 +642,22 @@ message ScalarUDFExprNode {optional bytes fun_definition = 3;}+message HigherOrderUDFExprNode {+ string fun_name = 1;+ repeated LogicalExprNode args = 2;+ optional bytes fun_definition = 3;+}++message Lambda {+ repeated string params = 1;+ LogicalExprNode body = 2;+}++message LambdaVariable {+ string name = 1;+ datafusion_common.Field field = 2;+}+message WindowExprNode {oneof window_function {// BuiltInWindowFunction built_in_function = 2;@@ -1007,6 +1027,10 @@ message PhysicalExprNode {PhysicalScalarSubqueryExprNode scalar_subquery = 22;PhysicalDynamicFilterNode dynamic_filter = 23;++ PhysicalHigherOrderUdfNode higher_order_udf = 24;+ PhysicalLambdaExprNode lambda = 25;+ PhysicalLambdaVariableExprNode lambda_variable = 26;}}@@ -1027,6 +1051,22 @@ message PhysicalScalarUdfNode {string return_field_name = 6;}+message PhysicalHigherOrderUdfNode {+ string name = 1;+ repeated PhysicalExprNode args = 2;+ optional bytes fun_definition = 3;+}++message PhysicalLambdaExprNode {+ repeated string params = 1;+ PhysicalExprNode body = 2;+}++message PhysicalLambdaVariableExprNode {+ uint32 index = 1;+ datafusion_common.Field field = 2;+}+message PhysicalAggregateExprNode {oneof AggregateFunction {string user_defined_aggr_function = 4;diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex f8e2103035..c334eac2f5 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -9342,6 +9342,137 @@ impl<'de> serde::Deserialize<'de> for HashRepartition {deserializer.deserialize_struct("datafusion.HashRepartition", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for HigherOrderUdfExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.fun_name.is_empty() {+ len += 1;+ }+ if !self.args.is_empty() {+ len += 1;+ }+ if self.fun_definition.is_some() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.HigherOrderUDFExprNode", len)?;+ if !self.fun_name.is_empty() {+ struct_ser.serialize_field("funName", &self.fun_name)?;+ }+ if !self.args.is_empty() {+ struct_ser.serialize_field("args", &self.args)?;+ }+ if let Some(v) = self.fun_definition.as_ref() {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("funDefinition", pbjson::private::base64::encode(&v).as_str())?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for HigherOrderUdfExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "fun_name",+ "funName",+ "args",+ "fun_definition",+ "funDefinition",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ FunName,+ Args,+ FunDefinition,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "funName" | "fun_name" => Ok(GeneratedField::FunName),+ "args" => Ok(GeneratedField::Args),+ "funDefinition" | "fun_definition" => Ok(GeneratedField::FunDefinition),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = HigherOrderUdfExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.HigherOrderUDFExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<HigherOrderUdfExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut fun_name__ = None;+ let mut args__ = None;+ let mut fun_definition__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::FunName => {+ if fun_name__.is_some() {+ return Err(serde::de::Error::duplicate_field("funName"));+ }+ fun_name__ = Some(map_.next_value()?);+ }+ GeneratedField::Args => {+ if args__.is_some() {+ return Err(serde::de::Error::duplicate_field("args"));+ }+ args__ = Some(map_.next_value()?);+ }+ GeneratedField::FunDefinition => {+ if fun_definition__.is_some() {+ return Err(serde::de::Error::duplicate_field("funDefinition"));+ }+ fun_definition__ =+ map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0)+ ;+ }+ }+ }+ Ok(HigherOrderUdfExprNode {+ fun_name: fun_name__.unwrap_or_default(),+ args: args__.unwrap_or_default(),+ fun_definition: fun_definition__,+ })+ }+ }+ deserializer.deserialize_struct("datafusion.HigherOrderUDFExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for ILikeNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -11321,7 +11452,7 @@ impl<'de> serde::Deserialize<'de> for JsonSinkExecNode {deserializer.deserialize_struct("datafusion.JsonSinkExecNode", FIELDS, GeneratedVisitor)}}-impl serde::Serialize for LikeNode {+impl serde::Serialize for Lambda {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>where@@ -11329,54 +11460,37 @@ impl serde::Serialize for LikeNode {{use serde::ser::SerializeStruct;let mut len = 0;- if self.negated {- len += 1;- }- if self.expr.is_some() {+ if !self.params.is_empty() {len += 1;}- if self.pattern.is_some() {- len += 1;- }- if !self.escape_char.is_empty() {+ if self.body.is_some() {len += 1;}- let mut struct_ser = serializer.serialize_struct("datafusion.LikeNode", len)?;- if self.negated {- struct_ser.serialize_field("negated", &self.negated)?;- }- if let Some(v) = self.expr.as_ref() {- struct_ser.serialize_field("expr", v)?;- }- if let Some(v) = self.pattern.as_ref() {- struct_ser.serialize_field("pattern", v)?;+ let mut struct_ser = serializer.serialize_struct("datafusion.Lambda", len)?;+ if !self.params.is_empty() {+ struct_ser.serialize_field("params", &self.params)?;}- if !self.escape_char.is_empty() {- struct_ser.serialize_field("escapeChar", &self.escape_char)?;+ if let Some(v) = self.body.as_ref() {+ struct_ser.serialize_field("body", v)?;}struct_ser.end()}}-impl<'de> serde::Deserialize<'de> for LikeNode {+impl<'de> serde::Deserialize<'de> for Lambda {#[allow(deprecated)]fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>whereD: serde::Deserializer<'de>,{const FIELDS: &[&str] = &[- "negated",- "expr",- "pattern",- "escape_char",- "escapeChar",+ "params",+ "body",];#[allow(clippy::enum_variant_names)]enum GeneratedField {- Negated,- Expr,- Pattern,- EscapeChar,+ Params,+ Body,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -11398,10 +11512,8 @@ impl<'de> serde::Deserialize<'de> for LikeNode {E: serde::de::Error,{match value {- "negated" => Ok(GeneratedField::Negated),- "expr" => Ok(GeneratedField::Expr),- "pattern" => Ok(GeneratedField::Pattern),- "escapeChar" | "escape_char" => Ok(GeneratedField::EscapeChar),+ "params" => Ok(GeneratedField::Params),+ "body" => Ok(GeneratedField::Body),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -11411,60 +11523,44 @@ impl<'de> serde::Deserialize<'de> for LikeNode {}struct GeneratedVisitor;impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {- type Value = LikeNode;+ type Value = Lambda;fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- formatter.write_str("struct datafusion.LikeNode")+ formatter.write_str("struct datafusion.Lambda")}- fn visit_map<V>(self, mut map_: V) -> std::result::Result<LikeNode, V::Error>+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<Lambda, V::Error>whereV: serde::de::MapAccess<'de>,{- let mut negated__ = None;- let mut expr__ = None;- let mut pattern__ = None;- let mut escape_char__ = None;+ let mut params__ = None;+ let mut body__ = None;while let Some(k) = map_.next_key()? {match k {- GeneratedField::Negated => {- if negated__.is_some() {- return Err(serde::de::Error::duplicate_field("negated"));- }- negated__ = Some(map_.next_value()?);- }- GeneratedField::Expr => {- if expr__.is_some() {- return Err(serde::de::Error::duplicate_field("expr"));- }- expr__ = map_.next_value()?;- }- GeneratedField::Pattern => {- if pattern__.is_some() {- return Err(serde::de::Error::duplicate_field("pattern"));+ GeneratedField::Params => {+ if params__.is_some() {+ return Err(serde::de::Error::duplicate_field("params"));}- pattern__ = map_.next_value()?;+ params__ = Some(map_.next_value()?);}- GeneratedField::EscapeChar => {- if escape_char__.is_some() {- return Err(serde::de::Error::duplicate_field("escapeChar"));+ GeneratedField::Body => {+ if body__.is_some() {+ return Err(serde::de::Error::duplicate_field("body"));}- escape_char__ = Some(map_.next_value()?);+ body__ = map_.next_value()?;}}}- Ok(LikeNode {- negated: negated__.unwrap_or_default(),- expr: expr__,- pattern: pattern__,- escape_char: escape_char__.unwrap_or_default(),+ Ok(Lambda {+ params: params__.unwrap_or_default(),+ body: body__,})}}- deserializer.deserialize_struct("datafusion.LikeNode", FIELDS, GeneratedVisitor)+ deserializer.deserialize_struct("datafusion.Lambda", FIELDS, GeneratedVisitor)}}-impl serde::Serialize for LimitNode {+impl serde::Serialize for LambdaVariable {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>where@@ -11472,49 +11568,37 @@ impl serde::Serialize for LimitNode {{use serde::ser::SerializeStruct;let mut len = 0;- if self.input.is_some() {- len += 1;- }- if self.skip != 0 {+ if !self.name.is_empty() {len += 1;}- if self.fetch != 0 {+ if self.field.is_some() {len += 1;}- let mut struct_ser = serializer.serialize_struct("datafusion.LimitNode", len)?;- if let Some(v) = self.input.as_ref() {- struct_ser.serialize_field("input", v)?;- }- if self.skip != 0 {- #[allow(clippy::needless_borrow)]- #[allow(clippy::needless_borrows_for_generic_args)]- struct_ser.serialize_field("skip", ToString::to_string(&self.skip).as_str())?;+ let mut struct_ser = serializer.serialize_struct("datafusion.LambdaVariable", len)?;+ if !self.name.is_empty() {+ struct_ser.serialize_field("name", &self.name)?;}- if self.fetch != 0 {- #[allow(clippy::needless_borrow)]- #[allow(clippy::needless_borrows_for_generic_args)]- struct_ser.serialize_field("fetch", ToString::to_string(&self.fetch).as_str())?;+ if let Some(v) = self.field.as_ref() {+ struct_ser.serialize_field("field", v)?;}struct_ser.end()}}-impl<'de> serde::Deserialize<'de> for LimitNode {+impl<'de> serde::Deserialize<'de> for LambdaVariable {#[allow(deprecated)]fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>whereD: serde::Deserializer<'de>,{const FIELDS: &[&str] = &[- "input",- "skip",- "fetch",+ "name",+ "field",];#[allow(clippy::enum_variant_names)]enum GeneratedField {- Input,- Skip,- Fetch,+ Name,+ Field,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -11536,9 +11620,8 @@ impl<'de> serde::Deserialize<'de> for LimitNode {E: serde::de::Error,{match value {- "input" => Ok(GeneratedField::Input),- "skip" => Ok(GeneratedField::Skip),- "fetch" => Ok(GeneratedField::Fetch),+ "name" => Ok(GeneratedField::Name),+ "field" => Ok(GeneratedField::Field),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -11548,48 +11631,312 @@ impl<'de> serde::Deserialize<'de> for LimitNode {}struct GeneratedVisitor;impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {- type Value = LimitNode;+ type Value = LambdaVariable;fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- formatter.write_str("struct datafusion.LimitNode")+ formatter.write_str("struct datafusion.LambdaVariable")}- fn visit_map<V>(self, mut map_: V) -> std::result::Result<LimitNode, V::Error>+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<LambdaVariable, V::Error>whereV: serde::de::MapAccess<'de>,{- let mut input__ = None;- let mut skip__ = None;- let mut fetch__ = None;+ let mut name__ = None;+ let mut field__ = None;while let Some(k) = map_.next_key()? {match k {- GeneratedField::Input => {- if input__.is_some() {- return Err(serde::de::Error::duplicate_field("input"));- }- input__ = map_.next_value()?;- }- GeneratedField::Skip => {- if skip__.is_some() {- return Err(serde::de::Error::duplicate_field("skip"));+ GeneratedField::Name => {+ if name__.is_some() {+ return Err(serde::de::Error::duplicate_field("name"));}- skip__ =- Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)- ;+ name__ = Some(map_.next_value()?);}- GeneratedField::Fetch => {- if fetch__.is_some() {- return Err(serde::de::Error::duplicate_field("fetch"));+ GeneratedField::Field => {+ if field__.is_some() {+ return Err(serde::de::Error::duplicate_field("field"));}- fetch__ =- Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)- ;+ field__ = map_.next_value()?;}}}- Ok(LimitNode {- input: input__,- skip: skip__.unwrap_or_default(),+ Ok(LambdaVariable {+ name: name__.unwrap_or_default(),+ field: field__,+ })+ }+ }+ deserializer.deserialize_struct("datafusion.LambdaVariable", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for LikeNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.negated {+ len += 1;+ }+ if self.expr.is_some() {+ len += 1;+ }+ if self.pattern.is_some() {+ len += 1;+ }+ if !self.escape_char.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.LikeNode", len)?;+ if self.negated {+ struct_ser.serialize_field("negated", &self.negated)?;+ }+ if let Some(v) = self.expr.as_ref() {+ struct_ser.serialize_field("expr", v)?;+ }+ if let Some(v) = self.pattern.as_ref() {+ struct_ser.serialize_field("pattern", v)?;+ }+ if !self.escape_char.is_empty() {+ struct_ser.serialize_field("escapeChar", &self.escape_char)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for LikeNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "negated",+ "expr",+ "pattern",+ "escape_char",+ "escapeChar",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Negated,+ Expr,+ Pattern,+ EscapeChar,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "negated" => Ok(GeneratedField::Negated),+ "expr" => Ok(GeneratedField::Expr),+ "pattern" => Ok(GeneratedField::Pattern),+ "escapeChar" | "escape_char" => Ok(GeneratedField::EscapeChar),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = LikeNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.LikeNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<LikeNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut negated__ = None;+ let mut expr__ = None;+ let mut pattern__ = None;+ let mut escape_char__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Negated => {+ if negated__.is_some() {+ return Err(serde::de::Error::duplicate_field("negated"));+ }+ negated__ = Some(map_.next_value()?);+ }+ GeneratedField::Expr => {+ if expr__.is_some() {+ return Err(serde::de::Error::duplicate_field("expr"));+ }+ expr__ = map_.next_value()?;+ }+ GeneratedField::Pattern => {+ if pattern__.is_some() {+ return Err(serde::de::Error::duplicate_field("pattern"));+ }+ pattern__ = map_.next_value()?;+ }+ GeneratedField::EscapeChar => {+ if escape_char__.is_some() {+ return Err(serde::de::Error::duplicate_field("escapeChar"));+ }+ escape_char__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(LikeNode {+ negated: negated__.unwrap_or_default(),+ expr: expr__,+ pattern: pattern__,+ escape_char: escape_char__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.LikeNode", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for LimitNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.input.is_some() {+ len += 1;+ }+ if self.skip != 0 {+ len += 1;+ }+ if self.fetch != 0 {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.LimitNode", len)?;+ if let Some(v) = self.input.as_ref() {+ struct_ser.serialize_field("input", v)?;+ }+ if self.skip != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("skip", ToString::to_string(&self.skip).as_str())?;+ }+ if self.fetch != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("fetch", ToString::to_string(&self.fetch).as_str())?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for LimitNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "input",+ "skip",+ "fetch",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Input,+ Skip,+ Fetch,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "input" => Ok(GeneratedField::Input),+ "skip" => Ok(GeneratedField::Skip),+ "fetch" => Ok(GeneratedField::Fetch),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = LimitNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.LimitNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<LimitNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut input__ = None;+ let mut skip__ = None;+ let mut fetch__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Input => {+ if input__.is_some() {+ return Err(serde::de::Error::duplicate_field("input"));+ }+ input__ = map_.next_value()?;+ }+ GeneratedField::Skip => {+ if skip__.is_some() {+ return Err(serde::de::Error::duplicate_field("skip"));+ }+ skip__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Fetch => {+ if fetch__.is_some() {+ return Err(serde::de::Error::duplicate_field("fetch"));+ }+ fetch__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ }+ }+ Ok(LimitNode {+ input: input__,+ skip: skip__.unwrap_or_default(),fetch: fetch__.unwrap_or_default(),})}@@ -12525,6 +12872,15 @@ impl serde::Serialize for LogicalExprNode {logical_expr_node::ExprType::ScalarSubqueryExpr(v) => {struct_ser.serialize_field("scalarSubqueryExpr", v)?;}+ logical_expr_node::ExprType::HigherOrderUdfExpr(v) => {+ struct_ser.serialize_field("higherOrderUdfExpr", v)?;+ }+ logical_expr_node::ExprType::Lambda(v) => {+ struct_ser.serialize_field("lambda", v)?;+ }+ logical_expr_node::ExprType::LambdaVariable(v) => {+ struct_ser.serialize_field("lambdaVariable", v)?;+ }}}struct_ser.end()@@ -12588,6 +12944,11 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {"unnest","scalar_subquery_expr","scalarSubqueryExpr",+ "higher_order_udf_expr",+ "higherOrderUdfExpr",+ "lambda",+ "lambda_variable",+ "lambdaVariable",];#[allow(clippy::enum_variant_names)]@@ -12624,6 +12985,9 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {Placeholder,Unnest,ScalarSubqueryExpr,+ HigherOrderUdfExpr,+ Lambda,+ LambdaVariable,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -12677,6 +13041,9 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {"placeholder" => Ok(GeneratedField::Placeholder),"unnest" => Ok(GeneratedField::Unnest),"scalarSubqueryExpr" | "scalar_subquery_expr" => Ok(GeneratedField::ScalarSubqueryExpr),+ "higherOrderUdfExpr" | "higher_order_udf_expr" => Ok(GeneratedField::HigherOrderUdfExpr),+ "lambda" => Ok(GeneratedField::Lambda),+ "lambdaVariable" | "lambda_variable" => Ok(GeneratedField::LambdaVariable),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -12921,6 +13288,27 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {return Err(serde::de::Error::duplicate_field("scalarSubqueryExpr"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::ScalarSubqueryExpr)+;+ }+ GeneratedField::HigherOrderUdfExpr => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("higherOrderUdfExpr"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::HigherOrderUdfExpr)+;+ }+ GeneratedField::Lambda => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("lambda"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::Lambda)+;+ }+ GeneratedField::LambdaVariable => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("lambdaVariable"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::LambdaVariable);}}@@ -17982,6 +18370,15 @@ impl serde::Serialize for PhysicalExprNode {physical_expr_node::ExprType::DynamicFilter(v) => {struct_ser.serialize_field("dynamicFilter", v)?;}+ physical_expr_node::ExprType::HigherOrderUdf(v) => {+ struct_ser.serialize_field("higherOrderUdf", v)?;+ }+ physical_expr_node::ExprType::Lambda(v) => {+ struct_ser.serialize_field("lambda", v)?;+ }+ physical_expr_node::ExprType::LambdaVariable(v) => {+ struct_ser.serialize_field("lambdaVariable", v)?;+ }}}struct_ser.end()@@ -18032,6 +18429,11 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"scalarSubquery","dynamic_filter","dynamicFilter",+ "higher_order_udf",+ "higherOrderUdf",+ "lambda",+ "lambda_variable",+ "lambdaVariable",];#[allow(clippy::enum_variant_names)]@@ -18058,6 +18460,9 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {HashExpr,ScalarSubquery,DynamicFilter,+ HigherOrderUdf,+ Lambda,+ LambdaVariable,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -18101,6 +18506,9 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),"scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery),"dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),+ "higherOrderUdf" | "higher_order_udf" => Ok(GeneratedField::HigherOrderUdf),+ "lambda" => Ok(GeneratedField::Lambda),+ "lambdaVariable" | "lambda_variable" => Ok(GeneratedField::LambdaVariable),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -18277,6 +18685,27 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {return Err(serde::de::Error::duplicate_field("dynamicFilter"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::DynamicFilter)+;+ }+ GeneratedField::HigherOrderUdf => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("higherOrderUdf"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::HigherOrderUdf)+;+ }+ GeneratedField::Lambda => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("lambda"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::Lambda)+;+ }+ GeneratedField::LambdaVariable => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("lambdaVariable"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::LambdaVariable);}}@@ -18634,17 +19063,131 @@ impl<'de> serde::Deserialize<'de> for PhysicalHashExprNode {}}}- Ok(PhysicalHashExprNode {- on_columns: on_columns__.unwrap_or_default(),- seed0: seed0__.unwrap_or_default(),- description: description__.unwrap_or_default(),+ Ok(PhysicalHashExprNode {+ on_columns: on_columns__.unwrap_or_default(),+ seed0: seed0__.unwrap_or_default(),+ description: description__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalHashExprNode", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for PhysicalHashRepartition {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.hash_expr.is_empty() {+ len += 1;+ }+ if self.partition_count != 0 {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalHashRepartition", len)?;+ if !self.hash_expr.is_empty() {+ struct_ser.serialize_field("hashExpr", &self.hash_expr)?;+ }+ if self.partition_count != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("partitionCount", ToString::to_string(&self.partition_count).as_str())?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalHashRepartition {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "hash_expr",+ "hashExpr",+ "partition_count",+ "partitionCount",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ HashExpr,+ PartitionCount,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),+ "partitionCount" | "partition_count" => Ok(GeneratedField::PartitionCount),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalHashRepartition;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalHashRepartition")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalHashRepartition, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut hash_expr__ = None;+ let mut partition_count__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::HashExpr => {+ if hash_expr__.is_some() {+ return Err(serde::de::Error::duplicate_field("hashExpr"));+ }+ hash_expr__ = Some(map_.next_value()?);+ }+ GeneratedField::PartitionCount => {+ if partition_count__.is_some() {+ return Err(serde::de::Error::duplicate_field("partitionCount"));+ }+ partition_count__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ }+ }+ Ok(PhysicalHashRepartition {+ hash_expr: hash_expr__.unwrap_or_default(),+ partition_count: partition_count__.unwrap_or_default(),})}}- deserializer.deserialize_struct("datafusion.PhysicalHashExprNode", FIELDS, GeneratedVisitor)+ deserializer.deserialize_struct("datafusion.PhysicalHashRepartition", FIELDS, GeneratedVisitor)}}-impl serde::Serialize for PhysicalHashRepartition {+impl serde::Serialize for PhysicalHigherOrderUdfNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>where@@ -18652,41 +19195,48 @@ impl serde::Serialize for PhysicalHashRepartition {{use serde::ser::SerializeStruct;let mut len = 0;- if !self.hash_expr.is_empty() {+ if !self.name.is_empty() {len += 1;}- if self.partition_count != 0 {+ if !self.args.is_empty() {len += 1;}- let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalHashRepartition", len)?;- if !self.hash_expr.is_empty() {- struct_ser.serialize_field("hashExpr", &self.hash_expr)?;+ if self.fun_definition.is_some() {+ len += 1;}- if self.partition_count != 0 {+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalHigherOrderUdfNode", len)?;+ if !self.name.is_empty() {+ struct_ser.serialize_field("name", &self.name)?;+ }+ if !self.args.is_empty() {+ struct_ser.serialize_field("args", &self.args)?;+ }+ if let Some(v) = self.fun_definition.as_ref() {#[allow(clippy::needless_borrow)]#[allow(clippy::needless_borrows_for_generic_args)]- struct_ser.serialize_field("partitionCount", ToString::to_string(&self.partition_count).as_str())?;+ struct_ser.serialize_field("funDefinition", pbjson::private::base64::encode(&v).as_str())?;}struct_ser.end()}}-impl<'de> serde::Deserialize<'de> for PhysicalHashRepartition {+impl<'de> serde::Deserialize<'de> for PhysicalHigherOrderUdfNode {#[allow(deprecated)]fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>whereD: serde::Deserializer<'de>,{const FIELDS: &[&str] = &[- "hash_expr",- "hashExpr",- "partition_count",- "partitionCount",+ "name",+ "args",+ "fun_definition",+ "funDefinition",];#[allow(clippy::enum_variant_names)]enum GeneratedField {- HashExpr,- PartitionCount,+ Name,+ Args,+ FunDefinition,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -18708,8 +19258,9 @@ impl<'de> serde::Deserialize<'de> for PhysicalHashRepartition {E: serde::de::Error,{match value {- "hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),- "partitionCount" | "partition_count" => Ok(GeneratedField::PartitionCount),+ "name" => Ok(GeneratedField::Name),+ "args" => Ok(GeneratedField::Args),+ "funDefinition" | "fun_definition" => Ok(GeneratedField::FunDefinition),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -18719,43 +19270,51 @@ impl<'de> serde::Deserialize<'de> for PhysicalHashRepartition {}struct GeneratedVisitor;impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {- type Value = PhysicalHashRepartition;+ type Value = PhysicalHigherOrderUdfNode;fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- formatter.write_str("struct datafusion.PhysicalHashRepartition")+ formatter.write_str("struct datafusion.PhysicalHigherOrderUdfNode")}- fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalHashRepartition, V::Error>+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalHigherOrderUdfNode, V::Error>whereV: serde::de::MapAccess<'de>,{- let mut hash_expr__ = None;- let mut partition_count__ = None;+ let mut name__ = None;+ let mut args__ = None;+ let mut fun_definition__ = None;while let Some(k) = map_.next_key()? {match k {- GeneratedField::HashExpr => {- if hash_expr__.is_some() {- return Err(serde::de::Error::duplicate_field("hashExpr"));+ GeneratedField::Name => {+ if name__.is_some() {+ return Err(serde::de::Error::duplicate_field("name"));}- hash_expr__ = Some(map_.next_value()?);+ name__ = Some(map_.next_value()?);}- GeneratedField::PartitionCount => {- if partition_count__.is_some() {- return Err(serde::de::Error::duplicate_field("partitionCount"));+ GeneratedField::Args => {+ if args__.is_some() {+ return Err(serde::de::Error::duplicate_field("args"));}- partition_count__ =- Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ args__ = Some(map_.next_value()?);+ }+ GeneratedField::FunDefinition => {+ if fun_definition__.is_some() {+ return Err(serde::de::Error::duplicate_field("funDefinition"));+ }+ fun_definition__ =+ map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0);}}}- Ok(PhysicalHashRepartition {- hash_expr: hash_expr__.unwrap_or_default(),- partition_count: partition_count__.unwrap_or_default(),+ Ok(PhysicalHigherOrderUdfNode {+ name: name__.unwrap_or_default(),+ args: args__.unwrap_or_default(),+ fun_definition: fun_definition__,})}}- deserializer.deserialize_struct("datafusion.PhysicalHashRepartition", FIELDS, GeneratedVisitor)+ deserializer.deserialize_struct("datafusion.PhysicalHigherOrderUdfNode", FIELDS, GeneratedVisitor)}}impl serde::Serialize for PhysicalInListNode {@@ -19065,6 +19624,224 @@ impl<'de> serde::Deserialize<'de> for PhysicalIsNull {deserializer.deserialize_struct("datafusion.PhysicalIsNull", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalLambdaExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.params.is_empty() {+ len += 1;+ }+ if self.body.is_some() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalLambdaExprNode", len)?;+ if !self.params.is_empty() {+ struct_ser.serialize_field("params", &self.params)?;+ }+ if let Some(v) = self.body.as_ref() {+ struct_ser.serialize_field("body", v)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalLambdaExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "params",+ "body",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Params,+ Body,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "params" => Ok(GeneratedField::Params),+ "body" => Ok(GeneratedField::Body),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalLambdaExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalLambdaExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalLambdaExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut params__ = None;+ let mut body__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Params => {+ if params__.is_some() {+ return Err(serde::de::Error::duplicate_field("params"));+ }+ params__ = Some(map_.next_value()?);+ }+ GeneratedField::Body => {+ if body__.is_some() {+ return Err(serde::de::Error::duplicate_field("body"));+ }+ body__ = map_.next_value()?;+ }+ }+ }+ Ok(PhysicalLambdaExprNode {+ params: params__.unwrap_or_default(),+ body: body__,+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalLambdaExprNode", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for PhysicalLambdaVariableExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.index != 0 {+ len += 1;+ }+ if self.field.is_some() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalLambdaVariableExprNode", len)?;+ if self.index != 0 {+ struct_ser.serialize_field("index", &self.index)?;+ }+ if let Some(v) = self.field.as_ref() {+ struct_ser.serialize_field("field", v)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalLambdaVariableExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "index",+ "field",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Index,+ Field,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "index" => Ok(GeneratedField::Index),+ "field" => Ok(GeneratedField::Field),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalLambdaVariableExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalLambdaVariableExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalLambdaVariableExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut index__ = None;+ let mut field__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Index => {+ if index__.is_some() {+ return Err(serde::de::Error::duplicate_field("index"));+ }+ index__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Field => {+ if field__.is_some() {+ return Err(serde::de::Error::duplicate_field("field"));+ }+ field__ = map_.next_value()?;+ }+ }+ }+ Ok(PhysicalLambdaVariableExprNode {+ index: index__.unwrap_or_default(),+ field: field__,+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalLambdaVariableExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalLikeExprNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 675ead23f4..db51edfd5d 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -721,7 +721,7 @@ pub struct SubqueryAliasNode {pub struct LogicalExprNode {#[prost(oneof = "logical_expr_node::ExprType",- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36"+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39")]pub expr_type: ::core::option::Option<logical_expr_node::ExprType>,}@@ -802,6 +802,12 @@ pub mod logical_expr_node {/// Subquery expressions#[prost(message, tag = "36")]ScalarSubqueryExpr(::prost::alloc::boxed::Box<super::ScalarSubqueryExprNode>),+ #[prost(message, tag = "37")]+ HigherOrderUdfExpr(super::HigherOrderUdfExprNode),+ #[prost(message, tag = "38")]+ Lambda(::prost::alloc::boxed::Box<super::Lambda>),+ #[prost(message, tag = "39")]+ LambdaVariable(super::LambdaVariable),}}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]@@ -991,6 +997,29 @@ pub struct ScalarUdfExprNode {pub fun_definition: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct HigherOrderUdfExprNode {+ #[prost(string, tag = "1")]+ pub fun_name: ::prost::alloc::string::String,+ #[prost(message, repeated, tag = "2")]+ pub args: ::prost::alloc::vec::Vec<LogicalExprNode>,+ #[prost(bytes = "vec", optional, tag = "3")]+ pub fun_definition: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct Lambda {+ #[prost(string, repeated, tag = "1")]+ pub params: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,+ #[prost(message, optional, boxed, tag = "2")]+ pub body: ::core::option::Option<::prost::alloc::boxed::Box<LogicalExprNode>>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct LambdaVariable {+ #[prost(string, tag = "1")]+ pub name: ::prost::alloc::string::String,+ #[prost(message, optional, tag = "2")]+ pub field: ::core::option::Option<super::datafusion_common::Field>,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct WindowExprNode {#[prost(message, repeated, tag = "4")]pub exprs: ::prost::alloc::vec::Vec<LogicalExprNode>,@@ -1475,7 +1504,7 @@ pub struct PhysicalExprNode {pub expr_id: ::core::option::Option<u64>,#[prost(oneof = "physical_expr_node::ExprType",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26")]pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,}@@ -1532,6 +1561,12 @@ pub mod physical_expr_node {ScalarSubquery(super::PhysicalScalarSubqueryExprNode),#[prost(message, tag = "23")]DynamicFilter(::prost::alloc::boxed::Box<super::PhysicalDynamicFilterNode>),+ #[prost(message, tag = "24")]+ HigherOrderUdf(super::PhysicalHigherOrderUdfNode),+ #[prost(message, tag = "25")]+ Lambda(::prost::alloc::boxed::Box<super::PhysicalLambdaExprNode>),+ #[prost(message, tag = "26")]+ LambdaVariable(super::PhysicalLambdaVariableExprNode),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -1563,6 +1598,29 @@ pub struct PhysicalScalarUdfNode {pub return_field_name: ::prost::alloc::string::String,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalHigherOrderUdfNode {+ #[prost(string, tag = "1")]+ pub name: ::prost::alloc::string::String,+ #[prost(message, repeated, tag = "2")]+ pub args: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ #[prost(bytes = "vec", optional, tag = "3")]+ pub fun_definition: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalLambdaExprNode {+ #[prost(string, repeated, tag = "1")]+ pub params: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,+ #[prost(message, optional, boxed, tag = "2")]+ pub body: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalExprNode>>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalLambdaVariableExprNode {+ #[prost(uint32, tag = "1")]+ pub index: u32,+ #[prost(message, optional, tag = "2")]+ pub field: ::core::option::Option<super::datafusion_common::Field>,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct PhysicalAggregateExprNode {#[prost(message, repeated, tag = "2")]pub expr: ::prost::alloc::vec::Vec<PhysicalExprNode>,diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex 372213e38f..6d9a73e06f 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -28,7 +28,9 @@ use datafusion_execution::registry::FunctionRegistry;use datafusion_expr::dml::{InsertOp, MergeIntoAction, MergeIntoClause, MergeIntoClauseKind, MergeIntoOp,};-use datafusion_expr::expr::{Alias, NullTreatment, Placeholder, Sort};+use datafusion_expr::expr::{+ Alias, Lambda, LambdaVariable, NullTreatment, Placeholder, Sort,+};use datafusion_expr::expr::{Unnest, WildcardOptions};use datafusion_expr::logical_plan::Subquery;use datafusion_expr::{@@ -718,6 +720,22 @@ pub fn parse_expr(parse_exprs(args, ctx, codec)?,)))}+ ExprType::HigherOrderUdfExpr(protobuf::HigherOrderUdfExprNode {+ fun_name,+ args,+ fun_definition,+ }) => {+ let hof_fn = match fun_definition {+ Some(buf) => codec.try_decode_higher_order_function(fun_name, buf)?,+ None => ctx+ .higher_order_function(fun_name.as_str())+ .or_else(|_| codec.try_decode_higher_order_function(fun_name, &[]))?,+ };+ Ok(Expr::HigherOrderFunction(expr::HigherOrderFunction::new(+ hof_fn,+ parse_exprs(args, ctx, codec)?,+ )))+ }ExprType::AggregateUdfExpr(pb) => {let agg_fn = match &pb.fun_definition {Some(buf) => codec.try_decode_udaf(&pb.fun_name, buf)?,@@ -791,6 +809,16 @@ pub fn parse_expr()?;Ok(Expr::ScalarSubquery(subquery))}+ ExprType::Lambda(lambda) => Ok(Expr::Lambda(Lambda::new(+ lambda.params.clone(),+ parse_required_expr(lambda.body.as_deref(), ctx, "body", codec)?,+ ))),+ ExprType::LambdaVariable(lambda_variable) => {+ Ok(Expr::LambdaVariable(LambdaVariable::new(+ lambda_variable.name.clone(),+ lambda_variable.field.as_ref().optional()?.map(Arc::new),+ )))+ }}}diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex b2b035af88..23ce254e99 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -26,8 +26,9 @@ use datafusion_expr::dml::{MergeIntoAction, MergeIntoClause, MergeIntoClauseKind, MergeIntoOp,};use datafusion_expr::expr::{- self, AggregateFunctionParams, Alias, Between, BinaryExpr, Cast, GroupingSet, InList,- Like, NullTreatment, Placeholder, ScalarFunction, Unnest,+ self, AggregateFunctionParams, Alias, Between, BinaryExpr, Cast, GroupingSet,+ HigherOrderFunction, InList, Lambda, LambdaVariable, Like, NullTreatment,+ Placeholder, ScalarFunction, Unnest,};use datafusion_expr::logical_plan::Subquery;use datafusion_expr::{@@ -413,6 +414,19 @@ pub fn serialize_expr(})),}}+ Expr::HigherOrderFunction(HigherOrderFunction { func, args }) => {+ let mut buf = Vec::new();+ let _ = codec.try_encode_higher_order_function(func.as_ref(), &mut buf);+ protobuf::LogicalExprNode {+ expr_type: Some(ExprType::HigherOrderUdfExpr(+ protobuf::HigherOrderUdfExprNode {+ fun_name: func.name().to_string(),+ fun_definition: (!buf.is_empty()).then_some(buf),+ args: serialize_exprs(args, codec)?,+ },+ )),+ }+ }Expr::Not(expr) => {let expr = Box::new(protobuf::Not {expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)),@@ -640,11 +654,22 @@ pub fn serialize_expr(.unwrap_or(HashMap::new()),})),},- Expr::HigherOrderFunction(_) | Expr::Lambda(_) | Expr::LambdaVariable(_) => {- return Err(Error::General(- "Proto serialization error: Lambda not implemented".to_string(),- ));- }+ Expr::Lambda(Lambda { params, body }) => protobuf::LogicalExprNode {+ expr_type: Some(ExprType::Lambda(Box::new(protobuf::Lambda {+ params: params.clone(),+ body: Some(Box::new(serialize_expr(body, codec)?)),+ }))),+ },+ Expr::LambdaVariable(LambdaVariable {+ name,+ field,+ spans: _,+ }) => protobuf::LogicalExprNode {+ expr_type: Some(ExprType::LambdaVariable(protobuf::LambdaVariable {+ name: name.clone(),+ field: field.as_deref().map(|v| v.try_into()).transpose()?,+ })),+ },};Ok(expr_node)diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 19e49f3cb8..b908b504bb 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -43,9 +43,12 @@ use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::WindowFunctionDefinition;use datafusion_expr::dml::InsertOp;use datafusion_expr::execution_props::SubqueryIndex;+use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;-use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};+use datafusion_physical_expr::{+ HigherOrderFunctionExpr, LexOrdering, PhysicalSortExpr, ScalarFunctionExpr,+};use datafusion_physical_plan::expressions::{BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,@@ -336,6 +339,31 @@ pub fn parse_physical_expr_with_converter(.with_nullable(e.nullable),)}+ ExprType::HigherOrderUdf(e) => {+ let func = match &e.fun_definition {+ Some(buf) => {+ ctx.codec().try_decode_higher_order_function(&e.name, buf)?+ }+ None => ctx+ .task_ctx()+ .higher_order_function(e.name.as_str())+ .or_else(|_| {+ ctx.codec().try_decode_higher_order_function(&e.name, &[])+ })?,+ };+ let func_def = Arc::clone(&func);++ let args = parse_physical_exprs(&e.args, ctx, input_schema, proto_converter)?;++ let config_options = Arc::clone(ctx.task_ctx().session_config().options());++ Arc::new(HigherOrderFunctionExpr::try_new_with_schema(+ func_def,+ args,+ input_schema,+ config_options,+ )?)+ }ExprType::LikeExpr(_) => LikeExpr::try_from_proto(proto, &decode_ctx)?,ExprType::HashExpr(_) => HashExpr::try_from_proto(proto, &decode_ctx)?,ExprType::ScalarSubquery(sq) => {@@ -371,6 +399,10 @@ pub fn parse_physical_expr_with_converter(ctx.codec().try_decode_expr(extension.expr.as_slice(), &inputs)? as _}+ ExprType::Lambda(_) => LambdaExpr::try_from_proto(proto, &decode_ctx)?,+ ExprType::LambdaVariable(_) => {+ LambdaVariable::try_from_proto(proto, &decode_ctx)?+ }};Ok(pexpr)diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 72f6e5af5b..0744a94dce 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -54,7 +54,7 @@ use datafusion_datasource_parquet::source::ParquetSource;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};-use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};+use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};use datafusion_functions_table::generate_series::{Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,};@@ -3946,6 +3946,24 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {Ok(())}+ fn try_decode_higher_order_function(+ &self,+ name: &str,+ _buf: &[u8],+ ) -> Result<Arc<HigherOrderUDF>> {+ not_impl_err!(+ "PhysicalExtensionCodec is not provided for higher order function {name}"+ )+ }++ fn try_encode_higher_order_function(+ &self,+ _node: &HigherOrderUDF,+ _buf: &mut Vec<u8>,+ ) -> Result<()> {+ Ok(())+ }+fn try_decode_expr(&self,_buf: &[u8],diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex e93bb9cc5f..4025b580e8 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -31,9 +31,9 @@ use datafusion_datasource_json::file_format::JsonSink;#[cfg(feature = "parquet")]use datafusion_datasource_parquet::file_format::ParquetSink;use datafusion_expr::WindowFrame;-use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};+use datafusion_physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};@@ -318,6 +318,19 @@ pub fn serialize_physical_expr_with_converter(},)),})+ } else if let Some(expr) = expr.downcast_ref::<HigherOrderFunctionExpr>() {+ let mut buf = Vec::new();+ codec.try_encode_higher_order_function(expr.fun(), &mut buf)?;+ Ok(protobuf::PhysicalExprNode {+ expr_id,+ expr_type: Some(protobuf::physical_expr_node::ExprType::HigherOrderUdf(+ protobuf::PhysicalHigherOrderUdfNode {+ name: expr.name().to_string(),+ args: serialize_physical_exprs(expr.args(), codec, proto_converter)?,+ fun_definition: (!buf.is_empty()).then_some(buf),+ },+ )),+ })} else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {Ok(protobuf::PhysicalExprNode {expr_id,diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rsindex 3abbaccf79..7a95ee0c29 100644--- a/datafusion/proto/tests/cases/mod.rs+++ b/datafusion/proto/tests/cases/mod.rs@@ -21,8 +21,10 @@ use datafusion::physical_expr::PhysicalExpr;use datafusion_common::plan_err;use datafusion_expr::function::AccumulatorArgs;use datafusion_expr::{- Accumulator, AggregateUDFImpl, LimitEffect, PartitionEvaluator, ScalarFunctionArgs,- ScalarUDFImpl, Signature, Volatility, WindowUDFImpl,+ Accumulator, AggregateUDFImpl, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs,+ HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, LimitEffect,+ PartitionEvaluator, ScalarFunctionArgs, ScalarUDFImpl, Signature, ValueOrLambda,+ Volatility, WindowUDFImpl,};use datafusion_functions_window_common::field::WindowUDFFieldArgs;use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;@@ -180,3 +182,66 @@ pub(in crate::cases) struct CustomUDWFNode {#[prost(string, tag = "1")]pub payload: String,}++#[derive(Debug, PartialEq, Eq, Hash)]+pub(in crate::cases) struct MyHigherOrderUDF {+ signature: HigherOrderSignature,+ pub payload: String,+}++impl MyHigherOrderUDF {+ pub fn new(payload: String) -> Self {+ Self {+ signature: HigherOrderSignature::any(2, Volatility::Immutable),+ payload,+ }+ }+}++impl HigherOrderUDFImpl for MyHigherOrderUDF {+ fn name(&self) -> &str {+ "higher_order_udf"+ }++ fn signature(&self) -> &HigherOrderSignature {+ &self.signature+ }++ fn lambda_parameters(+ &self,+ _step: usize,+ fields: &[ValueOrLambda<FieldRef, Option<FieldRef>>],+ ) -> datafusion_common::Result<LambdaParametersProgress> {+ let list = match fields.first() {+ Some(ValueOrLambda::Value(field)) => field,+ _ => return plan_err!("higher_order_udf expects a list as first argument"),+ };+ let element = match list.data_type() {+ DataType::List(field) | DataType::LargeList(field) => Arc::clone(field),+ other => {+ return plan_err!("higher_order_udf expected a list, got {other}");+ }+ };+ Ok(LambdaParametersProgress::Complete(vec![vec![element]]))+ }++ fn return_field_from_args(+ &self,+ _args: HigherOrderReturnFieldArgs,+ ) -> datafusion_common::Result<FieldRef> {+ Ok(Arc::new(Field::new("", DataType::Int64, true)))+ }++ fn invoke_with_args(+ &self,+ _args: HigherOrderFunctionArgs,+ ) -> datafusion_common::Result<ColumnarValue> {+ unimplemented!()+ }+}++#[derive(Clone, PartialEq, ::prost::Message)]+pub(in crate::cases) struct MyHigherOrderUdfNode {+ #[prost(string, tag = "1")]+ pub payload: String,+}diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 431b49dc8b..b3edc0f5ce 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -83,17 +83,17 @@ use datafusion_expr::dml::{MergeIntoAction, MergeIntoClause, MergeIntoClauseKind, MergeIntoOp,};use datafusion_expr::expr::{- self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, NullTreatment,- ScalarFunction, Unnest, WildcardOptions,+ self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, LambdaVariable, Like,+ NullTreatment, ScalarFunction, Unnest, WildcardOptions,};use datafusion_expr::logical_plan::{ExplainOption, Extension, UserDefinedLogicalNodeCore,};use datafusion_expr::{Accumulator, AggregateUDF, ColumnarValue, DmlStatement, ExprFunctionExt,- ExprSchemable, LimitEffect, Literal, LogicalPlan, LogicalPlanBuilder, Operator,- PartitionEvaluator, RangePartitioning, Repartition, ScalarUDF, Signature, TryCast,- Volatility, WindowFrame, WindowFrameBound, WindowFrameUnits,+ ExprSchemable, HigherOrderUDF, LimitEffect, Literal, LogicalPlan, LogicalPlanBuilder,+ Operator, PartitionEvaluator, RangePartitioning, Repartition, ScalarUDF, Signature,+ TryCast, Volatility, WindowFrame, WindowFrameBound, WindowFrameUnits,WindowFunctionDefinition, WindowUDF, WindowUDFImpl, WriteOp,};use datafusion_functions_aggregate::average::avg_udaf;@@ -118,7 +118,10 @@ use datafusion_proto::logical_plan::{};use datafusion_proto::protobuf;-use crate::cases::{MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf, MyRegexUdfNode};+use crate::cases::{+ MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF, MyHigherOrderUdfNode,+ MyRegexUdf, MyRegexUdfNode,+};#[cfg(feature = "json")]fn roundtrip_json_test(proto: &protobuf::LogicalExprNode) {@@ -1806,6 +1809,41 @@ impl LogicalExtensionCodec for UDFExtensionCodec {.map_err(|err| internal_datafusion_err!("failed to encode udf: {err}"))?;Ok(())}++ fn try_decode_higher_order_function(+ &self,+ name: &str,+ buf: &[u8],+ ) -> Result<Arc<HigherOrderUDF>> {+ if name == "higher_order_udf" {+ let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode higher_order_udf: {err}")+ })?;++ Ok(Arc::new(HigherOrderUDF::new_from_impl(+ MyHigherOrderUDF::new(proto.payload),+ )))+ } else {+ not_impl_err!("unrecognized higher order UDF implementation, cannot decode")+ }+ }++ fn try_encode_higher_order_function(+ &self,+ node: &HigherOrderUDF,+ buf: &mut Vec<u8>,+ ) -> Result<()> {+ let hof = (node.inner().as_ref() as &dyn Any)+ .downcast_ref::<MyHigherOrderUDF>()+ .unwrap();+ let proto = MyHigherOrderUdfNode {+ payload: hof.payload.clone(),+ };+ proto+ .encode(buf)+ .map_err(|err| internal_datafusion_err!("failed to encode hof: {err}"))?;+ Ok(())+ }}#[test]@@ -2867,6 +2905,115 @@ fn roundtrip_aggregate_udf_extension_codec() {roundtrip_json_test(&proto);}+fn dummy_higher_order_function_args() -> Vec<Expr> {+ let list = ScalarValue::List(ScalarValue::new_list_nullable(+ &[ScalarValue::Int32(Some(1))],+ &DataType::Int32,+ ));+ let lambda_var_with_field = Expr::LambdaVariable(LambdaVariable::new(+ "x".to_string(),+ Some(Arc::new(Field::new("x", DataType::Int32, true))),+ ));+ let lambda_var_without_field =+ Expr::LambdaVariable(LambdaVariable::new("x".into(), None));+ let lambda = lambda(["x"], lambda_var_with_field + lambda_var_without_field);+ vec![Expr::Literal(list, None), lambda]+}++#[test]+fn roundtrip_higher_order_function() {+ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));++ let test_expr = Expr::HigherOrderFunction(expr::HigherOrderFunction::new(+ Arc::clone(&hof),+ dummy_higher_order_function_args(),+ ));++ let ctx = SessionContext::new();+ ctx.register_higher_order_function(hof);++ roundtrip_expr_test(test_expr.clone(), ctx);++ // Now test loading the HOF without registering it in the context, but rather creating it+ // in the extension codec.+ #[derive(Debug)]+ struct DummyHigherOrderUDFExtensionCodec;++ impl LogicalExtensionCodec for DummyHigherOrderUDFExtensionCodec {+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[LogicalPlan],+ _ctx: &TaskContext,+ ) -> Result<Extension> {+ not_impl_err!("LogicalExtensionCodec is not provided")+ }++ fn try_encode(&self, _node: &Extension, _buf: &mut Vec<u8>) -> Result<()> {+ not_impl_err!("LogicalExtensionCodec is not provided")+ }++ fn try_decode_table_provider(+ &self,+ _buf: &[u8],+ _table_ref: &TableReference,+ _schema: SchemaRef,+ _ctx: &TaskContext,+ ) -> Result<Arc<dyn TableProvider>> {+ not_impl_err!("LogicalExtensionCodec is not provided")+ }++ fn try_encode_table_provider(+ &self,+ _table_ref: &TableReference,+ _node: Arc<dyn TableProvider>,+ _buf: &mut Vec<u8>,+ ) -> Result<()> {+ not_impl_err!("LogicalExtensionCodec is not provided")+ }++ fn try_decode_higher_order_function(+ &self,+ name: &str,+ _buf: &[u8],+ ) -> Result<Arc<HigherOrderUDF>> {+ if name == "higher_order_udf" {+ Ok(Arc::new(HigherOrderUDF::new_from_impl(+ MyHigherOrderUDF::new("payload".to_string()),+ )))+ } else {+ Err(internal_datafusion_err!("HOF {name} not found"))+ }+ }+ }++ let ctx = SessionContext::new();+ roundtrip_expr_test_with_codec(test_expr, ctx, &DummyHigherOrderUDFExtensionCodec)+}++#[test]+fn roundtrip_higher_order_udf_extension_codec() {+ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));++ let test_expr = Expr::HigherOrderFunction(expr::HigherOrderFunction::new(+ hof,+ dummy_higher_order_function_args(),+ ));++ let ctx = SessionContext::new();+ let proto = serialize_expr(&test_expr, &UDFExtensionCodec).expect("serialize expr");+ let round_trip =+ from_proto::parse_expr(&proto, ctx.task_ctx().as_ref(), &UDFExtensionCodec)+ .expect("parse expr");++ assert_eq!(format!("{:?}", test_expr), format!("{round_trip:?}"));+ roundtrip_json_test(&proto);+}+#[test]fn roundtrip_grouping_sets() {let test_expr = Expr::GroupingSet(GroupingSet::GroupingSets(vec![diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex bb99a1ecbf..6ede6fc0e9 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -52,7 +52,7 @@ use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctiuse datafusion::physical_expr::expressions::Literal;use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion::physical_expr::{- LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,+ HigherOrderFunctionExpr, LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,};use datafusion::physical_optimizer::PhysicalOptimizerRule;use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;@@ -108,7 +108,7 @@ use datafusion_datasource::{TableSchema, TableSchemaBuilder};use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};use datafusion_expr::dml::InsertOp;use datafusion_expr::{- Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue,+ Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, HigherOrderUDF,ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,WindowFrame, WindowFrameBound, WindowUDF,execution_props::{ScalarSubqueryResults, SubqueryIndex},@@ -140,9 +140,10 @@ use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};use prost::Message;use crate::cases::{- CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,- MyRegexUdfNode,+ CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF,+ MyHigherOrderUdfNode, MyRegexUdf, MyRegexUdfNode,};+use datafusion_physical_expr::expressions::{LambdaVariable, is_not_null, lambda};use datafusion_physical_expr::utils::reassign_expr_columns;/// Perform a serde roundtrip and assert that the string representation of the before and after plans@@ -1472,6 +1473,42 @@ impl PhysicalExtensionCodec for UDFExtensionCodec {}Ok(())}++ fn try_decode_higher_order_function(+ &self,+ name: &str,+ buf: &[u8],+ ) -> Result<Arc<HigherOrderUDF>> {+ if name == "higher_order_udf" {+ let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode higher_order_udf: {err}")+ })?;++ Ok(Arc::new(HigherOrderUDF::new_from_impl(+ MyHigherOrderUDF::new(proto.payload),+ )))+ } else {+ not_impl_err!("unrecognized higher order UDF implementation, cannot decode")+ }+ }++ fn try_encode_higher_order_function(+ &self,+ node: &HigherOrderUDF,+ buf: &mut Vec<u8>,+ ) -> Result<()> {+ if let Some(hof) = (node.inner().as_ref() as &dyn std::any::Any)+ .downcast_ref::<MyHigherOrderUDF>()+ {+ let proto = MyHigherOrderUdfNode {+ payload: hof.payload.clone(),+ };+ proto.encode(buf).map_err(|err| {+ internal_datafusion_err!("failed to encode hof: {err:?}")+ })?;+ }+ Ok(())+ }}#[test]@@ -1532,6 +1569,92 @@ fn roundtrip_scalar_udf_extension_codec() -> Result<()> {Ok(())}+#[test]+fn roundtrip_higher_order_udf() -> Result<()> {+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));+ let list_field = Field::new(+ "list_col",+ DataType::List(Arc::clone(&element_field)),+ false,+ );+ let schema = Arc::new(Schema::new(vec![list_field]));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));++ let expr = HigherOrderFunctionExpr::try_new_with_schema(+ Arc::clone(&hof),+ vec![+ col("list_col", &schema)?,+ lambda(+ ["v"],+ is_not_null(Arc::new(LambdaVariable::new(1, element_field)))?,+ )?,+ ],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?;++ let project = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(expr),+ alias: "a".to_string(),+ }],+ input,+ )?;++ let ctx = SessionContext::new();+ ctx.register_higher_order_function(hof);++ roundtrip_test_with_context(Arc::new(project), &ctx)+}++#[test]+fn roundtrip_higher_order_udf_extension_codec() -> Result<()> {+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));+ let list_field = Field::new(+ "list_col",+ DataType::List(Arc::clone(&element_field)),+ false,+ );+ let schema = Arc::new(Schema::new(vec![list_field]));+ let input = Arc::new(EmptyExec::new(schema.clone()));++ let lambda_body = Arc::new(LambdaVariable::new(1, Arc::clone(&element_field)));+ let lambda_expr = lambda(["v"], lambda_body)?;++ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));+ let hof_expr = Arc::new(HigherOrderFunctionExpr::try_new_with_schema(+ hof,+ vec![col("list_col", &schema)?, lambda_expr],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?);++ let project = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: hof_expr,+ alias: "out".to_string(),+ }],+ input,+ )?;++ let ctx = SessionContext::new();+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(+ Arc::new(project),+ &ctx,+ &UDFExtensionCodec,+ &proto_converter,+ )?;+ Ok(())+}+#[test]fn roundtrip_udwf_extension_codec() -> Result<()> {let field_a = Field::new("a", DataType::Int64, false);diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rsindex 850fd42ce1..a0a917e323 100644--- a/datafusion/proto/tests/cases/serialize.rs+++ b/datafusion/proto/tests/cases/serialize.rs@@ -22,14 +22,18 @@ use arrow::datatypes::{DataType, Field};use datafusion::execution::FunctionRegistry;use datafusion::prelude::SessionContext;-use datafusion_expr::expr::Placeholder;-use datafusion_expr::{ColumnarValue, col, create_udf, lit};+use datafusion_common::ScalarValue;+use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable, Placeholder};+use datafusion_expr::{ColumnarValue, HigherOrderUDF, col, create_udf, lambda, lit};use datafusion_expr::{Expr, Volatility};use datafusion_functions::string;use datafusion_proto::bytes::Serializeable;use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec;+use datafusion_proto::logical_plan::from_proto::parse_expr;use datafusion_proto::logical_plan::to_proto::serialize_expr;+use crate::cases::MyHigherOrderUDF;+#[test]#[should_panic(expected = "Error decoding expr as protobuf: failed to decode Protobuf message"@@ -298,3 +302,92 @@ fn test_expression_serialization_roundtrip() {name.split('(').next().unwrap().to_string()}}++/// return a `SessionContext` with `MyHigherOrderUDF` registered as a higher-order UDF+fn context_with_higher_order_function() -> SessionContext {+ let ctx = SessionContext::new();+ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));+ ctx.register_higher_order_function(hof);+ ctx+}++fn dummy_higher_order_function_call(hof: Arc<HigherOrderUDF>) -> Expr {+ let list = ScalarValue::List(ScalarValue::new_list_nullable(+ &[ScalarValue::Int32(Some(1))],+ &DataType::Int32,+ ));+ let lambda_var_with_field = Expr::LambdaVariable(LambdaVariable::new(+ "x".to_string(),+ Some(Arc::new(Field::new("x", DataType::Int32, true))),+ ));+ let lambda_var_without_field =+ Expr::LambdaVariable(LambdaVariable::new("x".into(), None));+ let lambda = lambda(["x"], lambda_var_with_field + lambda_var_without_field);+ Expr::HigherOrderFunction(HigherOrderFunction::new(+ hof,+ vec![Expr::Literal(list, None), lambda],+ ))+}++#[test]+fn hof_roundtrip_with_registry() {+ let ctx = context_with_higher_order_function();+ let hof = ctx+ .higher_order_function("higher_order_udf")+ .expect("could not find higher order udf");++ let expr = dummy_higher_order_function_call(hof);++ let bytes = expr.to_bytes().unwrap();+ let deserialized_expr =+ Expr::from_bytes_with_ctx(&bytes, ctx.task_ctx().as_ref()).unwrap();++ assert_eq!(expr, deserialized_expr);+}++#[test]+#[should_panic(+ expected = "LogicalExtensionCodec is not provided for higher order function higher_order_udf"+)]+fn hof_roundtrip_without_registry() {+ let ctx = context_with_higher_order_function();+ let hof = ctx+ .higher_order_function("higher_order_udf")+ .expect("could not find higher order udf");++ let expr = dummy_higher_order_function_call(hof);++ let bytes = expr.to_bytes().unwrap();+ Expr::from_bytes(&bytes).unwrap();+}++#[test]+fn test_higher_order_serialization_roundtrip() {+ let ctx = SessionContext::new();+ let list = ScalarValue::List(ScalarValue::new_list_nullable(+ &[ScalarValue::Int32(Some(1))],+ &DataType::Int32,+ ));+ let lambda_var_with_field = Expr::LambdaVariable(LambdaVariable::new(+ "x".to_string(),+ Some(Arc::new(Field::new("x", DataType::Int32, true))),+ ));+ let lambda_var_without_field =+ Expr::LambdaVariable(LambdaVariable::new("x".into(), None));+ let lambda = lambda(["x"], lambda_var_with_field + lambda_var_without_field);+ let args = vec![Expr::Literal(list, None), lambda];++ for function in datafusion::functions_nested::all_default_higher_order_functions() {+ let expr =+ Expr::HigherOrderFunction(HigherOrderFunction::new(function, args.clone()));++ let extension_codec = DefaultLogicalExtensionCodec {};+ let proto = serialize_expr(&expr, &extension_codec).unwrap();+ let deserialize =+ parse_expr(&proto, ctx.task_ctx().as_ref(), &extension_codec).unwrap();++ assert_eq!(deserialize, expr);+ }+}