Apache DataFusion SQL Query Engine
feat: support `null_treatment`, `distinct`, and `filter` for window functions in proto (#18024)
Khanh Duongcadf42955146
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex ee9ac0e790..11103472ae 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -518,6 +518,7 @@ message AggregateUDFExprNode {LogicalExprNode filter = 3;repeated SortExprNode order_by = 4;optional bytes fun_definition = 6;+ optional NullTreatment null_treatment = 7;}message ScalarUDFExprNode {@@ -538,6 +539,9 @@ message WindowExprNode {// repeated LogicalExprNode filter = 7;WindowFrame window_frame = 8;optional bytes fun_definition = 10;+ optional NullTreatment null_treatment = 11;+ bool distinct = 12;+ LogicalExprNode filter = 13;}message BetweenNode {@@ -622,6 +626,11 @@ message WindowFrameBound {datafusion_common.ScalarValue bound_value = 2;}+enum NullTreatment {+ RESPECT_NULLS = 0;+ IGNORE_NULLS = 1;+}+///////////////////////////////////////////////////////////////////////////////////////////////////// Arrow Data Types///////////////////////////////////////////////////////////////////////////////////////////////////@@ -1365,4 +1374,4 @@ message SortMergeJoinExecNode {JoinFilter filter = 5;repeated SortExprNode sort_options = 6;datafusion_common.NullEquality null_equality = 7;-}\ No newline at end of file+}diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 29967d8120..b34da2c312 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -596,6 +596,9 @@ impl serde::Serialize for AggregateUdfExprNode {if self.fun_definition.is_some() {len += 1;}+ if self.null_treatment.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.AggregateUDFExprNode", len)?;if !self.fun_name.is_empty() {struct_ser.serialize_field("funName", &self.fun_name)?;@@ -617,6 +620,11 @@ impl serde::Serialize for AggregateUdfExprNode {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("funDefinition", pbjson::private::base64::encode(&v).as_str())?;}+ if let Some(v) = self.null_treatment.as_ref() {+ let v = NullTreatment::try_from(*v)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", *v)))?;+ struct_ser.serialize_field("nullTreatment", &v)?;+ }struct_ser.end()}}@@ -636,6 +644,8 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {"orderBy","fun_definition","funDefinition",+ "null_treatment",+ "nullTreatment",];#[allow(clippy::enum_variant_names)]@@ -646,6 +656,7 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {Filter,OrderBy,FunDefinition,+ NullTreatment,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -673,6 +684,7 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {"filter" => Ok(GeneratedField::Filter),"orderBy" | "order_by" => Ok(GeneratedField::OrderBy),"funDefinition" | "fun_definition" => Ok(GeneratedField::FunDefinition),+ "nullTreatment" | "null_treatment" => Ok(GeneratedField::NullTreatment),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -698,6 +710,7 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {let mut filter__ = None;let mut order_by__ = None;let mut fun_definition__ = None;+ let mut null_treatment__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::FunName => {@@ -738,6 +751,12 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0);}+ GeneratedField::NullTreatment => {+ if null_treatment__.is_some() {+ return Err(serde::de::Error::duplicate_field("nullTreatment"));+ }+ null_treatment__ = map_.next_value::<::std::option::Option<NullTreatment>>()?.map(|x| x as i32);+ }}}Ok(AggregateUdfExprNode {@@ -747,6 +766,7 @@ impl<'de> serde::Deserialize<'de> for AggregateUdfExprNode {filter: filter__,order_by: order_by__.unwrap_or_default(),fun_definition: fun_definition__,+ null_treatment: null_treatment__,})}}@@ -13284,6 +13304,77 @@ impl<'de> serde::Deserialize<'de> for Not {deserializer.deserialize_struct("datafusion.Not", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for NullTreatment {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::RespectNulls => "RESPECT_NULLS",+ Self::IgnoreNulls => "IGNORE_NULLS",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for NullTreatment {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "RESPECT_NULLS",+ "IGNORE_NULLS",+ ];++ struct GeneratedVisitor;++ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = NullTreatment;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "RESPECT_NULLS" => Ok(NullTreatment::RespectNulls),+ "IGNORE_NULLS" => Ok(NullTreatment::IgnoreNulls),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for OptimizedLogicalPlanType {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -23514,6 +23605,15 @@ impl serde::Serialize for WindowExprNode {if self.fun_definition.is_some() {len += 1;}+ if self.null_treatment.is_some() {+ len += 1;+ }+ if self.distinct {+ len += 1;+ }+ if self.filter.is_some() {+ len += 1;+ }if self.window_function.is_some() {len += 1;}@@ -23535,6 +23635,17 @@ impl serde::Serialize for WindowExprNode {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("funDefinition", pbjson::private::base64::encode(&v).as_str())?;}+ if let Some(v) = self.null_treatment.as_ref() {+ let v = NullTreatment::try_from(*v)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", *v)))?;+ struct_ser.serialize_field("nullTreatment", &v)?;+ }+ if self.distinct {+ struct_ser.serialize_field("distinct", &self.distinct)?;+ }+ if let Some(v) = self.filter.as_ref() {+ struct_ser.serialize_field("filter", v)?;+ }if let Some(v) = self.window_function.as_ref() {match v {window_expr_node::WindowFunction::Udaf(v) => {@@ -23564,6 +23675,10 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {"windowFrame","fun_definition","funDefinition",+ "null_treatment",+ "nullTreatment",+ "distinct",+ "filter","udaf","udwf",];@@ -23575,6 +23690,9 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {OrderBy,WindowFrame,FunDefinition,+ NullTreatment,+ Distinct,+ Filter,Udaf,Udwf,}@@ -23603,6 +23721,9 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {"orderBy" | "order_by" => Ok(GeneratedField::OrderBy),"windowFrame" | "window_frame" => Ok(GeneratedField::WindowFrame),"funDefinition" | "fun_definition" => Ok(GeneratedField::FunDefinition),+ "nullTreatment" | "null_treatment" => Ok(GeneratedField::NullTreatment),+ "distinct" => Ok(GeneratedField::Distinct),+ "filter" => Ok(GeneratedField::Filter),"udaf" => Ok(GeneratedField::Udaf),"udwf" => Ok(GeneratedField::Udwf),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),@@ -23629,6 +23750,9 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {let mut order_by__ = None;let mut window_frame__ = None;let mut fun_definition__ = None;+ let mut null_treatment__ = None;+ let mut distinct__ = None;+ let mut filter__ = None;let mut window_function__ = None;while let Some(k) = map_.next_key()? {match k {@@ -23664,6 +23788,24 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0);}+ GeneratedField::NullTreatment => {+ if null_treatment__.is_some() {+ return Err(serde::de::Error::duplicate_field("nullTreatment"));+ }+ null_treatment__ = map_.next_value::<::std::option::Option<NullTreatment>>()?.map(|x| x as i32);+ }+ GeneratedField::Distinct => {+ if distinct__.is_some() {+ return Err(serde::de::Error::duplicate_field("distinct"));+ }+ distinct__ = Some(map_.next_value()?);+ }+ GeneratedField::Filter => {+ if filter__.is_some() {+ return Err(serde::de::Error::duplicate_field("filter"));+ }+ filter__ = map_.next_value()?;+ }GeneratedField::Udaf => {if window_function__.is_some() {return Err(serde::de::Error::duplicate_field("udaf"));@@ -23684,6 +23826,9 @@ impl<'de> serde::Deserialize<'de> for WindowExprNode {order_by: order_by__.unwrap_or_default(),window_frame: window_frame__,fun_definition: fun_definition__,+ null_treatment: null_treatment__,+ distinct: distinct__.unwrap_or_default(),+ filter: filter__,window_function: window_function__,})}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex d3b5f566e9..2e1c482db6 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -605,7 +605,7 @@ pub mod logical_expr_node {TryCast(::prost::alloc::boxed::Box<super::TryCastNode>),/// window expressions#[prost(message, tag = "18")]- WindowExpr(super::WindowExprNode),+ WindowExpr(::prost::alloc::boxed::Box<super::WindowExprNode>),/// AggregateUDF expressions#[prost(message, tag = "19")]AggregateUdfExpr(::prost::alloc::boxed::Box<super::AggregateUdfExprNode>),@@ -795,6 +795,8 @@ pub struct AggregateUdfExprNode {pub order_by: ::prost::alloc::vec::Vec<SortExprNode>,#[prost(bytes = "vec", optional, tag = "6")]pub fun_definition: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,+ #[prost(enumeration = "NullTreatment", optional, tag = "7")]+ pub null_treatment: ::core::option::Option<i32>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ScalarUdfExprNode {@@ -818,6 +820,12 @@ pub struct WindowExprNode {pub window_frame: ::core::option::Option<WindowFrame>,#[prost(bytes = "vec", optional, tag = "10")]pub fun_definition: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,+ #[prost(enumeration = "NullTreatment", optional, tag = "11")]+ pub null_treatment: ::core::option::Option<i32>,+ #[prost(bool, tag = "12")]+ pub distinct: bool,+ #[prost(message, optional, boxed, tag = "13")]+ pub filter: ::core::option::Option<::prost::alloc::boxed::Box<LogicalExprNode>>,#[prost(oneof = "window_expr_node::WindowFunction", tags = "3, 9")]pub window_function: ::core::option::Option<window_expr_node::WindowFunction>,}@@ -2129,6 +2137,32 @@ impl WindowFrameBoundType {}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]+pub enum NullTreatment {+ RespectNulls = 0,+ IgnoreNulls = 1,+}+impl NullTreatment {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::RespectNulls => "RESPECT_NULLS",+ Self::IgnoreNulls => "IGNORE_NULLS",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "RESPECT_NULLS" => Some(Self::RespectNulls),+ "IGNORE_NULLS" => Some(Self::IgnoreNulls),+ _ => None,+ }+ }+}+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]pub enum DateUnit {Day = 0,DateMillisecond = 1,diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex cbfa15183b..ec6415adc4 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -23,7 +23,7 @@ use datafusion_common::{RecursionUnnestOption, Result, ScalarValue, TableReference, UnnestOptions,};use datafusion_expr::dml::InsertOp;-use datafusion_expr::expr::{Alias, Placeholder, Sort};+use datafusion_expr::expr::{Alias, NullTreatment, Placeholder, Sort};use datafusion_expr::expr::{Unnest, WildcardOptions};use datafusion_expr::{expr::{self, InList, WindowFunction},@@ -243,6 +243,15 @@ impl From<protobuf::dml_node::Type> for WriteOp {}}+impl From<protobuf::NullTreatment> for NullTreatment {+ fn from(t: protobuf::NullTreatment) -> Self {+ match t {+ protobuf::NullTreatment::RespectNulls => NullTreatment::RespectNulls,+ protobuf::NullTreatment::IgnoreNulls => NullTreatment::IgnoreNulls,+ }+ }+}+pub fn parse_expr(proto: &protobuf::LogicalExprNode,registry: &dyn FunctionRegistry,@@ -301,9 +310,21 @@ pub fn parse_expr(exec_datafusion_err!("missing window frame during deserialization")})?;- // TODO: support null treatment, distinct, and filter in proto.- // See https://github.com/apache/datafusion/issues/17417- match window_function {+ let null_treatment = match expr.null_treatment {+ Some(null_treatment) => {+ let null_treatment = protobuf::NullTreatment::try_from(null_treatment)+ .map_err(|_| {+ proto_error(format!(+ "Received a WindowExprNode message with unknown NullTreatment {}",+ null_treatment+ ))+ })?;+ Some(NullTreatment::from(null_treatment))+ }+ None => None,+ };++ let agg_fn = match window_function {window_expr_node::WindowFunction::Udaf(udaf_name) => {let udaf_function = match &expr.fun_definition {Some(buf) => codec.try_decode_udaf(udaf_name, buf)?,@@ -311,17 +332,7 @@ pub fn parse_expr(.udaf(udaf_name).or_else(|_| codec.try_decode_udaf(udaf_name, &[]))?,};-- let args = parse_exprs(&expr.exprs, registry, codec)?;- Expr::from(WindowFunction::new(- expr::WindowFunctionDefinition::AggregateUDF(udaf_function),- args,- ))- .partition_by(partition_by)- .order_by(order_by)- .window_frame(window_frame)- .build()- .map_err(Error::DataFusionError)+ expr::WindowFunctionDefinition::AggregateUDF(udaf_function)}window_expr_node::WindowFunction::Udwf(udwf_name) => {let udwf_function = match &expr.fun_definition {@@ -330,19 +341,28 @@ pub fn parse_expr(.udwf(udwf_name).or_else(|_| codec.try_decode_udwf(udwf_name, &[]))?,};-- let args = parse_exprs(&expr.exprs, registry, codec)?;- Expr::from(WindowFunction::new(- expr::WindowFunctionDefinition::WindowUDF(udwf_function),- args,- ))- .partition_by(partition_by)- .order_by(order_by)- .window_frame(window_frame)- .build()- .map_err(Error::DataFusionError)+ expr::WindowFunctionDefinition::WindowUDF(udwf_function)}+ };++ let args = parse_exprs(&expr.exprs, registry, codec)?;+ let mut builder = Expr::from(WindowFunction::new(agg_fn, args))+ .partition_by(partition_by)+ .order_by(order_by)+ .window_frame(window_frame)+ .null_treatment(null_treatment);++ if expr.distinct {+ builder = builder.distinct();+ };++ if let Some(filter) =+ parse_optional_expr(expr.filter.as_deref(), registry, codec)?+ {+ builder = builder.filter(filter);}++ builder.build().map_err(Error::DataFusionError)}ExprType::Alias(alias) => Ok(Expr::Alias(Alias::new(parse_required_expr(alias.expr.as_deref(), registry, "expr", codec)?,@@ -571,6 +591,19 @@ pub fn parse_expr(.udaf(&pb.fun_name).or_else(|_| codec.try_decode_udaf(&pb.fun_name, &[]))?,};+ let null_treatment = match pb.null_treatment {+ Some(null_treatment) => {+ let null_treatment = protobuf::NullTreatment::try_from(null_treatment)+ .map_err(|_| {+ proto_error(format!(+ "Received an AggregateUdfExprNode message with unknown NullTreatment {}",+ null_treatment+ ))+ })?;+ Some(NullTreatment::from(null_treatment))+ }+ None => None,+ };Ok(Expr::AggregateFunction(expr::AggregateFunction::new_udf(agg_fn,@@ -578,7 +611,7 @@ pub fn parse_expr(pb.distinct,parse_optional_expr(pb.filter.as_deref(), registry, codec)?.map(Box::new),parse_sorts(&pb.order_by, registry, codec)?,- None,+ null_treatment,)))}diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex 1be3300008..6238c2f1cd 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -25,7 +25,7 @@ use datafusion_common::{NullEquality, TableReference, UnnestOptions};use datafusion_expr::dml::InsertOp;use datafusion_expr::expr::{self, AggregateFunctionParams, Alias, Between, BinaryExpr, Cast, GroupingSet, InList,- Like, Placeholder, ScalarFunction, Unnest,+ Like, NullTreatment, Placeholder, ScalarFunction, Unnest,};use datafusion_expr::WriteOp;use datafusion_expr::{@@ -314,11 +314,9 @@ pub fn serialize_expr(ref partition_by,ref order_by,ref window_frame,- // TODO: support null treatment, distinct, and filter in proto.- // See https://github.com/apache/datafusion/issues/17417- null_treatment: _,- distinct: _,- filter: _,+ ref null_treatment,+ ref distinct,+ ref filter,},} = window_fun.as_ref();let mut buf = Vec::new();@@ -342,16 +340,24 @@ pub fn serialize_expr(let window_frame: Option<protobuf::WindowFrame> =Some(window_frame.try_into()?);+let window_expr = protobuf::WindowExprNode {exprs: serialize_exprs(args, codec)?,window_function: Some(window_function),partition_by,order_by,window_frame,+ distinct: *distinct,+ filter: match filter {+ Some(e) => Some(Box::new(serialize_expr(e.as_ref(), codec)?)),+ None => None,+ },+ null_treatment: null_treatment+ .map(|nt| protobuf::NullTreatment::from(nt).into()),fun_definition,};protobuf::LogicalExprNode {- expr_type: Some(ExprType::WindowExpr(window_expr)),+ expr_type: Some(ExprType::WindowExpr(Box::new(window_expr))),}}Expr::AggregateFunction(expr::AggregateFunction {@@ -362,7 +368,7 @@ pub fn serialize_expr(ref distinct,ref filter,ref order_by,- null_treatment: _,+ ref null_treatment,},}) => {let mut buf = Vec::new();@@ -379,6 +385,8 @@ pub fn serialize_expr(},order_by: serialize_sorts(order_by, codec)?,fun_definition: (!buf.is_empty()).then_some(buf),+ null_treatment: null_treatment+ .map(|nt| protobuf::NullTreatment::from(nt).into()),},))),}@@ -722,3 +730,12 @@ impl From<&WriteOp> for protobuf::dml_node::Type {}}}++impl From<NullTreatment> for protobuf::NullTreatment {+ fn from(t: NullTreatment) -> Self {+ match t {+ NullTreatment::RespectNulls => protobuf::NullTreatment::RespectNulls,+ NullTreatment::IgnoreNulls => protobuf::NullTreatment::IgnoreNulls,+ }+ }+}diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex c5d4b49092..3d51038eba 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -73,8 +73,8 @@ use datafusion_common::{};use datafusion_expr::dml::CopyTo;use datafusion_expr::expr::{- self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, ScalarFunction,- Unnest, WildcardOptions,+ self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, NullTreatment,+ ScalarFunction, Unnest, WildcardOptions,};use datafusion_expr::logical_plan::{Extension, UserDefinedLogicalNodeCore};use datafusion_expr::{@@ -2190,7 +2190,11 @@ fn roundtrip_aggregate_udf() {Arc::new(vec![DataType::Float64, DataType::UInt32]),);- let test_expr = Expr::AggregateFunction(expr::AggregateFunction::new_udf(+ let ctx = SessionContext::new();+ ctx.register_udaf(dummy_agg.clone());++ // null_treatment absent+ let test_expr1 = Expr::AggregateFunction(expr::AggregateFunction::new_udf(Arc::new(dummy_agg.clone()),vec![lit(1.0_f64)],false,@@ -2199,10 +2203,29 @@ fn roundtrip_aggregate_udf() {None,));- let ctx = SessionContext::new();- ctx.register_udaf(dummy_agg);+ // null_treatment respect nulls+ let test_expr2 = Expr::AggregateFunction(expr::AggregateFunction::new_udf(+ Arc::new(dummy_agg.clone()),+ vec![lit(1.0_f64)],+ true,+ Some(Box::new(lit(true))),+ vec![],+ Some(NullTreatment::RespectNulls),+ ));- roundtrip_expr_test(test_expr, ctx);+ // null_treatment ignore nulls+ let test_expr3 = Expr::AggregateFunction(expr::AggregateFunction::new_udf(+ Arc::new(dummy_agg),+ vec![lit(1.0_f64)],+ true,+ Some(Box::new(lit(true))),+ vec![],+ Some(NullTreatment::IgnoreNulls),+ ));++ roundtrip_expr_test(test_expr1, ctx.clone());+ roundtrip_expr_test(test_expr2, ctx.clone());+ roundtrip_expr_test(test_expr3, ctx);}fn dummy_udf() -> ScalarUDF {@@ -2566,8 +2589,10 @@ fn roundtrip_window() {.window_frame(row_number_frame.clone()).build().unwrap();+ ctx.register_udwf(dummy_window_udf);- let text_expr7 = Expr::from(expr::WindowFunction::new(+ // 7. test with average udaf+ let test_expr7 = Expr::from(expr::WindowFunction::new(WindowFunctionDefinition::AggregateUDF(avg_udaf()),vec![col("col1")],))@@ -2575,7 +2600,53 @@ fn roundtrip_window() {.build().unwrap();- ctx.register_udwf(dummy_window_udf);+ // 8. test with respect nulls+ let test_expr8 = Expr::from(expr::WindowFunction::new(+ WindowFunctionDefinition::WindowUDF(rank_udwf()),+ vec![],+ ))+ .partition_by(vec![col("col1")])+ .order_by(vec![col("col2").sort(true, false)])+ .window_frame(WindowFrame::new(Some(false)))+ .null_treatment(NullTreatment::RespectNulls)+ .build()+ .unwrap();++ // 9. test with ignore nulls+ let test_expr9 = Expr::from(expr::WindowFunction::new(+ WindowFunctionDefinition::WindowUDF(rank_udwf()),+ vec![],+ ))+ .partition_by(vec![col("col1")])+ .order_by(vec![col("col2").sort(true, false)])+ .window_frame(WindowFrame::new(Some(false)))+ .null_treatment(NullTreatment::IgnoreNulls)+ .build()+ .unwrap();++ // 10. test with distinct is `true`+ let test_expr10 = Expr::from(expr::WindowFunction::new(+ WindowFunctionDefinition::WindowUDF(rank_udwf()),+ vec![],+ ))+ .partition_by(vec![col("col1")])+ .order_by(vec![col("col2").sort(true, false)])+ .window_frame(WindowFrame::new(Some(false)))+ .distinct()+ .build()+ .unwrap();++ // 11. test with filter+ let test_expr11 = Expr::from(expr::WindowFunction::new(+ WindowFunctionDefinition::WindowUDF(rank_udwf()),+ vec![],+ ))+ .partition_by(vec![col("col1")])+ .order_by(vec![col("col2").sort(true, false)])+ .window_frame(WindowFrame::new(Some(false)))+ .filter(col("col1").eq(lit(1)))+ .build()+ .unwrap();roundtrip_expr_test(test_expr1, ctx.clone());roundtrip_expr_test(test_expr2, ctx.clone());@@ -2583,7 +2654,11 @@ fn roundtrip_window() {roundtrip_expr_test(test_expr4, ctx.clone());roundtrip_expr_test(test_expr5, ctx.clone());roundtrip_expr_test(test_expr6, ctx.clone());- roundtrip_expr_test(text_expr7, ctx);+ roundtrip_expr_test(test_expr7, ctx.clone());+ roundtrip_expr_test(test_expr8, ctx.clone());+ roundtrip_expr_test(test_expr9, ctx.clone());+ roundtrip_expr_test(test_expr10, ctx.clone());+ roundtrip_expr_test(test_expr11, ctx);}#[tokio::test]