Apache DataFusion SQL Query Engine
preserve FilterExec batch size during ser/de (#19960)
Albert Skalt35e99b97bcb9
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 1edf96fe0c..50fae84b85 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -240,6 +240,11 @@ impl FilterExec {FilterExecBuilder::new(predicate, input).build()}+ /// Get a batch size+ pub fn batch_size(&self) -> usize {+ self.batch_size+ }+/// Set the default selectivitypub fn with_default_selectivity(mut self,diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 2b5e2368c1..2efef9f12e 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -1007,6 +1007,7 @@ message FilterExecNode {PhysicalExprNode expr = 2;uint32 default_filter_selectivity = 3;repeated uint32 projection = 9;+ uint32 batch_size = 10;}message FileGroup {diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 842dc7f632..00870d5ce1 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -6590,6 +6590,9 @@ impl serde::Serialize for FilterExecNode {if !self.projection.is_empty() {len += 1;}+ if self.batch_size != 0 {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.FilterExecNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -6603,6 +6606,9 @@ impl serde::Serialize for FilterExecNode {if !self.projection.is_empty() {struct_ser.serialize_field("projection", &self.projection)?;}+ if self.batch_size != 0 {+ struct_ser.serialize_field("batchSize", &self.batch_size)?;+ }struct_ser.end()}}@@ -6618,6 +6624,8 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {"default_filter_selectivity","defaultFilterSelectivity","projection",+ "batch_size",+ "batchSize",];#[allow(clippy::enum_variant_names)]@@ -6626,6 +6634,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {Expr,DefaultFilterSelectivity,Projection,+ BatchSize,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -6651,6 +6660,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {"expr" => Ok(GeneratedField::Expr),"defaultFilterSelectivity" | "default_filter_selectivity" => Ok(GeneratedField::DefaultFilterSelectivity),"projection" => Ok(GeneratedField::Projection),+ "batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -6674,6 +6684,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {let mut expr__ = None;let mut default_filter_selectivity__ = None;let mut projection__ = None;+ let mut batch_size__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -6705,6 +6716,14 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {.into_iter().map(|x| x.0).collect());}+ GeneratedField::BatchSize => {+ if batch_size__.is_some() {+ return Err(serde::de::Error::duplicate_field("batchSize"));+ }+ batch_size__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }}}Ok(FilterExecNode {@@ -6712,6 +6731,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {expr: expr__,default_filter_selectivity: default_filter_selectivity__.unwrap_or_default(),projection: projection__.unwrap_or_default(),+ batch_size: batch_size__.unwrap_or_default(),})}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 3a7b35509e..39d2604d45 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1546,6 +1546,8 @@ pub struct FilterExecNode {pub default_filter_selectivity: u32,#[prost(uint32, repeated, tag = "9")]pub projection: ::prost::alloc::vec::Vec<u32>,+ #[prost(uint32, tag = "10")]+ pub batch_size: u32,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct FileGroup {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex afb2d7ea51..54892597b5 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -590,6 +590,7 @@ impl protobuf::PhysicalPlanNode {let filter = FilterExecBuilder::new(predicate, input).apply_projection(projection)?+ .with_batch_size(filter.batch_size as usize).build()?;match filter_selectivity {Ok(filter_selectivity) => Ok(Arc::new(@@ -2121,6 +2122,7 @@ impl protobuf::PhysicalPlanNode {projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {v.iter().map(|x| *x as u32).collect::<Vec<u32>>()}),+ batch_size: exec.batch_size() as u32,},))),})