Skip to content

Apache DataFusion SQL Query Engine

preserve FilterExec batch size during ser/de (#19960)

Albert Skalt35e99b97bcb9
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 1edf96fe0c..50fae84b85 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -240,6 +240,11 @@ impl FilterExec {
FilterExecBuilder::new(predicate, input).build()
}
+ /// Get a batch size
+ pub fn batch_size(&self) -> usize {
+ self.batch_size
+ }
+
/// Set the default selectivity
pub fn with_default_selectivity(
mut self,
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index 2b5e2368c1..2efef9f12e 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1007,6 +1007,7 @@ message FilterExecNode {
PhysicalExprNode expr = 2;
uint32 default_filter_selectivity = 3;
repeated uint32 projection = 9;
+ uint32 batch_size = 10;
}
message FileGroup {
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 842dc7f632..00870d5ce1 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -6590,6 +6590,9 @@ impl serde::Serialize for FilterExecNode {
if !self.projection.is_empty() {
len += 1;
}
+ if self.batch_size != 0 {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.FilterExecNode", len)?;
if let Some(v) = self.input.as_ref() {
struct_ser.serialize_field("input", v)?;
@@ -6603,6 +6606,9 @@ impl serde::Serialize for FilterExecNode {
if !self.projection.is_empty() {
struct_ser.serialize_field("projection", &self.projection)?;
}
+ if self.batch_size != 0 {
+ struct_ser.serialize_field("batchSize", &self.batch_size)?;
+ }
struct_ser.end()
}
}
@@ -6618,6 +6624,8 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
"default_filter_selectivity",
"defaultFilterSelectivity",
"projection",
+ "batch_size",
+ "batchSize",
];
#[allow(clippy::enum_variant_names)]
@@ -6626,6 +6634,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
Expr,
DefaultFilterSelectivity,
Projection,
+ BatchSize,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -6651,6 +6660,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
"expr" => Ok(GeneratedField::Expr),
"defaultFilterSelectivity" | "default_filter_selectivity" => Ok(GeneratedField::DefaultFilterSelectivity),
"projection" => Ok(GeneratedField::Projection),
+ "batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -6674,6 +6684,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
let mut expr__ = None;
let mut default_filter_selectivity__ = None;
let mut projection__ = None;
+ let mut batch_size__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Input => {
@@ -6705,6 +6716,14 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
.into_iter().map(|x| x.0).collect())
;
}
+ GeneratedField::BatchSize => {
+ if batch_size__.is_some() {
+ return Err(serde::de::Error::duplicate_field("batchSize"));
+ }
+ batch_size__ =
+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
+ ;
+ }
}
}
Ok(FilterExecNode {
@@ -6712,6 +6731,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
expr: expr__,
default_filter_selectivity: default_filter_selectivity__.unwrap_or_default(),
projection: projection__.unwrap_or_default(),
+ batch_size: batch_size__.unwrap_or_default(),
})
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 3a7b35509e..39d2604d45 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1546,6 +1546,8 @@ pub struct FilterExecNode {
pub default_filter_selectivity: u32,
#[prost(uint32, repeated, tag = "9")]
pub projection: ::prost::alloc::vec::Vec<u32>,
+ #[prost(uint32, tag = "10")]
+ pub batch_size: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct FileGroup {
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index afb2d7ea51..54892597b5 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -590,6 +590,7 @@ impl protobuf::PhysicalPlanNode {
let filter = FilterExecBuilder::new(predicate, input)
.apply_projection(projection)?
+ .with_batch_size(filter.batch_size as usize)
.build()?;
match filter_selectivity {
Ok(filter_selectivity) => Ok(Arc::new(
@@ -2121,6 +2122,7 @@ impl protobuf::PhysicalPlanNode {
projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
}),
+ batch_size: exec.batch_size() as u32,
},
))),
})