Apache DataFusion SQL Query Engine
ser/de fetch in FilterExec (#20738)
Huaijin4dbb4498fc92
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 4622439b98..581e833f8c 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -725,6 +725,10 @@ impl ExecutionPlan for FilterExec {})}+ fn fetch(&self) -> Option<usize> {+ self.fetch+ }+fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {Some(Arc::new(Self {predicate: Arc::clone(&self.predicate),diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex e64bcdb41e..f91b4d1379 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -1034,6 +1034,7 @@ message FilterExecNode {uint32 default_filter_selectivity = 3;repeated uint32 projection = 9;uint32 batch_size = 10;+ optional uint32 fetch = 11;}message FileGroup {diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 15b9ba88f4..2b3be747e1 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -6930,6 +6930,9 @@ impl serde::Serialize for FilterExecNode {if self.batch_size != 0 {len += 1;}+ if self.fetch.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.FilterExecNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -6946,6 +6949,9 @@ impl serde::Serialize for FilterExecNode {if self.batch_size != 0 {struct_ser.serialize_field("batchSize", &self.batch_size)?;}+ if let Some(v) = self.fetch.as_ref() {+ struct_ser.serialize_field("fetch", v)?;+ }struct_ser.end()}}@@ -6963,6 +6969,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {"projection","batch_size","batchSize",+ "fetch",];#[allow(clippy::enum_variant_names)]@@ -6972,6 +6979,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {DefaultFilterSelectivity,Projection,BatchSize,+ Fetch,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -6998,6 +7006,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {"defaultFilterSelectivity" | "default_filter_selectivity" => Ok(GeneratedField::DefaultFilterSelectivity),"projection" => Ok(GeneratedField::Projection),"batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),+ "fetch" => Ok(GeneratedField::Fetch),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -7022,6 +7031,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {let mut default_filter_selectivity__ = None;let mut projection__ = None;let mut batch_size__ = None;+ let mut fetch__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -7061,6 +7071,14 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0);}+ GeneratedField::Fetch => {+ if fetch__.is_some() {+ return Err(serde::de::Error::duplicate_field("fetch"));+ }+ fetch__ =+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)+ ;+ }}}Ok(FilterExecNode {@@ -7069,6 +7087,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {default_filter_selectivity: default_filter_selectivity__.unwrap_or_default(),projection: projection__.unwrap_or_default(),batch_size: batch_size__.unwrap_or_default(),+ fetch: fetch__,})}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 48ad8dcfea..5a33d89027 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1577,6 +1577,8 @@ pub struct FilterExecNode {pub projection: ::prost::alloc::vec::Vec<u32>,#[prost(uint32, tag = "10")]pub batch_size: u32,+ #[prost(uint32, optional, tag = "11")]+ pub fetch: ::core::option::Option<u32>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct FileGroup {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex fce8ac658e..85406e31da 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -695,6 +695,7 @@ impl protobuf::PhysicalPlanNode {let filter = FilterExecBuilder::new(predicate, input).apply_projection(projection)?.with_batch_size(filter.batch_size as usize)+ .with_fetch(filter.fetch.map(|f| f as usize)).build()?;match filter_selectivity {Ok(filter_selectivity) => Ok(Arc::new(@@ -2334,6 +2335,7 @@ impl protobuf::PhysicalPlanNode {v.iter().map(|x| *x as u32).collect::<Vec<u32>>()}),batch_size: exec.batch_size() as u32,+ fetch: exec.fetch().map(|f| f as u32),},))),})diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex ccee240b94..66ca903e4e 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -795,6 +795,19 @@ fn roundtrip_filter_with_not_and_in_list() -> Result<()> {)?))}+#[test]+fn roundtrip_filter_with_fetch() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let predicate = col("a", &schema)?;+ let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))+ .with_fetch(Some(10))+ .build()?;+ assert_eq!(filter.fetch(), Some(10));+ roundtrip_test(Arc::new(filter))+}+#[test]fn roundtrip_sort() -> Result<()> {let field_a = Field::new("a", DataType::Boolean, false);