Skip to content

Apache DataFusion SQL Query Engine

ser/de fetch in FilterExec (#20738)

Huaijin4dbb4498fc92
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 4622439b98..581e833f8c 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -725,6 +725,10 @@ impl ExecutionPlan for FilterExec {
})
}
+ fn fetch(&self) -> Option<usize> {
+ self.fetch
+ }
+
fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
Some(Arc::new(Self {
predicate: Arc::clone(&self.predicate),
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index e64bcdb41e..f91b4d1379 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1034,6 +1034,7 @@ message FilterExecNode {
uint32 default_filter_selectivity = 3;
repeated uint32 projection = 9;
uint32 batch_size = 10;
+ optional uint32 fetch = 11;
}
message FileGroup {
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 15b9ba88f4..2b3be747e1 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -6930,6 +6930,9 @@ impl serde::Serialize for FilterExecNode {
if self.batch_size != 0 {
len += 1;
}
+ if self.fetch.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.FilterExecNode", len)?;
if let Some(v) = self.input.as_ref() {
struct_ser.serialize_field("input", v)?;
@@ -6946,6 +6949,9 @@ impl serde::Serialize for FilterExecNode {
if self.batch_size != 0 {
struct_ser.serialize_field("batchSize", &self.batch_size)?;
}
+ if let Some(v) = self.fetch.as_ref() {
+ struct_ser.serialize_field("fetch", v)?;
+ }
struct_ser.end()
}
}
@@ -6963,6 +6969,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
"projection",
"batch_size",
"batchSize",
+ "fetch",
];
#[allow(clippy::enum_variant_names)]
@@ -6972,6 +6979,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
DefaultFilterSelectivity,
Projection,
BatchSize,
+ Fetch,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -6998,6 +7006,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
"defaultFilterSelectivity" | "default_filter_selectivity" => Ok(GeneratedField::DefaultFilterSelectivity),
"projection" => Ok(GeneratedField::Projection),
"batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),
+ "fetch" => Ok(GeneratedField::Fetch),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -7022,6 +7031,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
let mut default_filter_selectivity__ = None;
let mut projection__ = None;
let mut batch_size__ = None;
+ let mut fetch__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Input => {
@@ -7061,6 +7071,14 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
;
}
+ GeneratedField::Fetch => {
+ if fetch__.is_some() {
+ return Err(serde::de::Error::duplicate_field("fetch"));
+ }
+ fetch__ =
+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)
+ ;
+ }
}
}
Ok(FilterExecNode {
@@ -7069,6 +7087,7 @@ impl<'de> serde::Deserialize<'de> for FilterExecNode {
default_filter_selectivity: default_filter_selectivity__.unwrap_or_default(),
projection: projection__.unwrap_or_default(),
batch_size: batch_size__.unwrap_or_default(),
+ fetch: fetch__,
})
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 48ad8dcfea..5a33d89027 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1577,6 +1577,8 @@ pub struct FilterExecNode {
pub projection: ::prost::alloc::vec::Vec<u32>,
#[prost(uint32, tag = "10")]
pub batch_size: u32,
+ #[prost(uint32, optional, tag = "11")]
+ pub fetch: ::core::option::Option<u32>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct FileGroup {
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index fce8ac658e..85406e31da 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -695,6 +695,7 @@ impl protobuf::PhysicalPlanNode {
let filter = FilterExecBuilder::new(predicate, input)
.apply_projection(projection)?
.with_batch_size(filter.batch_size as usize)
+ .with_fetch(filter.fetch.map(|f| f as usize))
.build()?;
match filter_selectivity {
Ok(filter_selectivity) => Ok(Arc::new(
@@ -2334,6 +2335,7 @@ impl protobuf::PhysicalPlanNode {
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
}),
batch_size: exec.batch_size() as u32,
+ fetch: exec.fetch().map(|f| f as u32),
},
))),
})
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index ccee240b94..66ca903e4e 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -795,6 +795,19 @@ fn roundtrip_filter_with_not_and_in_list() -> Result<()> {
)?))
}
+#[test]
+fn roundtrip_filter_with_fetch() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let predicate = col("a", &schema)?;
+ let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))
+ .with_fetch(Some(10))
+ .build()?;
+ assert_eq!(filter.fetch(), Some(10));
+ roundtrip_test(Arc::new(filter))
+}
+
#[test]
fn roundtrip_sort() -> Result<()> {
let field_a = Field::new("a", DataType::Boolean, false);