Skip to content

Apache DataFusion SQL Query Engine

test(proto): split roundtrip_physical_plan.rs by plan category (#24223)

Adrian Garcia Badaraccod443bab46289
diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rs
index 430b1405b1..4237b1c922 100644
--- a/datafusion/proto/tests/cases/mod.rs
+++ b/datafusion/proto/tests/cases/mod.rs
@@ -32,8 +32,8 @@ use std::fmt::Debug;
use std::hash::Hash;
use std::sync::Arc;
+mod plans;
mod roundtrip_logical_plan;
-mod roundtrip_physical_plan;
mod serialize;
mod stack_safety;
diff --git a/datafusion/proto/tests/cases/plans/aggregates.rs b/datafusion/proto/tests/cases/plans/aggregates.rs
new file mode 100644
index 0000000000..34b51b57b0
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/aggregates.rs
@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `AggregateExec`.
+
+use super::{roundtrip_test, roundtrip_test_with_context};
+use datafusion::arrow::array::ArrayRef;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::logical_expr::Volatility;
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
+use datafusion::physical_plan::PhysicalExpr;
+use datafusion::physical_plan::aggregates::{
+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
+};
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col, lit};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::Result;
+use datafusion_expr::{
+ Accumulator, AccumulatorFactoryFunction, AggregateUDF, Signature, SimpleAggregateUDF,
+};
+use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
+use datafusion_functions_aggregate::array_agg::array_agg_udaf;
+use datafusion_functions_aggregate::average::avg_udaf;
+use datafusion_functions_aggregate::nth_value::nth_value_udaf;
+use datafusion_functions_aggregate::string_agg::string_agg_udaf;
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_aggregate() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema)?, "unused".to_string())];
+
+ let avg_expr = AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("AVG(b)")
+ .build()?;
+ let nth_expr =
+ AggregateExprBuilder::new(nth_value_udaf(), vec![col("b", &schema)?, lit(1u64)])
+ .schema(Arc::clone(&schema))
+ .alias("NTH_VALUE(b, 1)")
+ .build()?;
+ let str_agg_expr =
+ AggregateExprBuilder::new(string_agg_udaf(), vec![col("b", &schema)?, lit(1u64)])
+ .schema(Arc::clone(&schema))
+ .alias("NTH_VALUE(b, 1)")
+ .build()?;
+
+ let test_cases = vec![
+ // AVG
+ vec![Arc::new(avg_expr)],
+ // NTH_VALUE
+ vec![Arc::new(nth_expr)],
+ // STRING_AGG
+ vec![Arc::new(str_agg_expr)],
+ ];
+
+ for aggregates in test_cases {
+ let schema = schema.clone();
+ roundtrip_test(Arc::new(AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new_single(groups.clone()),
+ aggregates,
+ vec![None],
+ Arc::new(EmptyExec::new(schema.clone())),
+ schema,
+ )?))?;
+ }
+
+ Ok(())
+}
+
+#[test]
+fn roundtrip_aggregate_with_limit() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema)?, "unused".to_string())];
+
+ let aggregates = vec![
+ AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("AVG(b)")
+ .build()
+ .map(Arc::new)?,
+ ];
+
+ let agg = AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new_single(groups.clone()),
+ aggregates,
+ vec![None],
+ Arc::new(EmptyExec::new(schema.clone())),
+ schema,
+ )?;
+ let agg = agg.with_limit_options(Some(LimitOptions::new_with_order(12, false)));
+ roundtrip_test(Arc::new(agg))
+}
+
+#[test]
+fn roundtrip_aggregate_with_approx_pencentile_cont() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema)?, "unused".to_string())];
+
+ let aggregates = vec![
+ AggregateExprBuilder::new(
+ approx_percentile_cont_udaf(),
+ vec![col("b", &schema)?, lit(0.5)],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY b)")
+ .build()
+ .map(Arc::new)?,
+ ];
+
+ let agg = AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new_single(groups.clone()),
+ aggregates,
+ vec![None],
+ Arc::new(EmptyExec::new(schema.clone())),
+ schema,
+ )?;
+ roundtrip_test(Arc::new(agg))
+}
+
+#[test]
+fn roundtrip_aggregate_with_sort() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema)?, "unused".to_string())];
+ let sort_exprs = vec![PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }];
+
+ let aggregates = vec![
+ AggregateExprBuilder::new(array_agg_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("ARRAY_AGG(b)")
+ .order_by(sort_exprs)
+ .build()
+ .map(Arc::new)?,
+ ];
+
+ let agg = AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new_single(groups.clone()),
+ aggregates,
+ vec![None],
+ Arc::new(EmptyExec::new(schema.clone())),
+ schema,
+ )?;
+ roundtrip_test(Arc::new(agg))
+}
+
+#[test]
+fn roundtrip_aggregate_udaf() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ #[derive(Debug)]
+ struct Example;
+ impl Accumulator for Example {
+ fn state(&mut self) -> Result<Vec<ScalarValue>> {
+ Ok(vec![ScalarValue::Int64(Some(0))])
+ }
+
+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+
+ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+
+ fn evaluate(&mut self) -> Result<ScalarValue> {
+ Ok(ScalarValue::Int64(Some(0)))
+ }
+
+ fn size(&self) -> usize {
+ 0
+ }
+ }
+
+ let return_type = DataType::Int64;
+ let accumulator: AccumulatorFactoryFunction = Arc::new(|_| Ok(Box::new(Example)));
+
+ let udaf = AggregateUDF::from(SimpleAggregateUDF::new_with_signature(
+ "example",
+ Signature::exact(vec![DataType::Int64], Volatility::Immutable),
+ return_type,
+ accumulator,
+ vec![Field::new("value", DataType::Int64, true).into()],
+ ));
+
+ let ctx = SessionContext::new();
+ ctx.register_udaf(udaf.clone());
+
+ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema)?, "unused".to_string())];
+
+ let aggregates = vec![
+ AggregateExprBuilder::new(Arc::new(udaf), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("example_agg")
+ .build()
+ .map(Arc::new)?,
+ ];
+
+ roundtrip_test_with_context(
+ Arc::new(AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new_single(groups.clone()),
+ aggregates,
+ vec![None],
+ Arc::new(EmptyExec::new(schema.clone())),
+ schema,
+ )?),
+ &ctx,
+ )
+}
diff --git a/datafusion/proto/tests/cases/plans/dispatch.rs b/datafusion/proto/tests/cases/plans/dispatch.rs
new file mode 100644
index 0000000000..5299733447
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/dispatch.rs
@@ -0,0 +1,367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Serde dispatch itself: which hook the central (de)serializer reaches,
+//! and how a custom converter or a deprecated shim participates.
+
+use super::roundtrip_test_and_return;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::{
+ DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,
+ SendableRecordBatchStream,
+};
+use datafusion::prelude::SessionContext;
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::{Result, exec_datafusion_err};
+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+ PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalPlanNodeExt,
+ PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};
+use std::fmt::Formatter;
+use std::sync::{Arc, RwLock};
+use std::vec;
+
+#[derive(Debug)]
+struct DowncastDelegatingExec {
+ inner: Arc<dyn ExecutionPlan>,
+}
+
+impl DowncastDelegatingExec {
+ fn new(inner: Arc<dyn ExecutionPlan>) -> Self {
+ Self { inner }
+ }
+}
+
+impl DisplayAs for DowncastDelegatingExec {
+ fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ self.inner.fmt_as(t, f)
+ }
+}
+
+impl ExecutionPlan for DowncastDelegatingExec {
+ fn name(&self) -> &str {
+ self.inner.name()
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ self.inner.properties()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ self.inner.children()
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ self.inner.apply_expressions(f)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let inner = Arc::clone(&self.inner).with_new_children(children)?;
+ Ok(Arc::new(Self::new(inner)))
+ }
+
+ fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {
+ Some(self.inner.as_ref())
+ }
+
+ fn execute(
+ &self,
+ partition: usize,
+ context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ self.inner.execute(partition, context)
+ }
+}
+
+#[test]
+fn serialize_uses_downcast_delegate() -> Result<()> {
+ let inner: Arc<dyn ExecutionPlan> =
+ Arc::new(EmptyExec::new(Arc::new(Schema::empty())));
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;
+
+ assert!(matches!(
+ proto.physical_plan_type,
+ Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))
+ ));
+
+ Ok(())
+}
+
+/// A wrapper delegating to a plan that serializes itself via the
+/// `try_to_proto` hook must serialize as its delegate: the wrapper's default
+/// hook returns `Ok(None)` and the delegate has no downcast-chain fallback.
+#[test]
+fn serialize_uses_downcast_delegate_for_self_serializing_plan() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
+ let inner: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: col("a", &schema)?,
+ alias: "a".to_string(),
+ }],
+ input,
+ )?);
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;
+
+ assert!(matches!(
+ proto.physical_plan_type,
+ Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(
+ _
+ ))
+ ));
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_physical_plan_node() {
+ use datafusion::prelude::*;
+ use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec,
+ };
+ use datafusion_proto::protobuf::PhysicalPlanNode;
+
+ let ctx = SessionContext::new();
+
+ ctx.register_parquet(
+ "pt",
+ &format!(
+ "{}/alltypes_plain.snappy.parquet",
+ datafusion_common::test_util::parquet_test_data()
+ ),
+ ParquetReadOptions::default(),
+ )
+ .await
+ .unwrap();
+
+ let plan = ctx
+ .sql("select id, string_col, timestamp_col from pt where id > 4 order by string_col")
+ .await
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap();
+
+ let node: PhysicalPlanNode =
+ PhysicalPlanNode::try_from_physical_plan(plan, &DefaultPhysicalExtensionCodec {})
+ .unwrap();
+
+ let plan = node
+ .try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})
+ .unwrap();
+
+ let _ = plan.execute(0, ctx.task_ctx()).unwrap();
+}
+
+/// The deprecated `try_into_projection_physical_plan` shim now delegates to
+/// [`ProjectionExec::try_from_proto`], which reads the enclosing
+/// `PhysicalPlanNode` rather than a `ProjectionExecNode`. Assert the shim still
+/// decodes the node passed as an argument, not `self`, so an out-of-tree caller
+/// that passes a projection unrelated to `self` keeps the old behaviour.
+#[test]
+fn deprecated_projection_shim_decodes_argument_not_self() -> Result<()> {
+ use datafusion_proto::protobuf::PhysicalPlanNode;
+ use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;
+
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
+ let projection = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr::new(
+ col("a", &schema)?,
+ "renamed".to_string(),
+ )],
+ input,
+ )?);
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let projection_node = PhysicalPlanNode::try_from_physical_plan_with_converter(
+ projection,
+ &codec,
+ &proto_converter,
+ )?;
+ let Some(PhysicalPlanType::Projection(projection_exec_node)) =
+ &projection_node.physical_plan_type
+ else {
+ panic!("expected a Projection node, got {projection_node:?}");
+ };
+
+ // `self` is deliberately a different plan variant than the argument.
+ let unrelated_node = PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::new(EmptyExec::new(Arc::new(schema))),
+ &codec,
+ &proto_converter,
+ )?;
+
+ let session_ctx = SessionContext::new();
+ let task_ctx = session_ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ #[expect(deprecated)]
+ let decoded = unrelated_node.try_into_projection_physical_plan(
+ projection_exec_node,
+ &decode_ctx,
+ &proto_converter,
+ )?;
+
+ let decoded = decoded
+ .downcast_ref::<ProjectionExec>()
+ .expect("decoded plan should be a ProjectionExec");
+ assert_eq!(decoded.expr().len(), 1);
+ assert_eq!(decoded.expr()[0].alias, "renamed");
+ Ok(())
+}
+
+#[test]
+fn custom_proto_converter_intercepts() -> Result<()> {
+ #[derive(Default)]
+ struct CustomConverterInterceptor {
+ num_proto_plans: RwLock<usize>,
+ num_physical_plans: RwLock<usize>,
+ num_proto_exprs: RwLock<usize>,
+ num_physical_exprs: RwLock<usize>,
+ }
+
+ impl PhysicalProtoConverterExtension for CustomConverterInterceptor {
+ fn proto_to_execution_plan(
+ &self,
+ proto: &PhysicalPlanNode,
+ ctx: &PhysicalPlanDecodeContext<'_>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ {
+ let mut counter = self
+ .num_proto_plans
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ self.default_proto_to_execution_plan(proto, ctx)
+ }
+
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalPlanNode>
+ where
+ Self: Sized,
+ {
+ {
+ let mut counter = self
+ .num_physical_plans
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::clone(plan),
+ codec,
+ self,
+ )
+ }
+
+ fn proto_to_physical_expr(
+ &self,
+ proto: &PhysicalExprNode,
+ input_schema: &Schema,
+ ctx: &PhysicalPlanDecodeContext<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>>
+ where
+ Self: Sized,
+ {
+ {
+ let mut counter = self
+ .num_proto_exprs
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ self.default_proto_to_physical_expr(proto, input_schema, ctx)
+ }
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalExprNode> {
+ {
+ let mut counter = self
+ .num_physical_exprs
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ serialize_physical_expr_with_converter(expr, codec, self)
+ }
+ }
+
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+
+ let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = CustomConverterInterceptor::default();
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+
+ assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/dynamic_filters.rs b/datafusion/proto/tests/cases/plans/dynamic_filters.rs
new file mode 100644
index 0000000000..cda649b4c5
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/dynamic_filters.rs
@@ -0,0 +1,1115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Dynamic filter expressions: their deduplication across a plan, and the
+//! plans that produce them.
+
+use super::{roundtrip_test_and_return, roundtrip_test_sql_with_context};
+use arrow::array::RecordBatch;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::datasource::empty::EmptyTable;
+use datafusion::datasource::listing::PartitionedFile;
+use datafusion::datasource::object_store::ObjectStoreUrl;
+use datafusion::datasource::physical_plan::{
+ FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,
+};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::{JoinType, Operator};
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;
+use datafusion::physical_plan::aggregates::{
+ AggregateExec, AggregateMode, PhysicalGroupBy,
+};
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{
+ BinaryExpr, Column, DynamicFilterPhysicalExpr, PhysicalSortExpr, lit,
+};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};
+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::{
+ DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,
+ SendableRecordBatchStream,
+};
+use datafusion::prelude::SessionContext;
+use datafusion_common::config::{ConfigOptions, TableParquetOptions};
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::{NullEquality, Result, internal_datafusion_err, internal_err};
+use datafusion_datasource::file::FileSource;
+use datafusion_expr::ColumnarValue;
+use datafusion_physical_expr::utils::reassign_expr_columns;
+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_proto_converter,
+ physical_plan_to_bytes_with_proto_converter,
+};
+use datafusion_proto::physical_plan::{
+ DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,
+ PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf::PhysicalExprNode;
+use prost::Message;
+use std::fmt::{Display, Formatter};
+use std::sync::Arc;
+use std::vec;
+
+/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.
+fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {
+ Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
+ lit(true),
+ )) as Arc<dyn PhysicalExpr>
+}
+
+/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.
+fn make_reassigned_dynamic_filter(
+ filter: Arc<dyn PhysicalExpr>,
+) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("b", DataType::Int64, false),
+ Field::new("a", DataType::Int64, false),
+ ]));
+ let reassigned = reassign_expr_columns(filter, &schema)?;
+ Ok((schema, reassigned))
+}
+
+/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the
+/// default serializer from `PhysicalExpr::expression_id`.
+fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {
+ expr.expr_id
+ .expect("expected PhysicalExprNode.expr_id to be populated")
+}
+
+/// Roundtrip a single physical expression shaped like so:
+///
+/// ```text
+/// BinaryExpr(AND)
+/// / \
+/// filter_expr_1 filter_expr_2
+/// ```
+///
+/// Returns filter_expr_1 and filter_expr_2 after deserialization.
+fn roundtrip_dynamic_filter_expr_pair(
+ filter_expr_1: Arc<dyn PhysicalExpr>,
+ filter_expr_2: Arc<dyn PhysicalExpr>,
+ schema: Arc<Schema>,
+) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {
+ let pair_expr = Arc::new(BinaryExpr::new(
+ Arc::clone(&filter_expr_1),
+ Operator::And,
+ Arc::clone(&filter_expr_2),
+ )) as Arc<dyn PhysicalExpr>;
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let deserialized_expr =
+ converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
+
+ let binary = deserialized_expr
+ .downcast_ref::<BinaryExpr>()
+ .expect("Expected BinaryExpr");
+
+ Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))
+}
+
+/// Roundtrip an execution plan shaped like so:
+///
+/// ```text
+/// FilterExec(dynamic_filter_1 on a@0)
+/// ProjectionExec(a := Column("a", source_index))
+/// DataSourceExec
+/// ParquetSource(predicate = dynamic_filter_2)
+/// ```
+///
+/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with
+/// different children.
+///
+/// Returns
+/// - `dynamic_filter_1` before serialization
+/// - `dynamic_filter_2` before serialization
+/// - `dynamic_filter_1` after serialization
+/// - `dynamic_filter_2` after serialization
+#[expect(clippy::type_complexity)]
+fn roundtrip_dynamic_filter_plan_pair() -> Result<(
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+)> {
+ let filter_expr_1 = make_dynamic_filter();
+ let (data_source_schema, filter_expr_2) =
+ make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;
+ let left_before = Arc::clone(&filter_expr_1);
+ let right_before = Arc::clone(&filter_expr_2);
+ let file_source = Arc::new(
+ ParquetSource::new(Arc::clone(&data_source_schema))
+ .with_predicate(Arc::clone(&filter_expr_2)),
+ );
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .build();
+ let data_source_exec =
+ DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;
+
+ let projection_exec = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,
+ alias: "a".to_string(),
+ }],
+ data_source_exec,
+ )?) as Arc<dyn ExecutionPlan>;
+ let filter_exec = Arc::new(FilterExec::try_new(
+ Arc::clone(&filter_expr_1),
+ projection_exec,
+ )?) as Arc<dyn ExecutionPlan>;
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;
+
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;
+
+ let outer_filter = deserialized_plan
+ .downcast_ref::<FilterExec>()
+ .expect("Expected outer FilterExec");
+ let left_filter = Arc::clone(outer_filter.predicate());
+ let projection = outer_filter.children()[0]
+ .downcast_ref::<ProjectionExec>()
+ .expect("Expected ProjectionExec");
+ let data_source = projection
+ .input()
+ .downcast_ref::<DataSourceExec>()
+ .expect("Expected DataSourceExec");
+ let scan_config = data_source
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .expect("Expected FileScanConfig");
+ let right_filter = scan_config
+ .file_source()
+ .filter()
+ .expect("Expected pushed-down predicate");
+
+ Ok((left_before, right_before, left_filter, right_filter))
+}
+
+/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible
+/// via the other. This helps assert that referential integrity is maintained after
+/// deserializing.
+fn assert_dynamic_filter_update_is_visible(
+ left_filter: &Arc<dyn PhysicalExpr>,
+ right_filter: &Arc<dyn PhysicalExpr>,
+) -> Result<()> {
+ let left_filter = left_filter
+ .downcast_ref::<DynamicFilterPhysicalExpr>()
+ .expect("Expected dynamic filter");
+ let right_filter = right_filter
+ .downcast_ref::<DynamicFilterPhysicalExpr>()
+ .expect("Expected dynamic filter");
+
+ // Sanity check that the filters have the same generation.
+ let original_generation = left_filter.snapshot_generation();
+ assert_eq!(original_generation, right_filter.snapshot_generation(),);
+
+ left_filter.update(lit(123_i64))?;
+
+ // Assert that both generations updated.
+ assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);
+ assert_eq!(
+ left_filter.snapshot_generation(),
+ right_filter.snapshot_generation(),
+ );
+
+ // Ensure both filters have the updated expr.
+ let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;
+ assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);
+ assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);
+
+ Ok(())
+}
+
+/// Extract the dynamic-filter predicate that was pushed down to the parquet
+/// scan at the bottom of the plan tree.
+fn parquet_source_predicate(child: &Arc<dyn ExecutionPlan>) -> Arc<dyn PhysicalExpr> {
+ let data_source = child
+ .downcast_ref::<DataSourceExec>()
+ .expect("Child should be DataSourceExec");
+ let (_, parquet_source) = data_source
+ .downcast_to_file_source::<ParquetSource>()
+ .expect("Should be ParquetSource");
+ parquet_source
+ .filter()
+ .expect("ParquetSource should have a predicate after roundtrip")
+}
+
+/// Assert that two dynamic filters are equal both structurally (Debug output)
+/// and by identity (`expression_id`).
+fn assert_dynamic_filters_equal(
+ expected: &Arc<dyn PhysicalExpr>,
+ actual: &Arc<dyn PhysicalExpr>,
+) {
+ // Structural.
+ let expected_dbg = format!("{expected:?}");
+ let actual_dbg = format!("{actual:?}");
+ if expected_dbg == actual_dbg {
+ return;
+ }
+
+ // Note that the `DeduplicatingDeserializer` routes every cache hit through
+ // `with_new_children`. This produces an equivalent expression, but with
+ // remapped children that are equal to the original. Handle that case here.
+ let rewritten = Arc::clone(expected)
+ .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())
+ .expect("with_new_children on a dynamic filter should not fail");
+ assert_eq!(format!("{rewritten:?}"), actual_dbg);
+}
+
+// Two clones of a dynamic filter expression should be deduped to the exact same expression.
+#[test]
+fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let filter_expr_1 = make_dynamic_filter();
+ let filter_expr_2 = Arc::clone(&filter_expr_1);
+
+ let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =
+ roundtrip_dynamic_filter_expr_pair(
+ Arc::clone(&filter_expr_1),
+ Arc::clone(&filter_expr_2),
+ schema,
+ )?;
+
+ // Assert the filters are not modified during roundtrip.
+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
+ assert_dynamic_filters_equal(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
+ );
+
+ // Assert referential integrity.
+ assert_dynamic_filter_update_is_visible(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
+ )?;
+
+ Ok(())
+}
+
+/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter
+/// with different children.
+#[test]
+fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {
+ let (
+ filter_expr_1,
+ filter_expr_2,
+ filter_expr_1_after_roundtrip,
+ filter_expr_2_after_roundtrip,
+ ) = roundtrip_dynamic_filter_plan_pair()?;
+
+ // Assert the filters are not modified during roundtrip.
+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
+
+ // Assert referential integrity.
+ assert_dynamic_filter_update_is_visible(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
+ )?;
+
+ Ok(())
+}
+
+#[test]
+fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {
+ let filter_expr = make_dynamic_filter();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DeduplicatingProtoConverter {};
+
+ let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
+ let expr_id1 = proto_expression_id(&proto1);
+
+ let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
+ let expr_id2 = proto_expression_id(&proto2);
+
+ assert_eq!(
+ expr_id1, expr_id2,
+ "Expected the same dynamic filter expression id across serializations"
+ );
+
+ Ok(())
+}
+
+/// Create a DataSourceExec backed by a ParquetSource that accepts filter pushdown,
+/// along with a ConfigOptions that enables all dynamic filter pushdown options.
+fn datasource_for_dynamic_filter_pushdown(
+ schema: &Arc<Schema>,
+) -> (Arc<dyn ExecutionPlan>, ConfigOptions) {
+ let mut parquet_options = TableParquetOptions::new();
+ parquet_options.global.pushdown_filters = true;
+ let source = Arc::new(
+ ParquetSource::new(Arc::clone(schema))
+ .with_table_parquet_options(parquet_options),
+ );
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)
+ .with_file(PartitionedFile::new("/path/to/file.parquet", 1024))
+ .build();
+
+ let mut config = ConfigOptions::default();
+ config.execution.parquet.pushdown_filters = true;
+ config.optimizer.enable_join_dynamic_filter_pushdown = true;
+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
+ config.optimizer.enable_topk_dynamic_filter_pushdown = true;
+
+ (DataSourceExec::from_data_source(scan_config), config)
+}
+
+/// Test that plan containing a HashJoinExec with dynamic filter pushdown
+/// can be serialized and deserialized while preserving references to the dynamic filter.
+#[test]
+fn test_hash_join_with_dynamic_filter_roundtrip() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));
+
+ let left_child = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let (right_child, config) = datasource_for_dynamic_filter_pushdown(&schema);
+
+ let on: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> = vec![(
+ Arc::new(Column::new("col", 0)),
+ Arc::new(Column::new("col", 0)),
+ )];
+
+ let hash_join = Arc::new(HashJoinExec::try_new(
+ left_child,
+ right_child,
+ on,
+ None,
+ &JoinType::Inner,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?) as Arc<dyn ExecutionPlan>;
+
+ // Run the optimizer rule for filter pushdown.
+ let optimizer = FilterPushdown::new_post_optimization();
+ let plan = optimizer.optimize(hash_join, &config)?;
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let deserialized = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;
+
+ // Extract the deserialized HashJoinExec and its dynamic filter.
+ let deserialized_join = deserialized
+ .downcast_ref::<HashJoinExec>()
+ .expect("Should be HashJoinExec");
+ let deserialized_hash_join_df = deserialized_join
+ .dynamic_expressions_produced()
+ .into_iter()
+ .next()
+ .expect("HashJoinExec should have a dynamic filter after roundtrip");
+
+ // Extract the dynamic filter pushed down to the probe side's ParquetSource.
+ let deserialized_predicate = parquet_source_predicate(deserialized_join.right());
+
+ // The HashJoinExec's dynamic filter and the probe side's predicate should
+ // refer to the same underlying expression.
+ let plan_df = deserialized_hash_join_df;
+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
+
+ Ok(())
+}
+
+/// returns a SessionContext with an empty `netflow` table registered
+fn netflow_context() -> Result<SessionContext> {
+ let ctx = SessionContext::new();
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("dst_geo_country_name", DataType::Utf8, true),
+ Field::new("dst_geo_city_name", DataType::Utf8, true),
+ Field::new("packets", DataType::UInt64, true),
+ Field::new("src_addr", DataType::Utf8, true),
+ Field::new("dst_addr", DataType::Utf8, true),
+ ]));
+
+ ctx.register_table("netflow", Arc::new(EmptyTable::new(schema)))?;
+
+ Ok(ctx)
+}
+
+/// Regression test for issue #18602:
+/// https://github.com/apache/datafusion/issues/18602
+///
+/// The physical filter expression here contains a long chain of `AND` predicates.
+/// Before linearizing `PhysicalBinaryExprNode`, encoding then decoding the protobuf
+/// could fail with `DecodeError: recursion limit reached`.
+#[tokio::test]
+async fn roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {
+ let ctx = netflow_context()?;
+ let sql = "SELECT \
+ dst_geo_country_name AS x_axis_1, \
+ dst_geo_city_name AS x_axis_2, \
+ sum(packets) AS y_axis_1 \
+ FROM netflow \
+ WHERE dst_geo_country_name IS NOT NULL \
+ AND src_addr NOT LIKE '10.201.%' \
+ AND dst_addr NOT LIKE '10.201.%' \
+ AND src_addr NOT LIKE '10.202.%' \
+ AND dst_addr NOT LIKE '10.202.%' \
+ AND src_addr NOT LIKE '10.203.%' \
+ AND dst_addr NOT LIKE '10.203.%' \
+ AND src_addr NOT LIKE '10.204.%' \
+ AND dst_addr NOT LIKE '10.204.%' \
+ AND src_addr NOT LIKE '172.16.186.%' \
+ AND dst_addr NOT LIKE '172.16.186.%' \
+ AND src_addr NOT LIKE '172.16.187.%' \
+ AND dst_addr NOT LIKE '172.16.187.%' \
+ AND src_addr NOT LIKE '172.16.188.%' \
+ AND dst_addr NOT LIKE '172.16.188.%' \
+ AND src_addr NOT LIKE '10.102.45.%' \
+ AND dst_addr NOT LIKE '10.102.45.%' \
+ AND src_addr NOT LIKE '172.25.210.%' \
+ AND dst_addr NOT LIKE '172.25.210.%' \
+ AND src_addr NOT LIKE '172.25.211.%' \
+ AND dst_addr NOT LIKE '172.25.211.%' \
+ AND src_addr NOT LIKE '141.226.101.%' \
+ AND dst_addr NOT LIKE '141.226.101.%' \
+ AND src_addr NOT LIKE '167.86.40.%' \
+ AND dst_addr NOT LIKE '167.86.40.%' \
+ AND src_addr NOT LIKE '66.22.38.%' \
+ AND dst_addr NOT LIKE '66.22.38.%' \
+ AND src_addr != '168.143.191.55' \
+ AND dst_addr != '168.143.191.55' \
+ AND src_addr != '82.112.107.142' \
+ AND dst_addr != '82.112.107.142' \
+ AND src_addr != '20.76.39.176' \
+ AND dst_addr != '20.76.39.176' \
+ AND src_addr != '162.159.129.83' \
+ AND dst_addr != '162.159.129.83' \
+ AND src_addr != '34.201.223.155' \
+ AND dst_addr != '34.201.223.155' \
+ AND src_addr != '34.201.223.156' \
+ AND dst_addr != '34.201.223.156' \
+ AND src_addr != '34.201.223.157' \
+ AND dst_addr != '34.201.223.157' \
+ AND src_addr != '134.201.223.157' \
+ AND dst_addr != '134.201.223.157' \
+ AND src_addr != '341.201.223.157' \
+ AND dst_addr != '341.201.223.157' \
+ GROUP BY x_axis_1, x_axis_2 \
+ ORDER BY y_axis_1 DESC \
+ LIMIT 20";
+
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+/// Test that plan containing a AggregateExec with dynamic filter pushdown
+/// can be serialized and deserialized while preserving references to the dynamic filter.
+#[test]
+fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+
+ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);
+
+ let agg = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::new_single(vec![]),
+ vec![
+ AggregateExprBuilder::new(
+ datafusion::functions_aggregate::min_max::min_udaf(),
+ vec![Arc::clone(&col_a)],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("min_a")
+ .build()
+ .map(Arc::new)?,
+ ],
+ vec![None],
+ child,
+ Arc::clone(&schema),
+ )?) as Arc<dyn ExecutionPlan>;
+
+ // Run the optimizer rule for filter pushdown.
+ let optimizer = FilterPushdown::new_post_optimization();
+ let plan = optimizer.optimize(agg, &config)?;
+
+ // Roundtrip with deduplication.
+ //
+ // Note: We don't use `roundtrip_test_and_return` here because there's a
+ // pre-existing issue with PhysicalGroupBy serialization where empty groups
+ // `[[]]` become `[]` after roundtrip. This behavior is unrelated to this test.
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&plan),
+ &codec,
+ &converter,
+ )?;
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &converter,
+ )?;
+
+ // Extract the deserialized AggregateExec and its dynamic filter.
+ let deserialized_agg = deserialized
+ .downcast_ref::<AggregateExec>()
+ .expect("Should be AggregateExec");
+ let deserialized_agg_df = deserialized_agg
+ .dynamic_expressions_produced()
+ .into_iter()
+ .next()
+ .expect("AggregateExec should have a dynamic filter after roundtrip");
+
+ // Extract the dynamic filter pushed down to the child ParquetSource.
+ let deserialized_predicate = parquet_source_predicate(deserialized_agg.input());
+
+ // The AggregateExec's dynamic filter and the child's predicate should
+ // refer to the same underlying expression.
+ let plan_df = deserialized_agg_df;
+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
+
+ Ok(())
+}
+
+#[test]
+fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::new_single(vec![]),
+ vec![
+ AggregateExprBuilder::new(
+ datafusion::functions_aggregate::min_max::min_udaf(),
+ vec![col_a],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("min_a")
+ .build()
+ .map(Arc::new)?,
+ ],
+ vec![None],
+ child,
+ Arc::clone(&schema),
+ )?) as Arc<dyn ExecutionPlan>;
+
+ let mut config = ConfigOptions::default();
+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
+ let optimizer = FilterPushdown::new_post_optimization();
+ let plan = optimizer.optimize(aggregate, &config)?;
+ assert!(
+ plan.downcast_ref::<AggregateExec>()
+ .expect("Should be AggregateExec")
+ .dynamic_expressions_produced()
+ .is_empty()
+ );
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DefaultPhysicalProtoConverter {};
+ let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &converter,
+ )?;
+
+ assert!(
+ deserialized
+ .downcast_ref::<AggregateExec>()
+ .expect("Should be AggregateExec")
+ .dynamic_expressions_produced()
+ .is_empty()
+ );
+ Ok(())
+}
+
+/// Test that plan containing a SortExec with dynamic filter pushdown
+/// can be serialized and deserialized while preserving references to the dynamic filter.
+#[test]
+fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+
+ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);
+
+ let sort = Arc::new(
+ SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr {
+ expr: Arc::clone(&col_a),
+ options: SortOptions::default(),
+ }])
+ .unwrap(),
+ child,
+ )
+ .with_fetch(Some(10)),
+ ) as Arc<dyn ExecutionPlan>;
+
+ // Verify the optimizer kept the dynamic filter on the SortExec.
+ let optimizer = FilterPushdown::new_post_optimization();
+ let plan = optimizer.optimize(sort, &config)?;
+
+ // Roundtrip with deduplication.
+ //
+ // Note: We don't use `roundtrip_test_and_return` here because
+ // `DeduplicatingDeserializer` rewrites cache hits via `with_new_children`,
+ // which sets `remapped_children: Some(...)` on the second encounter of a
+ // shared `DynamicFilterPhysicalExpr`. SortExec's `Debug` includes its
+ // dynamic filter, so the original-vs-deserialized structural equality check
+ // would fail purely on this artifact.
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&plan),
+ &codec,
+ &converter,
+ )?;
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &converter,
+ )?;
+
+ // Extract the deserialized SortExec and its dynamic filter.
+ let deserialized_sort = deserialized
+ .downcast_ref::<SortExec>()
+ .expect("Should be SortExec");
+ let deserialized_sort_df = deserialized_sort
+ .dynamic_expressions_produced()
+ .into_iter()
+ .next()
+ .expect("SortExec should have a dynamic filter after roundtrip");
+
+ // Extract the dynamic filter pushed down to the child ParquetSource.
+ let deserialized_predicate = parquet_source_predicate(deserialized_sort.input());
+
+ // The SortExec's dynamic filter and the child's predicate should
+ // refer to the same underlying expression.
+ let plan_df = deserialized_sort_df;
+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
+
+ Ok(())
+}
+
+/// A custom [`ExecutionPlan`] which stores [`PhysicalExpr`]s.
+struct CustomExecWithExprs {
+ exprs: Vec<Arc<dyn PhysicalExpr>>,
+ child: Arc<dyn ExecutionPlan>,
+}
+
+#[derive(Clone, PartialEq, Message)]
+struct CustomExecWithExprsProto {
+ #[prost(message, repeated, tag = "1")]
+ exprs: Vec<PhysicalExprNode>,
+}
+
+impl std::fmt::Debug for CustomExecWithExprs {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("CustomExecWithExprs")
+ .field("exprs", &self.exprs)
+ .field("child", &self.child)
+ .finish()
+ }
+}
+
+impl CustomExecWithExprs {
+ fn new(exprs: Vec<Arc<dyn PhysicalExpr>>, child: Arc<dyn ExecutionPlan>) -> Self {
+ Self { exprs, child }
+ }
+}
+
+impl DisplayAs for CustomExecWithExprs {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ write!(f, "CustomExecWithExprs")
+ }
+}
+
+impl ExecutionPlan for CustomExecWithExprs {
+ fn name(&self) -> &str {
+ "CustomExecWithExprs"
+ }
+
+ fn schema(&self) -> SchemaRef {
+ self.child.schema()
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ self.child.properties()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![&self.child]
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(&self.exprs, f)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unreachable!()
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unreachable!()
+ }
+}
+
+/// A [`PhysicalExtensionCodec`] for [`CustomExecWithExprs`].
+#[derive(Debug)]
+struct CustomExecWithExprsCodec {}
+
+impl PhysicalExtensionCodec for CustomExecWithExprsCodec {
+ fn try_decode(
+ &self,
+ buf: &[u8],
+ inputs: &[Arc<dyn ExecutionPlan>],
+ ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);
+ let input_schema = inputs[0].schema();
+ let proto = CustomExecWithExprsProto::decode(buf)
+ .map_err(|e| internal_datafusion_err!("Failed to decode custom exec: {e}"))?;
+ let exprs = proto
+ .exprs
+ .iter()
+ .map(|expr_proto| {
+ proto_converter.proto_to_physical_expr(
+ expr_proto,
+ input_schema.as_ref(),
+ &decode_ctx,
+ )
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(Arc::new(CustomExecWithExprs::new(exprs, inputs[0].clone())))
+ }
+
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ let custom = node
+ .downcast_ref::<CustomExecWithExprs>()
+ .ok_or_else(|| internal_datafusion_err!("Expected CustomExecWithExprs"))?;
+ let proto = CustomExecWithExprsProto {
+ exprs: custom
+ .exprs
+ .iter()
+ .map(|expr| proto_converter.physical_expr_to_proto(expr, self))
+ .collect::<Result<Vec<_>>>()?,
+ };
+ proto
+ .encode(buf)
+ .map_err(|e| internal_datafusion_err!("Failed to encode custom exec: {e}"))?;
+
+ Ok(())
+ }
+}
+
+/// Tests that a custom [`ExecutionPlan`] with [`PhysicalExpr`] can
+/// dedupe dynamic filters by using the proto converter in its
+/// [`PhysicalExtensionCodec`] implementation.
+#[test]
+fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {
+ // Create the plan:
+ //
+ // FilterExec(dynamic_filter)
+ // -> CustomExecWithExprs(exprs: [dynamic_filter])
+ // -> EmptyExec
+ //
+ // The same dynamic filter expression is saved in both the FilterExec and CustomExecWithExprs.
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
+ lit(true),
+ ));
+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> = dynamic_filter;
+
+ let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let custom_exec = Arc::new(CustomExecWithExprs::new(
+ vec![Arc::clone(&dynamic_filter_expr)],
+ empty,
+ ));
+ let filter_exec = Arc::new(FilterExec::try_new(
+ Arc::clone(&dynamic_filter_expr),
+ custom_exec,
+ )?) as Arc<dyn ExecutionPlan>;
+
+ // Roundtrip with DeduplicatingProtoConverter
+ let codec = CustomExecWithExprsCodec {};
+ let converter = DeduplicatingProtoConverter {};
+
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&filter_exec),
+ &codec,
+ &converter,
+ )?;
+
+ let ctx = SessionContext::new();
+ let deser_converter = DeduplicatingProtoConverter {};
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &deser_converter,
+ )?;
+
+ // Extract the deserialized FilterExec's dynamic filter
+ let deser_filter = deserialized
+ .downcast_ref::<FilterExec>()
+ .expect("Top-level should be FilterExec");
+ let deser_filter_df = deser_filter.predicate();
+
+ // Extract the deserialized custom node's dynamic filter
+ let deser_custom = deser_filter
+ .input()
+ .downcast_ref::<CustomExecWithExprs>()
+ .expect("FilterExec child should be CustomExecWithExprs");
+ assert_eq!(deser_custom.exprs.len(), 1, "Should have one expression");
+ let [deser_custom_df] = deser_custom.exprs.as_slice() else {
+ return internal_err!("Custom node should have one expression");
+ };
+
+ // Pass the un-remapped filter first so the helper's `with_new_children`
+ // rewrite can reconstruct the remapped form on the other side.
+ assert_dynamic_filters_equal(deser_custom_df, deser_filter_df);
+ assert_dynamic_filter_update_is_visible(deser_custom_df, deser_filter_df)?;
+
+ Ok(())
+}
+
+/// A custom `PhysicalExpr` whose extension codec embeds a nested
+/// `PhysicalExprNode` *inside its own blob* (rather than the standard
+/// `PhysicalExtensionExprNode.inputs` field). This is the case that only
+/// works if the expr-level codec methods receive the encode/decode context.
+#[derive(Debug)]
+struct WrapperExpr {
+ inner: Arc<dyn PhysicalExpr>,
+}
+
+impl Display for WrapperExpr {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "WrapperExpr({})", self.inner)
+ }
+}
+
+impl PartialEq for WrapperExpr {
+ fn eq(&self, other: &Self) -> bool {
+ self.inner.eq(&other.inner)
+ }
+}
+
+impl Eq for WrapperExpr {}
+
+impl std::hash::Hash for WrapperExpr {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.inner.hash(state);
+ }
+}
+
+impl PhysicalExpr for WrapperExpr {
+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+ self.inner.data_type(input_schema)
+ }
+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+ self.inner.nullable(input_schema)
+ }
+ fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {
+ internal_err!("WrapperExpr is not executable in this test")
+ }
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![&self.inner]
+ }
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(Arc::new(WrapperExpr {
+ inner: Arc::clone(&children[0]),
+ }))
+ }
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ Display::fmt(self, f)
+ }
+}
+
+/// Wire layout for [`WrapperExpr`]: a single nested `PhysicalExprNode`.
+#[derive(Clone, PartialEq, prost::Message)]
+struct WrapperExprProto {
+ #[prost(message, optional, boxed, tag = "1")]
+ inner: Option<Box<PhysicalExprNode>>,
+}
+
+#[derive(Debug)]
+struct WrapperCodec;
+
+impl PhysicalExtensionCodec for WrapperCodec {
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ _ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ internal_err!("not used")
+ }
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ internal_err!("not used")
+ }
+ fn try_decode_expr(
+ &self,
+ buf: &[u8],
+ _inputs: &[Arc<dyn PhysicalExpr>],
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ let proto = WrapperExprProto::decode(buf)
+ .map_err(|e| internal_datafusion_err!("decode WrapperExprProto: {e}"))?;
+ let inner_proto = proto
+ .inner
+ .ok_or_else(|| internal_datafusion_err!("missing inner"))?;
+ // Decode the nested expr through the context so it resolves against
+ // the real schema/registry AND participates in dedup — no fabricated
+ // `SessionContext` or hard-coded schema required.
+ let inner = ctx.decode(&inner_proto)?;
+ Ok(Arc::new(WrapperExpr { inner }))
+ }
+ fn try_encode_expr(
+ &self,
+ node: &Arc<dyn PhysicalExpr>,
+ buf: &mut Vec<u8>,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<()> {
+ let wrapper = node
+ .downcast_ref::<WrapperExpr>()
+ .ok_or_else(|| internal_datafusion_err!("not WrapperExpr"))?;
+ // Encode the nested expr through the context so an active
+ // `DeduplicatingProtoConverter` stamps a matching `expr_id`.
+ let inner_proto = ctx.encode_child(&wrapper.inner)?;
+ let proto = WrapperExprProto {
+ inner: Some(Box::new(inner_proto)),
+ };
+ proto
+ .encode(buf)
+ .map_err(|e| internal_datafusion_err!("encode WrapperExprProto: {e}"))?;
+ Ok(())
+ }
+}
+
+/// A `DynamicFilterPhysicalExpr` referenced both as a bare expression and
+/// nested inside a custom expression's codec blob must reconstruct to a
+/// single shared `Inner` after roundtrip.
+///
+/// This exercises the expr-level codec hooks receiving the encode/decode
+/// context: `try_encode_expr` routes its nested `PhysicalExprNode` through
+/// `ctx.encode_child` and `try_decode_expr` through `ctx.decode`, so the
+/// nested filter picks up the same `DeduplicatingProtoConverter` /
+/// `DeduplicatingDeserializer` cache as the bare reference. Without the
+/// context the nested expr would serialize with `expr_id: None` and decode
+/// into a distinct `Inner`, breaking heap-max propagation across the
+/// extension boundary in distributed execution.
+#[test]
+fn extension_codec_expr_participates_in_deduplication() -> Result<()> {
+ use prost::Message;
+
+ // A single composite expression holding TWO references to the same
+ // dynamic filter: bare on the left of an AND, wrapped on the right.
+ let dyn_filter = make_dynamic_filter();
+ let wrapper: Arc<dyn PhysicalExpr> = Arc::new(WrapperExpr {
+ inner: Arc::clone(&dyn_filter),
+ });
+ let composite: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::clone(&dyn_filter),
+ Operator::And,
+ Arc::clone(&wrapper),
+ ));
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let codec = WrapperCodec;
+ let converter = DeduplicatingProtoConverter {};
+
+ // Encode, then round-trip through prost bytes to mimic the wire.
+ let proto = converter.physical_expr_to_proto(&composite, &codec)?;
+ let bytes = proto.encode_to_vec();
+ let decoded_proto = PhysicalExprNode::decode(bytes.as_slice()).unwrap();
+
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let decoded =
+ converter.proto_to_physical_expr(&decoded_proto, &schema, &decode_ctx)?;
+
+ let binary = decoded
+ .downcast_ref::<BinaryExpr>()
+ .expect("must decode back to BinaryExpr");
+ let decoded_left = Arc::clone(binary.left());
+ let decoded_right = Arc::clone(binary.right());
+ let decoded_wrapper = decoded_right
+ .downcast_ref::<WrapperExpr>()
+ .expect("right side must decode back to WrapperExpr");
+
+ // The load-bearing check: an `update()` on the bare-side filter must be
+ // observable from the wrapped-side filter, proving both refs back the
+ // same `Inner`.
+ assert_dynamic_filter_update_is_visible(&decoded_left, &decoded_wrapper.inner)?;
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rs
new file mode 100644
index 0000000000..ed9745a4b1
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/exprs.rs
@@ -0,0 +1,399 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Physical expressions embedded in plans, including the binary
+//! expression linearization.
+
+use super::roundtrip_test;
+use arrow::datatypes::Fields;
+use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};
+use datafusion::logical_expr::Operator;
+use datafusion::physical_expr::expressions::Literal;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{
+ BinaryExpr, Column, binary, col, like, lit,
+};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
+use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::Result;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+ PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_date_time_interval() -> Result<()> {
+ let schema = Schema::new(vec![
+ Field::new("some_date", DataType::Date32, false),
+ Field::new(
+ "some_interval",
+ DataType::Interval(IntervalUnit::DayTime),
+ false,
+ ),
+ ]);
+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
+ let date_expr = col("some_date", &schema)?;
+ let literal_expr = col("some_interval", &schema)?;
+ let date_time_interval_expr =
+ binary(date_expr, Operator::Plus, literal_expr, &schema)?;
+ let plan = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: date_time_interval_expr,
+ alias: "result".to_string(),
+ }],
+ input,
+ )?);
+ roundtrip_test(plan)
+}
+
+#[test]
+fn roundtrip_like() -> Result<()> {
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("b", DataType::Utf8, false),
+ ]);
+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
+ let like_expr = like(
+ false,
+ false,
+ col("a", &schema)?,
+ col("b", &schema)?,
+ &schema,
+ )?;
+ let plan = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: like_expr,
+ alias: "result".to_string(),
+ }],
+ input,
+ )?);
+ roundtrip_test(plan)
+}
+
+/// Test that HashTableLookupExpr serializes to lit(true)
+///
+/// HashTableLookupExpr contains a runtime hash table that cannot be serialized.
+/// The serialization code replaces it with lit(true) which is safe because
+/// it's a performance optimization filter, not a correctness requirement.
+#[test]
+fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {
+ use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;
+ use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};
+
+ // Create a simple schema and input plan
+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ // Create a HashTableLookupExpr - it will be replaced with lit(true) during serialization
+ let hash_map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0))));
+ let on_columns = vec![col("col", &schema)?];
+ let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(
+ on_columns,
+ datafusion::physical_plan::joins::SeededRandomState::with_seed(0),
+ hash_map,
+ "test_lookup".to_string(),
+ ));
+
+ // Create a filter with the lookup expression
+ let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);
+
+ // Serialize
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let proto: PhysicalPlanNode =
+ PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)
+ .expect("serialization should succeed");
+
+ // Deserialize
+ let result: Arc<dyn ExecutionPlan> = proto
+ .try_into_physical_plan(&ctx.task_ctx(), &codec)
+ .expect("deserialization should succeed");
+
+ // The deserialized plan should have lit(true) instead of HashTableLookupExpr
+ // Verify the filter predicate is a Literal(true)
+ let result_filter = result.downcast_ref::<FilterExec>().unwrap();
+ let predicate = result_filter.predicate();
+ let literal = predicate.downcast_ref::<Literal>().unwrap();
+ assert_eq!(*literal.value(), ScalarValue::Boolean(Some(true)));
+
+ Ok(())
+}
+
+#[test]
+fn roundtrip_hash_expr() -> Result<()> {
+ use datafusion::physical_plan::joins::{HashExpr, SeededRandomState};
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int64, false),
+ Field::new("b", DataType::Utf8, false),
+ ]));
+
+ // Create a HashExpr with test columns and seeds
+ let on_columns = vec![col("a", &schema)?, col("b", &schema)?];
+ let hash_expr: Arc<dyn PhysicalExpr> = Arc::new(HashExpr::new(
+ on_columns,
+ SeededRandomState::with_seed(0), // arbitrary random seed for testing
+ "test_hash".to_string(),
+ ));
+
+ // Wrap in a filter by comparing hash value to a literal
+ // hash_expr > 0 is always boolean
+ let filter_expr = binary(hash_expr, Operator::Gt, lit(0u64), &schema)?;
+ let filter = Arc::new(FilterExec::try_new(
+ filter_expr,
+ Arc::new(EmptyExec::new(schema)),
+ )?);
+
+ // Confirm that the debug string contains the random state seeds
+ assert!(
+ format!("{filter:?}").contains("test_hash(a@0, b@1, [0])"),
+ "Debug string missing seeds: {filter:?}"
+ );
+ roundtrip_test(filter)
+}
+
+#[test]
+fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {
+ let data_type = DataType::Struct(Fields::from(vec![Field::new(
+ "item",
+ DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+ true,
+ )]));
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), true)]));
+ let scan = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let scalar = lit(ScalarValue::try_from(data_type)?);
+ let filter = Arc::new(FilterExec::try_new(
+ Arc::new(BinaryExpr::new(scalar, Operator::Eq, col("a", &schema)?)),
+ scan,
+ )?);
+
+ roundtrip_test(filter)
+}
+
+/// Test that a chain of the same operator (a AND b AND c) is linearized
+/// and roundtrips correctly.
+#[test]
+fn roundtrip_binary_expr_chain_same_op() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Boolean, false);
+ let field_c = Field::new("c", DataType::Boolean, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
+ let ab = binary(
+ col("a", &schema)?,
+ Operator::And,
+ col("b", &schema)?,
+ &schema,
+ )?;
+ let abc = binary(ab, Operator::And, col("c", &schema)?, &schema)?;
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ abc,
+ Arc::new(EmptyExec::new(schema)),
+ )?))
+}
+
+/// Test that mixed operators (a AND b OR c) are NOT linearized together —
+/// only chains of the same operator are flattened.
+#[test]
+fn roundtrip_binary_expr_mixed_ops() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Boolean, false);
+ let field_c = Field::new("c", DataType::Boolean, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
+ // (a AND b) OR c — AND and OR are different operators, so linearization stops
+ let a_and_b = binary(
+ col("a", &schema)?,
+ Operator::And,
+ col("b", &schema)?,
+ &schema,
+ )?;
+ let expr = binary(a_and_b, Operator::Or, col("c", &schema)?, &schema)?;
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ expr,
+ Arc::new(EmptyExec::new(schema)),
+ )?))
+}
+
+/// Test that a deeply nested chain of AND expressions (like many WHERE conditions)
+/// roundtrips correctly. This is the scenario from issue #18602.
+#[test]
+fn roundtrip_binary_expr_deeply_nested_and_chain() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let schema = Arc::new(Schema::new(vec![field_a]));
+
+ // Build a chain: a AND a AND a AND ... (100 times)
+ let col_a = col("a", &schema)?;
+ let mut expr = Arc::clone(&col_a);
+ for _ in 0..99 {
+ expr = binary(expr, Operator::And, Arc::clone(&col_a), &schema)?;
+ }
+
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ expr,
+ Arc::new(EmptyExec::new(schema)),
+ )?))
+}
+
+/// Test that a deeply nested chain of OR expressions roundtrips correctly.
+#[test]
+fn roundtrip_binary_expr_deeply_nested_or_chain() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let schema = Arc::new(Schema::new(vec![field_a]));
+
+ let col_a = col("a", &schema)?;
+ let mut expr = Arc::clone(&col_a);
+ for _ in 0..99 {
+ expr = binary(expr, Operator::Or, Arc::clone(&col_a), &schema)?;
+ }
+
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ expr,
+ Arc::new(EmptyExec::new(schema)),
+ )?))
+}
+
+/// Test that alternating AND/OR operators produce correct results —
+/// each sub-chain gets linearized independently.
+#[test]
+fn roundtrip_binary_expr_alternating_and_or() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Boolean, false);
+ let field_c = Field::new("c", DataType::Boolean, false);
+ let field_d = Field::new("d", DataType::Boolean, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c, field_d]));
+
+ // (a AND b) OR (c AND d)
+ let a_and_b = binary(
+ col("a", &schema)?,
+ Operator::And,
+ col("b", &schema)?,
+ &schema,
+ )?;
+ let c_and_d = binary(
+ col("c", &schema)?,
+ Operator::And,
+ col("d", &schema)?,
+ &schema,
+ )?;
+ let expr = binary(a_and_b, Operator::Or, c_and_d, &schema)?;
+
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ expr,
+ Arc::new(EmptyExec::new(schema)),
+ )?))
+}
+
+/// Verify that the linearized proto format has a flat operands list
+/// rather than deeply nested l/r fields.
+#[test]
+fn test_linearization_produces_flat_operands() -> Result<()> {
+ // Build: a AND a AND a AND a (4 operands, 3 levels of nesting)
+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+ let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(BinaryExpr::new(
+ Arc::clone(&col_a),
+ Operator::And,
+ Arc::clone(&col_a),
+ )),
+ Operator::And,
+ Arc::clone(&col_a),
+ )),
+ Operator::And,
+ Arc::clone(&col_a),
+ ));
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;
+
+ // The top-level should use the operands field with 4 entries
+ match &proto.expr_type {
+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {
+ assert!(
+ b.l.is_none(),
+ "l should be None when using linearized operands"
+ );
+ assert!(
+ b.r.is_none(),
+ "r should be None when using linearized operands"
+ );
+ assert_eq!(
+ b.operands.len(),
+ 4,
+ "Expected 4 linearized operands for a AND a AND a AND a"
+ );
+ assert_eq!(b.op, "And");
+ }
+ other => panic!("Expected BinaryExpr, got {other:?}"),
+ }
+
+ Ok(())
+}
+
+/// Test that linearization stops when encountering a different operator.
+/// For (a AND b) OR c, only the top-level OR should be represented, and
+/// the left-hand AND subtree should be a separate nested BinaryExpr.
+#[test]
+fn test_linearization_stops_at_different_op() -> Result<()> {
+ // (a AND b) OR c
+ let a_and_b: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::And,
+ Arc::new(Column::new("b", 1)),
+ ));
+ let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ a_and_b,
+ Operator::Or,
+ Arc::new(Column::new("c", 2)),
+ ));
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;
+
+ // The top-level OR should have only 2 operands (can't linearize through AND)
+ match &proto.expr_type {
+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {
+ assert_eq!(
+ b.operands.len(),
+ 2,
+ "Expected 2 operands for (a AND b) OR c"
+ );
+ assert_eq!(b.op, "Or");
+ // The first operand should be a nested AND BinaryExpr
+ match &b.operands[0].expr_type {
+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(inner)) => {
+ assert_eq!(inner.op, "And");
+ assert_eq!(inner.operands.len(), 2);
+ }
+ other => panic!("Expected inner BinaryExpr(AND), got {other:?}"),
+ }
+ }
+ other => panic!("Expected BinaryExpr, got {other:?}"),
+ }
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/filters.rs b/datafusion/proto/tests/cases/plans/filters.rs
new file mode 100644
index 0000000000..296923fcc7
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/filters.rs
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `FilterExec`.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::logical_expr::Operator;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{NotExpr, binary, col, in_list, lit};
+use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::Result;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_filter_with_not_and_in_list() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let field_c = Field::new("c", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
+ let not = Arc::new(NotExpr::new(col("a", &schema)?));
+ let in_list = in_list(
+ col("b", &schema)?,
+ vec![
+ lit(ScalarValue::Int64(Some(1))),
+ lit(ScalarValue::Int64(Some(2))),
+ ],
+ &false,
+ schema.as_ref(),
+ )?;
+ let and = binary(not, Operator::And, in_list, &schema)?;
+ roundtrip_test(Arc::new(FilterExec::try_new(
+ and,
+ Arc::new(EmptyExec::new(schema.clone())),
+ )?))
+}
+
+#[test]
+fn roundtrip_filter_with_fetch() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let predicate = col("a", &schema)?;
+ let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))
+ .with_fetch(Some(10))
+ .build()?;
+ assert_eq!(filter.fetch(), Some(10));
+ roundtrip_test(Arc::new(filter))
+}
+
+#[test]
+fn roundtrip_filter_projection_states() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Boolean, false),
+ Field::new("b", DataType::Int64, false),
+ ]));
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ for projection in [None, Some(vec![]), Some(vec![0])] {
+ let filter = FilterExecBuilder::new(
+ col("a", &schema)?,
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ )
+ .apply_projection(projection.clone())?
+ .with_default_selectivity(37)
+ .with_batch_size(1024)
+ .with_fetch(Some(5))
+ .build()?;
+
+ let result =
+ roundtrip_test_and_return(Arc::new(filter), &ctx, &codec, &proto_converter)?;
+ let result = result.downcast_ref::<FilterExec>().unwrap();
+ assert_eq!(result.projection().as_deref(), projection.as_deref());
+ assert_eq!(result.default_selectivity(), 37);
+ assert_eq!(result.batch_size(), 1024);
+ assert_eq!(result.fetch(), Some(5));
+ }
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/joins.rs b/datafusion/proto/tests/cases/plans/joins.rs
new file mode 100644
index 0000000000..941e883295
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/joins.rs
@@ -0,0 +1,458 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The join execs.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::logical_expr::{JoinType, Operator};
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{BinaryExpr, Column, PhysicalSortExpr};
+use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
+use datafusion::physical_plan::joins::{
+ HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,
+ StreamJoinPartitionMode, SymmetricHashJoinExec,
+};
+use datafusion::prelude::SessionContext;
+use datafusion_common::{JoinSide, NullEquality, Result};
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_hash_join() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_a]);
+ let on = vec![(
+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
+ )];
+
+ let schema_left = Arc::new(schema_left);
+ let schema_right = Arc::new(schema_right);
+ for join_type in &[
+ JoinType::Inner,
+ JoinType::Left,
+ JoinType::Right,
+ JoinType::Full,
+ JoinType::LeftAnti,
+ JoinType::RightAnti,
+ JoinType::LeftSemi,
+ JoinType::RightSemi,
+ ] {
+ for partition_mode in &[PartitionMode::Partitioned, PartitionMode::CollectLeft] {
+ roundtrip_test(Arc::new(HashJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ on.clone(),
+ None,
+ join_type,
+ None,
+ *partition_mode,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?))?;
+ }
+ }
+ Ok(())
+}
+
+#[test]
+fn roundtrip_nested_loop_join() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_a]);
+
+ let schema_left = Arc::new(schema_left);
+ let schema_right = Arc::new(schema_right);
+ for join_type in &[
+ JoinType::Inner,
+ JoinType::Left,
+ JoinType::Right,
+ JoinType::Full,
+ JoinType::LeftAnti,
+ JoinType::RightAnti,
+ JoinType::LeftSemi,
+ JoinType::RightSemi,
+ ] {
+ roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ None,
+ join_type,
+ Some(vec![0]),
+ )?))?;
+ }
+ Ok(())
+}
+
+/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",
+/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire
+/// representation. `try_embed_projection` (DataFusion 53+) produces
+/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,
+/// which previously round-tripped to `None` and caused downstream consumers (e.g.
+/// distributed Flight executors) to receive a different number of output
+/// columns than the planner declared. Verify all three states preserve.
+#[test]
+fn roundtrip_hash_join_projection_states() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
+ let schema_right = Arc::new(Schema::new(vec![field_a]));
+ let on = vec![(
+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
+ )];
+
+ for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
+ roundtrip_test(Arc::new(HashJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ on.clone(),
+ None,
+ &JoinType::Inner,
+ projection,
+ PartitionMode::Partitioned,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?))?;
+ }
+ Ok(())
+}
+
+/// Regression: `HashJoinExecNode` had no `fetch` field, so the row limit that
+/// the `limit_pushdown` physical optimizer rule pushes into the join via
+/// `ExecutionPlan::with_fetch` was silently dropped by serde. Because that rule
+/// also removes the enclosing `GlobalLimitExec` once the join absorbs the limit,
+/// a round-tripped plan had no limit left at all and a distributed executor
+/// returned more rows than the query asked for.
+///
+/// Note this cannot be covered by `roundtrip_test`: that helper compares
+/// `format!("{plan:?}")`, and `HashJoinExec`'s `Debug` output does not include
+/// `fetch`, so the before/after strings match even when the value is lost. The
+/// assertions below therefore inspect `fetch()` directly.
+#[test]
+fn roundtrip_hash_join_fetch() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
+ let schema_right = Arc::new(Schema::new(vec![field_a]));
+ let on = vec![(
+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
+ )];
+
+ // `usize::MAX` and `u32::MAX as usize` pin the decode-side `u64 -> usize`
+ // conversion: it is a checked `usize::try_from`, and a large fetch must
+ // survive the round trip exactly rather than being truncated or clamped.
+ // Both are representable on every target (on a 32-bit target `usize::MAX`
+ // is simply `u32::MAX`), so this stays portable. The truncating case
+ // itself -- a `u64` fetch above `usize::MAX` -- is only reachable on a
+ // 32-bit target and so is not exercised by this test on a 64-bit host.
+ for fetch in [None, Some(7), Some(u32::MAX as usize), Some(usize::MAX)] {
+ let join = HashJoinExec::try_new(
+ Arc::new(EmptyExec::new(Arc::clone(&schema_left))),
+ Arc::new(EmptyExec::new(Arc::clone(&schema_right))),
+ on.clone(),
+ None,
+ &JoinType::Inner,
+ None,
+ PartitionMode::Partitioned,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?;
+
+ let plan: Arc<dyn ExecutionPlan> = match fetch {
+ // This is how `limit_pushdown` installs the limit.
+ Some(fetch) => join
+ .with_fetch(Some(fetch))
+ .expect("HashJoinExec supports fetch"),
+ None => Arc::new(join),
+ };
+ assert_eq!(plan.fetch(), fetch);
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let deserialized =
+ roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
+
+ let deserialized_join = deserialized
+ .downcast_ref::<HashJoinExec>()
+ .expect("should be a HashJoinExec");
+ assert_eq!(deserialized_join.fetch(), fetch);
+ }
+ Ok(())
+}
+
+/// Same regression coverage for `NestedLoopJoinExec`, which shares the
+/// `repeated uint32 projection` proto field shape with `HashJoinExec`.
+#[test]
+fn roundtrip_nested_loop_join_projection_states() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
+ let schema_right = Arc::new(Schema::new(vec![field_a]));
+
+ for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
+ roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ None,
+ &JoinType::Inner,
+ projection,
+ )?))?;
+ }
+ Ok(())
+}
+
+#[test]
+fn roundtrip_sym_hash_join() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let field_a = Field::new("col_a", DataType::Int64, false);
+ let field_b = Field::new("col_b", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_b.clone()]);
+ let on = vec![(
+ Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,
+ Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,
+ )];
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("col_a", 0)),
+ Operator::Gt,
+ Arc::new(Column::new("col_b", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 0,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 0,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![field_a, field_b])),
+ );
+
+ let schema_left = Arc::new(schema_left);
+ let schema_right = Arc::new(schema_right);
+ let left_order: LexOrdering = [PhysicalSortExpr {
+ expr: Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)),
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ }]
+ .into();
+ let right_order: LexOrdering = [PhysicalSortExpr {
+ expr: Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }]
+ .into();
+ let ordering_cases = [
+ (None, None),
+ (Some(left_order.clone()), None),
+ (None, Some(right_order.clone())),
+ (Some(left_order), Some(right_order)),
+ ];
+ let ordering_options = |ordering: Option<&LexOrdering>| {
+ ordering
+ .map(|ordering| ordering.iter().map(|expr| expr.options).collect::<Vec<_>>())
+ };
+
+ for join_type in [
+ JoinType::Inner,
+ JoinType::Left,
+ JoinType::Right,
+ JoinType::Full,
+ JoinType::LeftAnti,
+ JoinType::RightAnti,
+ JoinType::LeftSemi,
+ JoinType::RightSemi,
+ JoinType::LeftMark,
+ JoinType::RightMark,
+ ] {
+ for null_equality in [
+ NullEquality::NullEqualsNothing,
+ NullEquality::NullEqualsNull,
+ ] {
+ for filter in [None, Some(filter.clone())] {
+ for partition_mode in [
+ StreamJoinPartitionMode::Partitioned,
+ StreamJoinPartitionMode::SinglePartition,
+ ] {
+ for (left_order, right_order) in &ordering_cases {
+ let result = roundtrip_test_and_return(
+ Arc::new(SymmetricHashJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ on.clone(),
+ filter.clone(),
+ &join_type,
+ null_equality,
+ left_order.clone(),
+ right_order.clone(),
+ partition_mode,
+ )?),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let result =
+ result.downcast_ref::<SymmetricHashJoinExec>().unwrap();
+ assert_eq!(result.join_type(), &join_type);
+ assert_eq!(result.null_equality(), null_equality);
+ assert_eq!(result.partition_mode(), partition_mode);
+ assert_eq!(
+ ordering_options(result.left_sort_exprs()),
+ ordering_options(left_order.as_ref())
+ );
+ assert_eq!(
+ ordering_options(result.right_sort_exprs()),
+ ordering_options(right_order.as_ref())
+ );
+ assert_eq!(
+ result.filter().map(JoinFilter::column_indices),
+ filter.as_ref().map(JoinFilter::column_indices)
+ );
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+}
+
+#[test]
+fn roundtrip_sort_merge_join() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let field_a = Field::new("col_a", DataType::Int64, false);
+ let field_b = Field::new("col_b", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_b.clone()]);
+ let on = vec![(
+ Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,
+ Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,
+ )];
+
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("col_a", 1)),
+ Operator::Gt,
+ Arc::new(Column::new("col_b", 0)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 0,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 0,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![field_a, field_b])),
+ );
+
+ let schema_left = Arc::new(schema_left);
+ let schema_right = Arc::new(schema_right);
+ let sort_options = vec![SortOptions {
+ descending: true,
+ nulls_first: false,
+ }];
+ for null_equality in [
+ NullEquality::NullEqualsNothing,
+ NullEquality::NullEqualsNull,
+ ] {
+ for filter in [None, Some(filter.clone())] {
+ for join_type in [
+ JoinType::Inner,
+ JoinType::Left,
+ JoinType::Right,
+ JoinType::Full,
+ JoinType::LeftAnti,
+ JoinType::RightAnti,
+ JoinType::LeftSemi,
+ JoinType::RightSemi,
+ JoinType::LeftMark,
+ JoinType::RightMark,
+ ] {
+ let result = roundtrip_test_and_return(
+ Arc::new(SortMergeJoinExec::try_new(
+ Arc::new(EmptyExec::new(schema_left.clone())),
+ Arc::new(EmptyExec::new(schema_right.clone())),
+ on.clone(),
+ filter.clone(),
+ join_type,
+ sort_options.clone(),
+ null_equality,
+ )?),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let result = result.downcast_ref::<SortMergeJoinExec>().unwrap();
+ assert_eq!(result.join_type(), join_type);
+ assert_eq!(result.null_equality(), null_equality);
+ assert_eq!(result.sort_options(), sort_options);
+ assert_eq!(
+ result.filter().as_ref().map(|f| f.column_indices()),
+ filter.as_ref().map(|f| f.column_indices())
+ );
+ }
+ }
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_logical_plan_sort_merge_join() -> Result<()> {
+ let ctx = SessionContext::new();
+ ctx.register_csv(
+ "t0",
+ "tests/testdata/test.csv",
+ datafusion::prelude::CsvReadOptions::default().has_header(true),
+ )
+ .await?;
+ ctx.register_csv(
+ "t1",
+ "tests/testdata/test.csv",
+ datafusion::prelude::CsvReadOptions::default().has_header(true),
+ )
+ .await?;
+
+ ctx.sql("SET datafusion.optimizer.prefer_hash_join = false")
+ .await?
+ .show()
+ .await?;
+
+ let query = "SELECT t1.* FROM t0 join t1 on t0.a = t1.a";
+ let plan = ctx.sql(query).await?.create_physical_plan().await?;
+ roundtrip_test(plan)
+}
diff --git a/datafusion/proto/tests/cases/plans/leaves.rs b/datafusion/proto/tests/cases/plans/leaves.rs
new file mode 100644
index 0000000000..afcab2dda2
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/leaves.rs
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Leaf plans: `EmptyExec` and `PlaceholderRowExec`.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use datafusion::arrow::datatypes::Schema;
+use datafusion::physical_plan::ExecutionPlanProperties;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use std::sync::Arc;
+
+#[test]
+fn roundtrip_empty() -> Result<()> {
+ roundtrip_test(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))
+}
+
+#[test]
+fn roundtrip_empty_with_partitions() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let plan = Arc::new(EmptyExec::new(Arc::new(Schema::empty())).with_partitions(4));
+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
+ assert_eq!(plan.output_partitioning().partition_count(), 4);
+ Ok(())
+}
+
+#[test]
+fn roundtrip_placeholder_row_with_partitions() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let plan =
+ Arc::new(PlaceholderRowExec::new(Arc::new(Schema::empty())).with_partitions(4));
+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
+ assert_eq!(plan.output_partitioning().partition_count(), 4);
+ Ok(())
+}
+
+/// Plans encoded before `partitions` was added carry no value for it, which
+/// decodes as zero and must be treated as the previous default of one.
+#[test]
+fn decode_empty_and_placeholder_row_without_partitions() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let schema: protobuf::Schema = (&Schema::empty()).try_into()?;
+
+ for physical_plan_type in [
+ protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {
+ schema: Some(schema.clone()),
+ partitions: 0,
+ }),
+ protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(
+ protobuf::PlaceholderRowExecNode {
+ schema: Some(schema.clone()),
+ partitions: 0,
+ },
+ ),
+ ] {
+ let node = PhysicalPlanNode {
+ physical_plan_type: Some(physical_plan_type),
+ };
+ let plan = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;
+ assert_eq!(plan.output_partitioning().partition_count(), 1);
+ }
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/limits.rs b/datafusion/proto/tests/cases/plans/limits.rs
new file mode 100644
index 0000000000..e1c33ff949
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/limits.rs
@@ -0,0 +1,236 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Plans that carry a row limit or shape the batch pipeline: the limit
+//! execs, the coalescing execs, `BufferExec` and `CooperativeExec`.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::datasource::listing::PartitionedFile;
+use datafusion::datasource::object_store::ObjectStoreUrl;
+use datafusion::datasource::physical_plan::{
+ FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,
+};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;
+use datafusion::physical_plan::buffer::BufferExec;
+#[expect(deprecated)]
+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_plan::coop::CooperativeExec;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
+use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_common::config::ConfigOptions;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_local_limit() -> Result<()> {
+ roundtrip_test(Arc::new(LocalLimitExec::new(
+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
+ 25,
+ )))
+}
+
+#[test]
+fn roundtrip_global_limit() -> Result<()> {
+ roundtrip_test(Arc::new(GlobalLimitExec::new(
+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
+ 0,
+ Some(25),
+ )))
+}
+
+#[test]
+fn roundtrip_global_skip_no_limit() -> Result<()> {
+ roundtrip_test(Arc::new(GlobalLimitExec::new(
+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
+ 10,
+ None, // no limit
+ )))
+}
+
+/// Sort key at index 1, so a decoder that misbinds column name vs index
+/// cannot pass.
+fn limit_test_schema() -> Arc<Schema> {
+ Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int64, false),
+ Field::new("b", DataType::Int64, false),
+ ]))
+}
+
+/// Non-default sort options, so a decode that falls back to defaults cannot
+/// pass.
+fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {
+ Ok(LexOrdering::new(vec![PhysicalSortExpr {
+ expr: col("b", schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ }]))
+}
+
+#[test]
+fn roundtrip_limit_with_required_ordering() -> Result<()> {
+ let schema = limit_test_schema();
+ let required_ordering = limit_required_ordering(&schema)?;
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ let mut global =
+ GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));
+ global.set_required_ordering(required_ordering.clone());
+ let decoded =
+ roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;
+ let decoded = decoded
+ .downcast_ref::<GlobalLimitExec>()
+ .expect("expected GlobalLimitExec");
+ assert_eq!(decoded.required_ordering(), &required_ordering);
+
+ let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);
+ local.set_required_ordering(required_ordering.clone());
+ let decoded =
+ roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;
+ let decoded = decoded
+ .downcast_ref::<LocalLimitExec>()
+ .expect("expected LocalLimitExec");
+ assert_eq!(decoded.required_ordering(), &required_ordering);
+ Ok(())
+}
+
+/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`
+/// whose sort node was optimized away is order-sensitive, so it must survive
+/// serde all the way into the scan's `preserve_order` flag.
+#[test]
+fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {
+ let file_schema = limit_test_schema();
+ let make_scan = || {
+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .build();
+ DataSourceExec::from_data_source(scan_config)
+ };
+ let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let decoded =
+ roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;
+
+ // Child replacement must not erase the decoded ordering before pushdown.
+ let rebuilt = decoded.with_new_children(vec![make_scan()])?;
+
+ let optimized =
+ LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;
+ let scan = optimized
+ .downcast_ref::<DataSourceExec>()
+ .expect("limit should be absorbed into the scan");
+ Ok(scan
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .expect("expected FileScanConfig")
+ .clone())
+ };
+
+ let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));
+ limit.set_required_ordering(limit_required_ordering(&file_schema)?);
+ let scan_config = scan_after_limit_pushdown(limit)?;
+ assert_eq!(scan_config.limit, Some(10));
+ assert!(scan_config.preserve_order);
+
+ let scan_config =
+ scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;
+ assert_eq!(scan_config.limit, Some(10));
+ assert!(!scan_config.preserve_order);
+ Ok(())
+}
+
+#[test]
+fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ #[expect(deprecated)]
+ roundtrip_test(Arc::new(CoalesceBatchesExec::new(
+ Arc::new(EmptyExec::new(schema.clone())),
+ 8096,
+ )))?;
+
+ #[expect(deprecated)]
+ roundtrip_test(Arc::new(
+ CoalesceBatchesExec::new(Arc::new(EmptyExec::new(schema)), 8096)
+ .with_fetch(Some(10)),
+ ))
+}
+
+#[test]
+fn roundtrip_coalesce_partitions_with_fetch() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ roundtrip_test(Arc::new(CoalescePartitionsExec::new(Arc::new(
+ EmptyExec::new(schema.clone()),
+ ))))?;
+
+ roundtrip_test(Arc::new(
+ CoalescePartitionsExec::new(Arc::new(EmptyExec::new(schema)))
+ .with_fetch(Some(10)),
+ ))
+}
+
+#[test]
+fn roundtrip_cooperative() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));
+ roundtrip_test(Arc::new(CooperativeExec::new(Arc::new(EmptyExec::new(
+ schema,
+ )))))
+}
+
+#[test]
+fn roundtrip_buffer() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let result = roundtrip_test_and_return(
+ Arc::new(BufferExec::new(Arc::new(EmptyExec::new(schema)), 4096)),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let result = result.downcast_ref::<BufferExec>().unwrap();
+ assert_eq!(result.capacity(), 4096);
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/misc.rs b/datafusion/proto/tests/cases/plans/misc.rs
new file mode 100644
index 0000000000..254387f14e
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/misc.rs
@@ -0,0 +1,415 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Plans that do not (yet) warrant a file of their own: unions, unnest,
+//! repartitioning and the analyze/explain execs.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use arrow::datatypes::{Fields, TimeUnit};
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_plan::analyze::AnalyzeExec;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::explain::ExplainExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
+use datafusion::physical_plan::metrics::MetricCategory;
+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion::physical_plan::repartition::RepartitionExec;
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::union::{InterleaveExec, UnionExec};
+use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};
+use datafusion::physical_plan::{
+ ExecutionPlan, Partitioning, RangePartitioning, SplitPoint,
+};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::display::{PlanType, StringifiedPlan};
+use datafusion_common::format::ExplainFormat;
+use datafusion_common::{DataFusionError, Result, UnnestOptions};
+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+ PhysicalPlanDecodeContext,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use prost::Message;
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_analyze() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("plan_type", DataType::Utf8, false),
+ Field::new("plan", DataType::Utf8, false),
+ ]));
+ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));
+ let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing];
+ let analyze = Arc::new(
+ AnalyzeExec::builder(true, true, input, Arc::clone(&schema))
+ .with_metric_categories(Some(metric_categories.clone()))
+ .with_format(ExplainFormat::Tree)
+ .build(),
+ );
+
+ let ctx = SessionContext::new();
+ let roundtripped = roundtrip_test_and_return(
+ analyze,
+ &ctx,
+ &DefaultPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
+ let roundtripped = roundtripped.downcast_ref::<AnalyzeExec>().unwrap();
+
+ assert_eq!(roundtripped.schema(), schema);
+ assert!(roundtripped.verbose());
+ assert!(roundtripped.show_statistics());
+ assert_eq!(
+ roundtripped.metric_categories(),
+ Some(metric_categories.as_slice())
+ );
+ assert_eq!(roundtripped.format(), &ExplainFormat::Tree);
+ assert!(
+ roundtripped
+ .input()
+ .downcast_ref::<PlaceholderRowExec>()
+ .is_some()
+ );
+ Ok(())
+}
+
+#[test]
+fn roundtrip_explain() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("plan_type", DataType::Utf8, false),
+ Field::new("plan", DataType::Utf8, false),
+ ]));
+ let stringified_plans = vec![
+ StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),
+ StringifiedPlan::new(
+ PlanType::AnalyzedLogicalPlan {
+ analyzer_name: "analyzer".to_string(),
+ },
+ "analyzed logical",
+ ),
+ StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),
+ StringifiedPlan::new(
+ PlanType::OptimizedLogicalPlan {
+ optimizer_name: "logical optimizer".to_string(),
+ },
+ "optimized logical",
+ ),
+ StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),
+ StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),
+ StringifiedPlan::new(
+ PlanType::InitialPhysicalPlanWithStats,
+ "initial physical with stats",
+ ),
+ StringifiedPlan::new(
+ PlanType::InitialPhysicalPlanWithSchema,
+ "initial physical with schema",
+ ),
+ StringifiedPlan::new(
+ PlanType::OptimizedPhysicalPlan {
+ optimizer_name: "physical optimizer".to_string(),
+ },
+ "optimized physical",
+ ),
+ StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),
+ StringifiedPlan::new(
+ PlanType::FinalPhysicalPlanWithStats,
+ "final physical with stats",
+ ),
+ StringifiedPlan::new(
+ PlanType::FinalPhysicalPlanWithSchema,
+ "final physical with schema",
+ ),
+ StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),
+ ];
+ let explain = Arc::new(ExplainExec::new(
+ Arc::clone(&schema),
+ stringified_plans.clone(),
+ true,
+ ));
+
+ let ctx = SessionContext::new();
+ let roundtripped = roundtrip_test_and_return(
+ explain,
+ &ctx,
+ &DefaultPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
+ let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();
+
+ assert_eq!(roundtripped.schema(), schema);
+ assert_eq!(roundtripped.stringified_plans(), stringified_plans);
+ assert!(roundtripped.verbose());
+ Ok(())
+}
+
+#[test]
+fn roundtrip_union() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_a]);
+ let left = EmptyExec::new(Arc::new(schema_left));
+ let right = EmptyExec::new(Arc::new(schema_right));
+ let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];
+ let union = UnionExec::try_new(inputs)?;
+ roundtrip_test(union)
+}
+
+#[test]
+fn roundtrip_repartition_preserve_order() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a]));
+ let sort_exprs: LexOrdering = [PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions::default(),
+ }]
+ .into();
+
+ // Create two sorted single-partition inputs, then union them to get
+ // a sorted input with 2 partitions.
+ let source1 = SortExec::new(
+ sort_exprs.clone(),
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ );
+ let source2 = SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)));
+ let union = UnionExec::try_new(vec![
+ Arc::new(source1) as Arc<dyn ExecutionPlan>,
+ Arc::new(source2) as Arc<dyn ExecutionPlan>,
+ ])?;
+
+ let repartition = RepartitionExec::try_new(union, Partitioning::RoundRobinBatch(10))?
+ .with_preserve_order();
+ assert!(repartition.preserve_order());
+
+ roundtrip_test(Arc::new(repartition))
+}
+
+#[test]
+fn roundtrip_range_partitioning() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let range_partitioning = Partitioning::Range(RangePartitioning::new(
+ [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),
+ vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],
+ ));
+ // RepartitionExec is used only to carry the partitioning through proto.
+ // Executing range repartitioning is intentionally unsupported.
+ let repartition = RepartitionExec::try_new(input, range_partitioning)?;
+
+ roundtrip_test(Arc::new(repartition))
+}
+
+/// `parse_protobuf_hash_partitioning` has no in-tree callers left; it delegates
+/// to the shared `Partitioning::try_from_proto`, so pin that it still decodes
+/// the hash message it is handed.
+#[test]
+fn parse_hash_partitioning_delegates_to_shared_decoder() -> Result<()> {
+ use datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;
+
+ let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let decode_ctx = PhysicalPlanDecodeContext::new(&task_ctx, &codec);
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ let hash_expr = serialize_physical_expr_with_converter(
+ &col("a", &schema)?,
+ &codec,
+ &proto_converter,
+ )?;
+ let hash = protobuf::PhysicalHashRepartition {
+ hash_expr: vec![hash_expr],
+ partition_count: 4,
+ };
+
+ let partitioning = parse_protobuf_hash_partitioning(
+ Some(&hash),
+ &decode_ctx,
+ &schema,
+ &proto_converter,
+ )?;
+ let Some(Partitioning::Hash(exprs, count)) = partitioning else {
+ panic!("expected hash partitioning, got {partitioning:?}");
+ };
+ assert_eq!(count, 4);
+ assert_eq!(exprs.len(), 1);
+ assert_eq!(exprs[0].to_string(), col("a", &schema)?.to_string());
+
+ // No message means no partitioning, as before.
+ assert!(
+ parse_protobuf_hash_partitioning(None, &decode_ctx, &schema, &proto_converter)?
+ .is_none()
+ );
+
+ // The count is a `u64` on the wire and a `usize` in memory, so decoding
+ // narrows it. A count that does not fit is the case that motivated routing
+ // this through the shared decoder: it used to `unwrap()` and panic, and now
+ // reports an error. Only a target narrower than 64 bits can reach that arm
+ // -- on a 64-bit target every `u64` fits, and the assertion there is that
+ // the largest possible count survives whole rather than being truncated.
+ let oversized = protobuf::PhysicalHashRepartition {
+ hash_expr: vec![serialize_physical_expr_with_converter(
+ &col("a", &schema)?,
+ &codec,
+ &proto_converter,
+ )?],
+ partition_count: u64::MAX,
+ };
+ let decoded = parse_protobuf_hash_partitioning(
+ Some(&oversized),
+ &decode_ctx,
+ &schema,
+ &proto_converter,
+ );
+
+ #[cfg(target_pointer_width = "64")]
+ {
+ let Some(Partitioning::Hash(_, count)) = decoded? else {
+ panic!("expected hash partitioning");
+ };
+ assert_eq!(count, usize::MAX);
+ }
+
+ #[cfg(not(target_pointer_width = "64"))]
+ assert!(
+ decoded
+ .unwrap_err()
+ .to_string()
+ .contains("Partition count 18446744073709551615 exceeds usize::MAX")
+ );
+
+ Ok(())
+}
+
+#[test]
+fn roundtrip_interleave() -> Result<()> {
+ let field_a = Field::new("col", DataType::Int64, false);
+ let schema_left = Schema::new(vec![field_a.clone()]);
+ let schema_right = Schema::new(vec![field_a]);
+ let partition = Partitioning::Hash(vec![], 3);
+ let left = RepartitionExec::try_new(
+ Arc::new(EmptyExec::new(Arc::new(schema_left))),
+ partition.clone(),
+ )?;
+ let right = RepartitionExec::try_new(
+ Arc::new(EmptyExec::new(Arc::new(schema_right))),
+ partition,
+ )?;
+ let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];
+ let interleave = InterleaveExec::try_new(inputs)?;
+ roundtrip_test(Arc::new(interleave))
+}
+
+#[test]
+fn roundtrip_unnest() -> Result<()> {
+ let fa = Field::new("a", DataType::Int64, true);
+ let fb0 = Field::new_list_field(DataType::Utf8, true);
+ let fb = Field::new_list("b", fb0.clone(), false);
+ let fc1 = Field::new("c1", DataType::Boolean, false);
+ let fc2 = Field::new("c2", DataType::Date64, true);
+ let fc = Field::new_struct("c", Fields::from(vec![fc1.clone(), fc2.clone()]), true);
+ let fd0 = Field::new_list_field(DataType::Float32, false);
+ let fd = Field::new_list("d", fd0.clone(), true);
+ let fe1 = Field::new("e1", DataType::UInt16, false);
+ let fe2 = Field::new("e2", DataType::Duration(TimeUnit::Millisecond), true);
+ let fe3 = Field::new("e3", DataType::Timestamp(TimeUnit::Millisecond, None), true);
+ let fe_fields = Fields::from(vec![fe1.clone(), fe2.clone(), fe3.clone()]);
+ let fe = Field::new_struct("e", fe_fields, false);
+
+ let fb0 = fb0.with_name("b");
+ let fd0 = fd0.with_name("d");
+ let input_schema = Arc::new(Schema::new(vec![fa.clone(), fb, fc, fd, fe]));
+ let output_schema =
+ Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));
+ let input = Arc::new(EmptyExec::new(input_schema));
+ let options = UnnestOptions {
+ null_handling: datafusion_common::NullHandling::Drop,
+ recursions: vec![datafusion_common::RecursionUnnestOption {
+ input_column: datafusion_common::Column::new_unqualified("b"),
+ output_column: datafusion_common::Column::new_unqualified("b"),
+ depth: 2,
+ }],
+ };
+ let unnest = UnnestExec::new(
+ input,
+ vec![
+ ListUnnest {
+ index_in_input_schema: 1,
+ depth: 1,
+ },
+ ListUnnest {
+ index_in_input_schema: 1,
+ depth: 2,
+ },
+ ListUnnest {
+ index_in_input_schema: 3,
+ depth: 2,
+ },
+ ],
+ vec![2, 4],
+ output_schema,
+ options.clone(),
+ )?;
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let result =
+ roundtrip_test_and_return(Arc::new(unnest), &ctx, &codec, &proto_converter)?;
+ let result = result.downcast_ref::<UnnestExec>().unwrap();
+ assert_eq!(result.options(), &options);
+
+ Ok(())
+}
+
+#[tokio::test]
+/// Tests that we can serialize an unoptimized "analyze" plan and it will work on the other end
+async fn analyze_roundtrip_unoptimized() -> Result<()> {
+ let ctx = SessionContext::new();
+
+ // No optimizations
+ let session_state =
+ datafusion::execution::SessionStateBuilder::new_from_existing(ctx.state())
+ .with_physical_optimizer_rules(vec![])
+ .build();
+
+ let logical_plan = session_state
+ .create_logical_plan("explain analyze select 1")
+ .await?;
+ let plan = session_state.create_physical_plan(&logical_plan).await?;
+
+ let node = PhysicalPlanNode::try_from_physical_plan(
+ plan.clone(),
+ &DefaultPhysicalExtensionCodec {},
+ )?;
+
+ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+ let unoptimized =
+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
+
+ let physical_planner =
+ datafusion::physical_planner::DefaultPhysicalPlanner::default();
+ physical_planner.optimize_physical_plan(unoptimized, &session_state, |_, _| {})?;
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rs
new file mode 100644
index 0000000000..8dad1eff67
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/mod.rs
@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Round trip tests for the physical plan protobuf representation.
+//!
+//! The tests are grouped by the kind of plan they cover; the shared
+//! round trip helpers live here.
+
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use datafusion_common::Result;
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_proto_converter,
+ physical_plan_to_bytes_with_proto_converter,
+};
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
+use std::sync::Arc;
+
+mod aggregates;
+mod dispatch;
+mod dynamic_filters;
+mod exprs;
+mod filters;
+mod joins;
+mod leaves;
+mod limits;
+mod misc;
+mod scalar_subquery;
+mod sinks;
+mod sorts;
+mod sources;
+mod tpch;
+mod udfs;
+mod windows;
+
+/// Perform a serde roundtrip and assert that the string representation of the before and after plans
+/// are identical. Note that this often isn't sufficient to guarantee that no information is
+/// lost during serde because the string representation of a plan often only shows a subset of state.
+fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+ Ok(())
+}
+
+/// Perform a serde roundtrip and assert that the string representation of the before and after plans
+/// are identical. Note that this often isn't sufficient to guarantee that no information is
+/// lost during serde because the string representation of a plan often only shows a subset of state.
+///
+/// This version of the roundtrip_test method returns the final plan after serde so that it can be inspected
+/// farther in tests.
+fn roundtrip_test_and_return(
+ exec_plan: Arc<dyn ExecutionPlan>,
+ ctx: &SessionContext,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&exec_plan),
+ codec,
+ proto_converter,
+ )?;
+ let result_exec_plan = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ codec,
+ proto_converter,
+ )?;
+
+ pretty_assertions::assert_eq!(
+ format!("{exec_plan:?}"),
+ format!("{result_exec_plan:?}")
+ );
+ Ok(result_exec_plan)
+}
+
+/// Perform a serde roundtrip and assert that the string representation of the before and after plans
+/// are identical. Note that this often isn't sufficient to guarantee that no information is
+/// lost during serde because the string representation of a plan often only shows a subset of state.
+///
+/// This version of the roundtrip_test function accepts a SessionContext, which is required when
+/// performing serde on some plans.
+fn roundtrip_test_with_context(
+ exec_plan: Arc<dyn ExecutionPlan>,
+ ctx: &SessionContext,
+) -> Result<()> {
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;
+ Ok(())
+}
+
+/// Perform a serde roundtrip for the specified sql query, and assert that
+/// query results are identical.
+async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;
+
+ roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;
+ Ok(())
+}
+
+/// returns a SessionContext with `alltypes_plain` registered
+async fn all_types_context() -> Result<SessionContext> {
+ let ctx = SessionContext::new();
+
+ let testdata = datafusion::test_util::parquet_test_data();
+ ctx.register_parquet(
+ "alltypes_plain",
+ &format!("{testdata}/alltypes_plain.parquet"),
+ ParquetReadOptions::default(),
+ )
+ .await?;
+
+ Ok(ctx)
+}
diff --git a/datafusion/proto/tests/cases/plans/scalar_subquery.rs b/datafusion/proto/tests/cases/plans/scalar_subquery.rs
new file mode 100644
index 0000000000..34d30aa03d
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/scalar_subquery.rs
@@ -0,0 +1,278 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `ScalarSubqueryExec` and the results it scopes to its subtree.
+
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::logical_expr::Operator;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{BinaryExpr, binary, col};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::scalar_subquery::{
+ ScalarSubqueryExec, ScalarSubqueryLink,
+};
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};
+use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_proto_converter,
+ physical_plan_to_bytes_with_proto_converter,
+};
+use datafusion_proto::physical_plan::{
+ DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,
+};
+use std::sync::Arc;
+use std::vec;
+
+/// Verify that ScalarSubqueryExpr nodes in the input plan are connected to the
+/// same shared results container as ScalarSubqueryExec after a proto round-trip.
+#[test]
+fn roundtrip_scalar_subquery_exec() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let results = ScalarSubqueryResults::new(1);
+
+ // Build the input plan: a filter whose predicate references the
+ // scalar subquery result via ScalarSubqueryExpr.
+ let sq_expr = Arc::new(ScalarSubqueryExpr::new(
+ DataType::Int64,
+ true,
+ SubqueryIndex::new(0),
+ results.clone(),
+ ));
+ let predicate = binary(col("a", &schema)?, Operator::Eq, sq_expr, &schema)?;
+ let filter =
+ FilterExec::try_new(predicate, Arc::new(EmptyExec::new(schema.clone())))?;
+
+ // Build a trivial subquery plan.
+ let subquery_plan =
+ Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(
+ "x",
+ DataType::Int64,
+ true,
+ )]))));
+
+ let exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
+ Arc::new(filter),
+ vec![ScalarSubqueryLink {
+ plan: subquery_plan,
+ index: SubqueryIndex::new(0),
+ }],
+ results,
+ ));
+
+ // Perform the round-trip using DeduplicatingProtoConverter, which
+ // creates a DeduplicatingDeserializer that threads scalar subquery
+ // results through expression deserialization.
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DeduplicatingProtoConverter {};
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&exec),
+ &codec,
+ &converter,
+ )?;
+ let ctx = SessionContext::new();
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &converter,
+ )?;
+
+ // Verify the deserialized ScalarSubqueryExec's results container is
+ // shared with the ScalarSubqueryExpr in the input plan.
+ let sq_exec = deserialized
+ .downcast_ref::<ScalarSubqueryExec>()
+ .expect("expected ScalarSubqueryExec");
+ let exec_results = sq_exec.results();
+
+ // Walk the input plan to find the ScalarSubqueryExpr and verify it
+ // points to the same results container.
+ let filter_exec = sq_exec
+ .input()
+ .downcast_ref::<FilterExec>()
+ .expect("expected FilterExec");
+ let binary_expr = filter_exec
+ .predicate()
+ .downcast_ref::<BinaryExpr>()
+ .expect("expected BinaryExpr");
+ let deserialized_sq_expr = binary_expr
+ .right()
+ .downcast_ref::<ScalarSubqueryExpr>()
+ .expect("expected ScalarSubqueryExpr");
+
+ assert!(
+ ScalarSubqueryResults::ptr_eq(exec_results, deserialized_sq_expr.results()),
+ "ScalarSubqueryExpr should share the same results container as ScalarSubqueryExec"
+ );
+ Ok(())
+}
+
+/// Verify that nested ScalarSubqueryExec nodes deserialize with distinct
+/// scoped results containers, and that each ScalarSubqueryExpr is wired to the
+/// container for its own surrounding ScalarSubqueryExec.
+#[test]
+fn roundtrip_nested_scalar_subquery_exec_scopes_results() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let subquery_schema =
+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
+
+ let inner_results = ScalarSubqueryResults::new(1);
+ let inner_sq_expr = Arc::new(ScalarSubqueryExpr::new(
+ DataType::Int64,
+ true,
+ SubqueryIndex::new(0),
+ inner_results.clone(),
+ ));
+ let inner_predicate =
+ binary(col("a", &schema)?, Operator::Eq, inner_sq_expr, &schema)?;
+ let inner_filter = Arc::new(FilterExec::try_new(
+ inner_predicate,
+ Arc::new(EmptyExec::new(schema.clone())),
+ )?);
+ let inner_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
+ inner_filter,
+ vec![ScalarSubqueryLink {
+ plan: Arc::new(EmptyExec::new(subquery_schema.clone())),
+ index: SubqueryIndex::new(0),
+ }],
+ inner_results,
+ ));
+
+ let outer_results = ScalarSubqueryResults::new(1);
+ let outer_sq_expr = Arc::new(ScalarSubqueryExpr::new(
+ DataType::Int64,
+ true,
+ SubqueryIndex::new(0),
+ outer_results.clone(),
+ ));
+ let outer_predicate =
+ binary(col("a", &schema)?, Operator::Eq, outer_sq_expr, &schema)?;
+ let outer_filter = Arc::new(FilterExec::try_new(outer_predicate, inner_exec)?);
+ let outer_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
+ outer_filter,
+ vec![ScalarSubqueryLink {
+ plan: Arc::new(EmptyExec::new(subquery_schema)),
+ index: SubqueryIndex::new(0),
+ }],
+ outer_results,
+ ));
+
+ let bytes = datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&outer_exec))?;
+ let ctx = SessionContext::new();
+ let deserialized = datafusion_proto::bytes::physical_plan_from_bytes(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ )?;
+
+ let outer_exec = deserialized
+ .downcast_ref::<ScalarSubqueryExec>()
+ .expect("expected outer ScalarSubqueryExec");
+ let outer_results = outer_exec.results();
+ let outer_filter = outer_exec
+ .input()
+ .downcast_ref::<FilterExec>()
+ .expect("expected outer FilterExec");
+ let outer_binary = outer_filter
+ .predicate()
+ .downcast_ref::<BinaryExpr>()
+ .expect("expected outer BinaryExpr");
+ let outer_sq_expr = outer_binary
+ .right()
+ .downcast_ref::<ScalarSubqueryExpr>()
+ .expect("expected outer ScalarSubqueryExpr");
+
+ let inner_exec = outer_filter
+ .input()
+ .downcast_ref::<ScalarSubqueryExec>()
+ .expect("expected inner ScalarSubqueryExec");
+ let inner_results = inner_exec.results();
+ let inner_filter = inner_exec
+ .input()
+ .downcast_ref::<FilterExec>()
+ .expect("expected inner FilterExec");
+ let inner_binary = inner_filter
+ .predicate()
+ .downcast_ref::<BinaryExpr>()
+ .expect("expected inner BinaryExpr");
+ let inner_sq_expr = inner_binary
+ .right()
+ .downcast_ref::<ScalarSubqueryExpr>()
+ .expect("expected inner ScalarSubqueryExpr");
+
+ assert!(
+ ScalarSubqueryResults::ptr_eq(outer_results, outer_sq_expr.results()),
+ "outer ScalarSubqueryExpr should use outer ScalarSubqueryExec results"
+ );
+ assert!(
+ ScalarSubqueryResults::ptr_eq(inner_results, inner_sq_expr.results()),
+ "inner ScalarSubqueryExpr should use inner ScalarSubqueryExec results"
+ );
+ assert!(
+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_results),
+ "nested ScalarSubqueryExec nodes should not share results containers"
+ );
+ assert!(
+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_sq_expr.results()),
+ "inner ScalarSubqueryExpr must not read from outer results"
+ );
+ assert!(
+ !ScalarSubqueryResults::ptr_eq(inner_results, outer_sq_expr.results()),
+ "outer ScalarSubqueryExpr must not read from inner results"
+ );
+
+ Ok(())
+}
+
+/// Verify that the default physical plan bytes round-trip preserves executable
+/// scalar subquery plans.
+#[tokio::test]
+async fn roundtrip_scalar_subquery_exec_with_default_converter_executes() -> Result<()> {
+ let ctx = SessionContext::new();
+ let sql = "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) AS s \
+ FROM (VALUES (2), (1)) AS t(x) \
+ ORDER BY s";
+
+ let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;
+ assert!(
+ format!("{initial_plan:?}").contains("ScalarSubqueryExec"),
+ "expected ScalarSubqueryExec in plan:\n{initial_plan:?}"
+ );
+
+ let bytes =
+ datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&initial_plan))?;
+ let roundtripped = datafusion_proto::bytes::physical_plan_from_bytes(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ )?;
+ assert!(
+ format!("{roundtripped:?}").contains("ScalarSubqueryExec"),
+ "expected ScalarSubqueryExec after roundtrip:\n{roundtripped:?}"
+ );
+
+ let batches = datafusion::physical_plan::common::collect(
+ roundtripped.execute(0, ctx.task_ctx())?,
+ )
+ .await?;
+ datafusion::assert_batches_eq!(
+ &["+----+", "| s |", "+----+", "| 21 |", "| 22 |", "+----+",],
+ &batches
+ );
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/sinks.rs b/datafusion/proto/tests/cases/plans/sinks.rs
new file mode 100644
index 0000000000..517659e9e9
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/sinks.rs
@@ -0,0 +1,329 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Data sinks and their file sink configurations.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use arrow::csv::WriterBuilder;
+use async_trait::async_trait;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::datasource::file_format::csv::CsvSink;
+use datafusion::datasource::file_format::json::JsonSink;
+use datafusion::datasource::file_format::parquet::ParquetSink;
+use datafusion::datasource::listing::{ListingTableUrl, PartitionedFile};
+use datafusion::datasource::object_store::ObjectStoreUrl;
+use datafusion::datasource::physical_plan::{FileGroup, FileOutputMode, FileSinkConfig};
+use datafusion::datasource::sink::{DataSink, DataSinkExec};
+use datafusion::execution::TaskContext;
+use datafusion::physical_expr::PhysicalSortRequirement;
+use datafusion::physical_plan::expressions::Column;
+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;
+use datafusion::physical_plan::{
+ DisplayAs, DisplayFormatType, ExecutionPlan, SendableRecordBatchStream,
+};
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_common::config::TableParquetOptions;
+use datafusion_common::file_options::csv_writer::CsvWriterOptions;
+use datafusion_common::file_options::json_writer::JsonWriterOptions;
+use datafusion_common::parsers::CompressionTypeVariant;
+use datafusion_expr::dml::InsertOp;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use std::fmt::Formatter;
+use std::sync::Arc;
+use std::vec;
+
+#[derive(Debug)]
+struct ProtoHookSink {
+ schema: SchemaRef,
+}
+
+impl DisplayAs for ProtoHookSink {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ write!(f, "ProtoHookSink")
+ }
+}
+
+#[async_trait]
+impl DataSink for ProtoHookSink {
+ fn schema(&self) -> &SchemaRef {
+ &self.schema
+ }
+
+ async fn write_all(
+ &self,
+ _data: SendableRecordBatchStream,
+ _context: &Arc<TaskContext>,
+ ) -> Result<u64> {
+ unreachable!("serialization test does not execute the sink")
+ }
+
+ fn try_to_proto(
+ &self,
+ exec: &DataSinkExec,
+ ctx: &ExecutionPlanEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalPlanNode>> {
+ let input = ctx.encode_child(exec.input())?;
+ let sort_order = exec.encode_sort_order(ctx)?;
+ assert!(matches!(
+ input.physical_plan_type,
+ Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))
+ ));
+ assert_eq!(
+ sort_order
+ .as_ref()
+ .map(|ordering| ordering.physical_sort_expr_nodes.len()),
+ Some(1)
+ );
+ assert_eq!(exec.schema().fields().len(), 1);
+
+ Ok(Some(PhysicalPlanNode {
+ physical_plan_type: Some(
+ protobuf::physical_plan_node::PhysicalPlanType::Empty(
+ protobuf::EmptyExecNode {
+ schema: Some(exec.schema().as_ref().try_into()?),
+ partitions: 1,
+ },
+ ),
+ ),
+ }))
+ }
+}
+
+#[test]
+fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {
+ let input_schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Int64,
+ false,
+ )]));
+ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&input_schema)));
+ let sink = Arc::new(ProtoHookSink {
+ schema: Arc::clone(&input_schema),
+ });
+ let sort_order = [PhysicalSortRequirement::new(
+ Arc::new(Column::new("value", 0)),
+ Some(SortOptions::default()),
+ )]
+ .into();
+ let plan = Arc::new(DataSinkExec::new(input, sink, Some(sort_order)));
+
+ let node = PhysicalPlanNode::try_from_physical_plan(
+ plan,
+ &DefaultPhysicalExtensionCodec {},
+ )?;
+
+ assert!(matches!(
+ node.physical_plan_type,
+ Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))
+ ));
+ Ok(())
+}
+
+#[test]
+fn file_sink_config_roundtrip_preserves_fields() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "partition",
+ DataType::Utf8,
+ false,
+ )]));
+ let config = FileSinkConfig {
+ original_url: "file:///tmp/output".to_string(),
+ object_store_url: ObjectStoreUrl::local_filesystem(),
+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp/output", 1)]),
+ table_paths: vec![ListingTableUrl::parse("file:///tmp/output")?],
+ output_schema: schema,
+ table_partition_cols: vec![("partition".to_string(), DataType::Utf8)],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: true,
+ file_extension: "parquet".to_string(),
+ file_output_mode: FileOutputMode::Directory,
+ };
+
+ let encoded = protobuf::FileSinkConfig::try_from(&config)?;
+ assert_eq!(encoded.insert_op(), protobuf::InsertOp::Overwrite);
+ assert_eq!(
+ encoded.file_output_mode(),
+ protobuf::FileOutputMode::Directory
+ );
+
+ let decoded = FileSinkConfig::try_from(&encoded)?;
+ assert_eq!(decoded.object_store_url, config.object_store_url);
+ assert_eq!(decoded.table_paths, config.table_paths);
+ assert_eq!(
+ decoded.output_schema.as_ref(),
+ config.output_schema.as_ref()
+ );
+ assert_eq!(decoded.table_partition_cols, config.table_partition_cols);
+ assert_eq!(decoded.insert_op, config.insert_op);
+ assert_eq!(
+ decoded.keep_partition_by_columns,
+ config.keep_partition_by_columns
+ );
+ assert_eq!(decoded.file_extension, config.file_extension);
+ assert_eq!(decoded.file_output_mode, config.file_output_mode);
+
+ let [decoded_file] = decoded.file_group.files() else {
+ panic!("expected one decoded output file");
+ };
+ let [config_file] = config.file_group.files() else {
+ panic!("expected one configured output file");
+ };
+ assert_eq!(
+ decoded_file.object_meta.location,
+ config_file.object_meta.location
+ );
+ assert_eq!(decoded_file.object_meta.size, config_file.object_meta.size);
+ Ok(())
+}
+
+#[test]
+fn roundtrip_json_sink() -> Result<()> {
+ let field_a = Field::new("plan_type", DataType::Utf8, false);
+ let field_b = Field::new("plan", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
+
+ let file_sink_config = FileSinkConfig {
+ original_url: String::default(),
+ object_store_url: ObjectStoreUrl::local_filesystem(),
+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
+ table_paths: vec![ListingTableUrl::parse("file:///")?],
+ output_schema: schema.clone(),
+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: true,
+ file_extension: "json".into(),
+ file_output_mode: FileOutputMode::SingleFile,
+ };
+ let data_sink = Arc::new(JsonSink::new(
+ file_sink_config,
+ JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED),
+ ));
+ let sort_order = [PhysicalSortRequirement::new(
+ Arc::new(Column::new("plan_type", 0)),
+ Some(SortOptions {
+ descending: true,
+ nulls_first: false,
+ }),
+ )]
+ .into();
+
+ roundtrip_test(Arc::new(DataSinkExec::new(
+ input,
+ data_sink,
+ Some(sort_order),
+ )))
+}
+
+#[test]
+fn roundtrip_csv_sink() -> Result<()> {
+ let field_a = Field::new("plan_type", DataType::Utf8, false);
+ let field_b = Field::new("plan", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
+
+ let file_sink_config = FileSinkConfig {
+ original_url: String::default(),
+ object_store_url: ObjectStoreUrl::local_filesystem(),
+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
+ table_paths: vec![ListingTableUrl::parse("file:///")?],
+ output_schema: schema.clone(),
+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: true,
+ file_extension: "csv".into(),
+ file_output_mode: FileOutputMode::Directory,
+ };
+ let data_sink = Arc::new(CsvSink::new(
+ file_sink_config,
+ CsvWriterOptions::new(WriterBuilder::default(), CompressionTypeVariant::ZSTD),
+ ));
+ let sort_order = [PhysicalSortRequirement::new(
+ Arc::new(Column::new("plan_type", 0)),
+ Some(SortOptions {
+ descending: true,
+ nulls_first: false,
+ }),
+ )]
+ .into();
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ let roundtrip_plan = roundtrip_test_and_return(
+ Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+
+ let roundtrip_plan = roundtrip_plan.downcast_ref::<DataSinkExec>().unwrap();
+ let csv_sink = roundtrip_plan.sink().downcast_ref::<CsvSink>().unwrap();
+ assert_eq!(
+ CompressionTypeVariant::ZSTD,
+ csv_sink.writer_options().compression
+ );
+
+ Ok(())
+}
+
+#[test]
+fn roundtrip_parquet_sink() -> Result<()> {
+ let field_a = Field::new("plan_type", DataType::Utf8, false);
+ let field_b = Field::new("plan", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
+
+ let file_sink_config = FileSinkConfig {
+ original_url: String::default(),
+ object_store_url: ObjectStoreUrl::local_filesystem(),
+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
+ table_paths: vec![ListingTableUrl::parse("file:///")?],
+ output_schema: schema.clone(),
+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: true,
+ file_extension: "parquet".into(),
+ file_output_mode: FileOutputMode::Automatic,
+ };
+ let data_sink = Arc::new(ParquetSink::new(
+ file_sink_config,
+ TableParquetOptions::default(),
+ ));
+ let sort_order = [PhysicalSortRequirement::new(
+ Arc::new(Column::new("plan_type", 0)),
+ Some(SortOptions {
+ descending: true,
+ nulls_first: false,
+ }),
+ )]
+ .into();
+
+ roundtrip_test(Arc::new(DataSinkExec::new(
+ input,
+ data_sink,
+ Some(sort_order),
+ )))
+}
diff --git a/datafusion/proto/tests/cases/plans/sorts.rs b/datafusion/proto/tests/cases/plans/sorts.rs
new file mode 100644
index 0000000000..1172775b1b
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/sorts.rs
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `SortExec` and `SortPreservingMergeExec`.
+
+use super::{roundtrip_test, roundtrip_test_and_return};
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_sort() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+ roundtrip_test(Arc::new(SortExec::new(
+ sort_exprs,
+ Arc::new(EmptyExec::new(schema)),
+ )))
+}
+
+#[test]
+fn roundtrip_sort_preserve_partitioning() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs: LexOrdering = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+
+ roundtrip_test(Arc::new(SortExec::new(
+ sort_exprs.clone(),
+ Arc::new(EmptyExec::new(schema.clone())),
+ )))?;
+
+ roundtrip_test(Arc::new(
+ SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)))
+ .with_preserve_partitioning(true),
+ ))
+}
+
+/// `SortExec::fetch` turns a sort into a top-k sort. Losing it during serde
+/// would silently widen the result set, so exercise the `Some(..)` state
+/// explicitly (`roundtrip_sort` only covers `None`).
+///
+/// `SortExec` currently derives `Debug`, so `roundtrip_test`'s
+/// `format!("{plan:?}")` comparison does observe `fetch`. The assertions below
+/// go through the accessor instead so that this coverage does not silently
+/// disappear if `SortExec` ever grows a hand-written `Debug` impl.
+#[test]
+fn roundtrip_sort_with_fetch() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs: LexOrdering = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ let roundtripped = roundtrip_test_and_return(
+ Arc::new(
+ SortExec::new(
+ sort_exprs.clone(),
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ )
+ .with_fetch(Some(7)),
+ ),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let roundtripped = roundtripped
+ .downcast_ref::<SortExec>()
+ .expect("should decode back into a SortExec");
+ assert_eq!(roundtripped.fetch(), Some(7));
+ assert_eq!(roundtripped.expr(), &sort_exprs);
+
+ // `fetch` combined with `preserve_partitioning`, since both share the same
+ // proto node.
+ let roundtripped = roundtrip_test_and_return(
+ Arc::new(
+ SortExec::new(sort_exprs.clone(), Arc::new(EmptyExec::new(schema)))
+ .with_fetch(Some(3))
+ .with_preserve_partitioning(true),
+ ),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let roundtripped = roundtripped
+ .downcast_ref::<SortExec>()
+ .expect("should decode back into a SortExec");
+ assert_eq!(roundtripped.fetch(), Some(3));
+ assert!(roundtripped.preserve_partitioning());
+ Ok(())
+}
+
+/// Round trip a [`SortPreservingMergeExec`], which had no dedicated round trip
+/// test at all.
+///
+/// Covers everything that is actually on the wire for this plan: the input, the
+/// sort expressions and `fetch` in both its `None` and `Some(..)` states.
+///
+/// Note that `SortPreservingMergeExec::enable_round_robin_repartition` is
+/// deliberately *not* asserted on here: it has no field in
+/// `SortPreservingMergeExecNode`, so it is not serialized and decoding always
+/// restores the `true` default from `SortPreservingMergeExec::new`. Asserting
+/// round trip equality on it would give a false sense of coverage.
+///
+/// `SortPreservingMergeExec` derives `Debug`, so `roundtrip_test`'s
+/// `format!("{plan:?}")` comparison does observe `expr` and `fetch`. The
+/// assertions below use the accessors so the coverage survives a future
+/// hand-written `Debug` impl.
+#[test]
+fn roundtrip_sort_preserving_merge() -> Result<()> {
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs: LexOrdering = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
+ // No fetch: `fetch` is encoded as -1 and must decode back to `None`.
+ let roundtripped = roundtrip_test_and_return(
+ Arc::new(SortPreservingMergeExec::new(
+ sort_exprs.clone(),
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ )),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let roundtripped = roundtripped
+ .downcast_ref::<SortPreservingMergeExec>()
+ .expect("should decode back into a SortPreservingMergeExec");
+ assert_eq!(roundtripped.fetch(), None);
+ assert_eq!(roundtripped.expr(), &sort_exprs);
+ assert_eq!(roundtripped.input().schema(), schema);
+
+ // With a fetch: dropping it would turn a bounded merge into an unbounded
+ // one and change the query result.
+ let roundtripped = roundtrip_test_and_return(
+ Arc::new(
+ SortPreservingMergeExec::new(
+ sort_exprs.clone(),
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ )
+ .with_fetch(Some(11)),
+ ),
+ &ctx,
+ &codec,
+ &proto_converter,
+ )?;
+ let roundtripped = roundtripped
+ .downcast_ref::<SortPreservingMergeExec>()
+ .expect("should decode back into a SortPreservingMergeExec");
+ assert_eq!(roundtripped.fetch(), Some(11));
+ assert_eq!(roundtripped.expr(), &sort_exprs);
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs
new file mode 100644
index 0000000000..04708dec64
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/sources.rs
@@ -0,0 +1,790 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Scans and data sources: file formats, `FileScanConfig`, listing tables
+//! and memory sources.
+
+use super::{
+ all_types_context, roundtrip_test, roundtrip_test_and_return,
+ roundtrip_test_sql_with_context,
+};
+use arrow::array::RecordBatch;
+use arrow::datatypes::Fields;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::datasource::empty::EmptyTable;
+use datafusion::datasource::file_format::json::JsonFormat;
+use datafusion::datasource::listing::{
+ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, PartitionedFile,
+};
+use datafusion::datasource::object_store::ObjectStoreUrl;
+use datafusion::datasource::physical_plan::{
+ ArrowSource, CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, JsonSource,
+ ParquetSource, wrap_partition_type_in_dict, wrap_partition_value_in_dict,
+};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::Operator;
+use datafusion::physical_expr::LexOrdering;
+use datafusion::physical_plan::expressions::{
+ BinaryExpr, Column, PhysicalSortExpr, col, lit,
+};
+use datafusion::physical_plan::filter::FilterExecBuilder;
+use datafusion::physical_plan::{
+ ExecutionPlan, Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, Statistics,
+ displayable,
+};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::config::TableParquetOptions;
+use datafusion_common::stats::Precision;
+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err};
+use datafusion_datasource::{TableSchema, TableSchemaBuilder};
+use datafusion_expr::ColumnarValue;
+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use prost::Message;
+use std::collections::HashMap;
+use std::fmt::{Display, Formatter};
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+
+ let predicate = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("col", 1)),
+ Operator::Eq,
+ lit("1"),
+ ));
+
+ let mut options = TableParquetOptions::new();
+ options.global.pushdown_filters = true;
+
+ let file_source = Arc::new(
+ ParquetSource::new(Arc::clone(&file_schema))
+ .with_table_parquet_options(options)
+ .with_predicate(predicate),
+ );
+
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .with_statistics(Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1024),
+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(
+ vec![Field::new("col", DataType::Utf8, false)],
+ ))),
+ })
+ .build();
+
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
+#[test]
+fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .with_statistics(Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1024),
+ column_statistics: Statistics::unknown_column(&file_schema),
+ })
+ .build();
+ let exec_plan = DataSourceExec::from_data_source(scan_config);
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let roundtripped =
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+
+ let data_source = roundtripped
+ .downcast_ref::<DataSourceExec>()
+ .ok_or_else(|| {
+ internal_datafusion_err!("Expected DataSourceExec after roundtrip")
+ })?;
+ let file_scan = data_source
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .ok_or_else(|| {
+ internal_datafusion_err!("Expected FileScanConfig after roundtrip")
+ })?;
+ let parquet_source = file_scan
+ .file_source()
+ .downcast_ref::<ParquetSource>()
+ .ok_or_else(|| {
+ internal_datafusion_err!("Expected ParquetSource after roundtrip")
+ })?;
+
+ assert!(
+ parquet_source.parquet_file_reader_factory().is_some(),
+ "Parquet reader factory should be attached after decoding from protobuf"
+ );
+ Ok(())
+}
+
+#[test]
+fn roundtrip_arrow_scan() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+
+ let table_schema = TableSchema::from(&file_schema);
+ let file_source = Arc::new(ArrowSource::new_file_source(table_schema));
+
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.arrow".to_string(),
+ 1024,
+ )])])
+ .with_statistics(Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1024),
+ column_statistics: Statistics::unknown_column(&file_schema),
+ })
+ .build();
+
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
+#[test]
+fn roundtrip_json_scan() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+ let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.json".to_string(),
+ 1024,
+ )])])
+ .build();
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
+#[cfg(feature = "avro")]
+#[test]
+fn roundtrip_avro_scan() -> Result<()> {
+ use datafusion_datasource_avro::source::AvroSource;
+
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+ let file_source = Arc::new(AvroSource::new(TableSchema::from(&file_schema)));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.avro".to_string(),
+ 1024,
+ )])])
+ .build();
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
+#[test]
+fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
+ use datafusion::common::config::CsvOptions;
+
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+ let table_schema = TableSchema::from(&file_schema);
+ let file_source =
+ Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {
+ has_header: Some(false),
+ delimiter: b'|',
+ quote: b'\'',
+ escape: Some(b'\\'),
+ comment: Some(b'#'),
+ newlines_in_values: Some(true),
+ truncated_rows: Some(true),
+ ..Default::default()
+ }));
+
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.csv".to_string(),
+ 1024,
+ )])])
+ .build();
+
+ let ctx = SessionContext::new();
+ let roundtripped = roundtrip_test_and_return(
+ DataSourceExec::from_data_source(scan_config),
+ &ctx,
+ &DefaultPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
+ let data_source = roundtripped
+ .downcast_ref::<DataSourceExec>()
+ .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;
+ let file_scan = data_source
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;
+ let csv_source = file_scan
+ .file_source()
+ .downcast_ref::<CsvSource>()
+ .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;
+
+ assert!(!csv_source.has_header());
+ assert_eq!(csv_source.delimiter(), b'|');
+ assert_eq!(csv_source.quote(), b'\'');
+ assert_eq!(csv_source.escape(), Some(b'\\'));
+ assert_eq!(csv_source.comment(), Some(b'#'));
+ assert!(csv_source.newlines_in_values());
+ assert!(csv_source.truncate_rows());
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {
+ let mut file_group =
+ PartitionedFile::new("/path/to/part=0/file.parquet".to_string(), 1024);
+ file_group.partition_values =
+ vec![wrap_partition_value_in_dict(ScalarValue::Int64(Some(0)))];
+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+
+ let table_schema = TableSchemaBuilder::from(&schema)
+ .with_table_partition_cols(vec![Arc::new(Field::new(
+ "part".to_string(),
+ wrap_partition_type_in_dict(DataType::Int16),
+ false,
+ ))])
+ .build();
+
+ let file_source = Arc::new(ParquetSource::new(table_schema.clone()));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_projection_indices(Some(vec![0, 1]))?
+ .with_file_group(FileGroup::new(vec![file_group]))
+ .build();
+
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
+#[test]
+fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+
+ let custom_predicate_expr = Arc::new(CustomPredicateExpr {
+ inner: Arc::new(Column::new("col", 1)),
+ });
+
+ let file_source = Arc::new(
+ ParquetSource::new(Arc::clone(&file_schema))
+ .with_predicate(custom_predicate_expr),
+ );
+
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .with_statistics(Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1024),
+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(
+ vec![Field::new("col", DataType::Utf8, false)],
+ ))),
+ })
+ .build();
+
+ #[derive(Debug, Clone, Eq)]
+ struct CustomPredicateExpr {
+ inner: Arc<dyn PhysicalExpr>,
+ }
+
+ // Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808
+ impl PartialEq for CustomPredicateExpr {
+ fn eq(&self, other: &Self) -> bool {
+ self.inner.eq(&other.inner)
+ }
+ }
+
+ impl std::hash::Hash for CustomPredicateExpr {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.inner.hash(state);
+ }
+ }
+
+ impl Display for CustomPredicateExpr {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "CustomPredicateExpr")
+ }
+ }
+
+ impl PhysicalExpr for CustomPredicateExpr {
+ fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+ unreachable!()
+ }
+
+ fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
+ unreachable!()
+ }
+
+ fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {
+ unreachable!()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![&self.inner]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(self)
+ }
+
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ Display::fmt(self, f)
+ }
+ }
+
+ #[derive(Debug)]
+ struct CustomPhysicalExtensionCodec;
+ impl PhysicalExtensionCodec for CustomPhysicalExtensionCodec {
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ _ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unreachable!()
+ }
+
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ unreachable!()
+ }
+
+ fn try_decode_expr(
+ &self,
+ buf: &[u8],
+ inputs: &[Arc<dyn PhysicalExpr>],
+ _ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ if buf == "CustomPredicateExpr".as_bytes() {
+ Ok(Arc::new(CustomPredicateExpr {
+ inner: inputs[0].clone(),
+ }))
+ } else {
+ internal_err!("Not supported")
+ }
+ }
+
+ fn try_encode_expr(
+ &self,
+ node: &Arc<dyn PhysicalExpr>,
+ buf: &mut Vec<u8>,
+ _ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<()> {
+ if node.downcast_ref::<CustomPredicateExpr>().is_some() {
+ buf.extend_from_slice("CustomPredicateExpr".as_bytes());
+ Ok(())
+ } else {
+ internal_err!("Not supported")
+ }
+ }
+ }
+
+ let exec_plan = DataSourceExec::from_data_source(scan_config);
+
+ let ctx = SessionContext::new();
+ roundtrip_test_and_return(
+ exec_plan,
+ &ctx,
+ &CustomPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_json_source() -> Result<()> {
+ let ctx = SessionContext::new();
+ ctx.register_json("t1", "../core/tests/data/1.json", Default::default())
+ .await?;
+ let plan = ctx.table("t1").await?.create_physical_plan().await?;
+ roundtrip_test(plan)
+}
+
+#[tokio::test]
+async fn roundtrip_coalesce() -> Result<()> {
+ let ctx = SessionContext::new();
+ ctx.register_table(
+ "t",
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([
+ Arc::new(Field::new("f", DataType::Int64, false)),
+ ]))))),
+ )?;
+ let df = ctx.sql("select coalesce(f) as f from t").await?;
+ let plan = df.create_physical_plan().await?;
+
+ let node = PhysicalPlanNode::try_from_physical_plan(
+ plan.clone(),
+ &DefaultPhysicalExtensionCodec {},
+ )?;
+ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+ let restored =
+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
+
+ assert_eq!(
+ plan.schema(),
+ restored.schema(),
+ "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",
+ displayable(plan.as_ref())
+ .set_show_schema(true)
+ .indent(true),
+ displayable(restored.as_ref())
+ .set_show_schema(true)
+ .indent(true),
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_generate_series() -> Result<()> {
+ let ctx = SessionContext::new();
+ ctx.register_table(
+ "t",
+ Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([
+ Arc::new(Field::new("f", DataType::Int64, false)),
+ ]))))),
+ )?;
+ let df = ctx.sql("select * from generate_series(1, 10000)").await?;
+ let plan = df.create_physical_plan().await?;
+
+ let node = PhysicalPlanNode::try_from_physical_plan(
+ plan.clone(),
+ &DefaultPhysicalExtensionCodec {},
+ )?;
+ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+ let restored =
+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
+
+ assert_eq!(
+ plan.schema(),
+ restored.schema(),
+ "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",
+ displayable(plan.as_ref())
+ .set_show_schema(true)
+ .indent(true),
+ displayable(restored.as_ref())
+ .set_show_schema(true)
+ .indent(true),
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_projection_source() -> Result<()> {
+ let schema = Arc::new(Schema::new(Fields::from([
+ Arc::new(Field::new("a", DataType::Utf8, false)),
+ Arc::new(Field::new("b", DataType::Utf8, false)),
+ Arc::new(Field::new("c", DataType::Int32, false)),
+ Arc::new(Field::new("d", DataType::Int32, false)),
+ ])));
+
+ let statistics = Statistics::new_unknown(&schema);
+
+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&schema)));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .with_statistics(statistics)
+ .with_projection_indices(Some(vec![0, 1, 2]))?
+ .build();
+
+ let filter = Arc::new(
+ FilterExecBuilder::new(
+ Arc::new(BinaryExpr::new(col("c", &schema)?, Operator::Eq, lit(1))),
+ DataSourceExec::from_data_source(scan_config),
+ )
+ .apply_projection(Some(vec![0, 1]))?
+ .build()?,
+ );
+
+ roundtrip_test(filter)
+}
+
+#[tokio::test]
+async fn roundtrip_parquet_select_star() -> Result<()> {
+ let ctx = all_types_context().await?;
+ let sql = "select * from alltypes_plain";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_parquet_select_projection() -> Result<()> {
+ let ctx = all_types_context().await?;
+ let sql = "select string_col, timestamp_col from alltypes_plain";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_parquet_select_star_predicate() -> Result<()> {
+ let ctx = all_types_context().await?;
+ let sql = "select * from alltypes_plain where id > 4";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_parquet_select_projection_predicate() -> Result<()> {
+ let ctx = all_types_context().await?;
+ let sql = "select string_col, timestamp_col from alltypes_plain where id > 4";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_empty_projection() -> Result<()> {
+ let ctx = all_types_context().await?;
+ let sql = "select 1 from alltypes_plain";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_memory_source_empty_projection() -> Result<()> {
+ // Memory scan: `Some(vec![])` must not decode back as `None`
+ let ctx = SessionContext::new();
+ let batch = RecordBatch::try_new(
+ Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("b", DataType::Int64, false),
+ ])),
+ vec![
+ Arc::new(arrow::array::StringArray::from(vec!["Tom"])),
+ Arc::new(arrow::array::Int64Array::from(vec![18i64])),
+ ],
+ )?;
+ ctx.register_batch("tmem", batch)?;
+ let sql = "select 1 from tmem";
+ roundtrip_test_sql_with_context(sql, &ctx).await
+}
+
+#[tokio::test]
+async fn roundtrip_memory_source() -> Result<()> {
+ let ctx = SessionContext::new();
+ let plan = ctx
+ .sql("select * from values ('Tom', 18)")
+ .await?
+ .create_physical_plan()
+ .await?;
+ roundtrip_test(plan)
+}
+
+#[tokio::test]
+async fn roundtrip_memory_source_sort_information_and_fetch() -> Result<()> {
+ use datafusion::datasource::memory::MemorySourceConfig;
+ use datafusion::datasource::source::DataSource as _;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("b", DataType::Int64, false),
+ ]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(arrow::array::StringArray::from(vec!["Tom", "Bob"])),
+ Arc::new(arrow::array::Int64Array::from(vec![18i64, 21i64])),
+ ],
+ )?;
+ let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(
+ col("b", &schema)?,
+ SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ )])
+ .unwrap();
+ let source = MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?
+ .with_limit(Some(1))
+ .with_show_sizes(false)
+ .try_with_sort_information(vec![ordering])?;
+ let exec_plan = DataSourceExec::from_data_source(source.clone());
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let decoded = roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+
+ // The string representation does not include every field; check the
+ // decoded source directly.
+ let decoded = decoded
+ .downcast_ref::<DataSourceExec>()
+ .expect("expected DataSourceExec");
+ let decoded_source = decoded
+ .data_source()
+ .downcast_ref::<MemorySourceConfig>()
+ .expect("expected MemorySourceConfig");
+ assert_eq!(decoded_source.partitions(), source.partitions());
+ assert_eq!(decoded_source.original_schema(), source.original_schema());
+ assert_eq!(decoded_source.projection(), source.projection());
+ assert_eq!(decoded_source.sort_information(), source.sort_information());
+ assert_eq!(decoded_source.fetch(), Some(1));
+ assert!(!decoded_source.show_sizes());
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {
+ let ctx = SessionContext::new();
+ let file_format = JsonFormat::default();
+ let table_partition_cols = vec![("part".to_owned(), DataType::Int64)];
+ let data = "../core/tests/data/partitioned_table_json";
+ let listing_table_url = ListingTableUrl::parse(data)?;
+ let listing_options = ListingOptions::new(Arc::new(file_format))
+ .with_table_partition_cols(table_partition_cols);
+
+ let config = ListingTableConfig::new(listing_table_url)
+ .with_listing_options(listing_options)
+ .infer_schema(&ctx.state())
+ .await?;
+
+ // Decorate metadata onto the inferred ListingTable schema
+ let schema_with_meta = config
+ .file_schema
+ .clone()
+ .map(|s| {
+ let mut meta: HashMap<String, String> = HashMap::new();
+ meta.insert("foo.bar".to_string(), "baz".to_string());
+ s.as_ref().clone().with_metadata(meta)
+ })
+ .expect("Must decorate metadata");
+
+ let config = config.with_schema(Arc::new(schema_with_meta));
+ ctx.register_table("hive_style", Arc::new(ListingTable::try_new(config)?))?;
+
+ let plan = ctx
+ .sql("select * from hive_style limit 1")
+ .await?
+ .create_physical_plan()
+ .await?;
+
+ roundtrip_test(plan)
+}
+
+fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanConfig> {
+ let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let result_plan =
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+
+ let data_source_exec = result_plan
+ .downcast_ref::<DataSourceExec>()
+ .expect("Expected DataSourceExec");
+ let file_scan_config = data_source_exec
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .expect("Expected FileScanConfig");
+ Ok(file_scan_config.clone())
+}
+
+#[test]
+fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
+ let output_partitioning =
+ Partitioning::Hash(vec![Arc::new(Column::new("col", 0))], 1);
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .with_output_partitioning(Some(output_partitioning.clone()))
+ .build();
+
+ assert_eq!(
+ roundtrip_file_scan_config(scan_config)?.output_partitioning,
+ Some(output_partitioning)
+ );
+
+ Ok(())
+}
+
+#[test]
+fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)]));
+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
+ let output_partitioning = Partitioning::Range(RangePartitioning::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "col", 0,
+ )))])
+ .unwrap(),
+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],
+ ));
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![
+ FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file-1.parquet".to_string(),
+ 1024,
+ )]),
+ FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file-2.parquet".to_string(),
+ 1024,
+ )]),
+ ])
+ .with_output_partitioning(Some(output_partitioning.clone()))
+ .build();
+
+ assert_eq!(
+ roundtrip_file_scan_config(scan_config)?.output_partitioning,
+ Some(output_partitioning)
+ );
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/tpch.rs b/datafusion/proto/tests/cases/plans/tpch.rs
new file mode 100644
index 0000000000..d24150b3e0
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/tpch.rs
@@ -0,0 +1,335 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! End to end round trips of the TPC-H queries, plus the human readable
+//! display of the plans they produce.
+
+use super::{roundtrip_test_and_return, roundtrip_test_sql_with_context};
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::functions_aggregate::first_last::first_value_udaf;
+use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
+use datafusion::physical_plan::aggregates::{
+ AggregateExec, AggregateMode, PhysicalGroupBy,
+};
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};
+use datafusion::prelude::SessionContext;
+use datafusion_common::{DataFusionError, Result};
+use datafusion_proto::physical_plan::{
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
+use datafusion_proto::protobuf::PhysicalPlanNode;
+use std::sync::Arc;
+use std::vec;
+
+/// Helper function to create a SessionContext with all TPC-H tables registered as external tables
+async fn tpch_context() -> Result<SessionContext> {
+ use datafusion_common::test_util::datafusion_test_data;
+
+ let ctx = SessionContext::new();
+ let test_data = datafusion_test_data();
+
+ // TPC-H table names
+ let tables = [
+ "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation",
+ "region",
+ ];
+
+ // Create external tables for all TPC-H tables
+ for table in &tables {
+ let table_sql = format!(
+ "CREATE EXTERNAL TABLE {table} STORED AS PARQUET LOCATION '{test_data}/tpch_{table}_small.parquet'"
+ );
+ ctx.sql(&table_sql).await.map_err(|e| {
+ DataFusionError::External(
+ format!("Failed to create {table} table: {e}").into(),
+ )
+ })?;
+ }
+
+ Ok(ctx)
+}
+
+/// Helper function to get TPC-H query SQL
+fn get_tpch_query_sql(query: usize) -> Result<Vec<String>> {
+ use std::fs;
+
+ if !(1..=22).contains(&query) {
+ return Err(DataFusionError::External(
+ format!("Invalid TPC-H query number: {query}").into(),
+ ));
+ }
+
+ let filename = format!("../../benchmarks/queries/q{query}.sql");
+ let contents = fs::read_to_string(&filename).map_err(|e| {
+ DataFusionError::External(
+ format!("Failed to read query file {filename}: {e}").into(),
+ )
+ })?;
+
+ Ok(contents
+ .split(';')
+ .map(|s| s.trim())
+ .filter(|s| !s.is_empty())
+ .map(|s| s.to_string())
+ .collect())
+}
+
+#[tokio::test]
+async fn test_serialize_deserialize_tpch_queries() -> Result<()> {
+ // Create context with TPC-H tables
+ let ctx = tpch_context().await?;
+
+ // repeat to run all 22 queries
+ for query in 1..=22 {
+ // run all statements in the query
+ let sql = get_tpch_query_sql(query)?;
+ for stmt in sql {
+ let logical_plan = ctx.sql(&stmt).await?.into_unoptimized_plan();
+ let optimized_plan = ctx.state().optimize(&logical_plan)?;
+ let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;
+
+ // serialize the physical plan
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let proto =
+ PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
+
+ // deserialize the physical plan
+ let _deserialized_plan =
+ proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
+ }
+ }
+
+ Ok(())
+}
+
+// Bugs: https://github.com/apache/datafusion/issues/16772
+#[tokio::test]
+async fn test_round_trip_tpch_queries() -> Result<()> {
+ // Create context with TPC-H tables
+ let ctx = tpch_context().await?;
+
+ // repeat to run all 22 queries
+ for query in 1..=22 {
+ // run all statements in the query
+ let sql = get_tpch_query_sql(query)?;
+ for stmt in sql {
+ roundtrip_test_sql_with_context(&stmt, &ctx).await?;
+ }
+ }
+
+ Ok(())
+}
+
+// Bug 1 of https://github.com/apache/datafusion/issues/16772
+/// Test that AggregateFunctionExpr human_display field is correctly preserved
+/// during serialization/deserialization roundtrip.
+///
+/// Test for issue where the human_display field (used for EXPLAIN output)
+/// was not being serialized to protobuf, causing it to be lost during roundtrip
+/// and resulting in empty or incorrect display strings in query plans.
+#[tokio::test]
+async fn test_round_trip_human_display() -> Result<()> {
+ // Create context with TPC-H tables
+ let ctx = tpch_context().await?;
+
+ let sql = "select r_name, count(1) from region group by r_name";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ let sql = "select r_name, count(*) from region group by r_name";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ let sql = "select r_name, count(r_name) from region group by r_name";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ let sql = "select count(*) as count_star from region";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ Ok(())
+}
+
+#[test]
+fn test_round_trip_aliased_reverse_human_display() -> Result<()> {
+ let aggregate_expr = roundtrip_first_value_aggregate(
+ "agg",
+ "first_value(b) ORDER BY [b ASC NULLS LAST]",
+ Some("agg"),
+ )?;
+ let reversed = aggregate_expr
+ .reverse_expr()
+ .expect("expected reverse expr");
+
+ assert_eq!(reversed.name(), "agg");
+ assert_eq!(reversed.human_display_alias(), Some("agg"));
+ assert_eq!(
+ reversed.human_display(),
+ Some("last_value(b) ORDER BY [b DESC NULLS FIRST]")
+ );
+
+ Ok(())
+}
+
+#[test]
+fn test_round_trip_human_display_alias_with_colon() -> Result<()> {
+ let aggregate_expr = roundtrip_first_value_aggregate(
+ "agg:one",
+ "first_value(b) ORDER BY [b ASC NULLS LAST]",
+ Some("agg:one"),
+ )?;
+
+ assert_eq!(aggregate_expr.name(), "agg:one");
+ assert_eq!(aggregate_expr.human_display_alias(), Some("agg:one"));
+ assert_eq!(
+ aggregate_expr.human_display(),
+ Some("first_value(b) ORDER BY [b ASC NULLS LAST]")
+ );
+
+ Ok(())
+}
+
+#[test]
+fn test_round_trip_non_aliased_human_display_ending_like_alias() -> Result<()> {
+ let aggregate_expr =
+ roundtrip_first_value_aggregate("agg", "first_value(b) as agg", None)?;
+
+ assert_eq!(aggregate_expr.name(), "agg");
+ assert_eq!(
+ aggregate_expr.human_display(),
+ Some("first_value(b) as agg")
+ );
+ assert_eq!(aggregate_expr.human_display_alias(), None);
+
+ Ok(())
+}
+
+fn roundtrip_first_value_aggregate(
+ alias: &str,
+ human_display: &str,
+ human_display_alias: Option<&str>,
+) -> Result<Arc<AggregateFunctionExpr>> {
+ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));
+ let mut builder =
+ AggregateExprBuilder::new(first_value_udaf(), vec![col("b", &schema)?])
+ .order_by(vec![PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions::new(false, false),
+ }])
+ .schema(Arc::clone(&schema))
+ .alias(alias)
+ .human_display(human_display);
+ if let Some(human_display_alias) = human_display_alias {
+ builder = builder.human_display_alias(human_display_alias);
+ }
+ let agg_expr = builder.build().map(Arc::new)?;
+
+ let plan = Arc::new(AggregateExec::try_new(
+ AggregateMode::Single,
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
+ vec![agg_expr],
+ vec![None],
+ Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ schema,
+ )?);
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let roundtrip_plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
+ let aggregate = roundtrip_plan
+ .as_ref()
+ .downcast_ref::<AggregateExec>()
+ .expect("expected AggregateExec after roundtrip");
+
+ Ok(Arc::clone(&aggregate.aggr_expr()[0]))
+}
+
+// Bug 2 of https://github.com/apache/datafusion/issues/16772
+/// Test that PhysicalGroupBy groups field is correctly serialized/deserialized
+/// for simple aggregates (no GROUP BY clause).
+///
+/// Test for issue where simple aggregates like "SELECT SUM(col1 * col2) FROM table"
+/// would incorrectly serialize groups as [[]] instead of [] during roundtrip serialization.
+/// The groups field should be empty ([]) when there are no GROUP BY expressions.
+#[tokio::test]
+async fn test_round_trip_groups_display() -> Result<()> {
+ // Create context with TPC-H tables
+ let ctx = tpch_context().await?;
+
+ let sql = "select sum(l_extendedprice * l_discount) as revenue from lineitem;";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ let sql = "select sum(l_extendedprice) as revenue from lineitem;";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ Ok(())
+}
+
+// Bug 3 of https://github.com/apache/datafusion/issues/16772
+/// Test that ScalarFunctionExpr return_field name is correctly preserved
+/// during serialization/deserialization roundtrip.
+///
+/// Test for issue where the return_field.name for scalar functions
+/// was not being serialized to protobuf, causing it to be lost during roundtrip
+/// and defaulting to a generic name like "f" instead of the proper function name.
+#[tokio::test]
+async fn test_round_trip_date_part_display() -> Result<()> {
+ // Create context with TPC-H tables
+ let ctx = tpch_context().await?;
+
+ let sql = "select extract(year from l_shipdate) as l_year from lineitem ";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ let sql = "select extract(month from l_shipdate) as l_year from lineitem ";
+ roundtrip_test_sql_with_context(sql, &ctx).await?;
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {
+ use datafusion_common::test_util::datafusion_test_data;
+
+ let ctx = SessionContext::new();
+
+ // Register the TPC-H part table using the local test data
+ let test_data = datafusion_test_data();
+ let table_sql = format!(
+ "CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '{test_data}/tpch_part_small.parquet'"
+ );
+ ctx.sql(&table_sql).await.map_err(|e| {
+ DataFusionError::External(format!("Failed to create part table: {e}").into())
+ })?;
+
+ // Test the exact problematic query
+ let sql =
+ "SELECT p_size FROM part WHERE p_size IN (14, 6, 5, 31) and p_partkey > 1000";
+
+ let logical_plan = ctx.sql(sql).await?.into_unoptimized_plan();
+ let optimized_plan = ctx.state().optimize(&logical_plan)?;
+ let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;
+
+ // Serialize the physical plan - bug may happen here already but not necessarily manifests
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
+
+ // This will fail with the bug, but should succeed when fixed
+ let _deserialized_plan = proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/udfs.rs b/datafusion/proto/tests/cases/plans/udfs.rs
new file mode 100644
index 0000000000..08d00030e0
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/udfs.rs
@@ -0,0 +1,573 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Plans carrying user defined functions, and the extension codec that
+//! (de)serializes them.
+
+use super::{roundtrip_test_and_return, roundtrip_test_with_context};
+use crate::cases::{
+ CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF,
+ MyHigherOrderUdfNode, MyRegexUdf, MyRegexUdfNode,
+};
+use datafusion::arrow::array::ArrayRef;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::{Operator, Volatility, create_udf};
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
+use datafusion::physical_expr::expressions::Literal;
+use datafusion::physical_expr::window::StandardWindowExpr;
+use datafusion::physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};
+use datafusion::physical_plan::aggregates::{
+ AggregateExec, AggregateMode, PhysicalGroupBy,
+};
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{BinaryExpr, PhysicalSortExpr, col, lit};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
+use datafusion::physical_plan::windows::{
+ BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,
+ create_udwf_window_expr,
+};
+use datafusion::physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr};
+use datafusion::prelude::SessionContext;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
+use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
+use datafusion_expr::{
+ AggregateUDF, ColumnarValue, HigherOrderUDF, ScalarFunctionArgs, ScalarUDF,
+ ScalarUDFImpl, Signature, WindowFrame, WindowFrameBound, WindowUDF,
+};
+use datafusion_functions_aggregate::min_max::max_udaf;
+use datafusion_physical_expr::expressions::{LambdaVariable, is_not_null, lambda};
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
+use prost::Message;
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_scalar_udf() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let scalar_fn = Arc::new(|args: &[ColumnarValue]| {
+ let ColumnarValue::Array(array) = &args[0] else {
+ panic!("should be array")
+ };
+ Ok(ColumnarValue::from(Arc::new(array.clone()) as ArrayRef))
+ });
+
+ let udf = create_udf(
+ "dummy",
+ vec![DataType::Int64],
+ DataType::Int64,
+ Volatility::Immutable,
+ scalar_fn.clone(),
+ );
+
+ let fun_def = Arc::new(udf.clone());
+
+ let expr = ScalarFunctionExpr::new(
+ "dummy",
+ fun_def,
+ vec![col("a", &schema)?],
+ Field::new("f", DataType::Int64, true).into(),
+ Arc::new(ConfigOptions::default()),
+ );
+
+ let project = ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: Arc::new(expr),
+ alias: "a".to_string(),
+ }],
+ input,
+ )?;
+
+ let ctx = SessionContext::new();
+
+ ctx.register_udf(udf);
+
+ roundtrip_test_with_context(Arc::new(project), &ctx)
+}
+
+#[derive(Debug)]
+struct UDFExtensionCodec;
+
+impl PhysicalExtensionCodec for UDFExtensionCodec {
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ _ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ not_impl_err!("No extension codec provided")
+ }
+
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ not_impl_err!("No extension codec provided")
+ }
+
+ fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {
+ if name == "regex_udf" {
+ let proto = MyRegexUdfNode::decode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to decode regex_udf: {err}")
+ })?;
+
+ Ok(Arc::new(ScalarUDF::from(MyRegexUdf::new(proto.pattern))))
+ } else {
+ not_impl_err!("unrecognized scalar UDF implementation, cannot decode")
+ }
+ }
+
+ fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {
+ let binding = node.inner();
+ if let Some(udf) = binding.downcast_ref::<MyRegexUdf>() {
+ let proto = MyRegexUdfNode {
+ pattern: udf.pattern.clone(),
+ };
+ proto
+ .encode(buf)
+ .map_err(|err| internal_datafusion_err!("failed to encode udf: {err}"))?;
+ }
+ Ok(())
+ }
+
+ fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result<Arc<AggregateUDF>> {
+ if name == "aggregate_udf" {
+ let proto = MyAggregateUdfNode::decode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to decode aggregate_udf: {err}")
+ })?;
+
+ Ok(Arc::new(AggregateUDF::from(MyAggregateUDF::new(
+ proto.result,
+ ))))
+ } else {
+ not_impl_err!("unrecognized scalar UDF implementation, cannot decode")
+ }
+ }
+
+ fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {
+ let binding = node.inner();
+ if let Some(udf) = binding.downcast_ref::<MyAggregateUDF>() {
+ let proto = MyAggregateUdfNode {
+ result: udf.result.clone(),
+ };
+ proto.encode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to encode udf: {err:?}")
+ })?;
+ }
+ Ok(())
+ }
+
+ fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result<Arc<WindowUDF>> {
+ if name == "custom_udwf" {
+ let proto = CustomUDWFNode::decode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to decode custom_udwf: {err}")
+ })?;
+
+ Ok(Arc::new(WindowUDF::from(CustomUDWF::new(proto.payload))))
+ } else {
+ not_impl_err!(
+ "unrecognized user-defined window function implementation, cannot decode"
+ )
+ }
+ }
+
+ fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec<u8>) -> Result<()> {
+ let binding = node.inner();
+ if let Some(udwf) = binding.downcast_ref::<CustomUDWF>() {
+ let proto = CustomUDWFNode {
+ payload: udwf.payload.clone(),
+ };
+ proto.encode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to encode udwf: {err:?}")
+ })?;
+ }
+ Ok(())
+ }
+
+ fn try_decode_higher_order_function(
+ &self,
+ name: &str,
+ buf: &[u8],
+ ) -> Result<Arc<HigherOrderUDF>> {
+ if name == "higher_order_udf" {
+ let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to decode higher_order_udf: {err}")
+ })?;
+
+ Ok(Arc::new(HigherOrderUDF::new_from_impl(
+ MyHigherOrderUDF::new(proto.payload),
+ )))
+ } else {
+ not_impl_err!("unrecognized higher order UDF implementation, cannot decode")
+ }
+ }
+
+ fn try_encode_higher_order_function(
+ &self,
+ node: &HigherOrderUDF,
+ buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ if let Some(hof) = (node.inner().as_ref() as &dyn std::any::Any)
+ .downcast_ref::<MyHigherOrderUDF>()
+ {
+ let proto = MyHigherOrderUdfNode {
+ payload: hof.payload.clone(),
+ };
+ proto.encode(buf).map_err(|err| {
+ internal_datafusion_err!("failed to encode hof: {err:?}")
+ })?;
+ }
+ Ok(())
+ }
+}
+
+#[test]
+fn roundtrip_scalar_udf_extension_codec() -> Result<()> {
+ let field_text = Field::new("text", DataType::Utf8, true);
+ let field_published = Field::new("published", DataType::Boolean, false);
+ let field_author = Field::new("author", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let udf_expr = Arc::new(ScalarFunctionExpr::new(
+ "regex_udf",
+ Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),
+ vec![col("text", &schema)?],
+ Field::new("f", DataType::Int64, true).into(),
+ Arc::new(ConfigOptions::default()),
+ ));
+
+ let filter = Arc::new(FilterExec::try_new(
+ Arc::new(BinaryExpr::new(
+ col("published", &schema)?,
+ Operator::And,
+ Arc::new(BinaryExpr::new(udf_expr.clone(), Operator::Gt, lit(0))),
+ )),
+ input,
+ )?);
+ let aggr_expr =
+ AggregateExprBuilder::new(max_udaf(), vec![udf_expr as Arc<dyn PhysicalExpr>])
+ .schema(schema.clone())
+ .alias("max")
+ .build()
+ .map(Arc::new)?;
+
+ let window = Arc::new(WindowAggExec::try_new(
+ vec![Arc::new(PlainAggregateWindowExpr::new(
+ aggr_expr.clone(),
+ &[col("author", &schema)?],
+ &[],
+ Arc::new(WindowFrame::new(None)),
+ None,
+ ))],
+ filter,
+ true,
+ )?);
+
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
+ vec![aggr_expr],
+ vec![None],
+ window,
+ schema,
+ )?);
+
+ let ctx = SessionContext::new();
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
+ Ok(())
+}
+
+#[test]
+fn roundtrip_higher_order_udf() -> Result<()> {
+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));
+ let list_field = Field::new(
+ "list_col",
+ DataType::List(Arc::clone(&element_field)),
+ false,
+ );
+ let schema = Arc::new(Schema::new(vec![list_field]));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(
+ "payload".to_string(),
+ )));
+
+ let expr = HigherOrderFunctionExpr::try_new_with_schema(
+ Arc::clone(&hof),
+ vec![
+ col("list_col", &schema)?,
+ lambda(
+ ["v"],
+ is_not_null(Arc::new(LambdaVariable::new(1, element_field)))?,
+ )?,
+ ],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?;
+
+ let project = ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: Arc::new(expr),
+ alias: "a".to_string(),
+ }],
+ input,
+ )?;
+
+ let ctx = SessionContext::new();
+ ctx.register_higher_order_function(hof);
+
+ roundtrip_test_with_context(Arc::new(project), &ctx)
+}
+
+#[test]
+fn roundtrip_higher_order_udf_extension_codec() -> Result<()> {
+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));
+ let list_field = Field::new(
+ "list_col",
+ DataType::List(Arc::clone(&element_field)),
+ false,
+ );
+ let schema = Arc::new(Schema::new(vec![list_field]));
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let lambda_body = Arc::new(LambdaVariable::new(1, Arc::clone(&element_field)));
+ let lambda_expr = lambda(["v"], lambda_body)?;
+
+ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(
+ "payload".to_string(),
+ )));
+ let hof_expr = Arc::new(HigherOrderFunctionExpr::try_new_with_schema(
+ hof,
+ vec![col("list_col", &schema)?, lambda_expr],
+ &schema,
+ Arc::new(ConfigOptions::default()),
+ )?);
+
+ let project = ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: hof_expr,
+ alias: "out".to_string(),
+ }],
+ input,
+ )?;
+
+ let ctx = SessionContext::new();
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(
+ Arc::new(project),
+ &ctx,
+ &UDFExtensionCodec,
+ &proto_converter,
+ )?;
+ Ok(())
+}
+
+#[test]
+fn roundtrip_udwf_extension_codec() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let custom_udwf = Arc::new(WindowUDF::from(CustomUDWF::new("payload".to_string())));
+ let udwf = create_udwf_window_expr(
+ &custom_udwf,
+ &[col("a", &schema)?],
+ schema.as_ref(),
+ "custom_udwf(a) PARTITION BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
+ false,
+ )?;
+
+ let window_frame = WindowFrame::new_bounds(
+ datafusion_expr::WindowFrameUnits::Range,
+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),
+ WindowFrameBound::CurrentRow,
+ );
+
+ let udwf_expr = Arc::new(StandardWindowExpr::new(
+ udwf,
+ &[col("b", &schema)?],
+ &[PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ }],
+ Arc::new(window_frame),
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+ let window = Arc::new(BoundedWindowAggExec::try_new(
+ vec![udwf_expr],
+ input,
+ InputOrderMode::Sorted,
+ true,
+ )?);
+
+ let ctx = SessionContext::new();
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;
+ Ok(())
+}
+
+#[test]
+fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {
+ let field_text = Field::new("text", DataType::Utf8, true);
+ let field_published = Field::new("published", DataType::Boolean, false);
+ let field_author = Field::new("author", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let udf_expr = Arc::new(ScalarFunctionExpr::new(
+ "regex_udf",
+ Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),
+ vec![col("text", &schema)?],
+ Field::new("f", DataType::Int64, true).into(),
+ Arc::new(ConfigOptions::default()),
+ ));
+
+ let udaf = Arc::new(AggregateUDF::from(MyAggregateUDF::new(
+ "result".to_string(),
+ )));
+ let aggr_args: Vec<Arc<dyn PhysicalExpr>> =
+ vec![Arc::new(Literal::new(ScalarValue::from(42)))];
+
+ let aggr_expr = AggregateExprBuilder::new(Arc::clone(&udaf), aggr_args.clone())
+ .schema(Arc::clone(&schema))
+ .alias("aggregate_udf")
+ .build()
+ .map(Arc::new)?;
+
+ let filter = Arc::new(FilterExec::try_new(
+ Arc::new(BinaryExpr::new(
+ col("published", &schema)?,
+ Operator::And,
+ Arc::new(BinaryExpr::new(udf_expr, Operator::Gt, lit(0))),
+ )),
+ input,
+ )?);
+
+ let window = Arc::new(WindowAggExec::try_new(
+ vec![Arc::new(PlainAggregateWindowExpr::new(
+ aggr_expr,
+ &[col("author", &schema)?],
+ &[],
+ Arc::new(WindowFrame::new(None)),
+ None,
+ ))],
+ filter,
+ true,
+ )?);
+
+ let aggr_expr = AggregateExprBuilder::new(udaf, aggr_args.clone())
+ .schema(Arc::clone(&schema))
+ .alias("aggregate_udf")
+ .distinct()
+ .ignore_nulls()
+ .build()
+ .map(Arc::new)?;
+
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Final,
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
+ vec![aggr_expr],
+ vec![None],
+ window,
+ schema,
+ )?);
+
+ let ctx = SessionContext::new();
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_async_func_exec() -> Result<()> {
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct TestAsyncUDF {
+ signature: Signature,
+ }
+
+ impl TestAsyncUDF {
+ fn new() -> Self {
+ Self {
+ signature: Signature::exact(vec![DataType::Int64], Volatility::Volatile),
+ }
+ }
+ }
+
+ impl ScalarUDFImpl for TestAsyncUDF {
+ fn name(&self) -> &str {
+ "test_async_udf"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Int64)
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ not_impl_err!("Must call from `invoke_async_with_args`")
+ }
+ }
+
+ #[async_trait::async_trait]
+ impl AsyncScalarUDFImpl for TestAsyncUDF {
+ async fn invoke_async_with_args(
+ &self,
+ args: ScalarFunctionArgs,
+ ) -> Result<ColumnarValue> {
+ Ok(args.args[0].clone())
+ }
+ }
+
+ let ctx = SessionContext::new();
+ let async_udf = AsyncScalarUDF::new(Arc::new(TestAsyncUDF::new()));
+ ctx.register_udf(async_udf.into_scalar_udf());
+
+ let physical_plan = ctx
+ .sql("select test_async_udf(1)")
+ .await?
+ .create_physical_plan()
+ .await?;
+
+ roundtrip_test_with_context(physical_plan, &ctx)?;
+
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/windows.rs b/datafusion/proto/tests/cases/plans/windows.rs
new file mode 100644
index 0000000000..cfd2532a88
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/windows.rs
@@ -0,0 +1,308 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The window execs and their window functions.
+
+use super::roundtrip_test;
+use datafusion::arrow::compute::kernels::sort::SortOptions;
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::functions_aggregate::count::count_udaf;
+use datafusion::functions_aggregate::sum::sum_udaf;
+use datafusion::functions_window::nth_value::nth_value_udwf;
+use datafusion::functions_window::row_number::row_number_udwf;
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
+use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
+use datafusion::physical_plan::InputOrderMode;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{PhysicalSortExpr, cast, col, lit};
+use datafusion::physical_plan::windows::{
+ BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,
+ create_udwf_window_expr,
+};
+use datafusion::scalar::ScalarValue;
+use datafusion_common::Result;
+use datafusion_expr::{WindowFrame, WindowFrameBound};
+use datafusion_functions_aggregate::average::avg_udaf;
+use std::sync::Arc;
+use std::vec;
+
+#[test]
+fn roundtrip_udwf() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let udwf_expr = Arc::new(StandardWindowExpr::new(
+ create_udwf_window_expr(
+ &row_number_udwf(),
+ &[],
+ &schema,
+ "row_number() PARTITION BY [a] ORDER BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
+ false,
+ )?,
+ &[
+ col("a", &schema)?
+ ],
+ &[
+ PhysicalSortExpr::new(col("b", &schema)?, SortOptions::new(true, true))
+ ],
+ Arc::new(WindowFrame::new(None)),
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(
+ vec![udwf_expr],
+ input,
+ InputOrderMode::Sorted,
+ true,
+ )?))
+}
+
+#[test]
+fn roundtrip_window() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ let window_frame = WindowFrame::new_bounds(
+ datafusion_expr::WindowFrameUnits::Range,
+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),
+ WindowFrameBound::CurrentRow,
+ );
+
+ let nth_value_window =
+ create_udwf_window_expr(
+ &nth_value_udwf(),
+ &[col("a", &schema)?,
+ lit(2)], schema.as_ref(),
+ "NTH_VALUE(a, 2) PARTITION BY [b] ORDER BY [a ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
+ false,
+ )?;
+ let udwf_expr = Arc::new(StandardWindowExpr::new(
+ nth_value_window,
+ &[col("b", &schema)?],
+ &[PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ }],
+ Arc::new(window_frame),
+ ));
+
+ let plain_aggr_window_expr = Arc::new(PlainAggregateWindowExpr::new(
+ AggregateExprBuilder::new(
+ avg_udaf(),
+ vec![cast(col("b", &schema)?, &schema, DataType::Float64)?],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("avg(b)")
+ .build()
+ .map(Arc::new)?,
+ &[],
+ &[],
+ Arc::new(WindowFrame::new(None)),
+ None,
+ ));
+
+ let window_frame = WindowFrame::new_bounds(
+ datafusion_expr::WindowFrameUnits::Range,
+ WindowFrameBound::CurrentRow,
+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),
+ );
+
+ let args = vec![cast(col("a", &schema)?, &schema, DataType::Float64)?];
+ let sum_expr = AggregateExprBuilder::new(sum_udaf(), args)
+ .schema(Arc::clone(&schema))
+ .alias("SUM(a) RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING")
+ .build()
+ .map(Arc::new)?;
+
+ let sliding_aggr_window_expr = Arc::new(SlidingAggregateWindowExpr::new(
+ sum_expr,
+ &[],
+ &[],
+ Arc::new(window_frame),
+ None,
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ roundtrip_test(Arc::new(WindowAggExec::try_new(
+ vec![plain_aggr_window_expr, sliding_aggr_window_expr, udwf_expr],
+ input,
+ false,
+ )?))
+}
+
+#[test]
+fn roundtrip_window_distinct() -> Result<()> {
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ // Create a distinct count window expression with unbounded frame (becomes PlainAggregateWindowExpr)
+ let distinct_count_expr = Arc::new(PlainAggregateWindowExpr::new(
+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("count(DISTINCT a)")
+ .distinct() // Enable distinct
+ .build()
+ .map(Arc::new)?,
+ &[col("b", &schema)?], // partition by b
+ &[], // no order by
+ Arc::new(WindowFrame::new(None)), // unbounded frame
+ None,
+ ));
+
+ // Create a distinct sum window expression with bounded frame (becomes SlidingAggregateWindowExpr)
+ let bounded_frame = WindowFrame::new_bounds(
+ datafusion_expr::WindowFrameUnits::Rows,
+ WindowFrameBound::Preceding(ScalarValue::UInt64(Some(1))),
+ WindowFrameBound::CurrentRow,
+ );
+
+ let distinct_sum_expr = Arc::new(SlidingAggregateWindowExpr::new(
+ AggregateExprBuilder::new(
+ sum_udaf(),
+ vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("sum(DISTINCT a)")
+ .distinct() // Enable distinct
+ .with_ignore_nulls(true) // Enable ignore nulls
+ .build()
+ .map(Arc::new)?,
+ &[], // no partition by
+ &[], // no order by
+ Arc::new(bounded_frame), // bounded frame
+ None,
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ roundtrip_test(Arc::new(WindowAggExec::try_new(
+ vec![distinct_count_expr, distinct_sum_expr],
+ input,
+ false,
+ )?))
+}
+
+#[test]
+fn test_distinct_window_serialization_end_to_end() -> Result<()> {
+ // Create a more comprehensive test that verifies distinct window functions
+ // work properly through the entire serialization/deserialization pipeline
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ // Test 1: DISTINCT COUNT with IGNORE NULLS
+ let distinct_count_ignore_nulls = Arc::new(PlainAggregateWindowExpr::new(
+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("count_distinct_ignore_nulls")
+ .distinct()
+ .with_ignore_nulls(true)
+ .build()
+ .map(Arc::new)?,
+ &[col("b", &schema)?],
+ &[],
+ Arc::new(WindowFrame::new(None)),
+ None,
+ ));
+
+ // Test 2: DISTINCT SUM (without ignore nulls)
+ let bounded_frame = WindowFrame::new_bounds(
+ datafusion_expr::WindowFrameUnits::Rows,
+ WindowFrameBound::Preceding(ScalarValue::UInt64(Some(2))),
+ WindowFrameBound::CurrentRow,
+ );
+
+ let distinct_sum = Arc::new(SlidingAggregateWindowExpr::new(
+ AggregateExprBuilder::new(
+ sum_udaf(),
+ vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("sum_distinct")
+ .distinct()
+ .build()
+ .map(Arc::new)?,
+ &[],
+ &[],
+ Arc::new(bounded_frame),
+ None,
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ let window_exec = Arc::new(WindowAggExec::try_new(
+ vec![distinct_count_ignore_nulls, distinct_sum],
+ input,
+ false,
+ )?);
+
+ // Perform the roundtrip test
+ roundtrip_test(window_exec)
+}
+
+/// Tests that `lead` window function with offset and default value args
+/// survives a protobuf round-trip. This is a regression test for a bug
+/// where `expressions()` (used during serialization) returns only the
+/// column expression for lead/lag, silently dropping the offset and
+/// default value literal args.
+#[test]
+fn roundtrip_lead_with_default_value() -> Result<()> {
+ use datafusion::functions_window::lead_lag::lead_udwf;
+
+ let field_a = Field::new("a", DataType::Int64, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+
+ // lead(a, 2, 42) — column a, offset 2, default value 42
+ let lead_window = create_udwf_window_expr(
+ &lead_udwf(),
+ &[col("a", &schema)?, lit(2i64), lit(42i64)],
+ schema.as_ref(),
+ "test lead with default".to_string(),
+ false,
+ )?;
+
+ let udwf_expr = Arc::new(StandardWindowExpr::new(
+ lead_window,
+ &[col("b", &schema)?],
+ &[PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ }],
+ Arc::new(WindowFrame::new(None)),
+ ));
+
+ let input = Arc::new(EmptyExec::new(schema.clone()));
+
+ roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(
+ vec![udwf_expr],
+ input,
+ InputOrderMode::Sorted,
+ true,
+ )?))
+}
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
deleted file mode 100644
index 4e33934c6b..0000000000
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ /dev/null
@@ -1,5785 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::collections::HashMap;
-use std::fmt::{Display, Formatter};
-use std::sync::{Arc, RwLock};
-use std::vec;
-
-use arrow::array::RecordBatch;
-use arrow::csv::WriterBuilder;
-use arrow::datatypes::{Fields, TimeUnit};
-use async_trait::async_trait;
-use datafusion::arrow::array::ArrayRef;
-use datafusion::arrow::compute::kernels::sort::SortOptions;
-use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, SchemaRef};
-use datafusion::datasource::empty::EmptyTable;
-use datafusion::datasource::file_format::csv::CsvSink;
-use datafusion::datasource::file_format::json::{JsonFormat, JsonSink};
-use datafusion::datasource::file_format::parquet::ParquetSink;
-use datafusion::datasource::listing::{
- ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, PartitionedFile,
-};
-use datafusion::datasource::object_store::ObjectStoreUrl;
-use datafusion::datasource::physical_plan::{
- ArrowSource, CsvSource, FileGroup, FileOutputMode, FileScanConfig,
- FileScanConfigBuilder, FileSinkConfig, JsonSource, ParquetSource,
- wrap_partition_type_in_dict, wrap_partition_value_in_dict,
-};
-use datafusion::datasource::sink::{DataSink, DataSinkExec};
-use datafusion::datasource::source::DataSourceExec;
-use datafusion::execution::TaskContext;
-use datafusion::functions_aggregate::count::count_udaf;
-use datafusion::functions_aggregate::first_last::first_value_udaf;
-use datafusion::functions_aggregate::sum::sum_udaf;
-use datafusion::functions_window::nth_value::nth_value_udwf;
-use datafusion::functions_window::row_number::row_number_udwf;
-use datafusion::logical_expr::{JoinType, Operator, Volatility, create_udf};
-use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
-use datafusion::physical_expr::expressions::Literal;
-use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
-use datafusion::physical_expr::{
- HigherOrderFunctionExpr, LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,
-};
-use datafusion::physical_optimizer::PhysicalOptimizerRule;
-use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;
-use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;
-use datafusion::physical_plan::aggregates::{
- AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
-};
-use datafusion::physical_plan::analyze::AnalyzeExec;
-use datafusion::physical_plan::buffer::BufferExec;
-#[expect(deprecated)]
-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
-use datafusion::physical_plan::coop::CooperativeExec;
-use datafusion::physical_plan::empty::EmptyExec;
-use datafusion::physical_plan::explain::ExplainExec;
-use datafusion::physical_plan::expressions::{
- BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,
- cast, col, in_list, like, lit,
-};
-use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
-use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
-use datafusion::physical_plan::joins::{
- HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,
- StreamJoinPartitionMode, SymmetricHashJoinExec,
-};
-use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
-use datafusion::physical_plan::metrics::MetricCategory;
-use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
-use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
-use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;
-use datafusion::physical_plan::repartition::RepartitionExec;
-use datafusion::physical_plan::scalar_subquery::{
- ScalarSubqueryExec, ScalarSubqueryLink,
-};
-use datafusion::physical_plan::sorts::sort::SortExec;
-use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use datafusion::physical_plan::union::{InterleaveExec, UnionExec};
-use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};
-use datafusion::physical_plan::windows::{
- BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,
- create_udwf_window_expr,
-};
-use datafusion::physical_plan::{
- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, InputOrderMode,
- Partitioning, PhysicalExpr, PlanProperties, RangePartitioning,
- SendableRecordBatchStream, SplitPoint, Statistics, displayable,
-};
-use datafusion::prelude::{ParquetReadOptions, SessionContext};
-use datafusion::scalar::ScalarValue;
-use datafusion_common::config::{ConfigOptions, TableParquetOptions};
-use datafusion_common::display::{PlanType, StringifiedPlan};
-use datafusion_common::file_options::csv_writer::CsvWriterOptions;
-use datafusion_common::file_options::json_writer::JsonWriterOptions;
-use datafusion_common::format::ExplainFormat;
-use datafusion_common::parsers::CompressionTypeVariant;
-use datafusion_common::stats::Precision;
-use datafusion_common::tree_node::TreeNodeRecursion;
-use datafusion_common::{
- DataFusionError, JoinSide, NullEquality, Result, UnnestOptions, exec_datafusion_err,
- internal_datafusion_err, internal_err, not_impl_err,
-};
-use datafusion_datasource::file::FileSource;
-use datafusion_datasource::{TableSchema, TableSchemaBuilder};
-use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
-use datafusion_expr::dml::InsertOp;
-use datafusion_expr::{
- Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, HigherOrderUDF,
- ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,
- WindowFrame, WindowFrameBound, WindowUDF,
- physical_planning_context::{ScalarSubqueryResults, SubqueryIndex},
-};
-use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
-use datafusion_functions_aggregate::array_agg::array_agg_udaf;
-use datafusion_functions_aggregate::average::avg_udaf;
-use datafusion_functions_aggregate::min_max::max_udaf;
-use datafusion_functions_aggregate::nth_value::nth_value_udaf;
-use datafusion_functions_aggregate::string_agg::string_agg_udaf;
-use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
-use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
-use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
-use datafusion_proto::bytes::{
- physical_plan_from_bytes_with_proto_converter,
- physical_plan_to_bytes_with_proto_converter,
-};
-use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
-use datafusion_proto::physical_plan::{
- AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,
- DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,
- PhysicalPlanNodeExt, PhysicalProtoConverterExtension,
-};
-use datafusion_proto::protobuf;
-use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};
-use prost::Message;
-
-use crate::cases::{
- CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF,
- MyHigherOrderUdfNode, MyRegexUdf, MyRegexUdfNode,
-};
-use datafusion_physical_expr::expressions::{LambdaVariable, is_not_null, lambda};
-use datafusion_physical_expr::utils::reassign_expr_columns;
-
-/// Perform a serde roundtrip and assert that the string representation of the before and after plans
-/// are identical. Note that this often isn't sufficient to guarantee that no information is
-/// lost during serde because the string representation of a plan often only shows a subset of state.
-fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
- Ok(())
-}
-
-/// Perform a serde roundtrip and assert that the string representation of the before and after plans
-/// are identical. Note that this often isn't sufficient to guarantee that no information is
-/// lost during serde because the string representation of a plan often only shows a subset of state.
-///
-/// This version of the roundtrip_test method returns the final plan after serde so that it can be inspected
-/// farther in tests.
-fn roundtrip_test_and_return(
- exec_plan: Arc<dyn ExecutionPlan>,
- ctx: &SessionContext,
- codec: &dyn PhysicalExtensionCodec,
- proto_converter: &dyn PhysicalProtoConverterExtension,
-) -> Result<Arc<dyn ExecutionPlan>> {
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan),
- codec,
- proto_converter,
- )?;
- let result_exec_plan = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- codec,
- proto_converter,
- )?;
-
- pretty_assertions::assert_eq!(
- format!("{exec_plan:?}"),
- format!("{result_exec_plan:?}")
- );
- Ok(result_exec_plan)
-}
-
-/// Perform a serde roundtrip and assert that the string representation of the before and after plans
-/// are identical. Note that this often isn't sufficient to guarantee that no information is
-/// lost during serde because the string representation of a plan often only shows a subset of state.
-///
-/// This version of the roundtrip_test function accepts a SessionContext, which is required when
-/// performing serde on some plans.
-fn roundtrip_test_with_context(
- exec_plan: Arc<dyn ExecutionPlan>,
- ctx: &SessionContext,
-) -> Result<()> {
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;
- Ok(())
-}
-
-/// Perform a serde roundtrip for the specified sql query, and assert that
-/// query results are identical.
-async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;
-
- roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;
- Ok(())
-}
-
-/// returns a SessionContext with `alltypes_plain` registered
-async fn all_types_context() -> Result<SessionContext> {
- let ctx = SessionContext::new();
-
- let testdata = datafusion::test_util::parquet_test_data();
- ctx.register_parquet(
- "alltypes_plain",
- &format!("{testdata}/alltypes_plain.parquet"),
- ParquetReadOptions::default(),
- )
- .await?;
-
- Ok(ctx)
-}
-
-#[test]
-fn roundtrip_empty() -> Result<()> {
- roundtrip_test(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))
-}
-
-#[test]
-fn roundtrip_empty_with_partitions() -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let plan = Arc::new(EmptyExec::new(Arc::new(Schema::empty())).with_partitions(4));
- let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
- assert_eq!(plan.output_partitioning().partition_count(), 4);
- Ok(())
-}
-
-#[test]
-fn roundtrip_placeholder_row_with_partitions() -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let plan =
- Arc::new(PlaceholderRowExec::new(Arc::new(Schema::empty())).with_partitions(4));
- let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
- assert_eq!(plan.output_partitioning().partition_count(), 4);
- Ok(())
-}
-
-/// Plans encoded before `partitions` was added carry no value for it, which
-/// decodes as zero and must be treated as the previous default of one.
-#[test]
-fn decode_empty_and_placeholder_row_without_partitions() -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let schema: protobuf::Schema = (&Schema::empty()).try_into()?;
-
- for physical_plan_type in [
- protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {
- schema: Some(schema.clone()),
- partitions: 0,
- }),
- protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(
- protobuf::PlaceholderRowExecNode {
- schema: Some(schema.clone()),
- partitions: 0,
- },
- ),
- ] {
- let node = PhysicalPlanNode {
- physical_plan_type: Some(physical_plan_type),
- };
- let plan = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;
- assert_eq!(plan.output_partitioning().partition_count(), 1);
- }
- Ok(())
-}
-
-#[derive(Debug)]
-struct DowncastDelegatingExec {
- inner: Arc<dyn ExecutionPlan>,
-}
-
-impl DowncastDelegatingExec {
- fn new(inner: Arc<dyn ExecutionPlan>) -> Self {
- Self { inner }
- }
-}
-
-impl DisplayAs for DowncastDelegatingExec {
- fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
- self.inner.fmt_as(t, f)
- }
-}
-
-impl ExecutionPlan for DowncastDelegatingExec {
- fn name(&self) -> &str {
- self.inner.name()
- }
-
- fn properties(&self) -> &Arc<PlanProperties> {
- self.inner.properties()
- }
-
- fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
- self.inner.children()
- }
-
- fn apply_expressions(
- &self,
- f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
- ) -> Result<TreeNodeRecursion> {
- self.inner.apply_expressions(f)
- }
-
- fn with_new_children(
- self: Arc<Self>,
- children: Vec<Arc<dyn ExecutionPlan>>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- let inner = Arc::clone(&self.inner).with_new_children(children)?;
- Ok(Arc::new(Self::new(inner)))
- }
-
- fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {
- Some(self.inner.as_ref())
- }
-
- fn execute(
- &self,
- partition: usize,
- context: Arc<TaskContext>,
- ) -> Result<SendableRecordBatchStream> {
- self.inner.execute(partition, context)
- }
-}
-
-#[test]
-fn serialize_uses_downcast_delegate() -> Result<()> {
- let inner: Arc<dyn ExecutionPlan> =
- Arc::new(EmptyExec::new(Arc::new(Schema::empty())));
- let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));
- let codec = DefaultPhysicalExtensionCodec {};
-
- let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;
-
- assert!(matches!(
- proto.physical_plan_type,
- Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))
- ));
-
- Ok(())
-}
-
-/// A wrapper delegating to a plan that serializes itself via the
-/// `try_to_proto` hook must serialize as its delegate: the wrapper's default
-/// hook returns `Ok(None)` and the delegate has no downcast-chain fallback.
-#[test]
-fn serialize_uses_downcast_delegate_for_self_serializing_plan() -> Result<()> {
- let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
- let inner: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: col("a", &schema)?,
- alias: "a".to_string(),
- }],
- input,
- )?);
- let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));
- let codec = DefaultPhysicalExtensionCodec {};
-
- let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;
-
- assert!(matches!(
- proto.physical_plan_type,
- Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(
- _
- ))
- ));
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_date_time_interval() -> Result<()> {
- let schema = Schema::new(vec![
- Field::new("some_date", DataType::Date32, false),
- Field::new(
- "some_interval",
- DataType::Interval(IntervalUnit::DayTime),
- false,
- ),
- ]);
- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
- let date_expr = col("some_date", &schema)?;
- let literal_expr = col("some_interval", &schema)?;
- let date_time_interval_expr =
- binary(date_expr, Operator::Plus, literal_expr, &schema)?;
- let plan = Arc::new(ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: date_time_interval_expr,
- alias: "result".to_string(),
- }],
- input,
- )?);
- roundtrip_test(plan)
-}
-
-#[test]
-fn roundtrip_local_limit() -> Result<()> {
- roundtrip_test(Arc::new(LocalLimitExec::new(
- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
- 25,
- )))
-}
-
-#[test]
-fn roundtrip_global_limit() -> Result<()> {
- roundtrip_test(Arc::new(GlobalLimitExec::new(
- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
- 0,
- Some(25),
- )))
-}
-
-#[test]
-fn roundtrip_global_skip_no_limit() -> Result<()> {
- roundtrip_test(Arc::new(GlobalLimitExec::new(
- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),
- 10,
- None, // no limit
- )))
-}
-
-/// Sort key at index 1, so a decoder that misbinds column name vs index
-/// cannot pass.
-fn limit_test_schema() -> Arc<Schema> {
- Arc::new(Schema::new(vec![
- Field::new("a", DataType::Int64, false),
- Field::new("b", DataType::Int64, false),
- ]))
-}
-
-/// Non-default sort options, so a decode that falls back to defaults cannot
-/// pass.
-fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {
- Ok(LexOrdering::new(vec![PhysicalSortExpr {
- expr: col("b", schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- }]))
-}
-
-#[test]
-fn roundtrip_limit_with_required_ordering() -> Result<()> {
- let schema = limit_test_schema();
- let required_ordering = limit_required_ordering(&schema)?;
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- let mut global =
- GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));
- global.set_required_ordering(required_ordering.clone());
- let decoded =
- roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;
- let decoded = decoded
- .downcast_ref::<GlobalLimitExec>()
- .expect("expected GlobalLimitExec");
- assert_eq!(decoded.required_ordering(), &required_ordering);
-
- let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);
- local.set_required_ordering(required_ordering.clone());
- let decoded =
- roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;
- let decoded = decoded
- .downcast_ref::<LocalLimitExec>()
- .expect("expected LocalLimitExec");
- assert_eq!(decoded.required_ordering(), &required_ordering);
- Ok(())
-}
-
-/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`
-/// whose sort node was optimized away is order-sensitive, so it must survive
-/// serde all the way into the scan's `preserve_order` flag.
-#[test]
-fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {
- let file_schema = limit_test_schema();
- let make_scan = || {
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .build();
- DataSourceExec::from_data_source(scan_config)
- };
- let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let decoded =
- roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;
-
- // Child replacement must not erase the decoded ordering before pushdown.
- let rebuilt = decoded.with_new_children(vec![make_scan()])?;
-
- let optimized =
- LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;
- let scan = optimized
- .downcast_ref::<DataSourceExec>()
- .expect("limit should be absorbed into the scan");
- Ok(scan
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .expect("expected FileScanConfig")
- .clone())
- };
-
- let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));
- limit.set_required_ordering(limit_required_ordering(&file_schema)?);
- let scan_config = scan_after_limit_pushdown(limit)?;
- assert_eq!(scan_config.limit, Some(10));
- assert!(scan_config.preserve_order);
-
- let scan_config =
- scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;
- assert_eq!(scan_config.limit, Some(10));
- assert!(!scan_config.preserve_order);
- Ok(())
-}
-
-#[test]
-fn roundtrip_hash_join() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_a]);
- let on = vec![(
- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
- )];
-
- let schema_left = Arc::new(schema_left);
- let schema_right = Arc::new(schema_right);
- for join_type in &[
- JoinType::Inner,
- JoinType::Left,
- JoinType::Right,
- JoinType::Full,
- JoinType::LeftAnti,
- JoinType::RightAnti,
- JoinType::LeftSemi,
- JoinType::RightSemi,
- ] {
- for partition_mode in &[PartitionMode::Partitioned, PartitionMode::CollectLeft] {
- roundtrip_test(Arc::new(HashJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- on.clone(),
- None,
- join_type,
- None,
- *partition_mode,
- NullEquality::NullEqualsNothing,
- false,
- )?))?;
- }
- }
- Ok(())
-}
-
-#[test]
-fn roundtrip_nested_loop_join() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_a]);
-
- let schema_left = Arc::new(schema_left);
- let schema_right = Arc::new(schema_right);
- for join_type in &[
- JoinType::Inner,
- JoinType::Left,
- JoinType::Right,
- JoinType::Full,
- JoinType::LeftAnti,
- JoinType::RightAnti,
- JoinType::LeftSemi,
- JoinType::RightSemi,
- ] {
- roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- None,
- join_type,
- Some(vec![0]),
- )?))?;
- }
- Ok(())
-}
-
-/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",
-/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire
-/// representation. `try_embed_projection` (DataFusion 53+) produces
-/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,
-/// which previously round-tripped to `None` and caused downstream consumers (e.g.
-/// distributed Flight executors) to receive a different number of output
-/// columns than the planner declared. Verify all three states preserve.
-#[test]
-fn roundtrip_hash_join_projection_states() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
- let schema_right = Arc::new(Schema::new(vec![field_a]));
- let on = vec![(
- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
- )];
-
- for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
- roundtrip_test(Arc::new(HashJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- on.clone(),
- None,
- &JoinType::Inner,
- projection,
- PartitionMode::Partitioned,
- NullEquality::NullEqualsNothing,
- false,
- )?))?;
- }
- Ok(())
-}
-
-/// Regression: `HashJoinExecNode` had no `fetch` field, so the row limit that
-/// the `limit_pushdown` physical optimizer rule pushes into the join via
-/// `ExecutionPlan::with_fetch` was silently dropped by serde. Because that rule
-/// also removes the enclosing `GlobalLimitExec` once the join absorbs the limit,
-/// a round-tripped plan had no limit left at all and a distributed executor
-/// returned more rows than the query asked for.
-///
-/// Note this cannot be covered by `roundtrip_test`: that helper compares
-/// `format!("{plan:?}")`, and `HashJoinExec`'s `Debug` output does not include
-/// `fetch`, so the before/after strings match even when the value is lost. The
-/// assertions below therefore inspect `fetch()` directly.
-#[test]
-fn roundtrip_hash_join_fetch() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
- let schema_right = Arc::new(Schema::new(vec![field_a]));
- let on = vec![(
- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,
- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,
- )];
-
- // `usize::MAX` and `u32::MAX as usize` pin the decode-side `u64 -> usize`
- // conversion: it is a checked `usize::try_from`, and a large fetch must
- // survive the round trip exactly rather than being truncated or clamped.
- // Both are representable on every target (on a 32-bit target `usize::MAX`
- // is simply `u32::MAX`), so this stays portable. The truncating case
- // itself -- a `u64` fetch above `usize::MAX` -- is only reachable on a
- // 32-bit target and so is not exercised by this test on a 64-bit host.
- for fetch in [None, Some(7), Some(u32::MAX as usize), Some(usize::MAX)] {
- let join = HashJoinExec::try_new(
- Arc::new(EmptyExec::new(Arc::clone(&schema_left))),
- Arc::new(EmptyExec::new(Arc::clone(&schema_right))),
- on.clone(),
- None,
- &JoinType::Inner,
- None,
- PartitionMode::Partitioned,
- NullEquality::NullEqualsNothing,
- false,
- )?;
-
- let plan: Arc<dyn ExecutionPlan> = match fetch {
- // This is how `limit_pushdown` installs the limit.
- Some(fetch) => join
- .with_fetch(Some(fetch))
- .expect("HashJoinExec supports fetch"),
- None => Arc::new(join),
- };
- assert_eq!(plan.fetch(), fetch);
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let deserialized =
- roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
-
- let deserialized_join = deserialized
- .downcast_ref::<HashJoinExec>()
- .expect("should be a HashJoinExec");
- assert_eq!(deserialized_join.fetch(), fetch);
- }
- Ok(())
-}
-
-/// Same regression coverage for `NestedLoopJoinExec`, which shares the
-/// `repeated uint32 projection` proto field shape with `HashJoinExec`.
-#[test]
-fn roundtrip_nested_loop_join_projection_states() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));
- let schema_right = Arc::new(Schema::new(vec![field_a]));
-
- for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {
- roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- None,
- &JoinType::Inner,
- projection,
- )?))?;
- }
- Ok(())
-}
-
-#[test]
-fn roundtrip_udwf() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let udwf_expr = Arc::new(StandardWindowExpr::new(
- create_udwf_window_expr(
- &row_number_udwf(),
- &[],
- &schema,
- "row_number() PARTITION BY [a] ORDER BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
- false,
- )?,
- &[
- col("a", &schema)?
- ],
- &[
- PhysicalSortExpr::new(col("b", &schema)?, SortOptions::new(true, true))
- ],
- Arc::new(WindowFrame::new(None)),
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(
- vec![udwf_expr],
- input,
- InputOrderMode::Sorted,
- true,
- )?))
-}
-
-#[test]
-fn roundtrip_window() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let window_frame = WindowFrame::new_bounds(
- datafusion_expr::WindowFrameUnits::Range,
- WindowFrameBound::Preceding(ScalarValue::Int64(None)),
- WindowFrameBound::CurrentRow,
- );
-
- let nth_value_window =
- create_udwf_window_expr(
- &nth_value_udwf(),
- &[col("a", &schema)?,
- lit(2)], schema.as_ref(),
- "NTH_VALUE(a, 2) PARTITION BY [b] ORDER BY [a ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
- false,
- )?;
- let udwf_expr = Arc::new(StandardWindowExpr::new(
- nth_value_window,
- &[col("b", &schema)?],
- &[PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: false,
- },
- }],
- Arc::new(window_frame),
- ));
-
- let plain_aggr_window_expr = Arc::new(PlainAggregateWindowExpr::new(
- AggregateExprBuilder::new(
- avg_udaf(),
- vec![cast(col("b", &schema)?, &schema, DataType::Float64)?],
- )
- .schema(Arc::clone(&schema))
- .alias("avg(b)")
- .build()
- .map(Arc::new)?,
- &[],
- &[],
- Arc::new(WindowFrame::new(None)),
- None,
- ));
-
- let window_frame = WindowFrame::new_bounds(
- datafusion_expr::WindowFrameUnits::Range,
- WindowFrameBound::CurrentRow,
- WindowFrameBound::Preceding(ScalarValue::Int64(None)),
- );
-
- let args = vec![cast(col("a", &schema)?, &schema, DataType::Float64)?];
- let sum_expr = AggregateExprBuilder::new(sum_udaf(), args)
- .schema(Arc::clone(&schema))
- .alias("SUM(a) RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING")
- .build()
- .map(Arc::new)?;
-
- let sliding_aggr_window_expr = Arc::new(SlidingAggregateWindowExpr::new(
- sum_expr,
- &[],
- &[],
- Arc::new(window_frame),
- None,
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- roundtrip_test(Arc::new(WindowAggExec::try_new(
- vec![plain_aggr_window_expr, sliding_aggr_window_expr, udwf_expr],
- input,
- false,
- )?))
-}
-
-#[test]
-fn roundtrip_window_distinct() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- // Create a distinct count window expression with unbounded frame (becomes PlainAggregateWindowExpr)
- let distinct_count_expr = Arc::new(PlainAggregateWindowExpr::new(
- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("count(DISTINCT a)")
- .distinct() // Enable distinct
- .build()
- .map(Arc::new)?,
- &[col("b", &schema)?], // partition by b
- &[], // no order by
- Arc::new(WindowFrame::new(None)), // unbounded frame
- None,
- ));
-
- // Create a distinct sum window expression with bounded frame (becomes SlidingAggregateWindowExpr)
- let bounded_frame = WindowFrame::new_bounds(
- datafusion_expr::WindowFrameUnits::Rows,
- WindowFrameBound::Preceding(ScalarValue::UInt64(Some(1))),
- WindowFrameBound::CurrentRow,
- );
-
- let distinct_sum_expr = Arc::new(SlidingAggregateWindowExpr::new(
- AggregateExprBuilder::new(
- sum_udaf(),
- vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],
- )
- .schema(Arc::clone(&schema))
- .alias("sum(DISTINCT a)")
- .distinct() // Enable distinct
- .with_ignore_nulls(true) // Enable ignore nulls
- .build()
- .map(Arc::new)?,
- &[], // no partition by
- &[], // no order by
- Arc::new(bounded_frame), // bounded frame
- None,
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- roundtrip_test(Arc::new(WindowAggExec::try_new(
- vec![distinct_count_expr, distinct_sum_expr],
- input,
- false,
- )?))
-}
-
-#[test]
-fn test_distinct_window_serialization_end_to_end() -> Result<()> {
- // Create a more comprehensive test that verifies distinct window functions
- // work properly through the entire serialization/deserialization pipeline
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- // Test 1: DISTINCT COUNT with IGNORE NULLS
- let distinct_count_ignore_nulls = Arc::new(PlainAggregateWindowExpr::new(
- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("count_distinct_ignore_nulls")
- .distinct()
- .with_ignore_nulls(true)
- .build()
- .map(Arc::new)?,
- &[col("b", &schema)?],
- &[],
- Arc::new(WindowFrame::new(None)),
- None,
- ));
-
- // Test 2: DISTINCT SUM (without ignore nulls)
- let bounded_frame = WindowFrame::new_bounds(
- datafusion_expr::WindowFrameUnits::Rows,
- WindowFrameBound::Preceding(ScalarValue::UInt64(Some(2))),
- WindowFrameBound::CurrentRow,
- );
-
- let distinct_sum = Arc::new(SlidingAggregateWindowExpr::new(
- AggregateExprBuilder::new(
- sum_udaf(),
- vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],
- )
- .schema(Arc::clone(&schema))
- .alias("sum_distinct")
- .distinct()
- .build()
- .map(Arc::new)?,
- &[],
- &[],
- Arc::new(bounded_frame),
- None,
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let window_exec = Arc::new(WindowAggExec::try_new(
- vec![distinct_count_ignore_nulls, distinct_sum],
- input,
- false,
- )?);
-
- // Perform the roundtrip test
- roundtrip_test(window_exec)
-}
-
-#[test]
-fn roundtrip_aggregate() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
- vec![(col("a", &schema)?, "unused".to_string())];
-
- let avg_expr = AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("AVG(b)")
- .build()?;
- let nth_expr =
- AggregateExprBuilder::new(nth_value_udaf(), vec![col("b", &schema)?, lit(1u64)])
- .schema(Arc::clone(&schema))
- .alias("NTH_VALUE(b, 1)")
- .build()?;
- let str_agg_expr =
- AggregateExprBuilder::new(string_agg_udaf(), vec![col("b", &schema)?, lit(1u64)])
- .schema(Arc::clone(&schema))
- .alias("NTH_VALUE(b, 1)")
- .build()?;
-
- let test_cases = vec![
- // AVG
- vec![Arc::new(avg_expr)],
- // NTH_VALUE
- vec![Arc::new(nth_expr)],
- // STRING_AGG
- vec![Arc::new(str_agg_expr)],
- ];
-
- for aggregates in test_cases {
- let schema = schema.clone();
- roundtrip_test(Arc::new(AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new_single(groups.clone()),
- aggregates,
- vec![None],
- Arc::new(EmptyExec::new(schema.clone())),
- schema,
- )?))?;
- }
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_aggregate_with_limit() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
- vec![(col("a", &schema)?, "unused".to_string())];
-
- let aggregates = vec![
- AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("AVG(b)")
- .build()
- .map(Arc::new)?,
- ];
-
- let agg = AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new_single(groups.clone()),
- aggregates,
- vec![None],
- Arc::new(EmptyExec::new(schema.clone())),
- schema,
- )?;
- let agg = agg.with_limit_options(Some(LimitOptions::new_with_order(12, false)));
- roundtrip_test(Arc::new(agg))
-}
-
-#[test]
-fn roundtrip_aggregate_with_approx_pencentile_cont() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
- vec![(col("a", &schema)?, "unused".to_string())];
-
- let aggregates = vec![
- AggregateExprBuilder::new(
- approx_percentile_cont_udaf(),
- vec![col("b", &schema)?, lit(0.5)],
- )
- .schema(Arc::clone(&schema))
- .alias("APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY b)")
- .build()
- .map(Arc::new)?,
- ];
-
- let agg = AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new_single(groups.clone()),
- aggregates,
- vec![None],
- Arc::new(EmptyExec::new(schema.clone())),
- schema,
- )?;
- roundtrip_test(Arc::new(agg))
-}
-
-#[test]
-fn roundtrip_aggregate_with_sort() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
- vec![(col("a", &schema)?, "unused".to_string())];
- let sort_exprs = vec![PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- }];
-
- let aggregates = vec![
- AggregateExprBuilder::new(array_agg_udaf(), vec![col("b", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("ARRAY_AGG(b)")
- .order_by(sort_exprs)
- .build()
- .map(Arc::new)?,
- ];
-
- let agg = AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new_single(groups.clone()),
- aggregates,
- vec![None],
- Arc::new(EmptyExec::new(schema.clone())),
- schema,
- )?;
- roundtrip_test(Arc::new(agg))
-}
-
-#[test]
-fn roundtrip_aggregate_udaf() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- #[derive(Debug)]
- struct Example;
- impl Accumulator for Example {
- fn state(&mut self) -> Result<Vec<ScalarValue>> {
- Ok(vec![ScalarValue::Int64(Some(0))])
- }
-
- fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
- Ok(())
- }
-
- fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
- Ok(())
- }
-
- fn evaluate(&mut self) -> Result<ScalarValue> {
- Ok(ScalarValue::Int64(Some(0)))
- }
-
- fn size(&self) -> usize {
- 0
- }
- }
-
- let return_type = DataType::Int64;
- let accumulator: AccumulatorFactoryFunction = Arc::new(|_| Ok(Box::new(Example)));
-
- let udaf = AggregateUDF::from(SimpleAggregateUDF::new_with_signature(
- "example",
- Signature::exact(vec![DataType::Int64], Volatility::Immutable),
- return_type,
- accumulator,
- vec![Field::new("value", DataType::Int64, true).into()],
- ));
-
- let ctx = SessionContext::new();
- ctx.register_udaf(udaf.clone());
-
- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =
- vec![(col("a", &schema)?, "unused".to_string())];
-
- let aggregates = vec![
- AggregateExprBuilder::new(Arc::new(udaf), vec![col("b", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("example_agg")
- .build()
- .map(Arc::new)?,
- ];
-
- roundtrip_test_with_context(
- Arc::new(AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new_single(groups.clone()),
- aggregates,
- vec![None],
- Arc::new(EmptyExec::new(schema.clone())),
- schema,
- )?),
- &ctx,
- )
-}
-
-#[test]
-fn roundtrip_filter_with_not_and_in_list() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let field_c = Field::new("c", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
- let not = Arc::new(NotExpr::new(col("a", &schema)?));
- let in_list = in_list(
- col("b", &schema)?,
- vec![
- lit(ScalarValue::Int64(Some(1))),
- lit(ScalarValue::Int64(Some(2))),
- ],
- &false,
- schema.as_ref(),
- )?;
- let and = binary(not, Operator::And, in_list, &schema)?;
- roundtrip_test(Arc::new(FilterExec::try_new(
- and,
- Arc::new(EmptyExec::new(schema.clone())),
- )?))
-}
-
-#[test]
-fn roundtrip_filter_with_fetch() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let predicate = col("a", &schema)?;
- let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))
- .with_fetch(Some(10))
- .build()?;
- assert_eq!(filter.fetch(), Some(10));
- roundtrip_test(Arc::new(filter))
-}
-
-#[test]
-fn roundtrip_filter_projection_states() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("a", DataType::Boolean, false),
- Field::new("b", DataType::Int64, false),
- ]));
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- for projection in [None, Some(vec![]), Some(vec![0])] {
- let filter = FilterExecBuilder::new(
- col("a", &schema)?,
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- )
- .apply_projection(projection.clone())?
- .with_default_selectivity(37)
- .with_batch_size(1024)
- .with_fetch(Some(5))
- .build()?;
-
- let result =
- roundtrip_test_and_return(Arc::new(filter), &ctx, &codec, &proto_converter)?;
- let result = result.downcast_ref::<FilterExec>().unwrap();
- assert_eq!(result.projection().as_deref(), projection.as_deref());
- assert_eq!(result.default_selectivity(), 37);
- assert_eq!(result.batch_size(), 1024);
- assert_eq!(result.fetch(), Some(5));
- }
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_sort() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let sort_exprs = [
- PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- },
- PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- },
- ]
- .into();
- roundtrip_test(Arc::new(SortExec::new(
- sort_exprs,
- Arc::new(EmptyExec::new(schema)),
- )))
-}
-
-#[test]
-fn roundtrip_sort_preserve_partitioning() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let sort_exprs: LexOrdering = [
- PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- },
- PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- },
- ]
- .into();
-
- roundtrip_test(Arc::new(SortExec::new(
- sort_exprs.clone(),
- Arc::new(EmptyExec::new(schema.clone())),
- )))?;
-
- roundtrip_test(Arc::new(
- SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)))
- .with_preserve_partitioning(true),
- ))
-}
-
-/// `SortExec::fetch` turns a sort into a top-k sort. Losing it during serde
-/// would silently widen the result set, so exercise the `Some(..)` state
-/// explicitly (`roundtrip_sort` only covers `None`).
-///
-/// `SortExec` currently derives `Debug`, so `roundtrip_test`'s
-/// `format!("{plan:?}")` comparison does observe `fetch`. The assertions below
-/// go through the accessor instead so that this coverage does not silently
-/// disappear if `SortExec` ever grows a hand-written `Debug` impl.
-#[test]
-fn roundtrip_sort_with_fetch() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let sort_exprs: LexOrdering = [
- PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- },
- PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- },
- ]
- .into();
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- let roundtripped = roundtrip_test_and_return(
- Arc::new(
- SortExec::new(
- sort_exprs.clone(),
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- )
- .with_fetch(Some(7)),
- ),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let roundtripped = roundtripped
- .downcast_ref::<SortExec>()
- .expect("should decode back into a SortExec");
- assert_eq!(roundtripped.fetch(), Some(7));
- assert_eq!(roundtripped.expr(), &sort_exprs);
-
- // `fetch` combined with `preserve_partitioning`, since both share the same
- // proto node.
- let roundtripped = roundtrip_test_and_return(
- Arc::new(
- SortExec::new(sort_exprs.clone(), Arc::new(EmptyExec::new(schema)))
- .with_fetch(Some(3))
- .with_preserve_partitioning(true),
- ),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let roundtripped = roundtripped
- .downcast_ref::<SortExec>()
- .expect("should decode back into a SortExec");
- assert_eq!(roundtripped.fetch(), Some(3));
- assert!(roundtripped.preserve_partitioning());
- Ok(())
-}
-
-/// Round trip a [`SortPreservingMergeExec`], which had no dedicated round trip
-/// test at all.
-///
-/// Covers everything that is actually on the wire for this plan: the input, the
-/// sort expressions and `fetch` in both its `None` and `Some(..)` states.
-///
-/// Note that `SortPreservingMergeExec::enable_round_robin_repartition` is
-/// deliberately *not* asserted on here: it has no field in
-/// `SortPreservingMergeExecNode`, so it is not serialized and decoding always
-/// restores the `true` default from `SortPreservingMergeExec::new`. Asserting
-/// round trip equality on it would give a false sense of coverage.
-///
-/// `SortPreservingMergeExec` derives `Debug`, so `roundtrip_test`'s
-/// `format!("{plan:?}")` comparison does observe `expr` and `fetch`. The
-/// assertions below use the accessors so the coverage survives a future
-/// hand-written `Debug` impl.
-#[test]
-fn roundtrip_sort_preserving_merge() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let sort_exprs: LexOrdering = [
- PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- },
- PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- },
- ]
- .into();
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- // No fetch: `fetch` is encoded as -1 and must decode back to `None`.
- let roundtripped = roundtrip_test_and_return(
- Arc::new(SortPreservingMergeExec::new(
- sort_exprs.clone(),
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- )),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let roundtripped = roundtripped
- .downcast_ref::<SortPreservingMergeExec>()
- .expect("should decode back into a SortPreservingMergeExec");
- assert_eq!(roundtripped.fetch(), None);
- assert_eq!(roundtripped.expr(), &sort_exprs);
- assert_eq!(roundtripped.input().schema(), schema);
-
- // With a fetch: dropping it would turn a bounded merge into an unbounded
- // one and change the query result.
- let roundtripped = roundtrip_test_and_return(
- Arc::new(
- SortPreservingMergeExec::new(
- sort_exprs.clone(),
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- )
- .with_fetch(Some(11)),
- ),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let roundtripped = roundtripped
- .downcast_ref::<SortPreservingMergeExec>()
- .expect("should decode back into a SortPreservingMergeExec");
- assert_eq!(roundtripped.fetch(), Some(11));
- assert_eq!(roundtripped.expr(), &sort_exprs);
- Ok(())
-}
-
-#[test]
-fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- #[expect(deprecated)]
- roundtrip_test(Arc::new(CoalesceBatchesExec::new(
- Arc::new(EmptyExec::new(schema.clone())),
- 8096,
- )))?;
-
- #[expect(deprecated)]
- roundtrip_test(Arc::new(
- CoalesceBatchesExec::new(Arc::new(EmptyExec::new(schema)), 8096)
- .with_fetch(Some(10)),
- ))
-}
-
-#[test]
-fn roundtrip_coalesce_partitions_with_fetch() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- roundtrip_test(Arc::new(CoalescePartitionsExec::new(Arc::new(
- EmptyExec::new(schema.clone()),
- ))))?;
-
- roundtrip_test(Arc::new(
- CoalescePartitionsExec::new(Arc::new(EmptyExec::new(schema)))
- .with_fetch(Some(10)),
- ))
-}
-
-#[test]
-fn roundtrip_cooperative() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));
- roundtrip_test(Arc::new(CooperativeExec::new(Arc::new(EmptyExec::new(
- schema,
- )))))
-}
-
-#[test]
-fn roundtrip_buffer() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let result = roundtrip_test_and_return(
- Arc::new(BufferExec::new(Arc::new(EmptyExec::new(schema)), 4096)),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let result = result.downcast_ref::<BufferExec>().unwrap();
- assert_eq!(result.capacity(), 4096);
- Ok(())
-}
-
-#[test]
-fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
-
- let predicate = Arc::new(BinaryExpr::new(
- Arc::new(Column::new("col", 1)),
- Operator::Eq,
- lit("1"),
- ));
-
- let mut options = TableParquetOptions::new();
- options.global.pushdown_filters = true;
-
- let file_source = Arc::new(
- ParquetSource::new(Arc::clone(&file_schema))
- .with_table_parquet_options(options)
- .with_predicate(predicate),
- );
-
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .with_statistics(Statistics {
- num_rows: Precision::Inexact(100),
- total_byte_size: Precision::Inexact(1024),
- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(
- vec![Field::new("col", DataType::Utf8, false)],
- ))),
- })
- .build();
-
- roundtrip_test(DataSourceExec::from_data_source(scan_config))
-}
-
-#[test]
-fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .with_statistics(Statistics {
- num_rows: Precision::Inexact(100),
- total_byte_size: Precision::Inexact(1024),
- column_statistics: Statistics::unknown_column(&file_schema),
- })
- .build();
- let exec_plan = DataSourceExec::from_data_source(scan_config);
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let roundtripped =
- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
-
- let data_source = roundtripped
- .downcast_ref::<DataSourceExec>()
- .ok_or_else(|| {
- internal_datafusion_err!("Expected DataSourceExec after roundtrip")
- })?;
- let file_scan = data_source
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .ok_or_else(|| {
- internal_datafusion_err!("Expected FileScanConfig after roundtrip")
- })?;
- let parquet_source = file_scan
- .file_source()
- .downcast_ref::<ParquetSource>()
- .ok_or_else(|| {
- internal_datafusion_err!("Expected ParquetSource after roundtrip")
- })?;
-
- assert!(
- parquet_source.parquet_file_reader_factory().is_some(),
- "Parquet reader factory should be attached after decoding from protobuf"
- );
- Ok(())
-}
-
-#[test]
-fn roundtrip_arrow_scan() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
-
- let table_schema = TableSchema::from(&file_schema);
- let file_source = Arc::new(ArrowSource::new_file_source(table_schema));
-
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.arrow".to_string(),
- 1024,
- )])])
- .with_statistics(Statistics {
- num_rows: Precision::Inexact(100),
- total_byte_size: Precision::Inexact(1024),
- column_statistics: Statistics::unknown_column(&file_schema),
- })
- .build();
-
- roundtrip_test(DataSourceExec::from_data_source(scan_config))
-}
-
-#[test]
-fn roundtrip_json_scan() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
- let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.json".to_string(),
- 1024,
- )])])
- .build();
- roundtrip_test(DataSourceExec::from_data_source(scan_config))
-}
-
-#[cfg(feature = "avro")]
-#[test]
-fn roundtrip_avro_scan() -> Result<()> {
- use datafusion_datasource_avro::source::AvroSource;
-
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
- let file_source = Arc::new(AvroSource::new(TableSchema::from(&file_schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.avro".to_string(),
- 1024,
- )])])
- .build();
- roundtrip_test(DataSourceExec::from_data_source(scan_config))
-}
-
-#[test]
-fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
- use datafusion::common::config::CsvOptions;
-
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
- let table_schema = TableSchema::from(&file_schema);
- let file_source =
- Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {
- has_header: Some(false),
- delimiter: b'|',
- quote: b'\'',
- escape: Some(b'\\'),
- comment: Some(b'#'),
- newlines_in_values: Some(true),
- truncated_rows: Some(true),
- ..Default::default()
- }));
-
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.csv".to_string(),
- 1024,
- )])])
- .build();
-
- let ctx = SessionContext::new();
- let roundtripped = roundtrip_test_and_return(
- DataSourceExec::from_data_source(scan_config),
- &ctx,
- &DefaultPhysicalExtensionCodec {},
- &DefaultPhysicalProtoConverter {},
- )?;
- let data_source = roundtripped
- .downcast_ref::<DataSourceExec>()
- .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;
- let file_scan = data_source
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;
- let csv_source = file_scan
- .file_source()
- .downcast_ref::<CsvSource>()
- .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;
-
- assert!(!csv_source.has_header());
- assert_eq!(csv_source.delimiter(), b'|');
- assert_eq!(csv_source.quote(), b'\'');
- assert_eq!(csv_source.escape(), Some(b'\\'));
- assert_eq!(csv_source.comment(), Some(b'#'));
- assert!(csv_source.newlines_in_values());
- assert!(csv_source.truncate_rows());
- Ok(())
-}
-#[tokio::test]
-async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {
- let mut file_group =
- PartitionedFile::new("/path/to/part=0/file.parquet".to_string(), 1024);
- file_group.partition_values =
- vec![wrap_partition_value_in_dict(ScalarValue::Int64(Some(0)))];
- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
-
- let table_schema = TableSchemaBuilder::from(&schema)
- .with_table_partition_cols(vec![Arc::new(Field::new(
- "part".to_string(),
- wrap_partition_type_in_dict(DataType::Int16),
- false,
- ))])
- .build();
-
- let file_source = Arc::new(ParquetSource::new(table_schema.clone()));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_projection_indices(Some(vec![0, 1]))?
- .with_file_group(FileGroup::new(vec![file_group]))
- .build();
-
- roundtrip_test(DataSourceExec::from_data_source(scan_config))
-}
-
-#[test]
-fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
-
- let custom_predicate_expr = Arc::new(CustomPredicateExpr {
- inner: Arc::new(Column::new("col", 1)),
- });
-
- let file_source = Arc::new(
- ParquetSource::new(Arc::clone(&file_schema))
- .with_predicate(custom_predicate_expr),
- );
-
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .with_statistics(Statistics {
- num_rows: Precision::Inexact(100),
- total_byte_size: Precision::Inexact(1024),
- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(
- vec![Field::new("col", DataType::Utf8, false)],
- ))),
- })
- .build();
-
- #[derive(Debug, Clone, Eq)]
- struct CustomPredicateExpr {
- inner: Arc<dyn PhysicalExpr>,
- }
-
- // Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808
- impl PartialEq for CustomPredicateExpr {
- fn eq(&self, other: &Self) -> bool {
- self.inner.eq(&other.inner)
- }
- }
-
- impl std::hash::Hash for CustomPredicateExpr {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.inner.hash(state);
- }
- }
-
- impl Display for CustomPredicateExpr {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(f, "CustomPredicateExpr")
- }
- }
-
- impl PhysicalExpr for CustomPredicateExpr {
- fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
- unreachable!()
- }
-
- fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
- unreachable!()
- }
-
- fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {
- unreachable!()
- }
-
- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
- vec![&self.inner]
- }
-
- fn with_new_children(
- self: Arc<Self>,
- _children: Vec<Arc<dyn PhysicalExpr>>,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- Ok(self)
- }
-
- fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- Display::fmt(self, f)
- }
- }
-
- #[derive(Debug)]
- struct CustomPhysicalExtensionCodec;
- impl PhysicalExtensionCodec for CustomPhysicalExtensionCodec {
- fn try_decode(
- &self,
- _buf: &[u8],
- _inputs: &[Arc<dyn ExecutionPlan>],
- _ctx: &TaskContext,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- unreachable!()
- }
-
- fn try_encode(
- &self,
- _node: Arc<dyn ExecutionPlan>,
- _buf: &mut Vec<u8>,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<()> {
- unreachable!()
- }
-
- fn try_decode_expr(
- &self,
- buf: &[u8],
- inputs: &[Arc<dyn PhysicalExpr>],
- _ctx: &PhysicalExprDecodeCtx<'_>,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- if buf == "CustomPredicateExpr".as_bytes() {
- Ok(Arc::new(CustomPredicateExpr {
- inner: inputs[0].clone(),
- }))
- } else {
- internal_err!("Not supported")
- }
- }
-
- fn try_encode_expr(
- &self,
- node: &Arc<dyn PhysicalExpr>,
- buf: &mut Vec<u8>,
- _ctx: &PhysicalExprEncodeCtx<'_>,
- ) -> Result<()> {
- if node.downcast_ref::<CustomPredicateExpr>().is_some() {
- buf.extend_from_slice("CustomPredicateExpr".as_bytes());
- Ok(())
- } else {
- internal_err!("Not supported")
- }
- }
- }
-
- let exec_plan = DataSourceExec::from_data_source(scan_config);
-
- let ctx = SessionContext::new();
- roundtrip_test_and_return(
- exec_plan,
- &ctx,
- &CustomPhysicalExtensionCodec {},
- &DefaultPhysicalProtoConverter {},
- )?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_scalar_udf() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let scalar_fn = Arc::new(|args: &[ColumnarValue]| {
- let ColumnarValue::Array(array) = &args[0] else {
- panic!("should be array")
- };
- Ok(ColumnarValue::from(Arc::new(array.clone()) as ArrayRef))
- });
-
- let udf = create_udf(
- "dummy",
- vec![DataType::Int64],
- DataType::Int64,
- Volatility::Immutable,
- scalar_fn.clone(),
- );
-
- let fun_def = Arc::new(udf.clone());
-
- let expr = ScalarFunctionExpr::new(
- "dummy",
- fun_def,
- vec![col("a", &schema)?],
- Field::new("f", DataType::Int64, true).into(),
- Arc::new(ConfigOptions::default()),
- );
-
- let project = ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: Arc::new(expr),
- alias: "a".to_string(),
- }],
- input,
- )?;
-
- let ctx = SessionContext::new();
-
- ctx.register_udf(udf);
-
- roundtrip_test_with_context(Arc::new(project), &ctx)
-}
-
-#[derive(Debug)]
-struct UDFExtensionCodec;
-
-impl PhysicalExtensionCodec for UDFExtensionCodec {
- fn try_decode(
- &self,
- _buf: &[u8],
- _inputs: &[Arc<dyn ExecutionPlan>],
- _ctx: &TaskContext,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- not_impl_err!("No extension codec provided")
- }
-
- fn try_encode(
- &self,
- _node: Arc<dyn ExecutionPlan>,
- _buf: &mut Vec<u8>,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<()> {
- not_impl_err!("No extension codec provided")
- }
-
- fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {
- if name == "regex_udf" {
- let proto = MyRegexUdfNode::decode(buf).map_err(|err| {
- internal_datafusion_err!("failed to decode regex_udf: {err}")
- })?;
-
- Ok(Arc::new(ScalarUDF::from(MyRegexUdf::new(proto.pattern))))
- } else {
- not_impl_err!("unrecognized scalar UDF implementation, cannot decode")
- }
- }
-
- fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {
- let binding = node.inner();
- if let Some(udf) = binding.downcast_ref::<MyRegexUdf>() {
- let proto = MyRegexUdfNode {
- pattern: udf.pattern.clone(),
- };
- proto
- .encode(buf)
- .map_err(|err| internal_datafusion_err!("failed to encode udf: {err}"))?;
- }
- Ok(())
- }
-
- fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result<Arc<AggregateUDF>> {
- if name == "aggregate_udf" {
- let proto = MyAggregateUdfNode::decode(buf).map_err(|err| {
- internal_datafusion_err!("failed to decode aggregate_udf: {err}")
- })?;
-
- Ok(Arc::new(AggregateUDF::from(MyAggregateUDF::new(
- proto.result,
- ))))
- } else {
- not_impl_err!("unrecognized scalar UDF implementation, cannot decode")
- }
- }
-
- fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {
- let binding = node.inner();
- if let Some(udf) = binding.downcast_ref::<MyAggregateUDF>() {
- let proto = MyAggregateUdfNode {
- result: udf.result.clone(),
- };
- proto.encode(buf).map_err(|err| {
- internal_datafusion_err!("failed to encode udf: {err:?}")
- })?;
- }
- Ok(())
- }
-
- fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result<Arc<WindowUDF>> {
- if name == "custom_udwf" {
- let proto = CustomUDWFNode::decode(buf).map_err(|err| {
- internal_datafusion_err!("failed to decode custom_udwf: {err}")
- })?;
-
- Ok(Arc::new(WindowUDF::from(CustomUDWF::new(proto.payload))))
- } else {
- not_impl_err!(
- "unrecognized user-defined window function implementation, cannot decode"
- )
- }
- }
-
- fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec<u8>) -> Result<()> {
- let binding = node.inner();
- if let Some(udwf) = binding.downcast_ref::<CustomUDWF>() {
- let proto = CustomUDWFNode {
- payload: udwf.payload.clone(),
- };
- proto.encode(buf).map_err(|err| {
- internal_datafusion_err!("failed to encode udwf: {err:?}")
- })?;
- }
- Ok(())
- }
-
- fn try_decode_higher_order_function(
- &self,
- name: &str,
- buf: &[u8],
- ) -> Result<Arc<HigherOrderUDF>> {
- if name == "higher_order_udf" {
- let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {
- internal_datafusion_err!("failed to decode higher_order_udf: {err}")
- })?;
-
- Ok(Arc::new(HigherOrderUDF::new_from_impl(
- MyHigherOrderUDF::new(proto.payload),
- )))
- } else {
- not_impl_err!("unrecognized higher order UDF implementation, cannot decode")
- }
- }
-
- fn try_encode_higher_order_function(
- &self,
- node: &HigherOrderUDF,
- buf: &mut Vec<u8>,
- ) -> Result<()> {
- if let Some(hof) = (node.inner().as_ref() as &dyn std::any::Any)
- .downcast_ref::<MyHigherOrderUDF>()
- {
- let proto = MyHigherOrderUdfNode {
- payload: hof.payload.clone(),
- };
- proto.encode(buf).map_err(|err| {
- internal_datafusion_err!("failed to encode hof: {err:?}")
- })?;
- }
- Ok(())
- }
-}
-
-#[test]
-fn roundtrip_scalar_udf_extension_codec() -> Result<()> {
- let field_text = Field::new("text", DataType::Utf8, true);
- let field_published = Field::new("published", DataType::Boolean, false);
- let field_author = Field::new("author", DataType::Utf8, false);
- let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let udf_expr = Arc::new(ScalarFunctionExpr::new(
- "regex_udf",
- Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),
- vec![col("text", &schema)?],
- Field::new("f", DataType::Int64, true).into(),
- Arc::new(ConfigOptions::default()),
- ));
-
- let filter = Arc::new(FilterExec::try_new(
- Arc::new(BinaryExpr::new(
- col("published", &schema)?,
- Operator::And,
- Arc::new(BinaryExpr::new(udf_expr.clone(), Operator::Gt, lit(0))),
- )),
- input,
- )?);
- let aggr_expr =
- AggregateExprBuilder::new(max_udaf(), vec![udf_expr as Arc<dyn PhysicalExpr>])
- .schema(schema.clone())
- .alias("max")
- .build()
- .map(Arc::new)?;
-
- let window = Arc::new(WindowAggExec::try_new(
- vec![Arc::new(PlainAggregateWindowExpr::new(
- aggr_expr.clone(),
- &[col("author", &schema)?],
- &[],
- Arc::new(WindowFrame::new(None)),
- None,
- ))],
- filter,
- true,
- )?);
-
- let aggregate = Arc::new(AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new(vec![], vec![], vec![], false),
- vec![aggr_expr],
- vec![None],
- window,
- schema,
- )?);
-
- let ctx = SessionContext::new();
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_higher_order_udf() -> Result<()> {
- let element_field = Arc::new(Field::new("v", DataType::Int32, true));
- let list_field = Field::new(
- "list_col",
- DataType::List(Arc::clone(&element_field)),
- false,
- );
- let schema = Arc::new(Schema::new(vec![list_field]));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(
- "payload".to_string(),
- )));
-
- let expr = HigherOrderFunctionExpr::try_new_with_schema(
- Arc::clone(&hof),
- vec![
- col("list_col", &schema)?,
- lambda(
- ["v"],
- is_not_null(Arc::new(LambdaVariable::new(1, element_field)))?,
- )?,
- ],
- &schema,
- Arc::new(ConfigOptions::default()),
- )?;
-
- let project = ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: Arc::new(expr),
- alias: "a".to_string(),
- }],
- input,
- )?;
-
- let ctx = SessionContext::new();
- ctx.register_higher_order_function(hof);
-
- roundtrip_test_with_context(Arc::new(project), &ctx)
-}
-
-#[test]
-fn roundtrip_higher_order_udf_extension_codec() -> Result<()> {
- let element_field = Arc::new(Field::new("v", DataType::Int32, true));
- let list_field = Field::new(
- "list_col",
- DataType::List(Arc::clone(&element_field)),
- false,
- );
- let schema = Arc::new(Schema::new(vec![list_field]));
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let lambda_body = Arc::new(LambdaVariable::new(1, Arc::clone(&element_field)));
- let lambda_expr = lambda(["v"], lambda_body)?;
-
- let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(
- "payload".to_string(),
- )));
- let hof_expr = Arc::new(HigherOrderFunctionExpr::try_new_with_schema(
- hof,
- vec![col("list_col", &schema)?, lambda_expr],
- &schema,
- Arc::new(ConfigOptions::default()),
- )?);
-
- let project = ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: hof_expr,
- alias: "out".to_string(),
- }],
- input,
- )?;
-
- let ctx = SessionContext::new();
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(
- Arc::new(project),
- &ctx,
- &UDFExtensionCodec,
- &proto_converter,
- )?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_udwf_extension_codec() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- let custom_udwf = Arc::new(WindowUDF::from(CustomUDWF::new("payload".to_string())));
- let udwf = create_udwf_window_expr(
- &custom_udwf,
- &[col("a", &schema)?],
- schema.as_ref(),
- "custom_udwf(a) PARTITION BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),
- false,
- )?;
-
- let window_frame = WindowFrame::new_bounds(
- datafusion_expr::WindowFrameUnits::Range,
- WindowFrameBound::Preceding(ScalarValue::Int64(None)),
- WindowFrameBound::CurrentRow,
- );
-
- let udwf_expr = Arc::new(StandardWindowExpr::new(
- udwf,
- &[col("b", &schema)?],
- &[PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: false,
- },
- }],
- Arc::new(window_frame),
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
- let window = Arc::new(BoundedWindowAggExec::try_new(
- vec![udwf_expr],
- input,
- InputOrderMode::Sorted,
- true,
- )?);
-
- let ctx = SessionContext::new();
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {
- let field_text = Field::new("text", DataType::Utf8, true);
- let field_published = Field::new("published", DataType::Boolean, false);
- let field_author = Field::new("author", DataType::Utf8, false);
- let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- let udf_expr = Arc::new(ScalarFunctionExpr::new(
- "regex_udf",
- Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),
- vec![col("text", &schema)?],
- Field::new("f", DataType::Int64, true).into(),
- Arc::new(ConfigOptions::default()),
- ));
-
- let udaf = Arc::new(AggregateUDF::from(MyAggregateUDF::new(
- "result".to_string(),
- )));
- let aggr_args: Vec<Arc<dyn PhysicalExpr>> =
- vec![Arc::new(Literal::new(ScalarValue::from(42)))];
-
- let aggr_expr = AggregateExprBuilder::new(Arc::clone(&udaf), aggr_args.clone())
- .schema(Arc::clone(&schema))
- .alias("aggregate_udf")
- .build()
- .map(Arc::new)?;
-
- let filter = Arc::new(FilterExec::try_new(
- Arc::new(BinaryExpr::new(
- col("published", &schema)?,
- Operator::And,
- Arc::new(BinaryExpr::new(udf_expr, Operator::Gt, lit(0))),
- )),
- input,
- )?);
-
- let window = Arc::new(WindowAggExec::try_new(
- vec![Arc::new(PlainAggregateWindowExpr::new(
- aggr_expr,
- &[col("author", &schema)?],
- &[],
- Arc::new(WindowFrame::new(None)),
- None,
- ))],
- filter,
- true,
- )?);
-
- let aggr_expr = AggregateExprBuilder::new(udaf, aggr_args.clone())
- .schema(Arc::clone(&schema))
- .alias("aggregate_udf")
- .distinct()
- .ignore_nulls()
- .build()
- .map(Arc::new)?;
-
- let aggregate = Arc::new(AggregateExec::try_new(
- AggregateMode::Final,
- PhysicalGroupBy::new(vec![], vec![], vec![], false),
- vec![aggr_expr],
- vec![None],
- window,
- schema,
- )?);
-
- let ctx = SessionContext::new();
- let proto_converter = DefaultPhysicalProtoConverter {};
- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_like() -> Result<()> {
- let schema = Schema::new(vec![
- Field::new("a", DataType::Utf8, false),
- Field::new("b", DataType::Utf8, false),
- ]);
- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
- let like_expr = like(
- false,
- false,
- col("a", &schema)?,
- col("b", &schema)?,
- &schema,
- )?;
- let plan = Arc::new(ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: like_expr,
- alias: "result".to_string(),
- }],
- input,
- )?);
- roundtrip_test(plan)
-}
-
-#[test]
-fn roundtrip_analyze() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("plan_type", DataType::Utf8, false),
- Field::new("plan", DataType::Utf8, false),
- ]));
- let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));
- let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing];
- let analyze = Arc::new(
- AnalyzeExec::builder(true, true, input, Arc::clone(&schema))
- .with_metric_categories(Some(metric_categories.clone()))
- .with_format(ExplainFormat::Tree)
- .build(),
- );
-
- let ctx = SessionContext::new();
- let roundtripped = roundtrip_test_and_return(
- analyze,
- &ctx,
- &DefaultPhysicalExtensionCodec {},
- &DefaultPhysicalProtoConverter {},
- )?;
- let roundtripped = roundtripped.downcast_ref::<AnalyzeExec>().unwrap();
-
- assert_eq!(roundtripped.schema(), schema);
- assert!(roundtripped.verbose());
- assert!(roundtripped.show_statistics());
- assert_eq!(
- roundtripped.metric_categories(),
- Some(metric_categories.as_slice())
- );
- assert_eq!(roundtripped.format(), &ExplainFormat::Tree);
- assert!(
- roundtripped
- .input()
- .downcast_ref::<PlaceholderRowExec>()
- .is_some()
- );
- Ok(())
-}
-
-#[test]
-fn roundtrip_explain() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("plan_type", DataType::Utf8, false),
- Field::new("plan", DataType::Utf8, false),
- ]));
- let stringified_plans = vec![
- StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),
- StringifiedPlan::new(
- PlanType::AnalyzedLogicalPlan {
- analyzer_name: "analyzer".to_string(),
- },
- "analyzed logical",
- ),
- StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),
- StringifiedPlan::new(
- PlanType::OptimizedLogicalPlan {
- optimizer_name: "logical optimizer".to_string(),
- },
- "optimized logical",
- ),
- StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),
- StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),
- StringifiedPlan::new(
- PlanType::InitialPhysicalPlanWithStats,
- "initial physical with stats",
- ),
- StringifiedPlan::new(
- PlanType::InitialPhysicalPlanWithSchema,
- "initial physical with schema",
- ),
- StringifiedPlan::new(
- PlanType::OptimizedPhysicalPlan {
- optimizer_name: "physical optimizer".to_string(),
- },
- "optimized physical",
- ),
- StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),
- StringifiedPlan::new(
- PlanType::FinalPhysicalPlanWithStats,
- "final physical with stats",
- ),
- StringifiedPlan::new(
- PlanType::FinalPhysicalPlanWithSchema,
- "final physical with schema",
- ),
- StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),
- ];
- let explain = Arc::new(ExplainExec::new(
- Arc::clone(&schema),
- stringified_plans.clone(),
- true,
- ));
-
- let ctx = SessionContext::new();
- let roundtripped = roundtrip_test_and_return(
- explain,
- &ctx,
- &DefaultPhysicalExtensionCodec {},
- &DefaultPhysicalProtoConverter {},
- )?;
- let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();
-
- assert_eq!(roundtripped.schema(), schema);
- assert_eq!(roundtripped.stringified_plans(), stringified_plans);
- assert!(roundtripped.verbose());
- Ok(())
-}
-
-#[derive(Debug)]
-struct ProtoHookSink {
- schema: SchemaRef,
-}
-
-impl DisplayAs for ProtoHookSink {
- fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
- write!(f, "ProtoHookSink")
- }
-}
-
-#[async_trait]
-impl DataSink for ProtoHookSink {
- fn schema(&self) -> &SchemaRef {
- &self.schema
- }
-
- async fn write_all(
- &self,
- _data: SendableRecordBatchStream,
- _context: &Arc<TaskContext>,
- ) -> Result<u64> {
- unreachable!("serialization test does not execute the sink")
- }
-
- fn try_to_proto(
- &self,
- exec: &DataSinkExec,
- ctx: &ExecutionPlanEncodeCtx<'_>,
- ) -> Result<Option<PhysicalPlanNode>> {
- let input = ctx.encode_child(exec.input())?;
- let sort_order = exec.encode_sort_order(ctx)?;
- assert!(matches!(
- input.physical_plan_type,
- Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))
- ));
- assert_eq!(
- sort_order
- .as_ref()
- .map(|ordering| ordering.physical_sort_expr_nodes.len()),
- Some(1)
- );
- assert_eq!(exec.schema().fields().len(), 1);
-
- Ok(Some(PhysicalPlanNode {
- physical_plan_type: Some(
- protobuf::physical_plan_node::PhysicalPlanType::Empty(
- protobuf::EmptyExecNode {
- schema: Some(exec.schema().as_ref().try_into()?),
- partitions: 1,
- },
- ),
- ),
- }))
- }
-}
-
-#[test]
-fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {
- let input_schema = Arc::new(Schema::new(vec![Field::new(
- "value",
- DataType::Int64,
- false,
- )]));
- let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&input_schema)));
- let sink = Arc::new(ProtoHookSink {
- schema: Arc::clone(&input_schema),
- });
- let sort_order = [PhysicalSortRequirement::new(
- Arc::new(Column::new("value", 0)),
- Some(SortOptions::default()),
- )]
- .into();
- let plan = Arc::new(DataSinkExec::new(input, sink, Some(sort_order)));
-
- let node = PhysicalPlanNode::try_from_physical_plan(
- plan,
- &DefaultPhysicalExtensionCodec {},
- )?;
-
- assert!(matches!(
- node.physical_plan_type,
- Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))
- ));
- Ok(())
-}
-
-#[test]
-fn file_sink_config_roundtrip_preserves_fields() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new(
- "partition",
- DataType::Utf8,
- false,
- )]));
- let config = FileSinkConfig {
- original_url: "file:///tmp/output".to_string(),
- object_store_url: ObjectStoreUrl::local_filesystem(),
- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp/output", 1)]),
- table_paths: vec![ListingTableUrl::parse("file:///tmp/output")?],
- output_schema: schema,
- table_partition_cols: vec![("partition".to_string(), DataType::Utf8)],
- insert_op: InsertOp::Overwrite,
- keep_partition_by_columns: true,
- file_extension: "parquet".to_string(),
- file_output_mode: FileOutputMode::Directory,
- };
-
- let encoded = protobuf::FileSinkConfig::try_from(&config)?;
- assert_eq!(encoded.insert_op(), protobuf::InsertOp::Overwrite);
- assert_eq!(
- encoded.file_output_mode(),
- protobuf::FileOutputMode::Directory
- );
-
- let decoded = FileSinkConfig::try_from(&encoded)?;
- assert_eq!(decoded.object_store_url, config.object_store_url);
- assert_eq!(decoded.table_paths, config.table_paths);
- assert_eq!(
- decoded.output_schema.as_ref(),
- config.output_schema.as_ref()
- );
- assert_eq!(decoded.table_partition_cols, config.table_partition_cols);
- assert_eq!(decoded.insert_op, config.insert_op);
- assert_eq!(
- decoded.keep_partition_by_columns,
- config.keep_partition_by_columns
- );
- assert_eq!(decoded.file_extension, config.file_extension);
- assert_eq!(decoded.file_output_mode, config.file_output_mode);
-
- let [decoded_file] = decoded.file_group.files() else {
- panic!("expected one decoded output file");
- };
- let [config_file] = config.file_group.files() else {
- panic!("expected one configured output file");
- };
- assert_eq!(
- decoded_file.object_meta.location,
- config_file.object_meta.location
- );
- assert_eq!(decoded_file.object_meta.size, config_file.object_meta.size);
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_json_source() -> Result<()> {
- let ctx = SessionContext::new();
- ctx.register_json("t1", "../core/tests/data/1.json", Default::default())
- .await?;
- let plan = ctx.table("t1").await?.create_physical_plan().await?;
- roundtrip_test(plan)
-}
-
-#[test]
-fn roundtrip_json_sink() -> Result<()> {
- let field_a = Field::new("plan_type", DataType::Utf8, false);
- let field_b = Field::new("plan", DataType::Utf8, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
-
- let file_sink_config = FileSinkConfig {
- original_url: String::default(),
- object_store_url: ObjectStoreUrl::local_filesystem(),
- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
- table_paths: vec![ListingTableUrl::parse("file:///")?],
- output_schema: schema.clone(),
- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
- insert_op: InsertOp::Overwrite,
- keep_partition_by_columns: true,
- file_extension: "json".into(),
- file_output_mode: FileOutputMode::SingleFile,
- };
- let data_sink = Arc::new(JsonSink::new(
- file_sink_config,
- JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED),
- ));
- let sort_order = [PhysicalSortRequirement::new(
- Arc::new(Column::new("plan_type", 0)),
- Some(SortOptions {
- descending: true,
- nulls_first: false,
- }),
- )]
- .into();
-
- roundtrip_test(Arc::new(DataSinkExec::new(
- input,
- data_sink,
- Some(sort_order),
- )))
-}
-
-#[test]
-fn roundtrip_csv_sink() -> Result<()> {
- let field_a = Field::new("plan_type", DataType::Utf8, false);
- let field_b = Field::new("plan", DataType::Utf8, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
-
- let file_sink_config = FileSinkConfig {
- original_url: String::default(),
- object_store_url: ObjectStoreUrl::local_filesystem(),
- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
- table_paths: vec![ListingTableUrl::parse("file:///")?],
- output_schema: schema.clone(),
- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
- insert_op: InsertOp::Overwrite,
- keep_partition_by_columns: true,
- file_extension: "csv".into(),
- file_output_mode: FileOutputMode::Directory,
- };
- let data_sink = Arc::new(CsvSink::new(
- file_sink_config,
- CsvWriterOptions::new(WriterBuilder::default(), CompressionTypeVariant::ZSTD),
- ));
- let sort_order = [PhysicalSortRequirement::new(
- Arc::new(Column::new("plan_type", 0)),
- Some(SortOptions {
- descending: true,
- nulls_first: false,
- }),
- )]
- .into();
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- let roundtrip_plan = roundtrip_test_and_return(
- Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),
- &ctx,
- &codec,
- &proto_converter,
- )?;
-
- let roundtrip_plan = roundtrip_plan.downcast_ref::<DataSinkExec>().unwrap();
- let csv_sink = roundtrip_plan.sink().downcast_ref::<CsvSink>().unwrap();
- assert_eq!(
- CompressionTypeVariant::ZSTD,
- csv_sink.writer_options().compression
- );
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_parquet_sink() -> Result<()> {
- let field_a = Field::new("plan_type", DataType::Utf8, false);
- let field_b = Field::new("plan", DataType::Utf8, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));
-
- let file_sink_config = FileSinkConfig {
- original_url: String::default(),
- object_store_url: ObjectStoreUrl::local_filesystem(),
- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
- table_paths: vec![ListingTableUrl::parse("file:///")?],
- output_schema: schema.clone(),
- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],
- insert_op: InsertOp::Overwrite,
- keep_partition_by_columns: true,
- file_extension: "parquet".into(),
- file_output_mode: FileOutputMode::Automatic,
- };
- let data_sink = Arc::new(ParquetSink::new(
- file_sink_config,
- TableParquetOptions::default(),
- ));
- let sort_order = [PhysicalSortRequirement::new(
- Arc::new(Column::new("plan_type", 0)),
- Some(SortOptions {
- descending: true,
- nulls_first: false,
- }),
- )]
- .into();
-
- roundtrip_test(Arc::new(DataSinkExec::new(
- input,
- data_sink,
- Some(sort_order),
- )))
-}
-
-#[test]
-fn roundtrip_sym_hash_join() -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let field_a = Field::new("col_a", DataType::Int64, false);
- let field_b = Field::new("col_b", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_b.clone()]);
- let on = vec![(
- Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,
- Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,
- )];
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("col_a", 0)),
- Operator::Gt,
- Arc::new(Column::new("col_b", 1)),
- )),
- vec![
- ColumnIndex {
- index: 0,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 0,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![field_a, field_b])),
- );
-
- let schema_left = Arc::new(schema_left);
- let schema_right = Arc::new(schema_right);
- let left_order: LexOrdering = [PhysicalSortExpr {
- expr: Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)),
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- }]
- .into();
- let right_order: LexOrdering = [PhysicalSortExpr {
- expr: Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)),
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- }]
- .into();
- let ordering_cases = [
- (None, None),
- (Some(left_order.clone()), None),
- (None, Some(right_order.clone())),
- (Some(left_order), Some(right_order)),
- ];
- let ordering_options = |ordering: Option<&LexOrdering>| {
- ordering
- .map(|ordering| ordering.iter().map(|expr| expr.options).collect::<Vec<_>>())
- };
-
- for join_type in [
- JoinType::Inner,
- JoinType::Left,
- JoinType::Right,
- JoinType::Full,
- JoinType::LeftAnti,
- JoinType::RightAnti,
- JoinType::LeftSemi,
- JoinType::RightSemi,
- JoinType::LeftMark,
- JoinType::RightMark,
- ] {
- for null_equality in [
- NullEquality::NullEqualsNothing,
- NullEquality::NullEqualsNull,
- ] {
- for filter in [None, Some(filter.clone())] {
- for partition_mode in [
- StreamJoinPartitionMode::Partitioned,
- StreamJoinPartitionMode::SinglePartition,
- ] {
- for (left_order, right_order) in &ordering_cases {
- let result = roundtrip_test_and_return(
- Arc::new(SymmetricHashJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- on.clone(),
- filter.clone(),
- &join_type,
- null_equality,
- left_order.clone(),
- right_order.clone(),
- partition_mode,
- )?),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let result =
- result.downcast_ref::<SymmetricHashJoinExec>().unwrap();
- assert_eq!(result.join_type(), &join_type);
- assert_eq!(result.null_equality(), null_equality);
- assert_eq!(result.partition_mode(), partition_mode);
- assert_eq!(
- ordering_options(result.left_sort_exprs()),
- ordering_options(left_order.as_ref())
- );
- assert_eq!(
- ordering_options(result.right_sort_exprs()),
- ordering_options(right_order.as_ref())
- );
- assert_eq!(
- result.filter().map(JoinFilter::column_indices),
- filter.as_ref().map(JoinFilter::column_indices)
- );
- }
- }
- }
- }
- }
- Ok(())
-}
-
-#[test]
-fn roundtrip_union() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_a]);
- let left = EmptyExec::new(Arc::new(schema_left));
- let right = EmptyExec::new(Arc::new(schema_right));
- let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];
- let union = UnionExec::try_new(inputs)?;
- roundtrip_test(union)
-}
-
-#[test]
-fn roundtrip_repartition_preserve_order() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
- let sort_exprs: LexOrdering = [PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions::default(),
- }]
- .into();
-
- // Create two sorted single-partition inputs, then union them to get
- // a sorted input with 2 partitions.
- let source1 = SortExec::new(
- sort_exprs.clone(),
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- );
- let source2 = SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)));
- let union = UnionExec::try_new(vec![
- Arc::new(source1) as Arc<dyn ExecutionPlan>,
- Arc::new(source2) as Arc<dyn ExecutionPlan>,
- ])?;
-
- let repartition = RepartitionExec::try_new(union, Partitioning::RoundRobinBatch(10))?
- .with_preserve_order();
- assert!(repartition.preserve_order());
-
- roundtrip_test(Arc::new(repartition))
-}
-
-#[test]
-fn roundtrip_range_partitioning() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let range_partitioning = Partitioning::Range(RangePartitioning::new(
- [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),
- vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],
- ));
- // RepartitionExec is used only to carry the partitioning through proto.
- // Executing range repartitioning is intentionally unsupported.
- let repartition = RepartitionExec::try_new(input, range_partitioning)?;
-
- roundtrip_test(Arc::new(repartition))
-}
-
-/// `parse_protobuf_hash_partitioning` has no in-tree callers left; it delegates
-/// to the shared `Partitioning::try_from_proto`, so pin that it still decodes
-/// the hash message it is handed.
-#[test]
-fn parse_hash_partitioning_delegates_to_shared_decoder() -> Result<()> {
- use datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;
-
- let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
- let ctx = SessionContext::new();
- let task_ctx = ctx.task_ctx();
- let codec = DefaultPhysicalExtensionCodec {};
- let decode_ctx = PhysicalPlanDecodeContext::new(&task_ctx, &codec);
- let proto_converter = DefaultPhysicalProtoConverter {};
-
- let hash_expr = serialize_physical_expr_with_converter(
- &col("a", &schema)?,
- &codec,
- &proto_converter,
- )?;
- let hash = protobuf::PhysicalHashRepartition {
- hash_expr: vec![hash_expr],
- partition_count: 4,
- };
-
- let partitioning = parse_protobuf_hash_partitioning(
- Some(&hash),
- &decode_ctx,
- &schema,
- &proto_converter,
- )?;
- let Some(Partitioning::Hash(exprs, count)) = partitioning else {
- panic!("expected hash partitioning, got {partitioning:?}");
- };
- assert_eq!(count, 4);
- assert_eq!(exprs.len(), 1);
- assert_eq!(exprs[0].to_string(), col("a", &schema)?.to_string());
-
- // No message means no partitioning, as before.
- assert!(
- parse_protobuf_hash_partitioning(None, &decode_ctx, &schema, &proto_converter)?
- .is_none()
- );
-
- // The count is a `u64` on the wire and a `usize` in memory, so decoding
- // narrows it. A count that does not fit is the case that motivated routing
- // this through the shared decoder: it used to `unwrap()` and panic, and now
- // reports an error. Only a target narrower than 64 bits can reach that arm
- // -- on a 64-bit target every `u64` fits, and the assertion there is that
- // the largest possible count survives whole rather than being truncated.
- let oversized = protobuf::PhysicalHashRepartition {
- hash_expr: vec![serialize_physical_expr_with_converter(
- &col("a", &schema)?,
- &codec,
- &proto_converter,
- )?],
- partition_count: u64::MAX,
- };
- let decoded = parse_protobuf_hash_partitioning(
- Some(&oversized),
- &decode_ctx,
- &schema,
- &proto_converter,
- );
-
- #[cfg(target_pointer_width = "64")]
- {
- let Some(Partitioning::Hash(_, count)) = decoded? else {
- panic!("expected hash partitioning");
- };
- assert_eq!(count, usize::MAX);
- }
-
- #[cfg(not(target_pointer_width = "64"))]
- assert!(
- decoded
- .unwrap_err()
- .to_string()
- .contains("Partition count 18446744073709551615 exceeds usize::MAX")
- );
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_interleave() -> Result<()> {
- let field_a = Field::new("col", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_a]);
- let partition = Partitioning::Hash(vec![], 3);
- let left = RepartitionExec::try_new(
- Arc::new(EmptyExec::new(Arc::new(schema_left))),
- partition.clone(),
- )?;
- let right = RepartitionExec::try_new(
- Arc::new(EmptyExec::new(Arc::new(schema_right))),
- partition,
- )?;
- let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];
- let interleave = InterleaveExec::try_new(inputs)?;
- roundtrip_test(Arc::new(interleave))
-}
-
-#[test]
-fn roundtrip_unnest() -> Result<()> {
- let fa = Field::new("a", DataType::Int64, true);
- let fb0 = Field::new_list_field(DataType::Utf8, true);
- let fb = Field::new_list("b", fb0.clone(), false);
- let fc1 = Field::new("c1", DataType::Boolean, false);
- let fc2 = Field::new("c2", DataType::Date64, true);
- let fc = Field::new_struct("c", Fields::from(vec![fc1.clone(), fc2.clone()]), true);
- let fd0 = Field::new_list_field(DataType::Float32, false);
- let fd = Field::new_list("d", fd0.clone(), true);
- let fe1 = Field::new("e1", DataType::UInt16, false);
- let fe2 = Field::new("e2", DataType::Duration(TimeUnit::Millisecond), true);
- let fe3 = Field::new("e3", DataType::Timestamp(TimeUnit::Millisecond, None), true);
- let fe_fields = Fields::from(vec![fe1.clone(), fe2.clone(), fe3.clone()]);
- let fe = Field::new_struct("e", fe_fields, false);
-
- let fb0 = fb0.with_name("b");
- let fd0 = fd0.with_name("d");
- let input_schema = Arc::new(Schema::new(vec![fa.clone(), fb, fc, fd, fe]));
- let output_schema =
- Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));
- let input = Arc::new(EmptyExec::new(input_schema));
- let options = UnnestOptions {
- null_handling: datafusion_common::NullHandling::Drop,
- recursions: vec![datafusion_common::RecursionUnnestOption {
- input_column: datafusion_common::Column::new_unqualified("b"),
- output_column: datafusion_common::Column::new_unqualified("b"),
- depth: 2,
- }],
- };
- let unnest = UnnestExec::new(
- input,
- vec![
- ListUnnest {
- index_in_input_schema: 1,
- depth: 1,
- },
- ListUnnest {
- index_in_input_schema: 1,
- depth: 2,
- },
- ListUnnest {
- index_in_input_schema: 3,
- depth: 2,
- },
- ],
- vec![2, 4],
- output_schema,
- options.clone(),
- )?;
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let result =
- roundtrip_test_and_return(Arc::new(unnest), &ctx, &codec, &proto_converter)?;
- let result = result.downcast_ref::<UnnestExec>().unwrap();
- assert_eq!(result.options(), &options);
-
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_coalesce() -> Result<()> {
- let ctx = SessionContext::new();
- ctx.register_table(
- "t",
- Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([
- Arc::new(Field::new("f", DataType::Int64, false)),
- ]))))),
- )?;
- let df = ctx.sql("select coalesce(f) as f from t").await?;
- let plan = df.create_physical_plan().await?;
-
- let node = PhysicalPlanNode::try_from_physical_plan(
- plan.clone(),
- &DefaultPhysicalExtensionCodec {},
- )?;
- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
- .map_err(|e| DataFusionError::External(Box::new(e)))?;
- let restored =
- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
-
- assert_eq!(
- plan.schema(),
- restored.schema(),
- "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",
- displayable(plan.as_ref())
- .set_show_schema(true)
- .indent(true),
- displayable(restored.as_ref())
- .set_show_schema(true)
- .indent(true),
- );
-
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_generate_series() -> Result<()> {
- let ctx = SessionContext::new();
- ctx.register_table(
- "t",
- Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([
- Arc::new(Field::new("f", DataType::Int64, false)),
- ]))))),
- )?;
- let df = ctx.sql("select * from generate_series(1, 10000)").await?;
- let plan = df.create_physical_plan().await?;
-
- let node = PhysicalPlanNode::try_from_physical_plan(
- plan.clone(),
- &DefaultPhysicalExtensionCodec {},
- )?;
- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
- .map_err(|e| DataFusionError::External(Box::new(e)))?;
- let restored =
- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
-
- assert_eq!(
- plan.schema(),
- restored.schema(),
- "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",
- displayable(plan.as_ref())
- .set_show_schema(true)
- .indent(true),
- displayable(restored.as_ref())
- .set_show_schema(true)
- .indent(true),
- );
-
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_projection_source() -> Result<()> {
- let schema = Arc::new(Schema::new(Fields::from([
- Arc::new(Field::new("a", DataType::Utf8, false)),
- Arc::new(Field::new("b", DataType::Utf8, false)),
- Arc::new(Field::new("c", DataType::Int32, false)),
- Arc::new(Field::new("d", DataType::Int32, false)),
- ])));
-
- let statistics = Statistics::new_unknown(&schema);
-
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&schema)));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .with_statistics(statistics)
- .with_projection_indices(Some(vec![0, 1, 2]))?
- .build();
-
- let filter = Arc::new(
- FilterExecBuilder::new(
- Arc::new(BinaryExpr::new(col("c", &schema)?, Operator::Eq, lit(1))),
- DataSourceExec::from_data_source(scan_config),
- )
- .apply_projection(Some(vec![0, 1]))?
- .build()?,
- );
-
- roundtrip_test(filter)
-}
-
-#[tokio::test]
-async fn roundtrip_parquet_select_star() -> Result<()> {
- let ctx = all_types_context().await?;
- let sql = "select * from alltypes_plain";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_parquet_select_projection() -> Result<()> {
- let ctx = all_types_context().await?;
- let sql = "select string_col, timestamp_col from alltypes_plain";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_parquet_select_star_predicate() -> Result<()> {
- let ctx = all_types_context().await?;
- let sql = "select * from alltypes_plain where id > 4";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_parquet_select_projection_predicate() -> Result<()> {
- let ctx = all_types_context().await?;
- let sql = "select string_col, timestamp_col from alltypes_plain where id > 4";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_empty_projection() -> Result<()> {
- let ctx = all_types_context().await?;
- let sql = "select 1 from alltypes_plain";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_memory_source_empty_projection() -> Result<()> {
- // Memory scan: `Some(vec![])` must not decode back as `None`
- let ctx = SessionContext::new();
- let batch = RecordBatch::try_new(
- Arc::new(Schema::new(vec![
- Field::new("a", DataType::Utf8, false),
- Field::new("b", DataType::Int64, false),
- ])),
- vec![
- Arc::new(arrow::array::StringArray::from(vec!["Tom"])),
- Arc::new(arrow::array::Int64Array::from(vec![18i64])),
- ],
- )?;
- ctx.register_batch("tmem", batch)?;
- let sql = "select 1 from tmem";
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-#[tokio::test]
-async fn roundtrip_physical_plan_node() {
- use datafusion::prelude::*;
- use datafusion_proto::physical_plan::{
- AsExecutionPlan, DefaultPhysicalExtensionCodec,
- };
- use datafusion_proto::protobuf::PhysicalPlanNode;
-
- let ctx = SessionContext::new();
-
- ctx.register_parquet(
- "pt",
- &format!(
- "{}/alltypes_plain.snappy.parquet",
- datafusion_common::test_util::parquet_test_data()
- ),
- ParquetReadOptions::default(),
- )
- .await
- .unwrap();
-
- let plan = ctx
- .sql("select id, string_col, timestamp_col from pt where id > 4 order by string_col")
- .await
- .unwrap()
- .create_physical_plan()
- .await
- .unwrap();
-
- let node: PhysicalPlanNode =
- PhysicalPlanNode::try_from_physical_plan(plan, &DefaultPhysicalExtensionCodec {})
- .unwrap();
-
- let plan = node
- .try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})
- .unwrap();
-
- let _ = plan.execute(0, ctx.task_ctx()).unwrap();
-}
-
-/// The deprecated `try_into_projection_physical_plan` shim now delegates to
-/// [`ProjectionExec::try_from_proto`], which reads the enclosing
-/// `PhysicalPlanNode` rather than a `ProjectionExecNode`. Assert the shim still
-/// decodes the node passed as an argument, not `self`, so an out-of-tree caller
-/// that passes a projection unrelated to `self` keeps the old behaviour.
-#[test]
-fn deprecated_projection_shim_decodes_argument_not_self() -> Result<()> {
- use datafusion_proto::protobuf::PhysicalPlanNode;
- use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;
-
- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));
- let projection = Arc::new(ProjectionExec::try_new(
- vec![ProjectionExpr::new(
- col("a", &schema)?,
- "renamed".to_string(),
- )],
- input,
- )?);
-
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let projection_node = PhysicalPlanNode::try_from_physical_plan_with_converter(
- projection,
- &codec,
- &proto_converter,
- )?;
- let Some(PhysicalPlanType::Projection(projection_exec_node)) =
- &projection_node.physical_plan_type
- else {
- panic!("expected a Projection node, got {projection_node:?}");
- };
-
- // `self` is deliberately a different plan variant than the argument.
- let unrelated_node = PhysicalPlanNode::try_from_physical_plan_with_converter(
- Arc::new(EmptyExec::new(Arc::new(schema))),
- &codec,
- &proto_converter,
- )?;
-
- let session_ctx = SessionContext::new();
- let task_ctx = session_ctx.task_ctx();
- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
- #[expect(deprecated)]
- let decoded = unrelated_node.try_into_projection_physical_plan(
- projection_exec_node,
- &decode_ctx,
- &proto_converter,
- )?;
-
- let decoded = decoded
- .downcast_ref::<ProjectionExec>()
- .expect("decoded plan should be a ProjectionExec");
- assert_eq!(decoded.expr().len(), 1);
- assert_eq!(decoded.expr()[0].alias, "renamed");
- Ok(())
-}
-
-/// Helper function to create a SessionContext with all TPC-H tables registered as external tables
-async fn tpch_context() -> Result<SessionContext> {
- use datafusion_common::test_util::datafusion_test_data;
-
- let ctx = SessionContext::new();
- let test_data = datafusion_test_data();
-
- // TPC-H table names
- let tables = [
- "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation",
- "region",
- ];
-
- // Create external tables for all TPC-H tables
- for table in &tables {
- let table_sql = format!(
- "CREATE EXTERNAL TABLE {table} STORED AS PARQUET LOCATION '{test_data}/tpch_{table}_small.parquet'"
- );
- ctx.sql(&table_sql).await.map_err(|e| {
- DataFusionError::External(
- format!("Failed to create {table} table: {e}").into(),
- )
- })?;
- }
-
- Ok(ctx)
-}
-
-/// Helper function to get TPC-H query SQL
-fn get_tpch_query_sql(query: usize) -> Result<Vec<String>> {
- use std::fs;
-
- if !(1..=22).contains(&query) {
- return Err(DataFusionError::External(
- format!("Invalid TPC-H query number: {query}").into(),
- ));
- }
-
- let filename = format!("../../benchmarks/queries/q{query}.sql");
- let contents = fs::read_to_string(&filename).map_err(|e| {
- DataFusionError::External(
- format!("Failed to read query file {filename}: {e}").into(),
- )
- })?;
-
- Ok(contents
- .split(';')
- .map(|s| s.trim())
- .filter(|s| !s.is_empty())
- .map(|s| s.to_string())
- .collect())
-}
-
-#[tokio::test]
-async fn test_serialize_deserialize_tpch_queries() -> Result<()> {
- // Create context with TPC-H tables
- let ctx = tpch_context().await?;
-
- // repeat to run all 22 queries
- for query in 1..=22 {
- // run all statements in the query
- let sql = get_tpch_query_sql(query)?;
- for stmt in sql {
- let logical_plan = ctx.sql(&stmt).await?.into_unoptimized_plan();
- let optimized_plan = ctx.state().optimize(&logical_plan)?;
- let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;
-
- // serialize the physical plan
- let codec = DefaultPhysicalExtensionCodec {};
-
- let proto =
- PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
-
- // deserialize the physical plan
- let _deserialized_plan =
- proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
- }
- }
-
- Ok(())
-}
-
-// Bugs: https://github.com/apache/datafusion/issues/16772
-#[tokio::test]
-async fn test_round_trip_tpch_queries() -> Result<()> {
- // Create context with TPC-H tables
- let ctx = tpch_context().await?;
-
- // repeat to run all 22 queries
- for query in 1..=22 {
- // run all statements in the query
- let sql = get_tpch_query_sql(query)?;
- for stmt in sql {
- roundtrip_test_sql_with_context(&stmt, &ctx).await?;
- }
- }
-
- Ok(())
-}
-
-// Bug 1 of https://github.com/apache/datafusion/issues/16772
-/// Test that AggregateFunctionExpr human_display field is correctly preserved
-/// during serialization/deserialization roundtrip.
-///
-/// Test for issue where the human_display field (used for EXPLAIN output)
-/// was not being serialized to protobuf, causing it to be lost during roundtrip
-/// and resulting in empty or incorrect display strings in query plans.
-#[tokio::test]
-async fn test_round_trip_human_display() -> Result<()> {
- // Create context with TPC-H tables
- let ctx = tpch_context().await?;
-
- let sql = "select r_name, count(1) from region group by r_name";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- let sql = "select r_name, count(*) from region group by r_name";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- let sql = "select r_name, count(r_name) from region group by r_name";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- let sql = "select count(*) as count_star from region";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- Ok(())
-}
-
-#[test]
-fn test_round_trip_aliased_reverse_human_display() -> Result<()> {
- let aggregate_expr = roundtrip_first_value_aggregate(
- "agg",
- "first_value(b) ORDER BY [b ASC NULLS LAST]",
- Some("agg"),
- )?;
- let reversed = aggregate_expr
- .reverse_expr()
- .expect("expected reverse expr");
-
- assert_eq!(reversed.name(), "agg");
- assert_eq!(reversed.human_display_alias(), Some("agg"));
- assert_eq!(
- reversed.human_display(),
- Some("last_value(b) ORDER BY [b DESC NULLS FIRST]")
- );
-
- Ok(())
-}
-
-#[test]
-fn test_round_trip_human_display_alias_with_colon() -> Result<()> {
- let aggregate_expr = roundtrip_first_value_aggregate(
- "agg:one",
- "first_value(b) ORDER BY [b ASC NULLS LAST]",
- Some("agg:one"),
- )?;
-
- assert_eq!(aggregate_expr.name(), "agg:one");
- assert_eq!(aggregate_expr.human_display_alias(), Some("agg:one"));
- assert_eq!(
- aggregate_expr.human_display(),
- Some("first_value(b) ORDER BY [b ASC NULLS LAST]")
- );
-
- Ok(())
-}
-
-#[test]
-fn test_round_trip_non_aliased_human_display_ending_like_alias() -> Result<()> {
- let aggregate_expr =
- roundtrip_first_value_aggregate("agg", "first_value(b) as agg", None)?;
-
- assert_eq!(aggregate_expr.name(), "agg");
- assert_eq!(
- aggregate_expr.human_display(),
- Some("first_value(b) as agg")
- );
- assert_eq!(aggregate_expr.human_display_alias(), None);
-
- Ok(())
-}
-
-fn roundtrip_first_value_aggregate(
- alias: &str,
- human_display: &str,
- human_display_alias: Option<&str>,
-) -> Result<Arc<AggregateFunctionExpr>> {
- let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));
- let mut builder =
- AggregateExprBuilder::new(first_value_udaf(), vec![col("b", &schema)?])
- .order_by(vec![PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions::new(false, false),
- }])
- .schema(Arc::clone(&schema))
- .alias(alias)
- .human_display(human_display);
- if let Some(human_display_alias) = human_display_alias {
- builder = builder.human_display_alias(human_display_alias);
- }
- let agg_expr = builder.build().map(Arc::new)?;
-
- let plan = Arc::new(AggregateExec::try_new(
- AggregateMode::Single,
- PhysicalGroupBy::new(vec![], vec![], vec![], false),
- vec![agg_expr],
- vec![None],
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- schema,
- )?);
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let roundtrip_plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;
- let aggregate = roundtrip_plan
- .as_ref()
- .downcast_ref::<AggregateExec>()
- .expect("expected AggregateExec after roundtrip");
-
- Ok(Arc::clone(&aggregate.aggr_expr()[0]))
-}
-
-// Bug 2 of https://github.com/apache/datafusion/issues/16772
-/// Test that PhysicalGroupBy groups field is correctly serialized/deserialized
-/// for simple aggregates (no GROUP BY clause).
-///
-/// Test for issue where simple aggregates like "SELECT SUM(col1 * col2) FROM table"
-/// would incorrectly serialize groups as [[]] instead of [] during roundtrip serialization.
-/// The groups field should be empty ([]) when there are no GROUP BY expressions.
-#[tokio::test]
-async fn test_round_trip_groups_display() -> Result<()> {
- // Create context with TPC-H tables
- let ctx = tpch_context().await?;
-
- let sql = "select sum(l_extendedprice * l_discount) as revenue from lineitem;";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- let sql = "select sum(l_extendedprice) as revenue from lineitem;";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- Ok(())
-}
-
-// Bug 3 of https://github.com/apache/datafusion/issues/16772
-/// Test that ScalarFunctionExpr return_field name is correctly preserved
-/// during serialization/deserialization roundtrip.
-///
-/// Test for issue where the return_field.name for scalar functions
-/// was not being serialized to protobuf, causing it to be lost during roundtrip
-/// and defaulting to a generic name like "f" instead of the proper function name.
-#[tokio::test]
-async fn test_round_trip_date_part_display() -> Result<()> {
- // Create context with TPC-H tables
- let ctx = tpch_context().await?;
-
- let sql = "select extract(year from l_shipdate) as l_year from lineitem ";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- let sql = "select extract(month from l_shipdate) as l_year from lineitem ";
- roundtrip_test_sql_with_context(sql, &ctx).await?;
-
- Ok(())
-}
-
-#[tokio::test]
-async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {
- use datafusion_common::test_util::datafusion_test_data;
-
- let ctx = SessionContext::new();
-
- // Register the TPC-H part table using the local test data
- let test_data = datafusion_test_data();
- let table_sql = format!(
- "CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '{test_data}/tpch_part_small.parquet'"
- );
- ctx.sql(&table_sql).await.map_err(|e| {
- DataFusionError::External(format!("Failed to create part table: {e}").into())
- })?;
-
- // Test the exact problematic query
- let sql =
- "SELECT p_size FROM part WHERE p_size IN (14, 6, 5, 31) and p_partkey > 1000";
-
- let logical_plan = ctx.sql(sql).await?.into_unoptimized_plan();
- let optimized_plan = ctx.state().optimize(&logical_plan)?;
- let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;
-
- // Serialize the physical plan - bug may happen here already but not necessarily manifests
- let codec = DefaultPhysicalExtensionCodec {};
-
- let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
-
- // This will fail with the bug, but should succeed when fixed
- let _deserialized_plan = proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;
- Ok(())
-}
-
-#[tokio::test]
-/// Tests that we can serialize an unoptimized "analyze" plan and it will work on the other end
-async fn analyze_roundtrip_unoptimized() -> Result<()> {
- let ctx = SessionContext::new();
-
- // No optimizations
- let session_state =
- datafusion::execution::SessionStateBuilder::new_from_existing(ctx.state())
- .with_physical_optimizer_rules(vec![])
- .build();
-
- let logical_plan = session_state
- .create_logical_plan("explain analyze select 1")
- .await?;
- let plan = session_state.create_physical_plan(&logical_plan).await?;
-
- let node = PhysicalPlanNode::try_from_physical_plan(
- plan.clone(),
- &DefaultPhysicalExtensionCodec {},
- )?;
-
- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())
- .map_err(|e| DataFusionError::External(Box::new(e)))?;
-
- let unoptimized =
- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;
-
- let physical_planner =
- datafusion::physical_planner::DefaultPhysicalPlanner::default();
- physical_planner.optimize_physical_plan(unoptimized, &session_state, |_, _| {})?;
- Ok(())
-}
-
-#[test]
-fn roundtrip_sort_merge_join() -> Result<()> {
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let field_a = Field::new("col_a", DataType::Int64, false);
- let field_b = Field::new("col_b", DataType::Int64, false);
- let schema_left = Schema::new(vec![field_a.clone()]);
- let schema_right = Schema::new(vec![field_b.clone()]);
- let on = vec![(
- Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,
- Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,
- )];
-
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("col_a", 1)),
- Operator::Gt,
- Arc::new(Column::new("col_b", 0)),
- )),
- vec![
- ColumnIndex {
- index: 0,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 0,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![field_a, field_b])),
- );
-
- let schema_left = Arc::new(schema_left);
- let schema_right = Arc::new(schema_right);
- let sort_options = vec![SortOptions {
- descending: true,
- nulls_first: false,
- }];
- for null_equality in [
- NullEquality::NullEqualsNothing,
- NullEquality::NullEqualsNull,
- ] {
- for filter in [None, Some(filter.clone())] {
- for join_type in [
- JoinType::Inner,
- JoinType::Left,
- JoinType::Right,
- JoinType::Full,
- JoinType::LeftAnti,
- JoinType::RightAnti,
- JoinType::LeftSemi,
- JoinType::RightSemi,
- JoinType::LeftMark,
- JoinType::RightMark,
- ] {
- let result = roundtrip_test_and_return(
- Arc::new(SortMergeJoinExec::try_new(
- Arc::new(EmptyExec::new(schema_left.clone())),
- Arc::new(EmptyExec::new(schema_right.clone())),
- on.clone(),
- filter.clone(),
- join_type,
- sort_options.clone(),
- null_equality,
- )?),
- &ctx,
- &codec,
- &proto_converter,
- )?;
- let result = result.downcast_ref::<SortMergeJoinExec>().unwrap();
- assert_eq!(result.join_type(), join_type);
- assert_eq!(result.null_equality(), null_equality);
- assert_eq!(result.sort_options(), sort_options);
- assert_eq!(
- result.filter().as_ref().map(|f| f.column_indices()),
- filter.as_ref().map(|f| f.column_indices())
- );
- }
- }
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_logical_plan_sort_merge_join() -> Result<()> {
- let ctx = SessionContext::new();
- ctx.register_csv(
- "t0",
- "tests/testdata/test.csv",
- datafusion::prelude::CsvReadOptions::default().has_header(true),
- )
- .await?;
- ctx.register_csv(
- "t1",
- "tests/testdata/test.csv",
- datafusion::prelude::CsvReadOptions::default().has_header(true),
- )
- .await?;
-
- ctx.sql("SET datafusion.optimizer.prefer_hash_join = false")
- .await?
- .show()
- .await?;
-
- let query = "SELECT t1.* FROM t0 join t1 on t0.a = t1.a";
- let plan = ctx.sql(query).await?.create_physical_plan().await?;
- roundtrip_test(plan)
-}
-
-#[tokio::test]
-async fn roundtrip_memory_source() -> Result<()> {
- let ctx = SessionContext::new();
- let plan = ctx
- .sql("select * from values ('Tom', 18)")
- .await?
- .create_physical_plan()
- .await?;
- roundtrip_test(plan)
-}
-
-#[tokio::test]
-async fn roundtrip_memory_source_sort_information_and_fetch() -> Result<()> {
- use datafusion::datasource::memory::MemorySourceConfig;
- use datafusion::datasource::source::DataSource as _;
-
- let schema = Arc::new(Schema::new(vec![
- Field::new("a", DataType::Utf8, false),
- Field::new("b", DataType::Int64, false),
- ]));
- let batch = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![
- Arc::new(arrow::array::StringArray::from(vec!["Tom", "Bob"])),
- Arc::new(arrow::array::Int64Array::from(vec![18i64, 21i64])),
- ],
- )?;
- let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(
- col("b", &schema)?,
- SortOptions {
- descending: true,
- nulls_first: false,
- },
- )])
- .unwrap();
- let source = MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?
- .with_limit(Some(1))
- .with_show_sizes(false)
- .try_with_sort_information(vec![ordering])?;
- let exec_plan = DataSourceExec::from_data_source(source.clone());
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let decoded = roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
-
- // The string representation does not include every field; check the
- // decoded source directly.
- let decoded = decoded
- .downcast_ref::<DataSourceExec>()
- .expect("expected DataSourceExec");
- let decoded_source = decoded
- .data_source()
- .downcast_ref::<MemorySourceConfig>()
- .expect("expected MemorySourceConfig");
- assert_eq!(decoded_source.partitions(), source.partitions());
- assert_eq!(decoded_source.original_schema(), source.original_schema());
- assert_eq!(decoded_source.projection(), source.projection());
- assert_eq!(decoded_source.sort_information(), source.sort_information());
- assert_eq!(decoded_source.fetch(), Some(1));
- assert!(!decoded_source.show_sizes());
- Ok(())
-}
-
-#[tokio::test]
-async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {
- let ctx = SessionContext::new();
- let file_format = JsonFormat::default();
- let table_partition_cols = vec![("part".to_owned(), DataType::Int64)];
- let data = "../core/tests/data/partitioned_table_json";
- let listing_table_url = ListingTableUrl::parse(data)?;
- let listing_options = ListingOptions::new(Arc::new(file_format))
- .with_table_partition_cols(table_partition_cols);
-
- let config = ListingTableConfig::new(listing_table_url)
- .with_listing_options(listing_options)
- .infer_schema(&ctx.state())
- .await?;
-
- // Decorate metadata onto the inferred ListingTable schema
- let schema_with_meta = config
- .file_schema
- .clone()
- .map(|s| {
- let mut meta: HashMap<String, String> = HashMap::new();
- meta.insert("foo.bar".to_string(), "baz".to_string());
- s.as_ref().clone().with_metadata(meta)
- })
- .expect("Must decorate metadata");
-
- let config = config.with_schema(Arc::new(schema_with_meta));
- ctx.register_table("hive_style", Arc::new(ListingTable::try_new(config)?))?;
-
- let plan = ctx
- .sql("select * from hive_style limit 1")
- .await?
- .create_physical_plan()
- .await?;
-
- roundtrip_test(plan)
-}
-
-#[tokio::test]
-async fn roundtrip_async_func_exec() -> Result<()> {
- #[derive(Debug, PartialEq, Eq, Hash)]
- struct TestAsyncUDF {
- signature: Signature,
- }
-
- impl TestAsyncUDF {
- fn new() -> Self {
- Self {
- signature: Signature::exact(vec![DataType::Int64], Volatility::Volatile),
- }
- }
- }
-
- impl ScalarUDFImpl for TestAsyncUDF {
- fn name(&self) -> &str {
- "test_async_udf"
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Int64)
- }
-
- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- not_impl_err!("Must call from `invoke_async_with_args`")
- }
- }
-
- #[async_trait::async_trait]
- impl AsyncScalarUDFImpl for TestAsyncUDF {
- async fn invoke_async_with_args(
- &self,
- args: ScalarFunctionArgs,
- ) -> Result<ColumnarValue> {
- Ok(args.args[0].clone())
- }
- }
-
- let ctx = SessionContext::new();
- let async_udf = AsyncScalarUDF::new(Arc::new(TestAsyncUDF::new()));
- ctx.register_udf(async_udf.into_scalar_udf());
-
- let physical_plan = ctx
- .sql("select test_async_udf(1)")
- .await?
- .create_physical_plan()
- .await?;
-
- roundtrip_test_with_context(physical_plan, &ctx)?;
-
- Ok(())
-}
-
-/// Test that HashTableLookupExpr serializes to lit(true)
-///
-/// HashTableLookupExpr contains a runtime hash table that cannot be serialized.
-/// The serialization code replaces it with lit(true) which is safe because
-/// it's a performance optimization filter, not a correctness requirement.
-#[test]
-fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {
- use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;
- use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};
-
- // Create a simple schema and input plan
- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- // Create a HashTableLookupExpr - it will be replaced with lit(true) during serialization
- let hash_map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0))));
- let on_columns = vec![col("col", &schema)?];
- let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(
- on_columns,
- datafusion::physical_plan::joins::SeededRandomState::with_seed(0),
- hash_map,
- "test_lookup".to_string(),
- ));
-
- // Create a filter with the lookup expression
- let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);
-
- // Serialize
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
-
- let proto: PhysicalPlanNode =
- PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)
- .expect("serialization should succeed");
-
- // Deserialize
- let result: Arc<dyn ExecutionPlan> = proto
- .try_into_physical_plan(&ctx.task_ctx(), &codec)
- .expect("deserialization should succeed");
-
- // The deserialized plan should have lit(true) instead of HashTableLookupExpr
- // Verify the filter predicate is a Literal(true)
- let result_filter = result.downcast_ref::<FilterExec>().unwrap();
- let predicate = result_filter.predicate();
- let literal = predicate.downcast_ref::<Literal>().unwrap();
- assert_eq!(*literal.value(), ScalarValue::Boolean(Some(true)));
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_hash_expr() -> Result<()> {
- use datafusion::physical_plan::joins::{HashExpr, SeededRandomState};
-
- let schema = Arc::new(Schema::new(vec![
- Field::new("a", DataType::Int64, false),
- Field::new("b", DataType::Utf8, false),
- ]));
-
- // Create a HashExpr with test columns and seeds
- let on_columns = vec![col("a", &schema)?, col("b", &schema)?];
- let hash_expr: Arc<dyn PhysicalExpr> = Arc::new(HashExpr::new(
- on_columns,
- SeededRandomState::with_seed(0), // arbitrary random seed for testing
- "test_hash".to_string(),
- ));
-
- // Wrap in a filter by comparing hash value to a literal
- // hash_expr > 0 is always boolean
- let filter_expr = binary(hash_expr, Operator::Gt, lit(0u64), &schema)?;
- let filter = Arc::new(FilterExec::try_new(
- filter_expr,
- Arc::new(EmptyExec::new(schema)),
- )?);
-
- // Confirm that the debug string contains the random state seeds
- assert!(
- format!("{filter:?}").contains("test_hash(a@0, b@1, [0])"),
- "Debug string missing seeds: {filter:?}"
- );
- roundtrip_test(filter)
-}
-
-#[test]
-fn custom_proto_converter_intercepts() -> Result<()> {
- #[derive(Default)]
- struct CustomConverterInterceptor {
- num_proto_plans: RwLock<usize>,
- num_physical_plans: RwLock<usize>,
- num_proto_exprs: RwLock<usize>,
- num_physical_exprs: RwLock<usize>,
- }
-
- impl PhysicalProtoConverterExtension for CustomConverterInterceptor {
- fn proto_to_execution_plan(
- &self,
- proto: &PhysicalPlanNode,
- ctx: &PhysicalPlanDecodeContext<'_>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- {
- let mut counter = self
- .num_proto_plans
- .write()
- .map_err(|err| exec_datafusion_err!("{err}"))?;
- *counter += 1;
- }
- self.default_proto_to_execution_plan(proto, ctx)
- }
-
- fn execution_plan_to_proto(
- &self,
- plan: &Arc<dyn ExecutionPlan>,
- codec: &dyn PhysicalExtensionCodec,
- ) -> Result<PhysicalPlanNode>
- where
- Self: Sized,
- {
- {
- let mut counter = self
- .num_physical_plans
- .write()
- .map_err(|err| exec_datafusion_err!("{err}"))?;
- *counter += 1;
- }
- PhysicalPlanNode::try_from_physical_plan_with_converter(
- Arc::clone(plan),
- codec,
- self,
- )
- }
-
- fn proto_to_physical_expr(
- &self,
- proto: &PhysicalExprNode,
- input_schema: &Schema,
- ctx: &PhysicalPlanDecodeContext<'_>,
- ) -> Result<Arc<dyn PhysicalExpr>>
- where
- Self: Sized,
- {
- {
- let mut counter = self
- .num_proto_exprs
- .write()
- .map_err(|err| exec_datafusion_err!("{err}"))?;
- *counter += 1;
- }
- self.default_proto_to_physical_expr(proto, input_schema, ctx)
- }
-
- fn physical_expr_to_proto(
- &self,
- expr: &Arc<dyn PhysicalExpr>,
- codec: &dyn PhysicalExtensionCodec,
- ) -> Result<PhysicalExprNode> {
- {
- let mut counter = self
- .num_physical_exprs
- .write()
- .map_err(|err| exec_datafusion_err!("{err}"))?;
- *counter += 1;
- }
- serialize_physical_expr_with_converter(expr, codec, self)
- }
- }
-
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
- let sort_exprs = [
- PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: true,
- nulls_first: false,
- },
- },
- PhysicalSortExpr {
- expr: col("b", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: true,
- },
- },
- ]
- .into();
-
- let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = CustomConverterInterceptor::default();
- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
-
- assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);
- assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);
- assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);
- assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {
- let data_type = DataType::Struct(Fields::from(vec![Field::new(
- "item",
- DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
- true,
- )]));
-
- let schema = Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), true)]));
- let scan = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let scalar = lit(ScalarValue::try_from(data_type)?);
- let filter = Arc::new(FilterExec::try_new(
- Arc::new(BinaryExpr::new(scalar, Operator::Eq, col("a", &schema)?)),
- scan,
- )?);
-
- roundtrip_test(filter)
-}
-
-/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.
-fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {
- Arc::new(DynamicFilterPhysicalExpr::new(
- vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
- lit(true),
- )) as Arc<dyn PhysicalExpr>
-}
-
-/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.
-fn make_reassigned_dynamic_filter(
- filter: Arc<dyn PhysicalExpr>,
-) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("b", DataType::Int64, false),
- Field::new("a", DataType::Int64, false),
- ]));
- let reassigned = reassign_expr_columns(filter, &schema)?;
- Ok((schema, reassigned))
-}
-
-/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the
-/// default serializer from `PhysicalExpr::expression_id`.
-fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {
- expr.expr_id
- .expect("expected PhysicalExprNode.expr_id to be populated")
-}
-
-/// Roundtrip a single physical expression shaped like so:
-///
-/// ```text
-/// BinaryExpr(AND)
-/// / \
-/// filter_expr_1 filter_expr_2
-/// ```
-///
-/// Returns filter_expr_1 and filter_expr_2 after deserialization.
-fn roundtrip_dynamic_filter_expr_pair(
- filter_expr_1: Arc<dyn PhysicalExpr>,
- filter_expr_2: Arc<dyn PhysicalExpr>,
- schema: Arc<Schema>,
-) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {
- let pair_expr = Arc::new(BinaryExpr::new(
- Arc::clone(&filter_expr_1),
- Operator::And,
- Arc::clone(&filter_expr_2),
- )) as Arc<dyn PhysicalExpr>;
-
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;
- let ctx = SessionContext::new();
- let task_ctx = ctx.task_ctx();
- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
- let deserialized_expr =
- converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
-
- let binary = deserialized_expr
- .downcast_ref::<BinaryExpr>()
- .expect("Expected BinaryExpr");
-
- Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))
-}
-
-/// Roundtrip an execution plan shaped like so:
-///
-/// ```text
-/// FilterExec(dynamic_filter_1 on a@0)
-/// ProjectionExec(a := Column("a", source_index))
-/// DataSourceExec
-/// ParquetSource(predicate = dynamic_filter_2)
-/// ```
-///
-/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with
-/// different children.
-///
-/// Returns
-/// - `dynamic_filter_1` before serialization
-/// - `dynamic_filter_2` before serialization
-/// - `dynamic_filter_1` after serialization
-/// - `dynamic_filter_2` after serialization
-#[expect(clippy::type_complexity)]
-fn roundtrip_dynamic_filter_plan_pair() -> Result<(
- Arc<dyn PhysicalExpr>,
- Arc<dyn PhysicalExpr>,
- Arc<dyn PhysicalExpr>,
- Arc<dyn PhysicalExpr>,
-)> {
- let filter_expr_1 = make_dynamic_filter();
- let (data_source_schema, filter_expr_2) =
- make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;
- let left_before = Arc::clone(&filter_expr_1);
- let right_before = Arc::clone(&filter_expr_2);
- let file_source = Arc::new(
- ParquetSource::new(Arc::clone(&data_source_schema))
- .with_predicate(Arc::clone(&filter_expr_2)),
- );
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .build();
- let data_source_exec =
- DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;
-
- let projection_exec = Arc::new(ProjectionExec::try_new(
- vec![ProjectionExpr {
- expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,
- alias: "a".to_string(),
- }],
- data_source_exec,
- )?) as Arc<dyn ExecutionPlan>;
- let filter_exec = Arc::new(FilterExec::try_new(
- Arc::clone(&filter_expr_1),
- projection_exec,
- )?) as Arc<dyn ExecutionPlan>;
-
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;
-
- let ctx = SessionContext::new();
- let task_ctx = ctx.task_ctx();
- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
- let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;
-
- let outer_filter = deserialized_plan
- .downcast_ref::<FilterExec>()
- .expect("Expected outer FilterExec");
- let left_filter = Arc::clone(outer_filter.predicate());
- let projection = outer_filter.children()[0]
- .downcast_ref::<ProjectionExec>()
- .expect("Expected ProjectionExec");
- let data_source = projection
- .input()
- .downcast_ref::<DataSourceExec>()
- .expect("Expected DataSourceExec");
- let scan_config = data_source
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .expect("Expected FileScanConfig");
- let right_filter = scan_config
- .file_source()
- .filter()
- .expect("Expected pushed-down predicate");
-
- Ok((left_before, right_before, left_filter, right_filter))
-}
-
-/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible
-/// via the other. This helps assert that referential integrity is maintained after
-/// deserializing.
-fn assert_dynamic_filter_update_is_visible(
- left_filter: &Arc<dyn PhysicalExpr>,
- right_filter: &Arc<dyn PhysicalExpr>,
-) -> Result<()> {
- let left_filter = left_filter
- .downcast_ref::<DynamicFilterPhysicalExpr>()
- .expect("Expected dynamic filter");
- let right_filter = right_filter
- .downcast_ref::<DynamicFilterPhysicalExpr>()
- .expect("Expected dynamic filter");
-
- // Sanity check that the filters have the same generation.
- let original_generation = left_filter.snapshot_generation();
- assert_eq!(original_generation, right_filter.snapshot_generation(),);
-
- left_filter.update(lit(123_i64))?;
-
- // Assert that both generations updated.
- assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);
- assert_eq!(
- left_filter.snapshot_generation(),
- right_filter.snapshot_generation(),
- );
-
- // Ensure both filters have the updated expr.
- let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;
- assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);
- assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);
-
- Ok(())
-}
-
-/// Extract the dynamic-filter predicate that was pushed down to the parquet
-/// scan at the bottom of the plan tree.
-fn parquet_source_predicate(child: &Arc<dyn ExecutionPlan>) -> Arc<dyn PhysicalExpr> {
- let data_source = child
- .downcast_ref::<DataSourceExec>()
- .expect("Child should be DataSourceExec");
- let (_, parquet_source) = data_source
- .downcast_to_file_source::<ParquetSource>()
- .expect("Should be ParquetSource");
- parquet_source
- .filter()
- .expect("ParquetSource should have a predicate after roundtrip")
-}
-
-/// Assert that two dynamic filters are equal both structurally (Debug output)
-/// and by identity (`expression_id`).
-fn assert_dynamic_filters_equal(
- expected: &Arc<dyn PhysicalExpr>,
- actual: &Arc<dyn PhysicalExpr>,
-) {
- // Structural.
- let expected_dbg = format!("{expected:?}");
- let actual_dbg = format!("{actual:?}");
- if expected_dbg == actual_dbg {
- return;
- }
-
- // Note that the `DeduplicatingDeserializer` routes every cache hit through
- // `with_new_children`. This produces an equivalent expression, but with
- // remapped children that are equal to the original. Handle that case here.
- let rewritten = Arc::clone(expected)
- .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())
- .expect("with_new_children on a dynamic filter should not fail");
- assert_eq!(format!("{rewritten:?}"), actual_dbg);
-}
-
-// Two clones of a dynamic filter expression should be deduped to the exact same expression.
-#[test]
-fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let filter_expr_1 = make_dynamic_filter();
- let filter_expr_2 = Arc::clone(&filter_expr_1);
-
- let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =
- roundtrip_dynamic_filter_expr_pair(
- Arc::clone(&filter_expr_1),
- Arc::clone(&filter_expr_2),
- schema,
- )?;
-
- // Assert the filters are not modified during roundtrip.
- assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
- assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
- assert_dynamic_filters_equal(
- &filter_expr_1_after_roundtrip,
- &filter_expr_2_after_roundtrip,
- );
-
- // Assert referential integrity.
- assert_dynamic_filter_update_is_visible(
- &filter_expr_1_after_roundtrip,
- &filter_expr_2_after_roundtrip,
- )?;
-
- Ok(())
-}
-
-/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter
-/// with different children.
-#[test]
-fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {
- let (
- filter_expr_1,
- filter_expr_2,
- filter_expr_1_after_roundtrip,
- filter_expr_2_after_roundtrip,
- ) = roundtrip_dynamic_filter_plan_pair()?;
-
- // Assert the filters are not modified during roundtrip.
- assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
- assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
-
- // Assert referential integrity.
- assert_dynamic_filter_update_is_visible(
- &filter_expr_1_after_roundtrip,
- &filter_expr_2_after_roundtrip,
- )?;
-
- Ok(())
-}
-
-#[test]
-fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {
- let filter_expr = make_dynamic_filter();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
-
- let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
- let expr_id1 = proto_expression_id(&proto1);
-
- let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
- let expr_id2 = proto_expression_id(&proto2);
-
- assert_eq!(
- expr_id1, expr_id2,
- "Expected the same dynamic filter expression id across serializations"
- );
-
- Ok(())
-}
-
-/// Tests that `lead` window function with offset and default value args
-/// survives a protobuf round-trip. This is a regression test for a bug
-/// where `expressions()` (used during serialization) returns only the
-/// column expression for lead/lag, silently dropping the offset and
-/// default value literal args.
-#[test]
-fn roundtrip_lead_with_default_value() -> Result<()> {
- use datafusion::functions_window::lead_lag::lead_udwf;
-
- let field_a = Field::new("a", DataType::Int64, false);
- let field_b = Field::new("b", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b]));
-
- // lead(a, 2, 42) — column a, offset 2, default value 42
- let lead_window = create_udwf_window_expr(
- &lead_udwf(),
- &[col("a", &schema)?, lit(2i64), lit(42i64)],
- schema.as_ref(),
- "test lead with default".to_string(),
- false,
- )?;
-
- let udwf_expr = Arc::new(StandardWindowExpr::new(
- lead_window,
- &[col("b", &schema)?],
- &[PhysicalSortExpr {
- expr: col("a", &schema)?,
- options: SortOptions {
- descending: false,
- nulls_first: false,
- },
- }],
- Arc::new(WindowFrame::new(None)),
- ));
-
- let input = Arc::new(EmptyExec::new(schema.clone()));
-
- roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(
- vec![udwf_expr],
- input,
- InputOrderMode::Sorted,
- true,
- )?))
-}
-
-/// Verify that ScalarSubqueryExpr nodes in the input plan are connected to the
-/// same shared results container as ScalarSubqueryExec after a proto round-trip.
-#[test]
-fn roundtrip_scalar_subquery_exec() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let results = ScalarSubqueryResults::new(1);
-
- // Build the input plan: a filter whose predicate references the
- // scalar subquery result via ScalarSubqueryExpr.
- let sq_expr = Arc::new(ScalarSubqueryExpr::new(
- DataType::Int64,
- true,
- SubqueryIndex::new(0),
- results.clone(),
- ));
- let predicate = binary(col("a", &schema)?, Operator::Eq, sq_expr, &schema)?;
- let filter =
- FilterExec::try_new(predicate, Arc::new(EmptyExec::new(schema.clone())))?;
-
- // Build a trivial subquery plan.
- let subquery_plan =
- Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(
- "x",
- DataType::Int64,
- true,
- )]))));
-
- let exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
- Arc::new(filter),
- vec![ScalarSubqueryLink {
- plan: subquery_plan,
- index: SubqueryIndex::new(0),
- }],
- results,
- ));
-
- // Perform the round-trip using DeduplicatingProtoConverter, which
- // creates a DeduplicatingDeserializer that threads scalar subquery
- // results through expression deserialization.
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec),
- &codec,
- &converter,
- )?;
- let ctx = SessionContext::new();
- let deserialized = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &converter,
- )?;
-
- // Verify the deserialized ScalarSubqueryExec's results container is
- // shared with the ScalarSubqueryExpr in the input plan.
- let sq_exec = deserialized
- .downcast_ref::<ScalarSubqueryExec>()
- .expect("expected ScalarSubqueryExec");
- let exec_results = sq_exec.results();
-
- // Walk the input plan to find the ScalarSubqueryExpr and verify it
- // points to the same results container.
- let filter_exec = sq_exec
- .input()
- .downcast_ref::<FilterExec>()
- .expect("expected FilterExec");
- let binary_expr = filter_exec
- .predicate()
- .downcast_ref::<BinaryExpr>()
- .expect("expected BinaryExpr");
- let deserialized_sq_expr = binary_expr
- .right()
- .downcast_ref::<ScalarSubqueryExpr>()
- .expect("expected ScalarSubqueryExpr");
-
- assert!(
- ScalarSubqueryResults::ptr_eq(exec_results, deserialized_sq_expr.results()),
- "ScalarSubqueryExpr should share the same results container as ScalarSubqueryExec"
- );
- Ok(())
-}
-
-/// Verify that nested ScalarSubqueryExec nodes deserialize with distinct
-/// scoped results containers, and that each ScalarSubqueryExpr is wired to the
-/// container for its own surrounding ScalarSubqueryExec.
-#[test]
-fn roundtrip_nested_scalar_subquery_exec_scopes_results() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let subquery_schema =
- Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
-
- let inner_results = ScalarSubqueryResults::new(1);
- let inner_sq_expr = Arc::new(ScalarSubqueryExpr::new(
- DataType::Int64,
- true,
- SubqueryIndex::new(0),
- inner_results.clone(),
- ));
- let inner_predicate =
- binary(col("a", &schema)?, Operator::Eq, inner_sq_expr, &schema)?;
- let inner_filter = Arc::new(FilterExec::try_new(
- inner_predicate,
- Arc::new(EmptyExec::new(schema.clone())),
- )?);
- let inner_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
- inner_filter,
- vec![ScalarSubqueryLink {
- plan: Arc::new(EmptyExec::new(subquery_schema.clone())),
- index: SubqueryIndex::new(0),
- }],
- inner_results,
- ));
-
- let outer_results = ScalarSubqueryResults::new(1);
- let outer_sq_expr = Arc::new(ScalarSubqueryExpr::new(
- DataType::Int64,
- true,
- SubqueryIndex::new(0),
- outer_results.clone(),
- ));
- let outer_predicate =
- binary(col("a", &schema)?, Operator::Eq, outer_sq_expr, &schema)?;
- let outer_filter = Arc::new(FilterExec::try_new(outer_predicate, inner_exec)?);
- let outer_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(
- outer_filter,
- vec![ScalarSubqueryLink {
- plan: Arc::new(EmptyExec::new(subquery_schema)),
- index: SubqueryIndex::new(0),
- }],
- outer_results,
- ));
-
- let bytes = datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&outer_exec))?;
- let ctx = SessionContext::new();
- let deserialized = datafusion_proto::bytes::physical_plan_from_bytes(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- )?;
-
- let outer_exec = deserialized
- .downcast_ref::<ScalarSubqueryExec>()
- .expect("expected outer ScalarSubqueryExec");
- let outer_results = outer_exec.results();
- let outer_filter = outer_exec
- .input()
- .downcast_ref::<FilterExec>()
- .expect("expected outer FilterExec");
- let outer_binary = outer_filter
- .predicate()
- .downcast_ref::<BinaryExpr>()
- .expect("expected outer BinaryExpr");
- let outer_sq_expr = outer_binary
- .right()
- .downcast_ref::<ScalarSubqueryExpr>()
- .expect("expected outer ScalarSubqueryExpr");
-
- let inner_exec = outer_filter
- .input()
- .downcast_ref::<ScalarSubqueryExec>()
- .expect("expected inner ScalarSubqueryExec");
- let inner_results = inner_exec.results();
- let inner_filter = inner_exec
- .input()
- .downcast_ref::<FilterExec>()
- .expect("expected inner FilterExec");
- let inner_binary = inner_filter
- .predicate()
- .downcast_ref::<BinaryExpr>()
- .expect("expected inner BinaryExpr");
- let inner_sq_expr = inner_binary
- .right()
- .downcast_ref::<ScalarSubqueryExpr>()
- .expect("expected inner ScalarSubqueryExpr");
-
- assert!(
- ScalarSubqueryResults::ptr_eq(outer_results, outer_sq_expr.results()),
- "outer ScalarSubqueryExpr should use outer ScalarSubqueryExec results"
- );
- assert!(
- ScalarSubqueryResults::ptr_eq(inner_results, inner_sq_expr.results()),
- "inner ScalarSubqueryExpr should use inner ScalarSubqueryExec results"
- );
- assert!(
- !ScalarSubqueryResults::ptr_eq(outer_results, inner_results),
- "nested ScalarSubqueryExec nodes should not share results containers"
- );
- assert!(
- !ScalarSubqueryResults::ptr_eq(outer_results, inner_sq_expr.results()),
- "inner ScalarSubqueryExpr must not read from outer results"
- );
- assert!(
- !ScalarSubqueryResults::ptr_eq(inner_results, outer_sq_expr.results()),
- "outer ScalarSubqueryExpr must not read from inner results"
- );
-
- Ok(())
-}
-
-/// Verify that the default physical plan bytes round-trip preserves executable
-/// scalar subquery plans.
-#[tokio::test]
-async fn roundtrip_scalar_subquery_exec_with_default_converter_executes() -> Result<()> {
- let ctx = SessionContext::new();
- let sql = "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) AS s \
- FROM (VALUES (2), (1)) AS t(x) \
- ORDER BY s";
-
- let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;
- assert!(
- format!("{initial_plan:?}").contains("ScalarSubqueryExec"),
- "expected ScalarSubqueryExec in plan:\n{initial_plan:?}"
- );
-
- let bytes =
- datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&initial_plan))?;
- let roundtripped = datafusion_proto::bytes::physical_plan_from_bytes(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- )?;
- assert!(
- format!("{roundtripped:?}").contains("ScalarSubqueryExec"),
- "expected ScalarSubqueryExec after roundtrip:\n{roundtripped:?}"
- );
-
- let batches = datafusion::physical_plan::common::collect(
- roundtripped.execute(0, ctx.task_ctx())?,
- )
- .await?;
- datafusion::assert_batches_eq!(
- &["+----+", "| s |", "+----+", "| 21 |", "| 22 |", "+----+",],
- &batches
- );
-
- Ok(())
-}
-
-/// Test that a chain of the same operator (a AND b AND c) is linearized
-/// and roundtrips correctly.
-#[test]
-fn roundtrip_binary_expr_chain_same_op() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Boolean, false);
- let field_c = Field::new("c", DataType::Boolean, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
- let ab = binary(
- col("a", &schema)?,
- Operator::And,
- col("b", &schema)?,
- &schema,
- )?;
- let abc = binary(ab, Operator::And, col("c", &schema)?, &schema)?;
- roundtrip_test(Arc::new(FilterExec::try_new(
- abc,
- Arc::new(EmptyExec::new(schema)),
- )?))
-}
-
-/// Test that mixed operators (a AND b OR c) are NOT linearized together —
-/// only chains of the same operator are flattened.
-#[test]
-fn roundtrip_binary_expr_mixed_ops() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Boolean, false);
- let field_c = Field::new("c", DataType::Boolean, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));
- // (a AND b) OR c — AND and OR are different operators, so linearization stops
- let a_and_b = binary(
- col("a", &schema)?,
- Operator::And,
- col("b", &schema)?,
- &schema,
- )?;
- let expr = binary(a_and_b, Operator::Or, col("c", &schema)?, &schema)?;
- roundtrip_test(Arc::new(FilterExec::try_new(
- expr,
- Arc::new(EmptyExec::new(schema)),
- )?))
-}
-
-/// Test that a deeply nested chain of AND expressions (like many WHERE conditions)
-/// roundtrips correctly. This is the scenario from issue #18602.
-#[test]
-fn roundtrip_binary_expr_deeply_nested_and_chain() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
-
- // Build a chain: a AND a AND a AND ... (100 times)
- let col_a = col("a", &schema)?;
- let mut expr = Arc::clone(&col_a);
- for _ in 0..99 {
- expr = binary(expr, Operator::And, Arc::clone(&col_a), &schema)?;
- }
-
- roundtrip_test(Arc::new(FilterExec::try_new(
- expr,
- Arc::new(EmptyExec::new(schema)),
- )?))
-}
-
-/// Test that a deeply nested chain of OR expressions roundtrips correctly.
-#[test]
-fn roundtrip_binary_expr_deeply_nested_or_chain() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
-
- let col_a = col("a", &schema)?;
- let mut expr = Arc::clone(&col_a);
- for _ in 0..99 {
- expr = binary(expr, Operator::Or, Arc::clone(&col_a), &schema)?;
- }
-
- roundtrip_test(Arc::new(FilterExec::try_new(
- expr,
- Arc::new(EmptyExec::new(schema)),
- )?))
-}
-
-/// Test that alternating AND/OR operators produce correct results —
-/// each sub-chain gets linearized independently.
-#[test]
-fn roundtrip_binary_expr_alternating_and_or() -> Result<()> {
- let field_a = Field::new("a", DataType::Boolean, false);
- let field_b = Field::new("b", DataType::Boolean, false);
- let field_c = Field::new("c", DataType::Boolean, false);
- let field_d = Field::new("d", DataType::Boolean, false);
- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c, field_d]));
-
- // (a AND b) OR (c AND d)
- let a_and_b = binary(
- col("a", &schema)?,
- Operator::And,
- col("b", &schema)?,
- &schema,
- )?;
- let c_and_d = binary(
- col("c", &schema)?,
- Operator::And,
- col("d", &schema)?,
- &schema,
- )?;
- let expr = binary(a_and_b, Operator::Or, c_and_d, &schema)?;
-
- roundtrip_test(Arc::new(FilterExec::try_new(
- expr,
- Arc::new(EmptyExec::new(schema)),
- )?))
-}
-
-/// Verify that the linearized proto format has a flat operands list
-/// rather than deeply nested l/r fields.
-#[test]
-fn test_linearization_produces_flat_operands() -> Result<()> {
- // Build: a AND a AND a AND a (4 operands, 3 levels of nesting)
- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
- let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- Arc::new(BinaryExpr::new(
- Arc::new(BinaryExpr::new(
- Arc::clone(&col_a),
- Operator::And,
- Arc::clone(&col_a),
- )),
- Operator::And,
- Arc::clone(&col_a),
- )),
- Operator::And,
- Arc::clone(&col_a),
- ));
-
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;
-
- // The top-level should use the operands field with 4 entries
- match &proto.expr_type {
- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {
- assert!(
- b.l.is_none(),
- "l should be None when using linearized operands"
- );
- assert!(
- b.r.is_none(),
- "r should be None when using linearized operands"
- );
- assert_eq!(
- b.operands.len(),
- 4,
- "Expected 4 linearized operands for a AND a AND a AND a"
- );
- assert_eq!(b.op, "And");
- }
- other => panic!("Expected BinaryExpr, got {other:?}"),
- }
-
- Ok(())
-}
-
-/// Test that linearization stops when encountering a different operator.
-/// For (a AND b) OR c, only the top-level OR should be represented, and
-/// the left-hand AND subtree should be a separate nested BinaryExpr.
-#[test]
-fn test_linearization_stops_at_different_op() -> Result<()> {
- // (a AND b) OR c
- let a_and_b: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- Arc::new(Column::new("a", 0)),
- Operator::And,
- Arc::new(Column::new("b", 1)),
- ));
- let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- a_and_b,
- Operator::Or,
- Arc::new(Column::new("c", 2)),
- ));
-
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;
-
- // The top-level OR should have only 2 operands (can't linearize through AND)
- match &proto.expr_type {
- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {
- assert_eq!(
- b.operands.len(),
- 2,
- "Expected 2 operands for (a AND b) OR c"
- );
- assert_eq!(b.op, "Or");
- // The first operand should be a nested AND BinaryExpr
- match &b.operands[0].expr_type {
- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(inner)) => {
- assert_eq!(inner.op, "And");
- assert_eq!(inner.operands.len(), 2);
- }
- other => panic!("Expected inner BinaryExpr(AND), got {other:?}"),
- }
- }
- other => panic!("Expected BinaryExpr, got {other:?}"),
- }
-
- Ok(())
-}
-
-/// Create a DataSourceExec backed by a ParquetSource that accepts filter pushdown,
-/// along with a ConfigOptions that enables all dynamic filter pushdown options.
-fn datasource_for_dynamic_filter_pushdown(
- schema: &Arc<Schema>,
-) -> (Arc<dyn ExecutionPlan>, ConfigOptions) {
- let mut parquet_options = TableParquetOptions::new();
- parquet_options.global.pushdown_filters = true;
- let source = Arc::new(
- ParquetSource::new(Arc::clone(schema))
- .with_table_parquet_options(parquet_options),
- );
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)
- .with_file(PartitionedFile::new("/path/to/file.parquet", 1024))
- .build();
-
- let mut config = ConfigOptions::default();
- config.execution.parquet.pushdown_filters = true;
- config.optimizer.enable_join_dynamic_filter_pushdown = true;
- config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
- config.optimizer.enable_topk_dynamic_filter_pushdown = true;
-
- (DataSourceExec::from_data_source(scan_config), config)
-}
-
-/// Test that plan containing a HashJoinExec with dynamic filter pushdown
-/// can be serialized and deserialized while preserving references to the dynamic filter.
-#[test]
-fn test_hash_join_with_dynamic_filter_roundtrip() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));
-
- let left_child = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let (right_child, config) = datasource_for_dynamic_filter_pushdown(&schema);
-
- let on: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> = vec![(
- Arc::new(Column::new("col", 0)),
- Arc::new(Column::new("col", 0)),
- )];
-
- let hash_join = Arc::new(HashJoinExec::try_new(
- left_child,
- right_child,
- on,
- None,
- &JoinType::Inner,
- None,
- PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- false,
- )?) as Arc<dyn ExecutionPlan>;
-
- // Run the optimizer rule for filter pushdown.
- let optimizer = FilterPushdown::new_post_optimization();
- let plan = optimizer.optimize(hash_join, &config)?;
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let deserialized = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;
-
- // Extract the deserialized HashJoinExec and its dynamic filter.
- let deserialized_join = deserialized
- .downcast_ref::<HashJoinExec>()
- .expect("Should be HashJoinExec");
- let deserialized_hash_join_df = deserialized_join
- .dynamic_expressions_produced()
- .into_iter()
- .next()
- .expect("HashJoinExec should have a dynamic filter after roundtrip");
-
- // Extract the dynamic filter pushed down to the probe side's ParquetSource.
- let deserialized_predicate = parquet_source_predicate(deserialized_join.right());
-
- // The HashJoinExec's dynamic filter and the probe side's predicate should
- // refer to the same underlying expression.
- let plan_df = deserialized_hash_join_df;
- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
-
- Ok(())
-}
-
-/// returns a SessionContext with an empty `netflow` table registered
-fn netflow_context() -> Result<SessionContext> {
- let ctx = SessionContext::new();
- let schema = Arc::new(Schema::new(vec![
- Field::new("dst_geo_country_name", DataType::Utf8, true),
- Field::new("dst_geo_city_name", DataType::Utf8, true),
- Field::new("packets", DataType::UInt64, true),
- Field::new("src_addr", DataType::Utf8, true),
- Field::new("dst_addr", DataType::Utf8, true),
- ]));
-
- ctx.register_table("netflow", Arc::new(EmptyTable::new(schema)))?;
-
- Ok(ctx)
-}
-
-/// Regression test for issue #18602:
-/// https://github.com/apache/datafusion/issues/18602
-///
-/// The physical filter expression here contains a long chain of `AND` predicates.
-/// Before linearizing `PhysicalBinaryExprNode`, encoding then decoding the protobuf
-/// could fail with `DecodeError: recursion limit reached`.
-#[tokio::test]
-async fn roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {
- let ctx = netflow_context()?;
- let sql = "SELECT \
- dst_geo_country_name AS x_axis_1, \
- dst_geo_city_name AS x_axis_2, \
- sum(packets) AS y_axis_1 \
- FROM netflow \
- WHERE dst_geo_country_name IS NOT NULL \
- AND src_addr NOT LIKE '10.201.%' \
- AND dst_addr NOT LIKE '10.201.%' \
- AND src_addr NOT LIKE '10.202.%' \
- AND dst_addr NOT LIKE '10.202.%' \
- AND src_addr NOT LIKE '10.203.%' \
- AND dst_addr NOT LIKE '10.203.%' \
- AND src_addr NOT LIKE '10.204.%' \
- AND dst_addr NOT LIKE '10.204.%' \
- AND src_addr NOT LIKE '172.16.186.%' \
- AND dst_addr NOT LIKE '172.16.186.%' \
- AND src_addr NOT LIKE '172.16.187.%' \
- AND dst_addr NOT LIKE '172.16.187.%' \
- AND src_addr NOT LIKE '172.16.188.%' \
- AND dst_addr NOT LIKE '172.16.188.%' \
- AND src_addr NOT LIKE '10.102.45.%' \
- AND dst_addr NOT LIKE '10.102.45.%' \
- AND src_addr NOT LIKE '172.25.210.%' \
- AND dst_addr NOT LIKE '172.25.210.%' \
- AND src_addr NOT LIKE '172.25.211.%' \
- AND dst_addr NOT LIKE '172.25.211.%' \
- AND src_addr NOT LIKE '141.226.101.%' \
- AND dst_addr NOT LIKE '141.226.101.%' \
- AND src_addr NOT LIKE '167.86.40.%' \
- AND dst_addr NOT LIKE '167.86.40.%' \
- AND src_addr NOT LIKE '66.22.38.%' \
- AND dst_addr NOT LIKE '66.22.38.%' \
- AND src_addr != '168.143.191.55' \
- AND dst_addr != '168.143.191.55' \
- AND src_addr != '82.112.107.142' \
- AND dst_addr != '82.112.107.142' \
- AND src_addr != '20.76.39.176' \
- AND dst_addr != '20.76.39.176' \
- AND src_addr != '162.159.129.83' \
- AND dst_addr != '162.159.129.83' \
- AND src_addr != '34.201.223.155' \
- AND dst_addr != '34.201.223.155' \
- AND src_addr != '34.201.223.156' \
- AND dst_addr != '34.201.223.156' \
- AND src_addr != '34.201.223.157' \
- AND dst_addr != '34.201.223.157' \
- AND src_addr != '134.201.223.157' \
- AND dst_addr != '134.201.223.157' \
- AND src_addr != '341.201.223.157' \
- AND dst_addr != '341.201.223.157' \
- GROUP BY x_axis_1, x_axis_2 \
- ORDER BY y_axis_1 DESC \
- LIMIT 20";
-
- roundtrip_test_sql_with_context(sql, &ctx).await
-}
-
-/// Test that plan containing a AggregateExec with dynamic filter pushdown
-/// can be serialized and deserialized while preserving references to the dynamic filter.
-#[test]
-fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
-
- let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);
-
- let agg = Arc::new(AggregateExec::try_new(
- AggregateMode::Partial,
- PhysicalGroupBy::new_single(vec![]),
- vec![
- AggregateExprBuilder::new(
- datafusion::functions_aggregate::min_max::min_udaf(),
- vec![Arc::clone(&col_a)],
- )
- .schema(Arc::clone(&schema))
- .alias("min_a")
- .build()
- .map(Arc::new)?,
- ],
- vec![None],
- child,
- Arc::clone(&schema),
- )?) as Arc<dyn ExecutionPlan>;
-
- // Run the optimizer rule for filter pushdown.
- let optimizer = FilterPushdown::new_post_optimization();
- let plan = optimizer.optimize(agg, &config)?;
-
- // Roundtrip with deduplication.
- //
- // Note: We don't use `roundtrip_test_and_return` here because there's a
- // pre-existing issue with PhysicalGroupBy serialization where empty groups
- // `[[]]` become `[]` after roundtrip. This behavior is unrelated to this test.
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&plan),
- &codec,
- &converter,
- )?;
- let deserialized = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &converter,
- )?;
-
- // Extract the deserialized AggregateExec and its dynamic filter.
- let deserialized_agg = deserialized
- .downcast_ref::<AggregateExec>()
- .expect("Should be AggregateExec");
- let deserialized_agg_df = deserialized_agg
- .dynamic_expressions_produced()
- .into_iter()
- .next()
- .expect("AggregateExec should have a dynamic filter after roundtrip");
-
- // Extract the dynamic filter pushed down to the child ParquetSource.
- let deserialized_predicate = parquet_source_predicate(deserialized_agg.input());
-
- // The AggregateExec's dynamic filter and the child's predicate should
- // refer to the same underlying expression.
- let plan_df = deserialized_agg_df;
- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
-
- Ok(())
-}
-
-#[test]
-fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
- let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let aggregate = Arc::new(AggregateExec::try_new(
- AggregateMode::Partial,
- PhysicalGroupBy::new_single(vec![]),
- vec![
- AggregateExprBuilder::new(
- datafusion::functions_aggregate::min_max::min_udaf(),
- vec![col_a],
- )
- .schema(Arc::clone(&schema))
- .alias("min_a")
- .build()
- .map(Arc::new)?,
- ],
- vec![None],
- child,
- Arc::clone(&schema),
- )?) as Arc<dyn ExecutionPlan>;
-
- let mut config = ConfigOptions::default();
- config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
- let optimizer = FilterPushdown::new_post_optimization();
- let plan = optimizer.optimize(aggregate, &config)?;
- assert!(
- plan.downcast_ref::<AggregateExec>()
- .expect("Should be AggregateExec")
- .dynamic_expressions_produced()
- .is_empty()
- );
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DefaultPhysicalProtoConverter {};
- let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;
- let deserialized = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &converter,
- )?;
-
- assert!(
- deserialized
- .downcast_ref::<AggregateExec>()
- .expect("Should be AggregateExec")
- .dynamic_expressions_produced()
- .is_empty()
- );
- Ok(())
-}
-
-/// Test that plan containing a SortExec with dynamic filter pushdown
-/// can be serialized and deserialized while preserving references to the dynamic filter.
-#[test]
-fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
-
- let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);
-
- let sort = Arc::new(
- SortExec::new(
- LexOrdering::new(vec![PhysicalSortExpr {
- expr: Arc::clone(&col_a),
- options: SortOptions::default(),
- }])
- .unwrap(),
- child,
- )
- .with_fetch(Some(10)),
- ) as Arc<dyn ExecutionPlan>;
-
- // Verify the optimizer kept the dynamic filter on the SortExec.
- let optimizer = FilterPushdown::new_post_optimization();
- let plan = optimizer.optimize(sort, &config)?;
-
- // Roundtrip with deduplication.
- //
- // Note: We don't use `roundtrip_test_and_return` here because
- // `DeduplicatingDeserializer` rewrites cache hits via `with_new_children`,
- // which sets `remapped_children: Some(...)` on the second encounter of a
- // shared `DynamicFilterPhysicalExpr`. SortExec's `Debug` includes its
- // dynamic filter, so the original-vs-deserialized structural equality check
- // would fail purely on this artifact.
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let converter = DeduplicatingProtoConverter {};
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&plan),
- &codec,
- &converter,
- )?;
- let deserialized = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &converter,
- )?;
-
- // Extract the deserialized SortExec and its dynamic filter.
- let deserialized_sort = deserialized
- .downcast_ref::<SortExec>()
- .expect("Should be SortExec");
- let deserialized_sort_df = deserialized_sort
- .dynamic_expressions_produced()
- .into_iter()
- .next()
- .expect("SortExec should have a dynamic filter after roundtrip");
-
- // Extract the dynamic filter pushed down to the child ParquetSource.
- let deserialized_predicate = parquet_source_predicate(deserialized_sort.input());
-
- // The SortExec's dynamic filter and the child's predicate should
- // refer to the same underlying expression.
- let plan_df = deserialized_sort_df;
- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);
- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;
-
- Ok(())
-}
-
-/// A custom [`ExecutionPlan`] which stores [`PhysicalExpr`]s.
-struct CustomExecWithExprs {
- exprs: Vec<Arc<dyn PhysicalExpr>>,
- child: Arc<dyn ExecutionPlan>,
-}
-
-#[derive(Clone, PartialEq, Message)]
-struct CustomExecWithExprsProto {
- #[prost(message, repeated, tag = "1")]
- exprs: Vec<PhysicalExprNode>,
-}
-
-impl std::fmt::Debug for CustomExecWithExprs {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("CustomExecWithExprs")
- .field("exprs", &self.exprs)
- .field("child", &self.child)
- .finish()
- }
-}
-
-impl CustomExecWithExprs {
- fn new(exprs: Vec<Arc<dyn PhysicalExpr>>, child: Arc<dyn ExecutionPlan>) -> Self {
- Self { exprs, child }
- }
-}
-
-impl DisplayAs for CustomExecWithExprs {
- fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
- write!(f, "CustomExecWithExprs")
- }
-}
-
-impl ExecutionPlan for CustomExecWithExprs {
- fn name(&self) -> &str {
- "CustomExecWithExprs"
- }
-
- fn schema(&self) -> SchemaRef {
- self.child.schema()
- }
-
- fn properties(&self) -> &Arc<PlanProperties> {
- self.child.properties()
- }
-
- fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
- vec![&self.child]
- }
-
- fn apply_expressions(
- &self,
- f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
- ) -> Result<TreeNodeRecursion> {
- datafusion_physical_plan::apply_expression_roots(&self.exprs, f)
- }
-
- fn with_new_children(
- self: Arc<Self>,
- _children: Vec<Arc<dyn ExecutionPlan>>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- unreachable!()
- }
-
- fn execute(
- &self,
- _partition: usize,
- _context: Arc<TaskContext>,
- ) -> Result<SendableRecordBatchStream> {
- unreachable!()
- }
-}
-
-/// A [`PhysicalExtensionCodec`] for [`CustomExecWithExprs`].
-#[derive(Debug)]
-struct CustomExecWithExprsCodec {}
-
-impl PhysicalExtensionCodec for CustomExecWithExprsCodec {
- fn try_decode(
- &self,
- buf: &[u8],
- inputs: &[Arc<dyn ExecutionPlan>],
- ctx: &TaskContext,
- proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);
- let input_schema = inputs[0].schema();
- let proto = CustomExecWithExprsProto::decode(buf)
- .map_err(|e| internal_datafusion_err!("Failed to decode custom exec: {e}"))?;
- let exprs = proto
- .exprs
- .iter()
- .map(|expr_proto| {
- proto_converter.proto_to_physical_expr(
- expr_proto,
- input_schema.as_ref(),
- &decode_ctx,
- )
- })
- .collect::<Result<Vec<_>>>()?;
-
- Ok(Arc::new(CustomExecWithExprs::new(exprs, inputs[0].clone())))
- }
-
- fn try_encode(
- &self,
- node: Arc<dyn ExecutionPlan>,
- buf: &mut Vec<u8>,
- proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<()> {
- let custom = node
- .downcast_ref::<CustomExecWithExprs>()
- .ok_or_else(|| internal_datafusion_err!("Expected CustomExecWithExprs"))?;
- let proto = CustomExecWithExprsProto {
- exprs: custom
- .exprs
- .iter()
- .map(|expr| proto_converter.physical_expr_to_proto(expr, self))
- .collect::<Result<Vec<_>>>()?,
- };
- proto
- .encode(buf)
- .map_err(|e| internal_datafusion_err!("Failed to encode custom exec: {e}"))?;
-
- Ok(())
- }
-}
-
-/// Tests that a custom [`ExecutionPlan`] with [`PhysicalExpr`] can
-/// dedupe dynamic filters by using the proto converter in its
-/// [`PhysicalExtensionCodec`] implementation.
-#[test]
-fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {
- // Create the plan:
- //
- // FilterExec(dynamic_filter)
- // -> CustomExecWithExprs(exprs: [dynamic_filter])
- // -> EmptyExec
- //
- // The same dynamic filter expression is saved in both the FilterExec and CustomExecWithExprs.
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
- vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
- lit(true),
- ));
- let dynamic_filter_expr: Arc<dyn PhysicalExpr> = dynamic_filter;
-
- let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let custom_exec = Arc::new(CustomExecWithExprs::new(
- vec![Arc::clone(&dynamic_filter_expr)],
- empty,
- ));
- let filter_exec = Arc::new(FilterExec::try_new(
- Arc::clone(&dynamic_filter_expr),
- custom_exec,
- )?) as Arc<dyn ExecutionPlan>;
-
- // Roundtrip with DeduplicatingProtoConverter
- let codec = CustomExecWithExprsCodec {};
- let converter = DeduplicatingProtoConverter {};
-
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&filter_exec),
- &codec,
- &converter,
- )?;
-
- let ctx = SessionContext::new();
- let deser_converter = DeduplicatingProtoConverter {};
- let deserialized = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &deser_converter,
- )?;
-
- // Extract the deserialized FilterExec's dynamic filter
- let deser_filter = deserialized
- .downcast_ref::<FilterExec>()
- .expect("Top-level should be FilterExec");
- let deser_filter_df = deser_filter.predicate();
-
- // Extract the deserialized custom node's dynamic filter
- let deser_custom = deser_filter
- .input()
- .downcast_ref::<CustomExecWithExprs>()
- .expect("FilterExec child should be CustomExecWithExprs");
- assert_eq!(deser_custom.exprs.len(), 1, "Should have one expression");
- let [deser_custom_df] = deser_custom.exprs.as_slice() else {
- return internal_err!("Custom node should have one expression");
- };
-
- // Pass the un-remapped filter first so the helper's `with_new_children`
- // rewrite can reconstruct the remapped form on the other side.
- assert_dynamic_filters_equal(deser_custom_df, deser_filter_df);
- assert_dynamic_filter_update_is_visible(deser_custom_df, deser_filter_df)?;
-
- Ok(())
-}
-
-fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanConfig> {
- let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
- let result_plan =
- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
-
- let data_source_exec = result_plan
- .downcast_ref::<DataSourceExec>()
- .expect("Expected DataSourceExec");
- let file_scan_config = data_source_exec
- .data_source()
- .downcast_ref::<FileScanConfig>()
- .expect("Expected FileScanConfig");
- Ok(file_scan_config.clone())
-}
-
-#[test]
-fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
- let output_partitioning =
- Partitioning::Hash(vec![Arc::new(Column::new("col", 0))], 1);
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file.parquet".to_string(),
- 1024,
- )])])
- .with_output_partitioning(Some(output_partitioning.clone()))
- .build();
-
- assert_eq!(
- roundtrip_file_scan_config(scan_config)?.output_partitioning,
- Some(output_partitioning)
- );
-
- Ok(())
-}
-
-#[test]
-fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {
- let file_schema =
- Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)]));
- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));
- let output_partitioning = Partitioning::Range(RangePartitioning::new(
- LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(
- "col", 0,
- )))])
- .unwrap(),
- vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],
- ));
- let scan_config =
- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
- .with_file_groups(vec![
- FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file-1.parquet".to_string(),
- 1024,
- )]),
- FileGroup::new(vec![PartitionedFile::new(
- "/path/to/file-2.parquet".to_string(),
- 1024,
- )]),
- ])
- .with_output_partitioning(Some(output_partitioning.clone()))
- .build();
-
- assert_eq!(
- roundtrip_file_scan_config(scan_config)?.output_partitioning,
- Some(output_partitioning)
- );
-
- Ok(())
-}
-
-/// A custom `PhysicalExpr` whose extension codec embeds a nested
-/// `PhysicalExprNode` *inside its own blob* (rather than the standard
-/// `PhysicalExtensionExprNode.inputs` field). This is the case that only
-/// works if the expr-level codec methods receive the encode/decode context.
-#[derive(Debug)]
-struct WrapperExpr {
- inner: Arc<dyn PhysicalExpr>,
-}
-
-impl Display for WrapperExpr {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(f, "WrapperExpr({})", self.inner)
- }
-}
-
-impl PartialEq for WrapperExpr {
- fn eq(&self, other: &Self) -> bool {
- self.inner.eq(&other.inner)
- }
-}
-impl Eq for WrapperExpr {}
-
-impl std::hash::Hash for WrapperExpr {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.inner.hash(state);
- }
-}
-
-impl PhysicalExpr for WrapperExpr {
- fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
- self.inner.data_type(input_schema)
- }
- fn nullable(&self, input_schema: &Schema) -> Result<bool> {
- self.inner.nullable(input_schema)
- }
- fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {
- internal_err!("WrapperExpr is not executable in this test")
- }
- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
- vec![&self.inner]
- }
- fn with_new_children(
- self: Arc<Self>,
- children: Vec<Arc<dyn PhysicalExpr>>,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- Ok(Arc::new(WrapperExpr {
- inner: Arc::clone(&children[0]),
- }))
- }
- fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- Display::fmt(self, f)
- }
-}
-
-/// Wire layout for [`WrapperExpr`]: a single nested `PhysicalExprNode`.
-#[derive(Clone, PartialEq, prost::Message)]
-struct WrapperExprProto {
- #[prost(message, optional, boxed, tag = "1")]
- inner: Option<Box<PhysicalExprNode>>,
-}
-
-#[derive(Debug)]
-struct WrapperCodec;
-
-impl PhysicalExtensionCodec for WrapperCodec {
- fn try_decode(
- &self,
- _buf: &[u8],
- _inputs: &[Arc<dyn ExecutionPlan>],
- _ctx: &TaskContext,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- internal_err!("not used")
- }
- fn try_encode(
- &self,
- _node: Arc<dyn ExecutionPlan>,
- _buf: &mut Vec<u8>,
- _proto_converter: &dyn PhysicalProtoConverterExtension,
- ) -> Result<()> {
- internal_err!("not used")
- }
- fn try_decode_expr(
- &self,
- buf: &[u8],
- _inputs: &[Arc<dyn PhysicalExpr>],
- ctx: &PhysicalExprDecodeCtx<'_>,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- let proto = WrapperExprProto::decode(buf)
- .map_err(|e| internal_datafusion_err!("decode WrapperExprProto: {e}"))?;
- let inner_proto = proto
- .inner
- .ok_or_else(|| internal_datafusion_err!("missing inner"))?;
- // Decode the nested expr through the context so it resolves against
- // the real schema/registry AND participates in dedup — no fabricated
- // `SessionContext` or hard-coded schema required.
- let inner = ctx.decode(&inner_proto)?;
- Ok(Arc::new(WrapperExpr { inner }))
- }
- fn try_encode_expr(
- &self,
- node: &Arc<dyn PhysicalExpr>,
- buf: &mut Vec<u8>,
- ctx: &PhysicalExprEncodeCtx<'_>,
- ) -> Result<()> {
- let wrapper = node
- .downcast_ref::<WrapperExpr>()
- .ok_or_else(|| internal_datafusion_err!("not WrapperExpr"))?;
- // Encode the nested expr through the context so an active
- // `DeduplicatingProtoConverter` stamps a matching `expr_id`.
- let inner_proto = ctx.encode_child(&wrapper.inner)?;
- let proto = WrapperExprProto {
- inner: Some(Box::new(inner_proto)),
- };
- proto
- .encode(buf)
- .map_err(|e| internal_datafusion_err!("encode WrapperExprProto: {e}"))?;
- Ok(())
- }
-}
-
-/// A `DynamicFilterPhysicalExpr` referenced both as a bare expression and
-/// nested inside a custom expression's codec blob must reconstruct to a
-/// single shared `Inner` after roundtrip.
-///
-/// This exercises the expr-level codec hooks receiving the encode/decode
-/// context: `try_encode_expr` routes its nested `PhysicalExprNode` through
-/// `ctx.encode_child` and `try_decode_expr` through `ctx.decode`, so the
-/// nested filter picks up the same `DeduplicatingProtoConverter` /
-/// `DeduplicatingDeserializer` cache as the bare reference. Without the
-/// context the nested expr would serialize with `expr_id: None` and decode
-/// into a distinct `Inner`, breaking heap-max propagation across the
-/// extension boundary in distributed execution.
-#[test]
-fn extension_codec_expr_participates_in_deduplication() -> Result<()> {
- use prost::Message;
-
- // A single composite expression holding TWO references to the same
- // dynamic filter: bare on the left of an AND, wrapped on the right.
- let dyn_filter = make_dynamic_filter();
- let wrapper: Arc<dyn PhysicalExpr> = Arc::new(WrapperExpr {
- inner: Arc::clone(&dyn_filter),
- });
- let composite: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- Arc::clone(&dyn_filter),
- Operator::And,
- Arc::clone(&wrapper),
- ));
-
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let codec = WrapperCodec;
- let converter = DeduplicatingProtoConverter {};
-
- // Encode, then round-trip through prost bytes to mimic the wire.
- let proto = converter.physical_expr_to_proto(&composite, &codec)?;
- let bytes = proto.encode_to_vec();
- let decoded_proto = PhysicalExprNode::decode(bytes.as_slice()).unwrap();
-
- let ctx = SessionContext::new();
- let task_ctx = ctx.task_ctx();
- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
- let decoded =
- converter.proto_to_physical_expr(&decoded_proto, &schema, &decode_ctx)?;
-
- let binary = decoded
- .downcast_ref::<BinaryExpr>()
- .expect("must decode back to BinaryExpr");
- let decoded_left = Arc::clone(binary.left());
- let decoded_right = Arc::clone(binary.right());
- let decoded_wrapper = decoded_right
- .downcast_ref::<WrapperExpr>()
- .expect("right side must decode back to WrapperExpr");
-
- // The load-bearing check: an `update()` on the bare-side filter must be
- // observable from the wrapped-side filter, proving both refs back the
- // same `Inner`.
- assert_dynamic_filter_update_is_visible(&decoded_left, &decoded_wrapper.inner)?;
-
- Ok(())
-}