Apache DataFusion SQL Query Engine
test(proto): split roundtrip_physical_plan.rs by plan category (#24223)
Adrian Garcia Badaraccod443bab46289
diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rsindex 430b1405b1..4237b1c922 100644--- a/datafusion/proto/tests/cases/mod.rs+++ b/datafusion/proto/tests/cases/mod.rs@@ -32,8 +32,8 @@ use std::fmt::Debug;use std::hash::Hash;use std::sync::Arc;+mod plans;mod roundtrip_logical_plan;-mod roundtrip_physical_plan;mod serialize;mod stack_safety;diff --git a/datafusion/proto/tests/cases/plans/aggregates.rs b/datafusion/proto/tests/cases/plans/aggregates.rsnew file mode 100644index 0000000000..34b51b57b0--- /dev/null+++ b/datafusion/proto/tests/cases/plans/aggregates.rs@@ -0,0 +1,256 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! `AggregateExec`.++use super::{roundtrip_test, roundtrip_test_with_context};+use datafusion::arrow::array::ArrayRef;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::logical_expr::Volatility;+use datafusion::physical_expr::aggregate::AggregateExprBuilder;+use datafusion::physical_plan::PhysicalExpr;+use datafusion::physical_plan::aggregates::{+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,+};+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col, lit};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::Result;+use datafusion_expr::{+ Accumulator, AccumulatorFactoryFunction, AggregateUDF, Signature, SimpleAggregateUDF,+};+use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;+use datafusion_functions_aggregate::array_agg::array_agg_udaf;+use datafusion_functions_aggregate::average::avg_udaf;+use datafusion_functions_aggregate::nth_value::nth_value_udaf;+use datafusion_functions_aggregate::string_agg::string_agg_udaf;+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_aggregate() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =+ vec![(col("a", &schema)?, "unused".to_string())];++ let avg_expr = AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("AVG(b)")+ .build()?;+ let nth_expr =+ AggregateExprBuilder::new(nth_value_udaf(), vec![col("b", &schema)?, lit(1u64)])+ .schema(Arc::clone(&schema))+ .alias("NTH_VALUE(b, 1)")+ .build()?;+ let str_agg_expr =+ AggregateExprBuilder::new(string_agg_udaf(), vec![col("b", &schema)?, lit(1u64)])+ .schema(Arc::clone(&schema))+ .alias("NTH_VALUE(b, 1)")+ .build()?;++ let test_cases = vec![+ // AVG+ vec![Arc::new(avg_expr)],+ // NTH_VALUE+ vec![Arc::new(nth_expr)],+ // STRING_AGG+ vec![Arc::new(str_agg_expr)],+ ];++ for aggregates in test_cases {+ let schema = schema.clone();+ roundtrip_test(Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(groups.clone()),+ aggregates,+ vec![None],+ Arc::new(EmptyExec::new(schema.clone())),+ schema,+ )?))?;+ }++ Ok(())+}++#[test]+fn roundtrip_aggregate_with_limit() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =+ vec![(col("a", &schema)?, "unused".to_string())];++ let aggregates = vec![+ AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("AVG(b)")+ .build()+ .map(Arc::new)?,+ ];++ let agg = AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(groups.clone()),+ aggregates,+ vec![None],+ Arc::new(EmptyExec::new(schema.clone())),+ schema,+ )?;+ let agg = agg.with_limit_options(Some(LimitOptions::new_with_order(12, false)));+ roundtrip_test(Arc::new(agg))+}++#[test]+fn roundtrip_aggregate_with_approx_pencentile_cont() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =+ vec![(col("a", &schema)?, "unused".to_string())];++ let aggregates = vec![+ AggregateExprBuilder::new(+ approx_percentile_cont_udaf(),+ vec![col("b", &schema)?, lit(0.5)],+ )+ .schema(Arc::clone(&schema))+ .alias("APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY b)")+ .build()+ .map(Arc::new)?,+ ];++ let agg = AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(groups.clone()),+ aggregates,+ vec![None],+ Arc::new(EmptyExec::new(schema.clone())),+ schema,+ )?;+ roundtrip_test(Arc::new(agg))+}++#[test]+fn roundtrip_aggregate_with_sort() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =+ vec![(col("a", &schema)?, "unused".to_string())];+ let sort_exprs = vec![PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }];++ let aggregates = vec![+ AggregateExprBuilder::new(array_agg_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("ARRAY_AGG(b)")+ .order_by(sort_exprs)+ .build()+ .map(Arc::new)?,+ ];++ let agg = AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(groups.clone()),+ aggregates,+ vec![None],+ Arc::new(EmptyExec::new(schema.clone())),+ schema,+ )?;+ roundtrip_test(Arc::new(agg))+}++#[test]+fn roundtrip_aggregate_udaf() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ #[derive(Debug)]+ struct Example;+ impl Accumulator for Example {+ fn state(&mut self) -> Result<Vec<ScalarValue>> {+ Ok(vec![ScalarValue::Int64(Some(0))])+ }++ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {+ Ok(())+ }++ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {+ Ok(())+ }++ fn evaluate(&mut self) -> Result<ScalarValue> {+ Ok(ScalarValue::Int64(Some(0)))+ }++ fn size(&self) -> usize {+ 0+ }+ }++ let return_type = DataType::Int64;+ let accumulator: AccumulatorFactoryFunction = Arc::new(|_| Ok(Box::new(Example)));++ let udaf = AggregateUDF::from(SimpleAggregateUDF::new_with_signature(+ "example",+ Signature::exact(vec![DataType::Int64], Volatility::Immutable),+ return_type,+ accumulator,+ vec![Field::new("value", DataType::Int64, true).into()],+ ));++ let ctx = SessionContext::new();+ ctx.register_udaf(udaf.clone());++ let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =+ vec![(col("a", &schema)?, "unused".to_string())];++ let aggregates = vec![+ AggregateExprBuilder::new(Arc::new(udaf), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("example_agg")+ .build()+ .map(Arc::new)?,+ ];++ roundtrip_test_with_context(+ Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(groups.clone()),+ aggregates,+ vec![None],+ Arc::new(EmptyExec::new(schema.clone())),+ schema,+ )?),+ &ctx,+ )+}diff --git a/datafusion/proto/tests/cases/plans/dispatch.rs b/datafusion/proto/tests/cases/plans/dispatch.rsnew file mode 100644index 0000000000..5299733447--- /dev/null+++ b/datafusion/proto/tests/cases/plans/dispatch.rs@@ -0,0 +1,367 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Serde dispatch itself: which hook the central (de)serializer reaches,+//! and how a custom converter or a deprecated shim participates.++use super::roundtrip_test_and_return;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::execution::TaskContext;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};+use datafusion::physical_plan::sorts::sort::SortExec;+use datafusion::physical_plan::{+ DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,+ SendableRecordBatchStream,+};+use datafusion::prelude::SessionContext;+use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{Result, exec_datafusion_err};+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+ PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalPlanNodeExt,+ PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};+use std::fmt::Formatter;+use std::sync::{Arc, RwLock};+use std::vec;++#[derive(Debug)]+struct DowncastDelegatingExec {+ inner: Arc<dyn ExecutionPlan>,+}++impl DowncastDelegatingExec {+ fn new(inner: Arc<dyn ExecutionPlan>) -> Self {+ Self { inner }+ }+}++impl DisplayAs for DowncastDelegatingExec {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {+ self.inner.fmt_as(t, f)+ }+}++impl ExecutionPlan for DowncastDelegatingExec {+ fn name(&self) -> &str {+ self.inner.name()+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.inner.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ self.inner.children()+ }++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ self.inner.apply_expressions(f)+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let inner = Arc::clone(&self.inner).with_new_children(children)?;+ Ok(Arc::new(Self::new(inner)))+ }++ fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {+ Some(self.inner.as_ref())+ }++ fn execute(+ &self,+ partition: usize,+ context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ self.inner.execute(partition, context)+ }+}++#[test]+fn serialize_uses_downcast_delegate() -> Result<()> {+ let inner: Arc<dyn ExecutionPlan> =+ Arc::new(EmptyExec::new(Arc::new(Schema::empty())));+ let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));+ let codec = DefaultPhysicalExtensionCodec {};++ let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;++ assert!(matches!(+ proto.physical_plan_type,+ Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))+ ));++ Ok(())+}++/// A wrapper delegating to a plan that serializes itself via the+/// `try_to_proto` hook must serialize as its delegate: the wrapper's default+/// hook returns `Ok(None)` and the delegate has no downcast-chain fallback.+#[test]+fn serialize_uses_downcast_delegate_for_self_serializing_plan() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));+ let inner: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: col("a", &schema)?,+ alias: "a".to_string(),+ }],+ input,+ )?);+ let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));+ let codec = DefaultPhysicalExtensionCodec {};++ let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;++ assert!(matches!(+ proto.physical_plan_type,+ Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(+ _+ ))+ ));++ Ok(())+}++#[tokio::test]+async fn roundtrip_physical_plan_node() {+ use datafusion::prelude::*;+ use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec,+ };+ use datafusion_proto::protobuf::PhysicalPlanNode;++ let ctx = SessionContext::new();++ ctx.register_parquet(+ "pt",+ &format!(+ "{}/alltypes_plain.snappy.parquet",+ datafusion_common::test_util::parquet_test_data()+ ),+ ParquetReadOptions::default(),+ )+ .await+ .unwrap();++ let plan = ctx+ .sql("select id, string_col, timestamp_col from pt where id > 4 order by string_col")+ .await+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap();++ let node: PhysicalPlanNode =+ PhysicalPlanNode::try_from_physical_plan(plan, &DefaultPhysicalExtensionCodec {})+ .unwrap();++ let plan = node+ .try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})+ .unwrap();++ let _ = plan.execute(0, ctx.task_ctx()).unwrap();+}++/// The deprecated `try_into_projection_physical_plan` shim now delegates to+/// [`ProjectionExec::try_from_proto`], which reads the enclosing+/// `PhysicalPlanNode` rather than a `ProjectionExecNode`. Assert the shim still+/// decodes the node passed as an argument, not `self`, so an out-of-tree caller+/// that passes a projection unrelated to `self` keeps the old behaviour.+#[test]+fn deprecated_projection_shim_decodes_argument_not_self() -> Result<()> {+ use datafusion_proto::protobuf::PhysicalPlanNode;+ use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));+ let projection = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr::new(+ col("a", &schema)?,+ "renamed".to_string(),+ )],+ input,+ )?);++ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let projection_node = PhysicalPlanNode::try_from_physical_plan_with_converter(+ projection,+ &codec,+ &proto_converter,+ )?;+ let Some(PhysicalPlanType::Projection(projection_exec_node)) =+ &projection_node.physical_plan_type+ else {+ panic!("expected a Projection node, got {projection_node:?}");+ };++ // `self` is deliberately a different plan variant than the argument.+ let unrelated_node = PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::new(EmptyExec::new(Arc::new(schema))),+ &codec,+ &proto_converter,+ )?;++ let session_ctx = SessionContext::new();+ let task_ctx = session_ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ #[expect(deprecated)]+ let decoded = unrelated_node.try_into_projection_physical_plan(+ projection_exec_node,+ &decode_ctx,+ &proto_converter,+ )?;++ let decoded = decoded+ .downcast_ref::<ProjectionExec>()+ .expect("decoded plan should be a ProjectionExec");+ assert_eq!(decoded.expr().len(), 1);+ assert_eq!(decoded.expr()[0].alias, "renamed");+ Ok(())+}++#[test]+fn custom_proto_converter_intercepts() -> Result<()> {+ #[derive(Default)]+ struct CustomConverterInterceptor {+ num_proto_plans: RwLock<usize>,+ num_physical_plans: RwLock<usize>,+ num_proto_exprs: RwLock<usize>,+ num_physical_exprs: RwLock<usize>,+ }++ impl PhysicalProtoConverterExtension for CustomConverterInterceptor {+ fn proto_to_execution_plan(+ &self,+ proto: &PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ {+ let mut counter = self+ .num_proto_plans+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ self.default_proto_to_execution_plan(proto, ctx)+ }++ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalPlanNode>+ where+ Self: Sized,+ {+ {+ let mut counter = self+ .num_physical_plans+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(plan),+ codec,+ self,+ )+ }++ fn proto_to_physical_expr(+ &self,+ proto: &PhysicalExprNode,+ input_schema: &Schema,+ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>>+ where+ Self: Sized,+ {+ {+ let mut counter = self+ .num_proto_exprs+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ self.default_proto_to_physical_expr(proto, input_schema, ctx)+ }++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalExprNode> {+ {+ let mut counter = self+ .num_physical_exprs+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ serialize_physical_expr_with_converter(expr, codec, self)+ }+ }++ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();++ let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = CustomConverterInterceptor::default();+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/dynamic_filters.rs b/datafusion/proto/tests/cases/plans/dynamic_filters.rsnew file mode 100644index 0000000000..cda649b4c5--- /dev/null+++ b/datafusion/proto/tests/cases/plans/dynamic_filters.rs@@ -0,0 +1,1115 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Dynamic filter expressions: their deduplication across a plan, and the+//! plans that produce them.++use super::{roundtrip_test_and_return, roundtrip_test_sql_with_context};+use arrow::array::RecordBatch;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion::datasource::empty::EmptyTable;+use datafusion::datasource::listing::PartitionedFile;+use datafusion::datasource::object_store::ObjectStoreUrl;+use datafusion::datasource::physical_plan::{+ FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,+};+use datafusion::datasource::source::DataSourceExec;+use datafusion::execution::TaskContext;+use datafusion::logical_expr::{JoinType, Operator};+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_expr::aggregate::AggregateExprBuilder;+use datafusion::physical_optimizer::PhysicalOptimizerRule;+use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;+use datafusion::physical_plan::aggregates::{+ AggregateExec, AggregateMode, PhysicalGroupBy,+};+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{+ BinaryExpr, Column, DynamicFilterPhysicalExpr, PhysicalSortExpr, lit,+};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};+use datafusion::physical_plan::sorts::sort::SortExec;+use datafusion::physical_plan::{+ DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,+ SendableRecordBatchStream,+};+use datafusion::prelude::SessionContext;+use datafusion_common::config::{ConfigOptions, TableParquetOptions};+use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{NullEquality, Result, internal_datafusion_err, internal_err};+use datafusion_datasource::file::FileSource;+use datafusion_expr::ColumnarValue;+use datafusion_physical_expr::utils::reassign_expr_columns;+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};+use datafusion_proto::physical_plan::{+ DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,+ PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::PhysicalExprNode;+use prost::Message;+use std::fmt::{Display, Formatter};+use std::sync::Arc;+use std::vec;++/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.+fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {+ Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],+ lit(true),+ )) as Arc<dyn PhysicalExpr>+}++/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.+fn make_reassigned_dynamic_filter(+ filter: Arc<dyn PhysicalExpr>,+) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("b", DataType::Int64, false),+ Field::new("a", DataType::Int64, false),+ ]));+ let reassigned = reassign_expr_columns(filter, &schema)?;+ Ok((schema, reassigned))+}++/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the+/// default serializer from `PhysicalExpr::expression_id`.+fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {+ expr.expr_id+ .expect("expected PhysicalExprNode.expr_id to be populated")+}++/// Roundtrip a single physical expression shaped like so:+///+/// ```text+/// BinaryExpr(AND)+/// / \+/// filter_expr_1 filter_expr_2+/// ```+///+/// Returns filter_expr_1 and filter_expr_2 after deserialization.+fn roundtrip_dynamic_filter_expr_pair(+ filter_expr_1: Arc<dyn PhysicalExpr>,+ filter_expr_2: Arc<dyn PhysicalExpr>,+ schema: Arc<Schema>,+) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {+ let pair_expr = Arc::new(BinaryExpr::new(+ Arc::clone(&filter_expr_1),+ Operator::And,+ Arc::clone(&filter_expr_2),+ )) as Arc<dyn PhysicalExpr>;++ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;+ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let deserialized_expr =+ converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;++ let binary = deserialized_expr+ .downcast_ref::<BinaryExpr>()+ .expect("Expected BinaryExpr");++ Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))+}++/// Roundtrip an execution plan shaped like so:+///+/// ```text+/// FilterExec(dynamic_filter_1 on a@0)+/// ProjectionExec(a := Column("a", source_index))+/// DataSourceExec+/// ParquetSource(predicate = dynamic_filter_2)+/// ```+///+/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with+/// different children.+///+/// Returns+/// - `dynamic_filter_1` before serialization+/// - `dynamic_filter_2` before serialization+/// - `dynamic_filter_1` after serialization+/// - `dynamic_filter_2` after serialization+#[expect(clippy::type_complexity)]+fn roundtrip_dynamic_filter_plan_pair() -> Result<(+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+)> {+ let filter_expr_1 = make_dynamic_filter();+ let (data_source_schema, filter_expr_2) =+ make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;+ let left_before = Arc::clone(&filter_expr_1);+ let right_before = Arc::clone(&filter_expr_2);+ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&data_source_schema))+ .with_predicate(Arc::clone(&filter_expr_2)),+ );+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .build();+ let data_source_exec =+ DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;++ let projection_exec = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,+ alias: "a".to_string(),+ }],+ data_source_exec,+ )?) as Arc<dyn ExecutionPlan>;+ let filter_exec = Arc::new(FilterExec::try_new(+ Arc::clone(&filter_expr_1),+ projection_exec,+ )?) as Arc<dyn ExecutionPlan>;++ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;++ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;++ let outer_filter = deserialized_plan+ .downcast_ref::<FilterExec>()+ .expect("Expected outer FilterExec");+ let left_filter = Arc::clone(outer_filter.predicate());+ let projection = outer_filter.children()[0]+ .downcast_ref::<ProjectionExec>()+ .expect("Expected ProjectionExec");+ let data_source = projection+ .input()+ .downcast_ref::<DataSourceExec>()+ .expect("Expected DataSourceExec");+ let scan_config = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ let right_filter = scan_config+ .file_source()+ .filter()+ .expect("Expected pushed-down predicate");++ Ok((left_before, right_before, left_filter, right_filter))+}++/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible+/// via the other. This helps assert that referential integrity is maintained after+/// deserializing.+fn assert_dynamic_filter_update_is_visible(+ left_filter: &Arc<dyn PhysicalExpr>,+ right_filter: &Arc<dyn PhysicalExpr>,+) -> Result<()> {+ let left_filter = left_filter+ .downcast_ref::<DynamicFilterPhysicalExpr>()+ .expect("Expected dynamic filter");+ let right_filter = right_filter+ .downcast_ref::<DynamicFilterPhysicalExpr>()+ .expect("Expected dynamic filter");++ // Sanity check that the filters have the same generation.+ let original_generation = left_filter.snapshot_generation();+ assert_eq!(original_generation, right_filter.snapshot_generation(),);++ left_filter.update(lit(123_i64))?;++ // Assert that both generations updated.+ assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);+ assert_eq!(+ left_filter.snapshot_generation(),+ right_filter.snapshot_generation(),+ );++ // Ensure both filters have the updated expr.+ let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;+ assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);+ assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);++ Ok(())+}++/// Extract the dynamic-filter predicate that was pushed down to the parquet+/// scan at the bottom of the plan tree.+fn parquet_source_predicate(child: &Arc<dyn ExecutionPlan>) -> Arc<dyn PhysicalExpr> {+ let data_source = child+ .downcast_ref::<DataSourceExec>()+ .expect("Child should be DataSourceExec");+ let (_, parquet_source) = data_source+ .downcast_to_file_source::<ParquetSource>()+ .expect("Should be ParquetSource");+ parquet_source+ .filter()+ .expect("ParquetSource should have a predicate after roundtrip")+}++/// Assert that two dynamic filters are equal both structurally (Debug output)+/// and by identity (`expression_id`).+fn assert_dynamic_filters_equal(+ expected: &Arc<dyn PhysicalExpr>,+ actual: &Arc<dyn PhysicalExpr>,+) {+ // Structural.+ let expected_dbg = format!("{expected:?}");+ let actual_dbg = format!("{actual:?}");+ if expected_dbg == actual_dbg {+ return;+ }++ // Note that the `DeduplicatingDeserializer` routes every cache hit through+ // `with_new_children`. This produces an equivalent expression, but with+ // remapped children that are equal to the original. Handle that case here.+ let rewritten = Arc::clone(expected)+ .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())+ .expect("with_new_children on a dynamic filter should not fail");+ assert_eq!(format!("{rewritten:?}"), actual_dbg);+}++// Two clones of a dynamic filter expression should be deduped to the exact same expression.+#[test]+fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let filter_expr_1 = make_dynamic_filter();+ let filter_expr_2 = Arc::clone(&filter_expr_1);++ let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =+ roundtrip_dynamic_filter_expr_pair(+ Arc::clone(&filter_expr_1),+ Arc::clone(&filter_expr_2),+ schema,+ )?;++ // Assert the filters are not modified during roundtrip.+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);+ assert_dynamic_filters_equal(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,+ );++ // Assert referential integrity.+ assert_dynamic_filter_update_is_visible(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,+ )?;++ Ok(())+}++/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter+/// with different children.+#[test]+fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {+ let (+ filter_expr_1,+ filter_expr_2,+ filter_expr_1_after_roundtrip,+ filter_expr_2_after_roundtrip,+ ) = roundtrip_dynamic_filter_plan_pair()?;++ // Assert the filters are not modified during roundtrip.+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);++ // Assert referential integrity.+ assert_dynamic_filter_update_is_visible(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,+ )?;++ Ok(())+}++#[test]+fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {+ let filter_expr = make_dynamic_filter();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DeduplicatingProtoConverter {};++ let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;+ let expr_id1 = proto_expression_id(&proto1);++ let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;+ let expr_id2 = proto_expression_id(&proto2);++ assert_eq!(+ expr_id1, expr_id2,+ "Expected the same dynamic filter expression id across serializations"+ );++ Ok(())+}++/// Create a DataSourceExec backed by a ParquetSource that accepts filter pushdown,+/// along with a ConfigOptions that enables all dynamic filter pushdown options.+fn datasource_for_dynamic_filter_pushdown(+ schema: &Arc<Schema>,+) -> (Arc<dyn ExecutionPlan>, ConfigOptions) {+ let mut parquet_options = TableParquetOptions::new();+ parquet_options.global.pushdown_filters = true;+ let source = Arc::new(+ ParquetSource::new(Arc::clone(schema))+ .with_table_parquet_options(parquet_options),+ );+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file(PartitionedFile::new("/path/to/file.parquet", 1024))+ .build();++ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_join_dynamic_filter_pushdown = true;+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;+ config.optimizer.enable_topk_dynamic_filter_pushdown = true;++ (DataSourceExec::from_data_source(scan_config), config)+}++/// Test that plan containing a HashJoinExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.+#[test]+fn test_hash_join_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));++ let left_child = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let (right_child, config) = datasource_for_dynamic_filter_pushdown(&schema);++ let on: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> = vec![(+ Arc::new(Column::new("col", 0)),+ Arc::new(Column::new("col", 0)),+ )];++ let hash_join = Arc::new(HashJoinExec::try_new(+ left_child,+ right_child,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?) as Arc<dyn ExecutionPlan>;++ // Run the optimizer rule for filter pushdown.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(hash_join, &config)?;++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let deserialized = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;++ // Extract the deserialized HashJoinExec and its dynamic filter.+ let deserialized_join = deserialized+ .downcast_ref::<HashJoinExec>()+ .expect("Should be HashJoinExec");+ let deserialized_hash_join_df = deserialized_join+ .dynamic_expressions_produced()+ .into_iter()+ .next()+ .expect("HashJoinExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the probe side's ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_join.right());++ // The HashJoinExec's dynamic filter and the probe side's predicate should+ // refer to the same underlying expression.+ let plan_df = deserialized_hash_join_df;+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;++ Ok(())+}++/// returns a SessionContext with an empty `netflow` table registered+fn netflow_context() -> Result<SessionContext> {+ let ctx = SessionContext::new();+ let schema = Arc::new(Schema::new(vec![+ Field::new("dst_geo_country_name", DataType::Utf8, true),+ Field::new("dst_geo_city_name", DataType::Utf8, true),+ Field::new("packets", DataType::UInt64, true),+ Field::new("src_addr", DataType::Utf8, true),+ Field::new("dst_addr", DataType::Utf8, true),+ ]));++ ctx.register_table("netflow", Arc::new(EmptyTable::new(schema)))?;++ Ok(ctx)+}++/// Regression test for issue #18602:+/// https://github.com/apache/datafusion/issues/18602+///+/// The physical filter expression here contains a long chain of `AND` predicates.+/// Before linearizing `PhysicalBinaryExprNode`, encoding then decoding the protobuf+/// could fail with `DecodeError: recursion limit reached`.+#[tokio::test]+async fn roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {+ let ctx = netflow_context()?;+ let sql = "SELECT \+ dst_geo_country_name AS x_axis_1, \+ dst_geo_city_name AS x_axis_2, \+ sum(packets) AS y_axis_1 \+ FROM netflow \+ WHERE dst_geo_country_name IS NOT NULL \+ AND src_addr NOT LIKE '10.201.%' \+ AND dst_addr NOT LIKE '10.201.%' \+ AND src_addr NOT LIKE '10.202.%' \+ AND dst_addr NOT LIKE '10.202.%' \+ AND src_addr NOT LIKE '10.203.%' \+ AND dst_addr NOT LIKE '10.203.%' \+ AND src_addr NOT LIKE '10.204.%' \+ AND dst_addr NOT LIKE '10.204.%' \+ AND src_addr NOT LIKE '172.16.186.%' \+ AND dst_addr NOT LIKE '172.16.186.%' \+ AND src_addr NOT LIKE '172.16.187.%' \+ AND dst_addr NOT LIKE '172.16.187.%' \+ AND src_addr NOT LIKE '172.16.188.%' \+ AND dst_addr NOT LIKE '172.16.188.%' \+ AND src_addr NOT LIKE '10.102.45.%' \+ AND dst_addr NOT LIKE '10.102.45.%' \+ AND src_addr NOT LIKE '172.25.210.%' \+ AND dst_addr NOT LIKE '172.25.210.%' \+ AND src_addr NOT LIKE '172.25.211.%' \+ AND dst_addr NOT LIKE '172.25.211.%' \+ AND src_addr NOT LIKE '141.226.101.%' \+ AND dst_addr NOT LIKE '141.226.101.%' \+ AND src_addr NOT LIKE '167.86.40.%' \+ AND dst_addr NOT LIKE '167.86.40.%' \+ AND src_addr NOT LIKE '66.22.38.%' \+ AND dst_addr NOT LIKE '66.22.38.%' \+ AND src_addr != '168.143.191.55' \+ AND dst_addr != '168.143.191.55' \+ AND src_addr != '82.112.107.142' \+ AND dst_addr != '82.112.107.142' \+ AND src_addr != '20.76.39.176' \+ AND dst_addr != '20.76.39.176' \+ AND src_addr != '162.159.129.83' \+ AND dst_addr != '162.159.129.83' \+ AND src_addr != '34.201.223.155' \+ AND dst_addr != '34.201.223.155' \+ AND src_addr != '34.201.223.156' \+ AND dst_addr != '34.201.223.156' \+ AND src_addr != '34.201.223.157' \+ AND dst_addr != '34.201.223.157' \+ AND src_addr != '134.201.223.157' \+ AND dst_addr != '134.201.223.157' \+ AND src_addr != '341.201.223.157' \+ AND dst_addr != '341.201.223.157' \+ GROUP BY x_axis_1, x_axis_2 \+ ORDER BY y_axis_1 DESC \+ LIMIT 20";++ roundtrip_test_sql_with_context(sql, &ctx).await+}++/// Test that plan containing a AggregateExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.+#[test]+fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));++ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);++ let agg = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![+ AggregateExprBuilder::new(+ datafusion::functions_aggregate::min_max::min_udaf(),+ vec![Arc::clone(&col_a)],+ )+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()+ .map(Arc::new)?,+ ],+ vec![None],+ child,+ Arc::clone(&schema),+ )?) as Arc<dyn ExecutionPlan>;++ // Run the optimizer rule for filter pushdown.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(agg, &config)?;++ // Roundtrip with deduplication.+ //+ // Note: We don't use `roundtrip_test_and_return` here because there's a+ // pre-existing issue with PhysicalGroupBy serialization where empty groups+ // `[[]]` become `[]` after roundtrip. This behavior is unrelated to this test.+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&plan),+ &codec,+ &converter,+ )?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Extract the deserialized AggregateExec and its dynamic filter.+ let deserialized_agg = deserialized+ .downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec");+ let deserialized_agg_df = deserialized_agg+ .dynamic_expressions_produced()+ .into_iter()+ .next()+ .expect("AggregateExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the child ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_agg.input());++ // The AggregateExec's dynamic filter and the child's predicate should+ // refer to the same underlying expression.+ let plan_df = deserialized_agg_df;+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;++ Ok(())+}++#[test]+fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![+ AggregateExprBuilder::new(+ datafusion::functions_aggregate::min_max::min_udaf(),+ vec![col_a],+ )+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()+ .map(Arc::new)?,+ ],+ vec![None],+ child,+ Arc::clone(&schema),+ )?) as Arc<dyn ExecutionPlan>;++ let mut config = ConfigOptions::default();+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(aggregate, &config)?;+ assert!(+ plan.downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec")+ .dynamic_expressions_produced()+ .is_empty()+ );++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DefaultPhysicalProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ assert!(+ deserialized+ .downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec")+ .dynamic_expressions_produced()+ .is_empty()+ );+ Ok(())+}++/// Test that plan containing a SortExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.+#[test]+fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));++ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);++ let sort = Arc::new(+ SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::clone(&col_a),+ options: SortOptions::default(),+ }])+ .unwrap(),+ child,+ )+ .with_fetch(Some(10)),+ ) as Arc<dyn ExecutionPlan>;++ // Verify the optimizer kept the dynamic filter on the SortExec.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(sort, &config)?;++ // Roundtrip with deduplication.+ //+ // Note: We don't use `roundtrip_test_and_return` here because+ // `DeduplicatingDeserializer` rewrites cache hits via `with_new_children`,+ // which sets `remapped_children: Some(...)` on the second encounter of a+ // shared `DynamicFilterPhysicalExpr`. SortExec's `Debug` includes its+ // dynamic filter, so the original-vs-deserialized structural equality check+ // would fail purely on this artifact.+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&plan),+ &codec,+ &converter,+ )?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Extract the deserialized SortExec and its dynamic filter.+ let deserialized_sort = deserialized+ .downcast_ref::<SortExec>()+ .expect("Should be SortExec");+ let deserialized_sort_df = deserialized_sort+ .dynamic_expressions_produced()+ .into_iter()+ .next()+ .expect("SortExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the child ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_sort.input());++ // The SortExec's dynamic filter and the child's predicate should+ // refer to the same underlying expression.+ let plan_df = deserialized_sort_df;+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;++ Ok(())+}++/// A custom [`ExecutionPlan`] which stores [`PhysicalExpr`]s.+struct CustomExecWithExprs {+ exprs: Vec<Arc<dyn PhysicalExpr>>,+ child: Arc<dyn ExecutionPlan>,+}++#[derive(Clone, PartialEq, Message)]+struct CustomExecWithExprsProto {+ #[prost(message, repeated, tag = "1")]+ exprs: Vec<PhysicalExprNode>,+}++impl std::fmt::Debug for CustomExecWithExprs {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("CustomExecWithExprs")+ .field("exprs", &self.exprs)+ .field("child", &self.child)+ .finish()+ }+}++impl CustomExecWithExprs {+ fn new(exprs: Vec<Arc<dyn PhysicalExpr>>, child: Arc<dyn ExecutionPlan>) -> Self {+ Self { exprs, child }+ }+}++impl DisplayAs for CustomExecWithExprs {+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {+ write!(f, "CustomExecWithExprs")+ }+}++impl ExecutionPlan for CustomExecWithExprs {+ fn name(&self) -> &str {+ "CustomExecWithExprs"+ }++ fn schema(&self) -> SchemaRef {+ self.child.schema()+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.child.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![&self.child]+ }++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(&self.exprs, f)+ }++ fn with_new_children(+ self: Arc<Self>,+ _children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ unreachable!()+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ unreachable!()+ }+}++/// A [`PhysicalExtensionCodec`] for [`CustomExecWithExprs`].+#[derive(Debug)]+struct CustomExecWithExprsCodec {}++impl PhysicalExtensionCodec for CustomExecWithExprsCodec {+ fn try_decode(+ &self,+ buf: &[u8],+ inputs: &[Arc<dyn ExecutionPlan>],+ ctx: &TaskContext,+ proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);+ let input_schema = inputs[0].schema();+ let proto = CustomExecWithExprsProto::decode(buf)+ .map_err(|e| internal_datafusion_err!("Failed to decode custom exec: {e}"))?;+ let exprs = proto+ .exprs+ .iter()+ .map(|expr_proto| {+ proto_converter.proto_to_physical_expr(+ expr_proto,+ input_schema.as_ref(),+ &decode_ctx,+ )+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(Arc::new(CustomExecWithExprs::new(exprs, inputs[0].clone())))+ }++ fn try_encode(+ &self,+ node: Arc<dyn ExecutionPlan>,+ buf: &mut Vec<u8>,+ proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ let custom = node+ .downcast_ref::<CustomExecWithExprs>()+ .ok_or_else(|| internal_datafusion_err!("Expected CustomExecWithExprs"))?;+ let proto = CustomExecWithExprsProto {+ exprs: custom+ .exprs+ .iter()+ .map(|expr| proto_converter.physical_expr_to_proto(expr, self))+ .collect::<Result<Vec<_>>>()?,+ };+ proto+ .encode(buf)+ .map_err(|e| internal_datafusion_err!("Failed to encode custom exec: {e}"))?;++ Ok(())+ }+}++/// Tests that a custom [`ExecutionPlan`] with [`PhysicalExpr`] can+/// dedupe dynamic filters by using the proto converter in its+/// [`PhysicalExtensionCodec`] implementation.+#[test]+fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {+ // Create the plan:+ //+ // FilterExec(dynamic_filter)+ // -> CustomExecWithExprs(exprs: [dynamic_filter])+ // -> EmptyExec+ //+ // The same dynamic filter expression is saved in both the FilterExec and CustomExecWithExprs.+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],+ lit(true),+ ));+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> = dynamic_filter;++ let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let custom_exec = Arc::new(CustomExecWithExprs::new(+ vec![Arc::clone(&dynamic_filter_expr)],+ empty,+ ));+ let filter_exec = Arc::new(FilterExec::try_new(+ Arc::clone(&dynamic_filter_expr),+ custom_exec,+ )?) as Arc<dyn ExecutionPlan>;++ // Roundtrip with DeduplicatingProtoConverter+ let codec = CustomExecWithExprsCodec {};+ let converter = DeduplicatingProtoConverter {};++ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&filter_exec),+ &codec,+ &converter,+ )?;++ let ctx = SessionContext::new();+ let deser_converter = DeduplicatingProtoConverter {};+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &deser_converter,+ )?;++ // Extract the deserialized FilterExec's dynamic filter+ let deser_filter = deserialized+ .downcast_ref::<FilterExec>()+ .expect("Top-level should be FilterExec");+ let deser_filter_df = deser_filter.predicate();++ // Extract the deserialized custom node's dynamic filter+ let deser_custom = deser_filter+ .input()+ .downcast_ref::<CustomExecWithExprs>()+ .expect("FilterExec child should be CustomExecWithExprs");+ assert_eq!(deser_custom.exprs.len(), 1, "Should have one expression");+ let [deser_custom_df] = deser_custom.exprs.as_slice() else {+ return internal_err!("Custom node should have one expression");+ };++ // Pass the un-remapped filter first so the helper's `with_new_children`+ // rewrite can reconstruct the remapped form on the other side.+ assert_dynamic_filters_equal(deser_custom_df, deser_filter_df);+ assert_dynamic_filter_update_is_visible(deser_custom_df, deser_filter_df)?;++ Ok(())+}++/// A custom `PhysicalExpr` whose extension codec embeds a nested+/// `PhysicalExprNode` *inside its own blob* (rather than the standard+/// `PhysicalExtensionExprNode.inputs` field). This is the case that only+/// works if the expr-level codec methods receive the encode/decode context.+#[derive(Debug)]+struct WrapperExpr {+ inner: Arc<dyn PhysicalExpr>,+}++impl Display for WrapperExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "WrapperExpr({})", self.inner)+ }+}++impl PartialEq for WrapperExpr {+ fn eq(&self, other: &Self) -> bool {+ self.inner.eq(&other.inner)+ }+}++impl Eq for WrapperExpr {}++impl std::hash::Hash for WrapperExpr {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.inner.hash(state);+ }+}++impl PhysicalExpr for WrapperExpr {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.inner.data_type(input_schema)+ }+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.inner.nullable(input_schema)+ }+ fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {+ internal_err!("WrapperExpr is not executable in this test")+ }+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.inner]+ }+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(WrapperExpr {+ inner: Arc::clone(&children[0]),+ }))+ }+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ Display::fmt(self, f)+ }+}++/// Wire layout for [`WrapperExpr`]: a single nested `PhysicalExprNode`.+#[derive(Clone, PartialEq, prost::Message)]+struct WrapperExprProto {+ #[prost(message, optional, boxed, tag = "1")]+ inner: Option<Box<PhysicalExprNode>>,+}++#[derive(Debug)]+struct WrapperCodec;++impl PhysicalExtensionCodec for WrapperCodec {+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ internal_err!("not used")+ }+ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ internal_err!("not used")+ }+ fn try_decode_expr(+ &self,+ buf: &[u8],+ _inputs: &[Arc<dyn PhysicalExpr>],+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ let proto = WrapperExprProto::decode(buf)+ .map_err(|e| internal_datafusion_err!("decode WrapperExprProto: {e}"))?;+ let inner_proto = proto+ .inner+ .ok_or_else(|| internal_datafusion_err!("missing inner"))?;+ // Decode the nested expr through the context so it resolves against+ // the real schema/registry AND participates in dedup — no fabricated+ // `SessionContext` or hard-coded schema required.+ let inner = ctx.decode(&inner_proto)?;+ Ok(Arc::new(WrapperExpr { inner }))+ }+ fn try_encode_expr(+ &self,+ node: &Arc<dyn PhysicalExpr>,+ buf: &mut Vec<u8>,+ ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<()> {+ let wrapper = node+ .downcast_ref::<WrapperExpr>()+ .ok_or_else(|| internal_datafusion_err!("not WrapperExpr"))?;+ // Encode the nested expr through the context so an active+ // `DeduplicatingProtoConverter` stamps a matching `expr_id`.+ let inner_proto = ctx.encode_child(&wrapper.inner)?;+ let proto = WrapperExprProto {+ inner: Some(Box::new(inner_proto)),+ };+ proto+ .encode(buf)+ .map_err(|e| internal_datafusion_err!("encode WrapperExprProto: {e}"))?;+ Ok(())+ }+}++/// A `DynamicFilterPhysicalExpr` referenced both as a bare expression and+/// nested inside a custom expression's codec blob must reconstruct to a+/// single shared `Inner` after roundtrip.+///+/// This exercises the expr-level codec hooks receiving the encode/decode+/// context: `try_encode_expr` routes its nested `PhysicalExprNode` through+/// `ctx.encode_child` and `try_decode_expr` through `ctx.decode`, so the+/// nested filter picks up the same `DeduplicatingProtoConverter` /+/// `DeduplicatingDeserializer` cache as the bare reference. Without the+/// context the nested expr would serialize with `expr_id: None` and decode+/// into a distinct `Inner`, breaking heap-max propagation across the+/// extension boundary in distributed execution.+#[test]+fn extension_codec_expr_participates_in_deduplication() -> Result<()> {+ use prost::Message;++ // A single composite expression holding TWO references to the same+ // dynamic filter: bare on the left of an AND, wrapped on the right.+ let dyn_filter = make_dynamic_filter();+ let wrapper: Arc<dyn PhysicalExpr> = Arc::new(WrapperExpr {+ inner: Arc::clone(&dyn_filter),+ });+ let composite: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::clone(&dyn_filter),+ Operator::And,+ Arc::clone(&wrapper),+ ));++ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let codec = WrapperCodec;+ let converter = DeduplicatingProtoConverter {};++ // Encode, then round-trip through prost bytes to mimic the wire.+ let proto = converter.physical_expr_to_proto(&composite, &codec)?;+ let bytes = proto.encode_to_vec();+ let decoded_proto = PhysicalExprNode::decode(bytes.as_slice()).unwrap();++ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let decoded =+ converter.proto_to_physical_expr(&decoded_proto, &schema, &decode_ctx)?;++ let binary = decoded+ .downcast_ref::<BinaryExpr>()+ .expect("must decode back to BinaryExpr");+ let decoded_left = Arc::clone(binary.left());+ let decoded_right = Arc::clone(binary.right());+ let decoded_wrapper = decoded_right+ .downcast_ref::<WrapperExpr>()+ .expect("right side must decode back to WrapperExpr");++ // The load-bearing check: an `update()` on the bare-side filter must be+ // observable from the wrapped-side filter, proving both refs back the+ // same `Inner`.+ assert_dynamic_filter_update_is_visible(&decoded_left, &decoded_wrapper.inner)?;++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rsnew file mode 100644index 0000000000..ed9745a4b1--- /dev/null+++ b/datafusion/proto/tests/cases/plans/exprs.rs@@ -0,0 +1,399 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Physical expressions embedded in plans, including the binary+//! expression linearization.++use super::roundtrip_test;+use arrow::datatypes::Fields;+use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};+use datafusion::logical_expr::Operator;+use datafusion::physical_expr::expressions::Literal;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{+ BinaryExpr, Column, binary, col, like, lit,+};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};+use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::Result;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+ PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::PhysicalPlanNode;+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_date_time_interval() -> Result<()> {+ let schema = Schema::new(vec![+ Field::new("some_date", DataType::Date32, false),+ Field::new(+ "some_interval",+ DataType::Interval(IntervalUnit::DayTime),+ false,+ ),+ ]);+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));+ let date_expr = col("some_date", &schema)?;+ let literal_expr = col("some_interval", &schema)?;+ let date_time_interval_expr =+ binary(date_expr, Operator::Plus, literal_expr, &schema)?;+ let plan = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: date_time_interval_expr,+ alias: "result".to_string(),+ }],+ input,+ )?);+ roundtrip_test(plan)+}++#[test]+fn roundtrip_like() -> Result<()> {+ let schema = Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Utf8, false),+ ]);+ let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));+ let like_expr = like(+ false,+ false,+ col("a", &schema)?,+ col("b", &schema)?,+ &schema,+ )?;+ let plan = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: like_expr,+ alias: "result".to_string(),+ }],+ input,+ )?);+ roundtrip_test(plan)+}++/// Test that HashTableLookupExpr serializes to lit(true)+///+/// HashTableLookupExpr contains a runtime hash table that cannot be serialized.+/// The serialization code replaces it with lit(true) which is safe because+/// it's a performance optimization filter, not a correctness requirement.+#[test]+fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {+ use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;+ use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};++ // Create a simple schema and input plan+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));+ let input = Arc::new(EmptyExec::new(schema.clone()));++ // Create a HashTableLookupExpr - it will be replaced with lit(true) during serialization+ let hash_map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0))));+ let on_columns = vec![col("col", &schema)?];+ let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(+ on_columns,+ datafusion::physical_plan::joins::SeededRandomState::with_seed(0),+ hash_map,+ "test_lookup".to_string(),+ ));++ // Create a filter with the lookup expression+ let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);++ // Serialize+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};++ let proto: PhysicalPlanNode =+ PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)+ .expect("serialization should succeed");++ // Deserialize+ let result: Arc<dyn ExecutionPlan> = proto+ .try_into_physical_plan(&ctx.task_ctx(), &codec)+ .expect("deserialization should succeed");++ // The deserialized plan should have lit(true) instead of HashTableLookupExpr+ // Verify the filter predicate is a Literal(true)+ let result_filter = result.downcast_ref::<FilterExec>().unwrap();+ let predicate = result_filter.predicate();+ let literal = predicate.downcast_ref::<Literal>().unwrap();+ assert_eq!(*literal.value(), ScalarValue::Boolean(Some(true)));++ Ok(())+}++#[test]+fn roundtrip_hash_expr() -> Result<()> {+ use datafusion::physical_plan::joins::{HashExpr, SeededRandomState};++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Utf8, false),+ ]));++ // Create a HashExpr with test columns and seeds+ let on_columns = vec![col("a", &schema)?, col("b", &schema)?];+ let hash_expr: Arc<dyn PhysicalExpr> = Arc::new(HashExpr::new(+ on_columns,+ SeededRandomState::with_seed(0), // arbitrary random seed for testing+ "test_hash".to_string(),+ ));++ // Wrap in a filter by comparing hash value to a literal+ // hash_expr > 0 is always boolean+ let filter_expr = binary(hash_expr, Operator::Gt, lit(0u64), &schema)?;+ let filter = Arc::new(FilterExec::try_new(+ filter_expr,+ Arc::new(EmptyExec::new(schema)),+ )?);++ // Confirm that the debug string contains the random state seeds+ assert!(+ format!("{filter:?}").contains("test_hash(a@0, b@1, [0])"),+ "Debug string missing seeds: {filter:?}"+ );+ roundtrip_test(filter)+}++#[test]+fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {+ let data_type = DataType::Struct(Fields::from(vec![Field::new(+ "item",+ DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),+ true,+ )]));++ let schema = Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), true)]));+ let scan = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let scalar = lit(ScalarValue::try_from(data_type)?);+ let filter = Arc::new(FilterExec::try_new(+ Arc::new(BinaryExpr::new(scalar, Operator::Eq, col("a", &schema)?)),+ scan,+ )?);++ roundtrip_test(filter)+}++/// Test that a chain of the same operator (a AND b AND c) is linearized+/// and roundtrips correctly.+#[test]+fn roundtrip_binary_expr_chain_same_op() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Boolean, false);+ let field_c = Field::new("c", DataType::Boolean, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));+ let ab = binary(+ col("a", &schema)?,+ Operator::And,+ col("b", &schema)?,+ &schema,+ )?;+ let abc = binary(ab, Operator::And, col("c", &schema)?, &schema)?;+ roundtrip_test(Arc::new(FilterExec::try_new(+ abc,+ Arc::new(EmptyExec::new(schema)),+ )?))+}++/// Test that mixed operators (a AND b OR c) are NOT linearized together —+/// only chains of the same operator are flattened.+#[test]+fn roundtrip_binary_expr_mixed_ops() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Boolean, false);+ let field_c = Field::new("c", DataType::Boolean, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));+ // (a AND b) OR c — AND and OR are different operators, so linearization stops+ let a_and_b = binary(+ col("a", &schema)?,+ Operator::And,+ col("b", &schema)?,+ &schema,+ )?;+ let expr = binary(a_and_b, Operator::Or, col("c", &schema)?, &schema)?;+ roundtrip_test(Arc::new(FilterExec::try_new(+ expr,+ Arc::new(EmptyExec::new(schema)),+ )?))+}++/// Test that a deeply nested chain of AND expressions (like many WHERE conditions)+/// roundtrips correctly. This is the scenario from issue #18602.+#[test]+fn roundtrip_binary_expr_deeply_nested_and_chain() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let schema = Arc::new(Schema::new(vec![field_a]));++ // Build a chain: a AND a AND a AND ... (100 times)+ let col_a = col("a", &schema)?;+ let mut expr = Arc::clone(&col_a);+ for _ in 0..99 {+ expr = binary(expr, Operator::And, Arc::clone(&col_a), &schema)?;+ }++ roundtrip_test(Arc::new(FilterExec::try_new(+ expr,+ Arc::new(EmptyExec::new(schema)),+ )?))+}++/// Test that a deeply nested chain of OR expressions roundtrips correctly.+#[test]+fn roundtrip_binary_expr_deeply_nested_or_chain() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let schema = Arc::new(Schema::new(vec![field_a]));++ let col_a = col("a", &schema)?;+ let mut expr = Arc::clone(&col_a);+ for _ in 0..99 {+ expr = binary(expr, Operator::Or, Arc::clone(&col_a), &schema)?;+ }++ roundtrip_test(Arc::new(FilterExec::try_new(+ expr,+ Arc::new(EmptyExec::new(schema)),+ )?))+}++/// Test that alternating AND/OR operators produce correct results —+/// each sub-chain gets linearized independently.+#[test]+fn roundtrip_binary_expr_alternating_and_or() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Boolean, false);+ let field_c = Field::new("c", DataType::Boolean, false);+ let field_d = Field::new("d", DataType::Boolean, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c, field_d]));++ // (a AND b) OR (c AND d)+ let a_and_b = binary(+ col("a", &schema)?,+ Operator::And,+ col("b", &schema)?,+ &schema,+ )?;+ let c_and_d = binary(+ col("c", &schema)?,+ Operator::And,+ col("d", &schema)?,+ &schema,+ )?;+ let expr = binary(a_and_b, Operator::Or, c_and_d, &schema)?;++ roundtrip_test(Arc::new(FilterExec::try_new(+ expr,+ Arc::new(EmptyExec::new(schema)),+ )?))+}++/// Verify that the linearized proto format has a flat operands list+/// rather than deeply nested l/r fields.+#[test]+fn test_linearization_produces_flat_operands() -> Result<()> {+ // Build: a AND a AND a AND a (4 operands, 3 levels of nesting)+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));+ let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(BinaryExpr::new(+ Arc::new(BinaryExpr::new(+ Arc::clone(&col_a),+ Operator::And,+ Arc::clone(&col_a),+ )),+ Operator::And,+ Arc::clone(&col_a),+ )),+ Operator::And,+ Arc::clone(&col_a),+ ));++ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;++ // The top-level should use the operands field with 4 entries+ match &proto.expr_type {+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {+ assert!(+ b.l.is_none(),+ "l should be None when using linearized operands"+ );+ assert!(+ b.r.is_none(),+ "r should be None when using linearized operands"+ );+ assert_eq!(+ b.operands.len(),+ 4,+ "Expected 4 linearized operands for a AND a AND a AND a"+ );+ assert_eq!(b.op, "And");+ }+ other => panic!("Expected BinaryExpr, got {other:?}"),+ }++ Ok(())+}++/// Test that linearization stops when encountering a different operator.+/// For (a AND b) OR c, only the top-level OR should be represented, and+/// the left-hand AND subtree should be a separate nested BinaryExpr.+#[test]+fn test_linearization_stops_at_different_op() -> Result<()> {+ // (a AND b) OR c+ let a_and_b: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::And,+ Arc::new(Column::new("b", 1)),+ ));+ let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ a_and_b,+ Operator::Or,+ Arc::new(Column::new("c", 2)),+ ));++ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;++ // The top-level OR should have only 2 operands (can't linearize through AND)+ match &proto.expr_type {+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {+ assert_eq!(+ b.operands.len(),+ 2,+ "Expected 2 operands for (a AND b) OR c"+ );+ assert_eq!(b.op, "Or");+ // The first operand should be a nested AND BinaryExpr+ match &b.operands[0].expr_type {+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(inner)) => {+ assert_eq!(inner.op, "And");+ assert_eq!(inner.operands.len(), 2);+ }+ other => panic!("Expected inner BinaryExpr(AND), got {other:?}"),+ }+ }+ other => panic!("Expected BinaryExpr, got {other:?}"),+ }++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/filters.rs b/datafusion/proto/tests/cases/plans/filters.rsnew file mode 100644index 0000000000..296923fcc7--- /dev/null+++ b/datafusion/proto/tests/cases/plans/filters.rs@@ -0,0 +1,103 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! `FilterExec`.++use super::{roundtrip_test, roundtrip_test_and_return};+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::logical_expr::Operator;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{NotExpr, binary, col, in_list, lit};+use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::Result;+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_filter_with_not_and_in_list() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let field_c = Field::new("c", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));+ let not = Arc::new(NotExpr::new(col("a", &schema)?));+ let in_list = in_list(+ col("b", &schema)?,+ vec![+ lit(ScalarValue::Int64(Some(1))),+ lit(ScalarValue::Int64(Some(2))),+ ],+ &false,+ schema.as_ref(),+ )?;+ let and = binary(not, Operator::And, in_list, &schema)?;+ roundtrip_test(Arc::new(FilterExec::try_new(+ and,+ Arc::new(EmptyExec::new(schema.clone())),+ )?))+}++#[test]+fn roundtrip_filter_with_fetch() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let predicate = col("a", &schema)?;+ let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))+ .with_fetch(Some(10))+ .build()?;+ assert_eq!(filter.fetch(), Some(10));+ roundtrip_test(Arc::new(filter))+}++#[test]+fn roundtrip_filter_projection_states() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Boolean, false),+ Field::new("b", DataType::Int64, false),+ ]));+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};++ for projection in [None, Some(vec![]), Some(vec![0])] {+ let filter = FilterExecBuilder::new(+ col("a", &schema)?,+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ )+ .apply_projection(projection.clone())?+ .with_default_selectivity(37)+ .with_batch_size(1024)+ .with_fetch(Some(5))+ .build()?;++ let result =+ roundtrip_test_and_return(Arc::new(filter), &ctx, &codec, &proto_converter)?;+ let result = result.downcast_ref::<FilterExec>().unwrap();+ assert_eq!(result.projection().as_deref(), projection.as_deref());+ assert_eq!(result.default_selectivity(), 37);+ assert_eq!(result.batch_size(), 1024);+ assert_eq!(result.fetch(), Some(5));+ }++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/joins.rs b/datafusion/proto/tests/cases/plans/joins.rsnew file mode 100644index 0000000000..941e883295--- /dev/null+++ b/datafusion/proto/tests/cases/plans/joins.rs@@ -0,0 +1,458 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! The join execs.++use super::{roundtrip_test, roundtrip_test_and_return};+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::logical_expr::{JoinType, Operator};+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{BinaryExpr, Column, PhysicalSortExpr};+use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};+use datafusion::physical_plan::joins::{+ HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,+ StreamJoinPartitionMode, SymmetricHashJoinExec,+};+use datafusion::prelude::SessionContext;+use datafusion_common::{JoinSide, NullEquality, Result};+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_hash_join() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_a]);+ let on = vec![(+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,+ )];++ let schema_left = Arc::new(schema_left);+ let schema_right = Arc::new(schema_right);+ for join_type in &[+ JoinType::Inner,+ JoinType::Left,+ JoinType::Right,+ JoinType::Full,+ JoinType::LeftAnti,+ JoinType::RightAnti,+ JoinType::LeftSemi,+ JoinType::RightSemi,+ ] {+ for partition_mode in &[PartitionMode::Partitioned, PartitionMode::CollectLeft] {+ roundtrip_test(Arc::new(HashJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ on.clone(),+ None,+ join_type,+ None,+ *partition_mode,+ NullEquality::NullEqualsNothing,+ false,+ )?))?;+ }+ }+ Ok(())+}++#[test]+fn roundtrip_nested_loop_join() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_a]);++ let schema_left = Arc::new(schema_left);+ let schema_right = Arc::new(schema_right);+ for join_type in &[+ JoinType::Inner,+ JoinType::Left,+ JoinType::Right,+ JoinType::Full,+ JoinType::LeftAnti,+ JoinType::RightAnti,+ JoinType::LeftSemi,+ JoinType::RightSemi,+ ] {+ roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ None,+ join_type,+ Some(vec![0]),+ )?))?;+ }+ Ok(())+}++/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",+/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire+/// representation. `try_embed_projection` (DataFusion 53+) produces+/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,+/// which previously round-tripped to `None` and caused downstream consumers (e.g.+/// distributed Flight executors) to receive a different number of output+/// columns than the planner declared. Verify all three states preserve.+#[test]+fn roundtrip_hash_join_projection_states() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));+ let schema_right = Arc::new(Schema::new(vec![field_a]));+ let on = vec![(+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,+ )];++ for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {+ roundtrip_test(Arc::new(HashJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ on.clone(),+ None,+ &JoinType::Inner,+ projection,+ PartitionMode::Partitioned,+ NullEquality::NullEqualsNothing,+ false,+ )?))?;+ }+ Ok(())+}++/// Regression: `HashJoinExecNode` had no `fetch` field, so the row limit that+/// the `limit_pushdown` physical optimizer rule pushes into the join via+/// `ExecutionPlan::with_fetch` was silently dropped by serde. Because that rule+/// also removes the enclosing `GlobalLimitExec` once the join absorbs the limit,+/// a round-tripped plan had no limit left at all and a distributed executor+/// returned more rows than the query asked for.+///+/// Note this cannot be covered by `roundtrip_test`: that helper compares+/// `format!("{plan:?}")`, and `HashJoinExec`'s `Debug` output does not include+/// `fetch`, so the before/after strings match even when the value is lost. The+/// assertions below therefore inspect `fetch()` directly.+#[test]+fn roundtrip_hash_join_fetch() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));+ let schema_right = Arc::new(Schema::new(vec![field_a]));+ let on = vec![(+ Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,+ Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,+ )];++ // `usize::MAX` and `u32::MAX as usize` pin the decode-side `u64 -> usize`+ // conversion: it is a checked `usize::try_from`, and a large fetch must+ // survive the round trip exactly rather than being truncated or clamped.+ // Both are representable on every target (on a 32-bit target `usize::MAX`+ // is simply `u32::MAX`), so this stays portable. The truncating case+ // itself -- a `u64` fetch above `usize::MAX` -- is only reachable on a+ // 32-bit target and so is not exercised by this test on a 64-bit host.+ for fetch in [None, Some(7), Some(u32::MAX as usize), Some(usize::MAX)] {+ let join = HashJoinExec::try_new(+ Arc::new(EmptyExec::new(Arc::clone(&schema_left))),+ Arc::new(EmptyExec::new(Arc::clone(&schema_right))),+ on.clone(),+ None,+ &JoinType::Inner,+ None,+ PartitionMode::Partitioned,+ NullEquality::NullEqualsNothing,+ false,+ )?;++ let plan: Arc<dyn ExecutionPlan> = match fetch {+ // This is how `limit_pushdown` installs the limit.+ Some(fetch) => join+ .with_fetch(Some(fetch))+ .expect("HashJoinExec supports fetch"),+ None => Arc::new(join),+ };+ assert_eq!(plan.fetch(), fetch);++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let deserialized =+ roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;++ let deserialized_join = deserialized+ .downcast_ref::<HashJoinExec>()+ .expect("should be a HashJoinExec");+ assert_eq!(deserialized_join.fetch(), fetch);+ }+ Ok(())+}++/// Same regression coverage for `NestedLoopJoinExec`, which shares the+/// `repeated uint32 projection` proto field shape with `HashJoinExec`.+#[test]+fn roundtrip_nested_loop_join_projection_states() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));+ let schema_right = Arc::new(Schema::new(vec![field_a]));++ for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {+ roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ None,+ &JoinType::Inner,+ projection,+ )?))?;+ }+ Ok(())+}++#[test]+fn roundtrip_sym_hash_join() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let field_a = Field::new("col_a", DataType::Int64, false);+ let field_b = Field::new("col_b", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_b.clone()]);+ let on = vec![(+ Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,+ Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,+ )];+ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("col_a", 0)),+ Operator::Gt,+ Arc::new(Column::new("col_b", 1)),+ )),+ vec![+ ColumnIndex {+ index: 0,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 0,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![field_a, field_b])),+ );++ let schema_left = Arc::new(schema_left);+ let schema_right = Arc::new(schema_right);+ let left_order: LexOrdering = [PhysicalSortExpr {+ expr: Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)),+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ }]+ .into();+ let right_order: LexOrdering = [PhysicalSortExpr {+ expr: Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }]+ .into();+ let ordering_cases = [+ (None, None),+ (Some(left_order.clone()), None),+ (None, Some(right_order.clone())),+ (Some(left_order), Some(right_order)),+ ];+ let ordering_options = |ordering: Option<&LexOrdering>| {+ ordering+ .map(|ordering| ordering.iter().map(|expr| expr.options).collect::<Vec<_>>())+ };++ for join_type in [+ JoinType::Inner,+ JoinType::Left,+ JoinType::Right,+ JoinType::Full,+ JoinType::LeftAnti,+ JoinType::RightAnti,+ JoinType::LeftSemi,+ JoinType::RightSemi,+ JoinType::LeftMark,+ JoinType::RightMark,+ ] {+ for null_equality in [+ NullEquality::NullEqualsNothing,+ NullEquality::NullEqualsNull,+ ] {+ for filter in [None, Some(filter.clone())] {+ for partition_mode in [+ StreamJoinPartitionMode::Partitioned,+ StreamJoinPartitionMode::SinglePartition,+ ] {+ for (left_order, right_order) in &ordering_cases {+ let result = roundtrip_test_and_return(+ Arc::new(SymmetricHashJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ on.clone(),+ filter.clone(),+ &join_type,+ null_equality,+ left_order.clone(),+ right_order.clone(),+ partition_mode,+ )?),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let result =+ result.downcast_ref::<SymmetricHashJoinExec>().unwrap();+ assert_eq!(result.join_type(), &join_type);+ assert_eq!(result.null_equality(), null_equality);+ assert_eq!(result.partition_mode(), partition_mode);+ assert_eq!(+ ordering_options(result.left_sort_exprs()),+ ordering_options(left_order.as_ref())+ );+ assert_eq!(+ ordering_options(result.right_sort_exprs()),+ ordering_options(right_order.as_ref())+ );+ assert_eq!(+ result.filter().map(JoinFilter::column_indices),+ filter.as_ref().map(JoinFilter::column_indices)+ );+ }+ }+ }+ }+ }+ Ok(())+}++#[test]+fn roundtrip_sort_merge_join() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let field_a = Field::new("col_a", DataType::Int64, false);+ let field_b = Field::new("col_b", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_b.clone()]);+ let on = vec![(+ Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,+ Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,+ )];++ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("col_a", 1)),+ Operator::Gt,+ Arc::new(Column::new("col_b", 0)),+ )),+ vec![+ ColumnIndex {+ index: 0,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 0,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![field_a, field_b])),+ );++ let schema_left = Arc::new(schema_left);+ let schema_right = Arc::new(schema_right);+ let sort_options = vec![SortOptions {+ descending: true,+ nulls_first: false,+ }];+ for null_equality in [+ NullEquality::NullEqualsNothing,+ NullEquality::NullEqualsNull,+ ] {+ for filter in [None, Some(filter.clone())] {+ for join_type in [+ JoinType::Inner,+ JoinType::Left,+ JoinType::Right,+ JoinType::Full,+ JoinType::LeftAnti,+ JoinType::RightAnti,+ JoinType::LeftSemi,+ JoinType::RightSemi,+ JoinType::LeftMark,+ JoinType::RightMark,+ ] {+ let result = roundtrip_test_and_return(+ Arc::new(SortMergeJoinExec::try_new(+ Arc::new(EmptyExec::new(schema_left.clone())),+ Arc::new(EmptyExec::new(schema_right.clone())),+ on.clone(),+ filter.clone(),+ join_type,+ sort_options.clone(),+ null_equality,+ )?),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let result = result.downcast_ref::<SortMergeJoinExec>().unwrap();+ assert_eq!(result.join_type(), join_type);+ assert_eq!(result.null_equality(), null_equality);+ assert_eq!(result.sort_options(), sort_options);+ assert_eq!(+ result.filter().as_ref().map(|f| f.column_indices()),+ filter.as_ref().map(|f| f.column_indices())+ );+ }+ }+ }+ Ok(())+}++#[tokio::test]+async fn roundtrip_logical_plan_sort_merge_join() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_csv(+ "t0",+ "tests/testdata/test.csv",+ datafusion::prelude::CsvReadOptions::default().has_header(true),+ )+ .await?;+ ctx.register_csv(+ "t1",+ "tests/testdata/test.csv",+ datafusion::prelude::CsvReadOptions::default().has_header(true),+ )+ .await?;++ ctx.sql("SET datafusion.optimizer.prefer_hash_join = false")+ .await?+ .show()+ .await?;++ let query = "SELECT t1.* FROM t0 join t1 on t0.a = t1.a";+ let plan = ctx.sql(query).await?.create_physical_plan().await?;+ roundtrip_test(plan)+}diff --git a/datafusion/proto/tests/cases/plans/leaves.rs b/datafusion/proto/tests/cases/plans/leaves.rsnew file mode 100644index 0000000000..afcab2dda2--- /dev/null+++ b/datafusion/proto/tests/cases/plans/leaves.rs@@ -0,0 +1,89 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Leaf plans: `EmptyExec` and `PlaceholderRowExec`.++use super::{roundtrip_test, roundtrip_test_and_return};+use datafusion::arrow::datatypes::Schema;+use datafusion::physical_plan::ExecutionPlanProperties;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion::prelude::SessionContext;+use datafusion_common::Result;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::PhysicalPlanNode;+use std::sync::Arc;++#[test]+fn roundtrip_empty() -> Result<()> {+ roundtrip_test(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))+}++#[test]+fn roundtrip_empty_with_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let plan = Arc::new(EmptyExec::new(Arc::new(Schema::empty())).with_partitions(4));+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;+ assert_eq!(plan.output_partitioning().partition_count(), 4);+ Ok(())+}++#[test]+fn roundtrip_placeholder_row_with_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let plan =+ Arc::new(PlaceholderRowExec::new(Arc::new(Schema::empty())).with_partitions(4));+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;+ assert_eq!(plan.output_partitioning().partition_count(), 4);+ Ok(())+}++/// Plans encoded before `partitions` was added carry no value for it, which+/// decodes as zero and must be treated as the previous default of one.+#[test]+fn decode_empty_and_placeholder_row_without_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let schema: protobuf::Schema = (&Schema::empty()).try_into()?;++ for physical_plan_type in [+ protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {+ schema: Some(schema.clone()),+ partitions: 0,+ }),+ protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(+ protobuf::PlaceholderRowExecNode {+ schema: Some(schema.clone()),+ partitions: 0,+ },+ ),+ ] {+ let node = PhysicalPlanNode {+ physical_plan_type: Some(physical_plan_type),+ };+ let plan = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;+ assert_eq!(plan.output_partitioning().partition_count(), 1);+ }+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/limits.rs b/datafusion/proto/tests/cases/plans/limits.rsnew file mode 100644index 0000000000..e1c33ff949--- /dev/null+++ b/datafusion/proto/tests/cases/plans/limits.rs@@ -0,0 +1,236 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Plans that carry a row limit or shape the batch pipeline: the limit+//! execs, the coalescing execs, `BufferExec` and `CooperativeExec`.++use super::{roundtrip_test, roundtrip_test_and_return};+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::datasource::listing::PartitionedFile;+use datafusion::datasource::object_store::ObjectStoreUrl;+use datafusion::datasource::physical_plan::{+ FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource,+};+use datafusion::datasource::source::DataSourceExec;+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_optimizer::PhysicalOptimizerRule;+use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;+use datafusion::physical_plan::buffer::BufferExec;+#[expect(deprecated)]+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;+use datafusion::physical_plan::coop::CooperativeExec;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};+use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};+use datafusion::prelude::SessionContext;+use datafusion_common::Result;+use datafusion_common::config::ConfigOptions;+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_local_limit() -> Result<()> {+ roundtrip_test(Arc::new(LocalLimitExec::new(+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),+ 25,+ )))+}++#[test]+fn roundtrip_global_limit() -> Result<()> {+ roundtrip_test(Arc::new(GlobalLimitExec::new(+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),+ 0,+ Some(25),+ )))+}++#[test]+fn roundtrip_global_skip_no_limit() -> Result<()> {+ roundtrip_test(Arc::new(GlobalLimitExec::new(+ Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),+ 10,+ None, // no limit+ )))+}++/// Sort key at index 1, so a decoder that misbinds column name vs index+/// cannot pass.+fn limit_test_schema() -> Arc<Schema> {+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, false),+ ]))+}++/// Non-default sort options, so a decode that falls back to defaults cannot+/// pass.+fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {+ Ok(LexOrdering::new(vec![PhysicalSortExpr {+ expr: col("b", schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ }]))+}++#[test]+fn roundtrip_limit_with_required_ordering() -> Result<()> {+ let schema = limit_test_schema();+ let required_ordering = limit_required_ordering(&schema)?;+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};++ let mut global =+ GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));+ global.set_required_ordering(required_ordering.clone());+ let decoded =+ roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;+ let decoded = decoded+ .downcast_ref::<GlobalLimitExec>()+ .expect("expected GlobalLimitExec");+ assert_eq!(decoded.required_ordering(), &required_ordering);++ let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);+ local.set_required_ordering(required_ordering.clone());+ let decoded =+ roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;+ let decoded = decoded+ .downcast_ref::<LocalLimitExec>()+ .expect("expected LocalLimitExec");+ assert_eq!(decoded.required_ordering(), &required_ordering);+ Ok(())+}++/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`+/// whose sort node was optimized away is order-sensitive, so it must survive+/// serde all the way into the scan's `preserve_order` flag.+#[test]+fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {+ let file_schema = limit_test_schema();+ let make_scan = || {+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .build();+ DataSourceExec::from_data_source(scan_config)+ };+ let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let decoded =+ roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;++ // Child replacement must not erase the decoded ordering before pushdown.+ let rebuilt = decoded.with_new_children(vec![make_scan()])?;++ let optimized =+ LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;+ let scan = optimized+ .downcast_ref::<DataSourceExec>()+ .expect("limit should be absorbed into the scan");+ Ok(scan+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .expect("expected FileScanConfig")+ .clone())+ };++ let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));+ limit.set_required_ordering(limit_required_ordering(&file_schema)?);+ let scan_config = scan_after_limit_pushdown(limit)?;+ assert_eq!(scan_config.limit, Some(10));+ assert!(scan_config.preserve_order);++ let scan_config =+ scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;+ assert_eq!(scan_config.limit, Some(10));+ assert!(!scan_config.preserve_order);+ Ok(())+}++#[test]+fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ #[expect(deprecated)]+ roundtrip_test(Arc::new(CoalesceBatchesExec::new(+ Arc::new(EmptyExec::new(schema.clone())),+ 8096,+ )))?;++ #[expect(deprecated)]+ roundtrip_test(Arc::new(+ CoalesceBatchesExec::new(Arc::new(EmptyExec::new(schema)), 8096)+ .with_fetch(Some(10)),+ ))+}++#[test]+fn roundtrip_coalesce_partitions_with_fetch() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ roundtrip_test(Arc::new(CoalescePartitionsExec::new(Arc::new(+ EmptyExec::new(schema.clone()),+ ))))?;++ roundtrip_test(Arc::new(+ CoalescePartitionsExec::new(Arc::new(EmptyExec::new(schema)))+ .with_fetch(Some(10)),+ ))+}++#[test]+fn roundtrip_cooperative() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));+ roundtrip_test(Arc::new(CooperativeExec::new(Arc::new(EmptyExec::new(+ schema,+ )))))+}++#[test]+fn roundtrip_buffer() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let result = roundtrip_test_and_return(+ Arc::new(BufferExec::new(Arc::new(EmptyExec::new(schema)), 4096)),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let result = result.downcast_ref::<BufferExec>().unwrap();+ assert_eq!(result.capacity(), 4096);+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/misc.rs b/datafusion/proto/tests/cases/plans/misc.rsnew file mode 100644index 0000000000..254387f14e--- /dev/null+++ b/datafusion/proto/tests/cases/plans/misc.rs@@ -0,0 +1,415 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Plans that do not (yet) warrant a file of their own: unions, unnest,+//! repartitioning and the analyze/explain execs.++use super::{roundtrip_test, roundtrip_test_and_return};+use arrow::datatypes::{Fields, TimeUnit};+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_plan::analyze::AnalyzeExec;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::explain::ExplainExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};+use datafusion::physical_plan::metrics::MetricCategory;+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion::physical_plan::repartition::RepartitionExec;+use datafusion::physical_plan::sorts::sort::SortExec;+use datafusion::physical_plan::union::{InterleaveExec, UnionExec};+use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};+use datafusion::physical_plan::{+ ExecutionPlan, Partitioning, RangePartitioning, SplitPoint,+};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::display::{PlanType, StringifiedPlan};+use datafusion_common::format::ExplainFormat;+use datafusion_common::{DataFusionError, Result, UnnestOptions};+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+ PhysicalPlanDecodeContext,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::PhysicalPlanNode;+use prost::Message;+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_analyze() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("plan_type", DataType::Utf8, false),+ Field::new("plan", DataType::Utf8, false),+ ]));+ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));+ let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing];+ let analyze = Arc::new(+ AnalyzeExec::builder(true, true, input, Arc::clone(&schema))+ .with_metric_categories(Some(metric_categories.clone()))+ .with_format(ExplainFormat::Tree)+ .build(),+ );++ let ctx = SessionContext::new();+ let roundtripped = roundtrip_test_and_return(+ analyze,+ &ctx,+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ let roundtripped = roundtripped.downcast_ref::<AnalyzeExec>().unwrap();++ assert_eq!(roundtripped.schema(), schema);+ assert!(roundtripped.verbose());+ assert!(roundtripped.show_statistics());+ assert_eq!(+ roundtripped.metric_categories(),+ Some(metric_categories.as_slice())+ );+ assert_eq!(roundtripped.format(), &ExplainFormat::Tree);+ assert!(+ roundtripped+ .input()+ .downcast_ref::<PlaceholderRowExec>()+ .is_some()+ );+ Ok(())+}++#[test]+fn roundtrip_explain() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("plan_type", DataType::Utf8, false),+ Field::new("plan", DataType::Utf8, false),+ ]));+ let stringified_plans = vec![+ StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),+ StringifiedPlan::new(+ PlanType::AnalyzedLogicalPlan {+ analyzer_name: "analyzer".to_string(),+ },+ "analyzed logical",+ ),+ StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),+ StringifiedPlan::new(+ PlanType::OptimizedLogicalPlan {+ optimizer_name: "logical optimizer".to_string(),+ },+ "optimized logical",+ ),+ StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),+ StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),+ StringifiedPlan::new(+ PlanType::InitialPhysicalPlanWithStats,+ "initial physical with stats",+ ),+ StringifiedPlan::new(+ PlanType::InitialPhysicalPlanWithSchema,+ "initial physical with schema",+ ),+ StringifiedPlan::new(+ PlanType::OptimizedPhysicalPlan {+ optimizer_name: "physical optimizer".to_string(),+ },+ "optimized physical",+ ),+ StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),+ StringifiedPlan::new(+ PlanType::FinalPhysicalPlanWithStats,+ "final physical with stats",+ ),+ StringifiedPlan::new(+ PlanType::FinalPhysicalPlanWithSchema,+ "final physical with schema",+ ),+ StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),+ ];+ let explain = Arc::new(ExplainExec::new(+ Arc::clone(&schema),+ stringified_plans.clone(),+ true,+ ));++ let ctx = SessionContext::new();+ let roundtripped = roundtrip_test_and_return(+ explain,+ &ctx,+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();++ assert_eq!(roundtripped.schema(), schema);+ assert_eq!(roundtripped.stringified_plans(), stringified_plans);+ assert!(roundtripped.verbose());+ Ok(())+}++#[test]+fn roundtrip_union() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_a]);+ let left = EmptyExec::new(Arc::new(schema_left));+ let right = EmptyExec::new(Arc::new(schema_right));+ let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];+ let union = UnionExec::try_new(inputs)?;+ roundtrip_test(union)+}++#[test]+fn roundtrip_repartition_preserve_order() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a]));+ let sort_exprs: LexOrdering = [PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions::default(),+ }]+ .into();++ // Create two sorted single-partition inputs, then union them to get+ // a sorted input with 2 partitions.+ let source1 = SortExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ );+ let source2 = SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)));+ let union = UnionExec::try_new(vec![+ Arc::new(source1) as Arc<dyn ExecutionPlan>,+ Arc::new(source2) as Arc<dyn ExecutionPlan>,+ ])?;++ let repartition = RepartitionExec::try_new(union, Partitioning::RoundRobinBatch(10))?+ .with_preserve_order();+ assert!(repartition.preserve_order());++ roundtrip_test(Arc::new(repartition))+}++#[test]+fn roundtrip_range_partitioning() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let range_partitioning = Partitioning::Range(RangePartitioning::new(+ [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),+ vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],+ ));+ // RepartitionExec is used only to carry the partitioning through proto.+ // Executing range repartitioning is intentionally unsupported.+ let repartition = RepartitionExec::try_new(input, range_partitioning)?;++ roundtrip_test(Arc::new(repartition))+}++/// `parse_protobuf_hash_partitioning` has no in-tree callers left; it delegates+/// to the shared `Partitioning::try_from_proto`, so pin that it still decodes+/// the hash message it is handed.+#[test]+fn parse_hash_partitioning_delegates_to_shared_decoder() -> Result<()> {+ use datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;++ let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);+ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let codec = DefaultPhysicalExtensionCodec {};+ let decode_ctx = PhysicalPlanDecodeContext::new(&task_ctx, &codec);+ let proto_converter = DefaultPhysicalProtoConverter {};++ let hash_expr = serialize_physical_expr_with_converter(+ &col("a", &schema)?,+ &codec,+ &proto_converter,+ )?;+ let hash = protobuf::PhysicalHashRepartition {+ hash_expr: vec![hash_expr],+ partition_count: 4,+ };++ let partitioning = parse_protobuf_hash_partitioning(+ Some(&hash),+ &decode_ctx,+ &schema,+ &proto_converter,+ )?;+ let Some(Partitioning::Hash(exprs, count)) = partitioning else {+ panic!("expected hash partitioning, got {partitioning:?}");+ };+ assert_eq!(count, 4);+ assert_eq!(exprs.len(), 1);+ assert_eq!(exprs[0].to_string(), col("a", &schema)?.to_string());++ // No message means no partitioning, as before.+ assert!(+ parse_protobuf_hash_partitioning(None, &decode_ctx, &schema, &proto_converter)?+ .is_none()+ );++ // The count is a `u64` on the wire and a `usize` in memory, so decoding+ // narrows it. A count that does not fit is the case that motivated routing+ // this through the shared decoder: it used to `unwrap()` and panic, and now+ // reports an error. Only a target narrower than 64 bits can reach that arm+ // -- on a 64-bit target every `u64` fits, and the assertion there is that+ // the largest possible count survives whole rather than being truncated.+ let oversized = protobuf::PhysicalHashRepartition {+ hash_expr: vec![serialize_physical_expr_with_converter(+ &col("a", &schema)?,+ &codec,+ &proto_converter,+ )?],+ partition_count: u64::MAX,+ };+ let decoded = parse_protobuf_hash_partitioning(+ Some(&oversized),+ &decode_ctx,+ &schema,+ &proto_converter,+ );++ #[cfg(target_pointer_width = "64")]+ {+ let Some(Partitioning::Hash(_, count)) = decoded? else {+ panic!("expected hash partitioning");+ };+ assert_eq!(count, usize::MAX);+ }++ #[cfg(not(target_pointer_width = "64"))]+ assert!(+ decoded+ .unwrap_err()+ .to_string()+ .contains("Partition count 18446744073709551615 exceeds usize::MAX")+ );++ Ok(())+}++#[test]+fn roundtrip_interleave() -> Result<()> {+ let field_a = Field::new("col", DataType::Int64, false);+ let schema_left = Schema::new(vec![field_a.clone()]);+ let schema_right = Schema::new(vec![field_a]);+ let partition = Partitioning::Hash(vec![], 3);+ let left = RepartitionExec::try_new(+ Arc::new(EmptyExec::new(Arc::new(schema_left))),+ partition.clone(),+ )?;+ let right = RepartitionExec::try_new(+ Arc::new(EmptyExec::new(Arc::new(schema_right))),+ partition,+ )?;+ let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];+ let interleave = InterleaveExec::try_new(inputs)?;+ roundtrip_test(Arc::new(interleave))+}++#[test]+fn roundtrip_unnest() -> Result<()> {+ let fa = Field::new("a", DataType::Int64, true);+ let fb0 = Field::new_list_field(DataType::Utf8, true);+ let fb = Field::new_list("b", fb0.clone(), false);+ let fc1 = Field::new("c1", DataType::Boolean, false);+ let fc2 = Field::new("c2", DataType::Date64, true);+ let fc = Field::new_struct("c", Fields::from(vec![fc1.clone(), fc2.clone()]), true);+ let fd0 = Field::new_list_field(DataType::Float32, false);+ let fd = Field::new_list("d", fd0.clone(), true);+ let fe1 = Field::new("e1", DataType::UInt16, false);+ let fe2 = Field::new("e2", DataType::Duration(TimeUnit::Millisecond), true);+ let fe3 = Field::new("e3", DataType::Timestamp(TimeUnit::Millisecond, None), true);+ let fe_fields = Fields::from(vec![fe1.clone(), fe2.clone(), fe3.clone()]);+ let fe = Field::new_struct("e", fe_fields, false);++ let fb0 = fb0.with_name("b");+ let fd0 = fd0.with_name("d");+ let input_schema = Arc::new(Schema::new(vec![fa.clone(), fb, fc, fd, fe]));+ let output_schema =+ Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));+ let input = Arc::new(EmptyExec::new(input_schema));+ let options = UnnestOptions {+ null_handling: datafusion_common::NullHandling::Drop,+ recursions: vec![datafusion_common::RecursionUnnestOption {+ input_column: datafusion_common::Column::new_unqualified("b"),+ output_column: datafusion_common::Column::new_unqualified("b"),+ depth: 2,+ }],+ };+ let unnest = UnnestExec::new(+ input,+ vec![+ ListUnnest {+ index_in_input_schema: 1,+ depth: 1,+ },+ ListUnnest {+ index_in_input_schema: 1,+ depth: 2,+ },+ ListUnnest {+ index_in_input_schema: 3,+ depth: 2,+ },+ ],+ vec![2, 4],+ output_schema,+ options.clone(),+ )?;+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let result =+ roundtrip_test_and_return(Arc::new(unnest), &ctx, &codec, &proto_converter)?;+ let result = result.downcast_ref::<UnnestExec>().unwrap();+ assert_eq!(result.options(), &options);++ Ok(())+}++#[tokio::test]+/// Tests that we can serialize an unoptimized "analyze" plan and it will work on the other end+async fn analyze_roundtrip_unoptimized() -> Result<()> {+ let ctx = SessionContext::new();++ // No optimizations+ let session_state =+ datafusion::execution::SessionStateBuilder::new_from_existing(ctx.state())+ .with_physical_optimizer_rules(vec![])+ .build();++ let logical_plan = session_state+ .create_logical_plan("explain analyze select 1")+ .await?;+ let plan = session_state.create_physical_plan(&logical_plan).await?;++ let node = PhysicalPlanNode::try_from_physical_plan(+ plan.clone(),+ &DefaultPhysicalExtensionCodec {},+ )?;++ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())+ .map_err(|e| DataFusionError::External(Box::new(e)))?;++ let unoptimized =+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;++ let physical_planner =+ datafusion::physical_planner::DefaultPhysicalPlanner::default();+ physical_planner.optimize_physical_plan(unoptimized, &session_state, |_, _| {})?;+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rsnew file mode 100644index 0000000000..8dad1eff67--- /dev/null+++ b/datafusion/proto/tests/cases/plans/mod.rs@@ -0,0 +1,135 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Round trip tests for the physical plan protobuf representation.+//!+//! The tests are grouped by the kind of plan they cover; the shared+//! round trip helpers live here.++use datafusion::physical_plan::ExecutionPlan;+use datafusion::prelude::{ParquetReadOptions, SessionContext};+use datafusion_common::Result;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,+};+use std::sync::Arc;++mod aggregates;+mod dispatch;+mod dynamic_filters;+mod exprs;+mod filters;+mod joins;+mod leaves;+mod limits;+mod misc;+mod scalar_subquery;+mod sinks;+mod sorts;+mod sources;+mod tpch;+mod udfs;+mod windows;++/// Perform a serde roundtrip and assert that the string representation of the before and after plans+/// are identical. Note that this often isn't sufficient to guarantee that no information is+/// lost during serde because the string representation of a plan often only shows a subset of state.+fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;+ Ok(())+}++/// Perform a serde roundtrip and assert that the string representation of the before and after plans+/// are identical. Note that this often isn't sufficient to guarantee that no information is+/// lost during serde because the string representation of a plan often only shows a subset of state.+///+/// This version of the roundtrip_test method returns the final plan after serde so that it can be inspected+/// farther in tests.+fn roundtrip_test_and_return(+ exec_plan: Arc<dyn ExecutionPlan>,+ ctx: &SessionContext,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,+) -> Result<Arc<dyn ExecutionPlan>> {+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&exec_plan),+ codec,+ proto_converter,+ )?;+ let result_exec_plan = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ codec,+ proto_converter,+ )?;++ pretty_assertions::assert_eq!(+ format!("{exec_plan:?}"),+ format!("{result_exec_plan:?}")+ );+ Ok(result_exec_plan)+}++/// Perform a serde roundtrip and assert that the string representation of the before and after plans+/// are identical. Note that this often isn't sufficient to guarantee that no information is+/// lost during serde because the string representation of a plan often only shows a subset of state.+///+/// This version of the roundtrip_test function accepts a SessionContext, which is required when+/// performing serde on some plans.+fn roundtrip_test_with_context(+ exec_plan: Arc<dyn ExecutionPlan>,+ ctx: &SessionContext,+) -> Result<()> {+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;+ Ok(())+}++/// Perform a serde roundtrip for the specified sql query, and assert that+/// query results are identical.+async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;++ roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;+ Ok(())+}++/// returns a SessionContext with `alltypes_plain` registered+async fn all_types_context() -> Result<SessionContext> {+ let ctx = SessionContext::new();++ let testdata = datafusion::test_util::parquet_test_data();+ ctx.register_parquet(+ "alltypes_plain",+ &format!("{testdata}/alltypes_plain.parquet"),+ ParquetReadOptions::default(),+ )+ .await?;++ Ok(ctx)+}diff --git a/datafusion/proto/tests/cases/plans/scalar_subquery.rs b/datafusion/proto/tests/cases/plans/scalar_subquery.rsnew file mode 100644index 0000000000..34d30aa03d--- /dev/null+++ b/datafusion/proto/tests/cases/plans/scalar_subquery.rs@@ -0,0 +1,278 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! `ScalarSubqueryExec` and the results it scopes to its subtree.++use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::logical_expr::Operator;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{BinaryExpr, binary, col};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::scalar_subquery::{+ ScalarSubqueryExec, ScalarSubqueryLink,+};+use datafusion::prelude::SessionContext;+use datafusion_common::Result;+use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};+use datafusion_proto::physical_plan::{+ DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,+};+use std::sync::Arc;+use std::vec;++/// Verify that ScalarSubqueryExpr nodes in the input plan are connected to the+/// same shared results container as ScalarSubqueryExec after a proto round-trip.+#[test]+fn roundtrip_scalar_subquery_exec() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let results = ScalarSubqueryResults::new(1);++ // Build the input plan: a filter whose predicate references the+ // scalar subquery result via ScalarSubqueryExpr.+ let sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ results.clone(),+ ));+ let predicate = binary(col("a", &schema)?, Operator::Eq, sq_expr, &schema)?;+ let filter =+ FilterExec::try_new(predicate, Arc::new(EmptyExec::new(schema.clone())))?;++ // Build a trivial subquery plan.+ let subquery_plan =+ Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(+ "x",+ DataType::Int64,+ true,+ )]))));++ let exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ Arc::new(filter),+ vec![ScalarSubqueryLink {+ plan: subquery_plan,+ index: SubqueryIndex::new(0),+ }],+ results,+ ));++ // Perform the round-trip using DeduplicatingProtoConverter, which+ // creates a DeduplicatingDeserializer that threads scalar subquery+ // results through expression deserialization.+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&exec),+ &codec,+ &converter,+ )?;+ let ctx = SessionContext::new();+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Verify the deserialized ScalarSubqueryExec's results container is+ // shared with the ScalarSubqueryExpr in the input plan.+ let sq_exec = deserialized+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected ScalarSubqueryExec");+ let exec_results = sq_exec.results();++ // Walk the input plan to find the ScalarSubqueryExpr and verify it+ // points to the same results container.+ let filter_exec = sq_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected FilterExec");+ let binary_expr = filter_exec+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected BinaryExpr");+ let deserialized_sq_expr = binary_expr+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected ScalarSubqueryExpr");++ assert!(+ ScalarSubqueryResults::ptr_eq(exec_results, deserialized_sq_expr.results()),+ "ScalarSubqueryExpr should share the same results container as ScalarSubqueryExec"+ );+ Ok(())+}++/// Verify that nested ScalarSubqueryExec nodes deserialize with distinct+/// scoped results containers, and that each ScalarSubqueryExpr is wired to the+/// container for its own surrounding ScalarSubqueryExec.+#[test]+fn roundtrip_nested_scalar_subquery_exec_scopes_results() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let subquery_schema =+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));++ let inner_results = ScalarSubqueryResults::new(1);+ let inner_sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ inner_results.clone(),+ ));+ let inner_predicate =+ binary(col("a", &schema)?, Operator::Eq, inner_sq_expr, &schema)?;+ let inner_filter = Arc::new(FilterExec::try_new(+ inner_predicate,+ Arc::new(EmptyExec::new(schema.clone())),+ )?);+ let inner_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ inner_filter,+ vec![ScalarSubqueryLink {+ plan: Arc::new(EmptyExec::new(subquery_schema.clone())),+ index: SubqueryIndex::new(0),+ }],+ inner_results,+ ));++ let outer_results = ScalarSubqueryResults::new(1);+ let outer_sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ outer_results.clone(),+ ));+ let outer_predicate =+ binary(col("a", &schema)?, Operator::Eq, outer_sq_expr, &schema)?;+ let outer_filter = Arc::new(FilterExec::try_new(outer_predicate, inner_exec)?);+ let outer_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ outer_filter,+ vec![ScalarSubqueryLink {+ plan: Arc::new(EmptyExec::new(subquery_schema)),+ index: SubqueryIndex::new(0),+ }],+ outer_results,+ ));++ let bytes = datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&outer_exec))?;+ let ctx = SessionContext::new();+ let deserialized = datafusion_proto::bytes::physical_plan_from_bytes(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ )?;++ let outer_exec = deserialized+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected outer ScalarSubqueryExec");+ let outer_results = outer_exec.results();+ let outer_filter = outer_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected outer FilterExec");+ let outer_binary = outer_filter+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected outer BinaryExpr");+ let outer_sq_expr = outer_binary+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected outer ScalarSubqueryExpr");++ let inner_exec = outer_filter+ .input()+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected inner ScalarSubqueryExec");+ let inner_results = inner_exec.results();+ let inner_filter = inner_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected inner FilterExec");+ let inner_binary = inner_filter+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected inner BinaryExpr");+ let inner_sq_expr = inner_binary+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected inner ScalarSubqueryExpr");++ assert!(+ ScalarSubqueryResults::ptr_eq(outer_results, outer_sq_expr.results()),+ "outer ScalarSubqueryExpr should use outer ScalarSubqueryExec results"+ );+ assert!(+ ScalarSubqueryResults::ptr_eq(inner_results, inner_sq_expr.results()),+ "inner ScalarSubqueryExpr should use inner ScalarSubqueryExec results"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_results),+ "nested ScalarSubqueryExec nodes should not share results containers"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_sq_expr.results()),+ "inner ScalarSubqueryExpr must not read from outer results"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(inner_results, outer_sq_expr.results()),+ "outer ScalarSubqueryExpr must not read from inner results"+ );++ Ok(())+}++/// Verify that the default physical plan bytes round-trip preserves executable+/// scalar subquery plans.+#[tokio::test]+async fn roundtrip_scalar_subquery_exec_with_default_converter_executes() -> Result<()> {+ let ctx = SessionContext::new();+ let sql = "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) AS s \+ FROM (VALUES (2), (1)) AS t(x) \+ ORDER BY s";++ let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;+ assert!(+ format!("{initial_plan:?}").contains("ScalarSubqueryExec"),+ "expected ScalarSubqueryExec in plan:\n{initial_plan:?}"+ );++ let bytes =+ datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&initial_plan))?;+ let roundtripped = datafusion_proto::bytes::physical_plan_from_bytes(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ )?;+ assert!(+ format!("{roundtripped:?}").contains("ScalarSubqueryExec"),+ "expected ScalarSubqueryExec after roundtrip:\n{roundtripped:?}"+ );++ let batches = datafusion::physical_plan::common::collect(+ roundtripped.execute(0, ctx.task_ctx())?,+ )+ .await?;+ datafusion::assert_batches_eq!(+ &["+----+", "| s |", "+----+", "| 21 |", "| 22 |", "+----+",],+ &batches+ );++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/sinks.rs b/datafusion/proto/tests/cases/plans/sinks.rsnew file mode 100644index 0000000000..517659e9e9--- /dev/null+++ b/datafusion/proto/tests/cases/plans/sinks.rs@@ -0,0 +1,329 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Data sinks and their file sink configurations.++use super::{roundtrip_test, roundtrip_test_and_return};+use arrow::csv::WriterBuilder;+use async_trait::async_trait;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion::datasource::file_format::csv::CsvSink;+use datafusion::datasource::file_format::json::JsonSink;+use datafusion::datasource::file_format::parquet::ParquetSink;+use datafusion::datasource::listing::{ListingTableUrl, PartitionedFile};+use datafusion::datasource::object_store::ObjectStoreUrl;+use datafusion::datasource::physical_plan::{FileGroup, FileOutputMode, FileSinkConfig};+use datafusion::datasource::sink::{DataSink, DataSinkExec};+use datafusion::execution::TaskContext;+use datafusion::physical_expr::PhysicalSortRequirement;+use datafusion::physical_plan::expressions::Column;+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;+use datafusion::physical_plan::{+ DisplayAs, DisplayFormatType, ExecutionPlan, SendableRecordBatchStream,+};+use datafusion::prelude::SessionContext;+use datafusion_common::Result;+use datafusion_common::config::TableParquetOptions;+use datafusion_common::file_options::csv_writer::CsvWriterOptions;+use datafusion_common::file_options::json_writer::JsonWriterOptions;+use datafusion_common::parsers::CompressionTypeVariant;+use datafusion_expr::dml::InsertOp;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::PhysicalPlanNode;+use std::fmt::Formatter;+use std::sync::Arc;+use std::vec;++#[derive(Debug)]+struct ProtoHookSink {+ schema: SchemaRef,+}++impl DisplayAs for ProtoHookSink {+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {+ write!(f, "ProtoHookSink")+ }+}++#[async_trait]+impl DataSink for ProtoHookSink {+ fn schema(&self) -> &SchemaRef {+ &self.schema+ }++ async fn write_all(+ &self,+ _data: SendableRecordBatchStream,+ _context: &Arc<TaskContext>,+ ) -> Result<u64> {+ unreachable!("serialization test does not execute the sink")+ }++ fn try_to_proto(+ &self,+ exec: &DataSinkExec,+ ctx: &ExecutionPlanEncodeCtx<'_>,+ ) -> Result<Option<PhysicalPlanNode>> {+ let input = ctx.encode_child(exec.input())?;+ let sort_order = exec.encode_sort_order(ctx)?;+ assert!(matches!(+ input.physical_plan_type,+ Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))+ ));+ assert_eq!(+ sort_order+ .as_ref()+ .map(|ordering| ordering.physical_sort_expr_nodes.len()),+ Some(1)+ );+ assert_eq!(exec.schema().fields().len(), 1);++ Ok(Some(PhysicalPlanNode {+ physical_plan_type: Some(+ protobuf::physical_plan_node::PhysicalPlanType::Empty(+ protobuf::EmptyExecNode {+ schema: Some(exec.schema().as_ref().try_into()?),+ partitions: 1,+ },+ ),+ ),+ }))+ }+}++#[test]+fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {+ let input_schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Int64,+ false,+ )]));+ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&input_schema)));+ let sink = Arc::new(ProtoHookSink {+ schema: Arc::clone(&input_schema),+ });+ let sort_order = [PhysicalSortRequirement::new(+ Arc::new(Column::new("value", 0)),+ Some(SortOptions::default()),+ )]+ .into();+ let plan = Arc::new(DataSinkExec::new(input, sink, Some(sort_order)));++ let node = PhysicalPlanNode::try_from_physical_plan(+ plan,+ &DefaultPhysicalExtensionCodec {},+ )?;++ assert!(matches!(+ node.physical_plan_type,+ Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))+ ));+ Ok(())+}++#[test]+fn file_sink_config_roundtrip_preserves_fields() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "partition",+ DataType::Utf8,+ false,+ )]));+ let config = FileSinkConfig {+ original_url: "file:///tmp/output".to_string(),+ object_store_url: ObjectStoreUrl::local_filesystem(),+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp/output", 1)]),+ table_paths: vec![ListingTableUrl::parse("file:///tmp/output")?],+ output_schema: schema,+ table_partition_cols: vec![("partition".to_string(), DataType::Utf8)],+ insert_op: InsertOp::Overwrite,+ keep_partition_by_columns: true,+ file_extension: "parquet".to_string(),+ file_output_mode: FileOutputMode::Directory,+ };++ let encoded = protobuf::FileSinkConfig::try_from(&config)?;+ assert_eq!(encoded.insert_op(), protobuf::InsertOp::Overwrite);+ assert_eq!(+ encoded.file_output_mode(),+ protobuf::FileOutputMode::Directory+ );++ let decoded = FileSinkConfig::try_from(&encoded)?;+ assert_eq!(decoded.object_store_url, config.object_store_url);+ assert_eq!(decoded.table_paths, config.table_paths);+ assert_eq!(+ decoded.output_schema.as_ref(),+ config.output_schema.as_ref()+ );+ assert_eq!(decoded.table_partition_cols, config.table_partition_cols);+ assert_eq!(decoded.insert_op, config.insert_op);+ assert_eq!(+ decoded.keep_partition_by_columns,+ config.keep_partition_by_columns+ );+ assert_eq!(decoded.file_extension, config.file_extension);+ assert_eq!(decoded.file_output_mode, config.file_output_mode);++ let [decoded_file] = decoded.file_group.files() else {+ panic!("expected one decoded output file");+ };+ let [config_file] = config.file_group.files() else {+ panic!("expected one configured output file");+ };+ assert_eq!(+ decoded_file.object_meta.location,+ config_file.object_meta.location+ );+ assert_eq!(decoded_file.object_meta.size, config_file.object_meta.size);+ Ok(())+}++#[test]+fn roundtrip_json_sink() -> Result<()> {+ let field_a = Field::new("plan_type", DataType::Utf8, false);+ let field_b = Field::new("plan", DataType::Utf8, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));++ let file_sink_config = FileSinkConfig {+ original_url: String::default(),+ object_store_url: ObjectStoreUrl::local_filesystem(),+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),+ table_paths: vec![ListingTableUrl::parse("file:///")?],+ output_schema: schema.clone(),+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],+ insert_op: InsertOp::Overwrite,+ keep_partition_by_columns: true,+ file_extension: "json".into(),+ file_output_mode: FileOutputMode::SingleFile,+ };+ let data_sink = Arc::new(JsonSink::new(+ file_sink_config,+ JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED),+ ));+ let sort_order = [PhysicalSortRequirement::new(+ Arc::new(Column::new("plan_type", 0)),+ Some(SortOptions {+ descending: true,+ nulls_first: false,+ }),+ )]+ .into();++ roundtrip_test(Arc::new(DataSinkExec::new(+ input,+ data_sink,+ Some(sort_order),+ )))+}++#[test]+fn roundtrip_csv_sink() -> Result<()> {+ let field_a = Field::new("plan_type", DataType::Utf8, false);+ let field_b = Field::new("plan", DataType::Utf8, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));++ let file_sink_config = FileSinkConfig {+ original_url: String::default(),+ object_store_url: ObjectStoreUrl::local_filesystem(),+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),+ table_paths: vec![ListingTableUrl::parse("file:///")?],+ output_schema: schema.clone(),+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],+ insert_op: InsertOp::Overwrite,+ keep_partition_by_columns: true,+ file_extension: "csv".into(),+ file_output_mode: FileOutputMode::Directory,+ };+ let data_sink = Arc::new(CsvSink::new(+ file_sink_config,+ CsvWriterOptions::new(WriterBuilder::default(), CompressionTypeVariant::ZSTD),+ ));+ let sort_order = [PhysicalSortRequirement::new(+ Arc::new(Column::new("plan_type", 0)),+ Some(SortOptions {+ descending: true,+ nulls_first: false,+ }),+ )]+ .into();++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};++ let roundtrip_plan = roundtrip_test_and_return(+ Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),+ &ctx,+ &codec,+ &proto_converter,+ )?;++ let roundtrip_plan = roundtrip_plan.downcast_ref::<DataSinkExec>().unwrap();+ let csv_sink = roundtrip_plan.sink().downcast_ref::<CsvSink>().unwrap();+ assert_eq!(+ CompressionTypeVariant::ZSTD,+ csv_sink.writer_options().compression+ );++ Ok(())+}++#[test]+fn roundtrip_parquet_sink() -> Result<()> {+ let field_a = Field::new("plan_type", DataType::Utf8, false);+ let field_b = Field::new("plan", DataType::Utf8, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let input = Arc::new(PlaceholderRowExec::new(schema.clone()));++ let file_sink_config = FileSinkConfig {+ original_url: String::default(),+ object_store_url: ObjectStoreUrl::local_filesystem(),+ file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),+ table_paths: vec![ListingTableUrl::parse("file:///")?],+ output_schema: schema.clone(),+ table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],+ insert_op: InsertOp::Overwrite,+ keep_partition_by_columns: true,+ file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,+ };+ let data_sink = Arc::new(ParquetSink::new(+ file_sink_config,+ TableParquetOptions::default(),+ ));+ let sort_order = [PhysicalSortRequirement::new(+ Arc::new(Column::new("plan_type", 0)),+ Some(SortOptions {+ descending: true,+ nulls_first: false,+ }),+ )]+ .into();++ roundtrip_test(Arc::new(DataSinkExec::new(+ input,+ data_sink,+ Some(sort_order),+ )))+}diff --git a/datafusion/proto/tests/cases/plans/sorts.rs b/datafusion/proto/tests/cases/plans/sorts.rsnew file mode 100644index 0000000000..1172775b1b--- /dev/null+++ b/datafusion/proto/tests/cases/plans/sorts.rs@@ -0,0 +1,251 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! `SortExec` and `SortPreservingMergeExec`.++use super::{roundtrip_test, roundtrip_test_and_return};+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};+use datafusion::physical_plan::sorts::sort::SortExec;+use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;+use datafusion::prelude::SessionContext;+use datafusion_common::Result;+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_sort() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();+ roundtrip_test(Arc::new(SortExec::new(+ sort_exprs,+ Arc::new(EmptyExec::new(schema)),+ )))+}++#[test]+fn roundtrip_sort_preserve_partitioning() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs: LexOrdering = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();++ roundtrip_test(Arc::new(SortExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(schema.clone())),+ )))?;++ roundtrip_test(Arc::new(+ SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)))+ .with_preserve_partitioning(true),+ ))+}++/// `SortExec::fetch` turns a sort into a top-k sort. Losing it during serde+/// would silently widen the result set, so exercise the `Some(..)` state+/// explicitly (`roundtrip_sort` only covers `None`).+///+/// `SortExec` currently derives `Debug`, so `roundtrip_test`'s+/// `format!("{plan:?}")` comparison does observe `fetch`. The assertions below+/// go through the accessor instead so that this coverage does not silently+/// disappear if `SortExec` ever grows a hand-written `Debug` impl.+#[test]+fn roundtrip_sort_with_fetch() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs: LexOrdering = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};++ let roundtripped = roundtrip_test_and_return(+ Arc::new(+ SortExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ )+ .with_fetch(Some(7)),+ ),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let roundtripped = roundtripped+ .downcast_ref::<SortExec>()+ .expect("should decode back into a SortExec");+ assert_eq!(roundtripped.fetch(), Some(7));+ assert_eq!(roundtripped.expr(), &sort_exprs);++ // `fetch` combined with `preserve_partitioning`, since both share the same+ // proto node.+ let roundtripped = roundtrip_test_and_return(+ Arc::new(+ SortExec::new(sort_exprs.clone(), Arc::new(EmptyExec::new(schema)))+ .with_fetch(Some(3))+ .with_preserve_partitioning(true),+ ),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let roundtripped = roundtripped+ .downcast_ref::<SortExec>()+ .expect("should decode back into a SortExec");+ assert_eq!(roundtripped.fetch(), Some(3));+ assert!(roundtripped.preserve_partitioning());+ Ok(())+}++/// Round trip a [`SortPreservingMergeExec`], which had no dedicated round trip+/// test at all.+///+/// Covers everything that is actually on the wire for this plan: the input, the+/// sort expressions and `fetch` in both its `None` and `Some(..)` states.+///+/// Note that `SortPreservingMergeExec::enable_round_robin_repartition` is+/// deliberately *not* asserted on here: it has no field in+/// `SortPreservingMergeExecNode`, so it is not serialized and decoding always+/// restores the `true` default from `SortPreservingMergeExec::new`. Asserting+/// round trip equality on it would give a false sense of coverage.+///+/// `SortPreservingMergeExec` derives `Debug`, so `roundtrip_test`'s+/// `format!("{plan:?}")` comparison does observe `expr` and `fetch`. The+/// assertions below use the accessors so the coverage survives a future+/// hand-written `Debug` impl.+#[test]+fn roundtrip_sort_preserving_merge() -> Result<()> {+ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs: LexOrdering = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};++ // No fetch: `fetch` is encoded as -1 and must decode back to `None`.+ let roundtripped = roundtrip_test_and_return(+ Arc::new(SortPreservingMergeExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ )),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let roundtripped = roundtripped+ .downcast_ref::<SortPreservingMergeExec>()+ .expect("should decode back into a SortPreservingMergeExec");+ assert_eq!(roundtripped.fetch(), None);+ assert_eq!(roundtripped.expr(), &sort_exprs);+ assert_eq!(roundtripped.input().schema(), schema);++ // With a fetch: dropping it would turn a bounded merge into an unbounded+ // one and change the query result.+ let roundtripped = roundtrip_test_and_return(+ Arc::new(+ SortPreservingMergeExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ )+ .with_fetch(Some(11)),+ ),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ let roundtripped = roundtripped+ .downcast_ref::<SortPreservingMergeExec>()+ .expect("should decode back into a SortPreservingMergeExec");+ assert_eq!(roundtripped.fetch(), Some(11));+ assert_eq!(roundtripped.expr(), &sort_exprs);+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsnew file mode 100644index 0000000000..04708dec64--- /dev/null+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -0,0 +1,790 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Scans and data sources: file formats, `FileScanConfig`, listing tables+//! and memory sources.++use super::{+ all_types_context, roundtrip_test, roundtrip_test_and_return,+ roundtrip_test_sql_with_context,+};+use arrow::array::RecordBatch;+use arrow::datatypes::Fields;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::datasource::empty::EmptyTable;+use datafusion::datasource::file_format::json::JsonFormat;+use datafusion::datasource::listing::{+ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, PartitionedFile,+};+use datafusion::datasource::object_store::ObjectStoreUrl;+use datafusion::datasource::physical_plan::{+ ArrowSource, CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, JsonSource,+ ParquetSource, wrap_partition_type_in_dict, wrap_partition_value_in_dict,+};+use datafusion::datasource::source::DataSourceExec;+use datafusion::execution::TaskContext;+use datafusion::logical_expr::Operator;+use datafusion::physical_expr::LexOrdering;+use datafusion::physical_plan::expressions::{+ BinaryExpr, Column, PhysicalSortExpr, col, lit,+};+use datafusion::physical_plan::filter::FilterExecBuilder;+use datafusion::physical_plan::{+ ExecutionPlan, Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, Statistics,+ displayable,+};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::config::TableParquetOptions;+use datafusion_common::stats::Precision;+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err};+use datafusion_datasource::{TableSchema, TableSchemaBuilder};+use datafusion_expr::ColumnarValue;+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::PhysicalPlanNode;+use prost::Message;+use std::collections::HashMap;+use std::fmt::{Display, Formatter};+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));++ let predicate = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("col", 1)),+ Operator::Eq,+ lit("1"),+ ));++ let mut options = TableParquetOptions::new();+ options.global.pushdown_filters = true;++ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&file_schema))+ .with_table_parquet_options(options)+ .with_predicate(predicate),+ );++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(+ vec![Field::new("col", DataType::Utf8, false)],+ ))),+ })+ .build();++ roundtrip_test(DataSourceExec::from_data_source(scan_config))+}++#[test]+fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&file_schema),+ })+ .build();+ let exec_plan = DataSourceExec::from_data_source(scan_config);++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let roundtripped =+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ let data_source = roundtripped+ .downcast_ref::<DataSourceExec>()+ .ok_or_else(|| {+ internal_datafusion_err!("Expected DataSourceExec after roundtrip")+ })?;+ let file_scan = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .ok_or_else(|| {+ internal_datafusion_err!("Expected FileScanConfig after roundtrip")+ })?;+ let parquet_source = file_scan+ .file_source()+ .downcast_ref::<ParquetSource>()+ .ok_or_else(|| {+ internal_datafusion_err!("Expected ParquetSource after roundtrip")+ })?;++ assert!(+ parquet_source.parquet_file_reader_factory().is_some(),+ "Parquet reader factory should be attached after decoding from protobuf"+ );+ Ok(())+}++#[test]+fn roundtrip_arrow_scan() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));++ let table_schema = TableSchema::from(&file_schema);+ let file_source = Arc::new(ArrowSource::new_file_source(table_schema));++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.arrow".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&file_schema),+ })+ .build();++ roundtrip_test(DataSourceExec::from_data_source(scan_config))+}++#[test]+fn roundtrip_json_scan() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.json".to_string(),+ 1024,+ )])])+ .build();+ roundtrip_test(DataSourceExec::from_data_source(scan_config))+}++#[cfg(feature = "avro")]+#[test]+fn roundtrip_avro_scan() -> Result<()> {+ use datafusion_datasource_avro::source::AvroSource;++ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(AvroSource::new(TableSchema::from(&file_schema)));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.avro".to_string(),+ 1024,+ )])])+ .build();+ roundtrip_test(DataSourceExec::from_data_source(scan_config))+}++#[test]+fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {+ use datafusion::common::config::CsvOptions;++ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let table_schema = TableSchema::from(&file_schema);+ let file_source =+ Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {+ has_header: Some(false),+ delimiter: b'|',+ quote: b'\'',+ escape: Some(b'\\'),+ comment: Some(b'#'),+ newlines_in_values: Some(true),+ truncated_rows: Some(true),+ ..Default::default()+ }));++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.csv".to_string(),+ 1024,+ )])])+ .build();++ let ctx = SessionContext::new();+ let roundtripped = roundtrip_test_and_return(+ DataSourceExec::from_data_source(scan_config),+ &ctx,+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ let data_source = roundtripped+ .downcast_ref::<DataSourceExec>()+ .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;+ let file_scan = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;+ let csv_source = file_scan+ .file_source()+ .downcast_ref::<CsvSource>()+ .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;++ assert!(!csv_source.has_header());+ assert_eq!(csv_source.delimiter(), b'|');+ assert_eq!(csv_source.quote(), b'\'');+ assert_eq!(csv_source.escape(), Some(b'\\'));+ assert_eq!(csv_source.comment(), Some(b'#'));+ assert!(csv_source.newlines_in_values());+ assert!(csv_source.truncate_rows());+ Ok(())+}++#[tokio::test]+async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {+ let mut file_group =+ PartitionedFile::new("/path/to/part=0/file.parquet".to_string(), 1024);+ file_group.partition_values =+ vec![wrap_partition_value_in_dict(ScalarValue::Int64(Some(0)))];+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));++ let table_schema = TableSchemaBuilder::from(&schema)+ .with_table_partition_cols(vec![Arc::new(Field::new(+ "part".to_string(),+ wrap_partition_type_in_dict(DataType::Int16),+ false,+ ))])+ .build();++ let file_source = Arc::new(ParquetSource::new(table_schema.clone()));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_projection_indices(Some(vec![0, 1]))?+ .with_file_group(FileGroup::new(vec![file_group]))+ .build();++ roundtrip_test(DataSourceExec::from_data_source(scan_config))+}++#[test]+fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));++ let custom_predicate_expr = Arc::new(CustomPredicateExpr {+ inner: Arc::new(Column::new("col", 1)),+ });++ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&file_schema))+ .with_predicate(custom_predicate_expr),+ );++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(+ vec![Field::new("col", DataType::Utf8, false)],+ ))),+ })+ .build();++ #[derive(Debug, Clone, Eq)]+ struct CustomPredicateExpr {+ inner: Arc<dyn PhysicalExpr>,+ }++ // Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808+ impl PartialEq for CustomPredicateExpr {+ fn eq(&self, other: &Self) -> bool {+ self.inner.eq(&other.inner)+ }+ }++ impl std::hash::Hash for CustomPredicateExpr {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.inner.hash(state);+ }+ }++ impl Display for CustomPredicateExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "CustomPredicateExpr")+ }+ }++ impl PhysicalExpr for CustomPredicateExpr {+ fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {+ unreachable!()+ }++ fn nullable(&self, _input_schema: &Schema) -> Result<bool> {+ unreachable!()+ }++ fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {+ unreachable!()+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.inner]+ }++ fn with_new_children(+ self: Arc<Self>,+ _children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(self)+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ Display::fmt(self, f)+ }+ }++ #[derive(Debug)]+ struct CustomPhysicalExtensionCodec;+ impl PhysicalExtensionCodec for CustomPhysicalExtensionCodec {+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ unreachable!()+ }++ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ unreachable!()+ }++ fn try_decode_expr(+ &self,+ buf: &[u8],+ inputs: &[Arc<dyn PhysicalExpr>],+ _ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ if buf == "CustomPredicateExpr".as_bytes() {+ Ok(Arc::new(CustomPredicateExpr {+ inner: inputs[0].clone(),+ }))+ } else {+ internal_err!("Not supported")+ }+ }++ fn try_encode_expr(+ &self,+ node: &Arc<dyn PhysicalExpr>,+ buf: &mut Vec<u8>,+ _ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<()> {+ if node.downcast_ref::<CustomPredicateExpr>().is_some() {+ buf.extend_from_slice("CustomPredicateExpr".as_bytes());+ Ok(())+ } else {+ internal_err!("Not supported")+ }+ }+ }++ let exec_plan = DataSourceExec::from_data_source(scan_config);++ let ctx = SessionContext::new();+ roundtrip_test_and_return(+ exec_plan,+ &ctx,+ &CustomPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ Ok(())+}++#[tokio::test]+async fn roundtrip_json_source() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_json("t1", "../core/tests/data/1.json", Default::default())+ .await?;+ let plan = ctx.table("t1").await?.create_physical_plan().await?;+ roundtrip_test(plan)+}++#[tokio::test]+async fn roundtrip_coalesce() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_table(+ "t",+ Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([+ Arc::new(Field::new("f", DataType::Int64, false)),+ ]))))),+ )?;+ let df = ctx.sql("select coalesce(f) as f from t").await?;+ let plan = df.create_physical_plan().await?;++ let node = PhysicalPlanNode::try_from_physical_plan(+ plan.clone(),+ &DefaultPhysicalExtensionCodec {},+ )?;+ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())+ .map_err(|e| DataFusionError::External(Box::new(e)))?;+ let restored =+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;++ assert_eq!(+ plan.schema(),+ restored.schema(),+ "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",+ displayable(plan.as_ref())+ .set_show_schema(true)+ .indent(true),+ displayable(restored.as_ref())+ .set_show_schema(true)+ .indent(true),+ );++ Ok(())+}++#[tokio::test]+async fn roundtrip_generate_series() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_table(+ "t",+ Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([+ Arc::new(Field::new("f", DataType::Int64, false)),+ ]))))),+ )?;+ let df = ctx.sql("select * from generate_series(1, 10000)").await?;+ let plan = df.create_physical_plan().await?;++ let node = PhysicalPlanNode::try_from_physical_plan(+ plan.clone(),+ &DefaultPhysicalExtensionCodec {},+ )?;+ let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())+ .map_err(|e| DataFusionError::External(Box::new(e)))?;+ let restored =+ node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;++ assert_eq!(+ plan.schema(),+ restored.schema(),+ "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",+ displayable(plan.as_ref())+ .set_show_schema(true)+ .indent(true),+ displayable(restored.as_ref())+ .set_show_schema(true)+ .indent(true),+ );++ Ok(())+}++#[tokio::test]+async fn roundtrip_projection_source() -> Result<()> {+ let schema = Arc::new(Schema::new(Fields::from([+ Arc::new(Field::new("a", DataType::Utf8, false)),+ Arc::new(Field::new("b", DataType::Utf8, false)),+ Arc::new(Field::new("c", DataType::Int32, false)),+ Arc::new(Field::new("d", DataType::Int32, false)),+ ])));++ let statistics = Statistics::new_unknown(&schema);++ let file_source = Arc::new(ParquetSource::new(Arc::clone(&schema)));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(statistics)+ .with_projection_indices(Some(vec![0, 1, 2]))?+ .build();++ let filter = Arc::new(+ FilterExecBuilder::new(+ Arc::new(BinaryExpr::new(col("c", &schema)?, Operator::Eq, lit(1))),+ DataSourceExec::from_data_source(scan_config),+ )+ .apply_projection(Some(vec![0, 1]))?+ .build()?,+ );++ roundtrip_test(filter)+}++#[tokio::test]+async fn roundtrip_parquet_select_star() -> Result<()> {+ let ctx = all_types_context().await?;+ let sql = "select * from alltypes_plain";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_parquet_select_projection() -> Result<()> {+ let ctx = all_types_context().await?;+ let sql = "select string_col, timestamp_col from alltypes_plain";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_parquet_select_star_predicate() -> Result<()> {+ let ctx = all_types_context().await?;+ let sql = "select * from alltypes_plain where id > 4";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_parquet_select_projection_predicate() -> Result<()> {+ let ctx = all_types_context().await?;+ let sql = "select string_col, timestamp_col from alltypes_plain where id > 4";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_empty_projection() -> Result<()> {+ let ctx = all_types_context().await?;+ let sql = "select 1 from alltypes_plain";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_memory_source_empty_projection() -> Result<()> {+ // Memory scan: `Some(vec![])` must not decode back as `None`+ let ctx = SessionContext::new();+ let batch = RecordBatch::try_new(+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Int64, false),+ ])),+ vec![+ Arc::new(arrow::array::StringArray::from(vec!["Tom"])),+ Arc::new(arrow::array::Int64Array::from(vec![18i64])),+ ],+ )?;+ ctx.register_batch("tmem", batch)?;+ let sql = "select 1 from tmem";+ roundtrip_test_sql_with_context(sql, &ctx).await+}++#[tokio::test]+async fn roundtrip_memory_source() -> Result<()> {+ let ctx = SessionContext::new();+ let plan = ctx+ .sql("select * from values ('Tom', 18)")+ .await?+ .create_physical_plan()+ .await?;+ roundtrip_test(plan)+}++#[tokio::test]+async fn roundtrip_memory_source_sort_information_and_fetch() -> Result<()> {+ use datafusion::datasource::memory::MemorySourceConfig;+ use datafusion::datasource::source::DataSource as _;++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Int64, false),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(arrow::array::StringArray::from(vec!["Tom", "Bob"])),+ Arc::new(arrow::array::Int64Array::from(vec![18i64, 21i64])),+ ],+ )?;+ let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(+ col("b", &schema)?,+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ )])+ .unwrap();+ let source = MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .with_limit(Some(1))+ .with_show_sizes(false)+ .try_with_sort_information(vec![ordering])?;+ let exec_plan = DataSourceExec::from_data_source(source.clone());++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let decoded = roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ // The string representation does not include every field; check the+ // decoded source directly.+ let decoded = decoded+ .downcast_ref::<DataSourceExec>()+ .expect("expected DataSourceExec");+ let decoded_source = decoded+ .data_source()+ .downcast_ref::<MemorySourceConfig>()+ .expect("expected MemorySourceConfig");+ assert_eq!(decoded_source.partitions(), source.partitions());+ assert_eq!(decoded_source.original_schema(), source.original_schema());+ assert_eq!(decoded_source.projection(), source.projection());+ assert_eq!(decoded_source.sort_information(), source.sort_information());+ assert_eq!(decoded_source.fetch(), Some(1));+ assert!(!decoded_source.show_sizes());+ Ok(())+}++#[tokio::test]+async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {+ let ctx = SessionContext::new();+ let file_format = JsonFormat::default();+ let table_partition_cols = vec![("part".to_owned(), DataType::Int64)];+ let data = "../core/tests/data/partitioned_table_json";+ let listing_table_url = ListingTableUrl::parse(data)?;+ let listing_options = ListingOptions::new(Arc::new(file_format))+ .with_table_partition_cols(table_partition_cols);++ let config = ListingTableConfig::new(listing_table_url)+ .with_listing_options(listing_options)+ .infer_schema(&ctx.state())+ .await?;++ // Decorate metadata onto the inferred ListingTable schema+ let schema_with_meta = config+ .file_schema+ .clone()+ .map(|s| {+ let mut meta: HashMap<String, String> = HashMap::new();+ meta.insert("foo.bar".to_string(), "baz".to_string());+ s.as_ref().clone().with_metadata(meta)+ })+ .expect("Must decorate metadata");++ let config = config.with_schema(Arc::new(schema_with_meta));+ ctx.register_table("hive_style", Arc::new(ListingTable::try_new(config)?))?;++ let plan = ctx+ .sql("select * from hive_style limit 1")+ .await?+ .create_physical_plan()+ .await?;++ roundtrip_test(plan)+}++fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanConfig> {+ let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let result_plan =+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ let data_source_exec = result_plan+ .downcast_ref::<DataSourceExec>()+ .expect("Expected DataSourceExec");+ let file_scan_config = data_source_exec+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ Ok(file_scan_config.clone())+}++#[test]+fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning =+ Partitioning::Hash(vec![Arc::new(Column::new("col", 0))], 1);+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();++ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );++ Ok(())+}++#[test]+fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning = Partitioning::Range(RangePartitioning::new(+ LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(+ "col", 0,+ )))])+ .unwrap(),+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],+ ));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-1.parquet".to_string(),+ 1024,+ )]),+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-2.parquet".to_string(),+ 1024,+ )]),+ ])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();++ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/tpch.rs b/datafusion/proto/tests/cases/plans/tpch.rsnew file mode 100644index 0000000000..d24150b3e0--- /dev/null+++ b/datafusion/proto/tests/cases/plans/tpch.rs@@ -0,0 +1,335 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! End to end round trips of the TPC-H queries, plus the human readable+//! display of the plans they produce.++use super::{roundtrip_test_and_return, roundtrip_test_sql_with_context};+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::functions_aggregate::first_last::first_value_udaf;+use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};+use datafusion::physical_plan::aggregates::{+ AggregateExec, AggregateMode, PhysicalGroupBy,+};+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};+use datafusion::prelude::SessionContext;+use datafusion_common::{DataFusionError, Result};+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+};+use datafusion_proto::protobuf::PhysicalPlanNode;+use std::sync::Arc;+use std::vec;++/// Helper function to create a SessionContext with all TPC-H tables registered as external tables+async fn tpch_context() -> Result<SessionContext> {+ use datafusion_common::test_util::datafusion_test_data;++ let ctx = SessionContext::new();+ let test_data = datafusion_test_data();++ // TPC-H table names+ let tables = [+ "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation",+ "region",+ ];++ // Create external tables for all TPC-H tables+ for table in &tables {+ let table_sql = format!(+ "CREATE EXTERNAL TABLE {table} STORED AS PARQUET LOCATION '{test_data}/tpch_{table}_small.parquet'"+ );+ ctx.sql(&table_sql).await.map_err(|e| {+ DataFusionError::External(+ format!("Failed to create {table} table: {e}").into(),+ )+ })?;+ }++ Ok(ctx)+}++/// Helper function to get TPC-H query SQL+fn get_tpch_query_sql(query: usize) -> Result<Vec<String>> {+ use std::fs;++ if !(1..=22).contains(&query) {+ return Err(DataFusionError::External(+ format!("Invalid TPC-H query number: {query}").into(),+ ));+ }++ let filename = format!("../../benchmarks/queries/q{query}.sql");+ let contents = fs::read_to_string(&filename).map_err(|e| {+ DataFusionError::External(+ format!("Failed to read query file {filename}: {e}").into(),+ )+ })?;++ Ok(contents+ .split(';')+ .map(|s| s.trim())+ .filter(|s| !s.is_empty())+ .map(|s| s.to_string())+ .collect())+}++#[tokio::test]+async fn test_serialize_deserialize_tpch_queries() -> Result<()> {+ // Create context with TPC-H tables+ let ctx = tpch_context().await?;++ // repeat to run all 22 queries+ for query in 1..=22 {+ // run all statements in the query+ let sql = get_tpch_query_sql(query)?;+ for stmt in sql {+ let logical_plan = ctx.sql(&stmt).await?.into_unoptimized_plan();+ let optimized_plan = ctx.state().optimize(&logical_plan)?;+ let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;++ // serialize the physical plan+ let codec = DefaultPhysicalExtensionCodec {};++ let proto =+ PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;++ // deserialize the physical plan+ let _deserialized_plan =+ proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ }+ }++ Ok(())+}++// Bugs: https://github.com/apache/datafusion/issues/16772+#[tokio::test]+async fn test_round_trip_tpch_queries() -> Result<()> {+ // Create context with TPC-H tables+ let ctx = tpch_context().await?;++ // repeat to run all 22 queries+ for query in 1..=22 {+ // run all statements in the query+ let sql = get_tpch_query_sql(query)?;+ for stmt in sql {+ roundtrip_test_sql_with_context(&stmt, &ctx).await?;+ }+ }++ Ok(())+}++// Bug 1 of https://github.com/apache/datafusion/issues/16772+/// Test that AggregateFunctionExpr human_display field is correctly preserved+/// during serialization/deserialization roundtrip.+///+/// Test for issue where the human_display field (used for EXPLAIN output)+/// was not being serialized to protobuf, causing it to be lost during roundtrip+/// and resulting in empty or incorrect display strings in query plans.+#[tokio::test]+async fn test_round_trip_human_display() -> Result<()> {+ // Create context with TPC-H tables+ let ctx = tpch_context().await?;++ let sql = "select r_name, count(1) from region group by r_name";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ let sql = "select r_name, count(*) from region group by r_name";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ let sql = "select r_name, count(r_name) from region group by r_name";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ let sql = "select count(*) as count_star from region";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ Ok(())+}++#[test]+fn test_round_trip_aliased_reverse_human_display() -> Result<()> {+ let aggregate_expr = roundtrip_first_value_aggregate(+ "agg",+ "first_value(b) ORDER BY [b ASC NULLS LAST]",+ Some("agg"),+ )?;+ let reversed = aggregate_expr+ .reverse_expr()+ .expect("expected reverse expr");++ assert_eq!(reversed.name(), "agg");+ assert_eq!(reversed.human_display_alias(), Some("agg"));+ assert_eq!(+ reversed.human_display(),+ Some("last_value(b) ORDER BY [b DESC NULLS FIRST]")+ );++ Ok(())+}++#[test]+fn test_round_trip_human_display_alias_with_colon() -> Result<()> {+ let aggregate_expr = roundtrip_first_value_aggregate(+ "agg:one",+ "first_value(b) ORDER BY [b ASC NULLS LAST]",+ Some("agg:one"),+ )?;++ assert_eq!(aggregate_expr.name(), "agg:one");+ assert_eq!(aggregate_expr.human_display_alias(), Some("agg:one"));+ assert_eq!(+ aggregate_expr.human_display(),+ Some("first_value(b) ORDER BY [b ASC NULLS LAST]")+ );++ Ok(())+}++#[test]+fn test_round_trip_non_aliased_human_display_ending_like_alias() -> Result<()> {+ let aggregate_expr =+ roundtrip_first_value_aggregate("agg", "first_value(b) as agg", None)?;++ assert_eq!(aggregate_expr.name(), "agg");+ assert_eq!(+ aggregate_expr.human_display(),+ Some("first_value(b) as agg")+ );+ assert_eq!(aggregate_expr.human_display_alias(), None);++ Ok(())+}++fn roundtrip_first_value_aggregate(+ alias: &str,+ human_display: &str,+ human_display_alias: Option<&str>,+) -> Result<Arc<AggregateFunctionExpr>> {+ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));+ let mut builder =+ AggregateExprBuilder::new(first_value_udaf(), vec![col("b", &schema)?])+ .order_by(vec![PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions::new(false, false),+ }])+ .schema(Arc::clone(&schema))+ .alias(alias)+ .human_display(human_display);+ if let Some(human_display_alias) = human_display_alias {+ builder = builder.human_display_alias(human_display_alias);+ }+ let agg_expr = builder.build().map(Arc::new)?;++ let plan = Arc::new(AggregateExec::try_new(+ AggregateMode::Single,+ PhysicalGroupBy::new(vec![], vec![], vec![], false),+ vec![agg_expr],+ vec![None],+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ schema,+ )?);++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let roundtrip_plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;+ let aggregate = roundtrip_plan+ .as_ref()+ .downcast_ref::<AggregateExec>()+ .expect("expected AggregateExec after roundtrip");++ Ok(Arc::clone(&aggregate.aggr_expr()[0]))+}++// Bug 2 of https://github.com/apache/datafusion/issues/16772+/// Test that PhysicalGroupBy groups field is correctly serialized/deserialized+/// for simple aggregates (no GROUP BY clause).+///+/// Test for issue where simple aggregates like "SELECT SUM(col1 * col2) FROM table"+/// would incorrectly serialize groups as [[]] instead of [] during roundtrip serialization.+/// The groups field should be empty ([]) when there are no GROUP BY expressions.+#[tokio::test]+async fn test_round_trip_groups_display() -> Result<()> {+ // Create context with TPC-H tables+ let ctx = tpch_context().await?;++ let sql = "select sum(l_extendedprice * l_discount) as revenue from lineitem;";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ let sql = "select sum(l_extendedprice) as revenue from lineitem;";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ Ok(())+}++// Bug 3 of https://github.com/apache/datafusion/issues/16772+/// Test that ScalarFunctionExpr return_field name is correctly preserved+/// during serialization/deserialization roundtrip.+///+/// Test for issue where the return_field.name for scalar functions+/// was not being serialized to protobuf, causing it to be lost during roundtrip+/// and defaulting to a generic name like "f" instead of the proper function name.+#[tokio::test]+async fn test_round_trip_date_part_display() -> Result<()> {+ // Create context with TPC-H tables+ let ctx = tpch_context().await?;++ let sql = "select extract(year from l_shipdate) as l_year from lineitem ";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ let sql = "select extract(month from l_shipdate) as l_year from lineitem ";+ roundtrip_test_sql_with_context(sql, &ctx).await?;++ Ok(())+}++#[tokio::test]+async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {+ use datafusion_common::test_util::datafusion_test_data;++ let ctx = SessionContext::new();++ // Register the TPC-H part table using the local test data+ let test_data = datafusion_test_data();+ let table_sql = format!(+ "CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '{test_data}/tpch_part_small.parquet'"+ );+ ctx.sql(&table_sql).await.map_err(|e| {+ DataFusionError::External(format!("Failed to create part table: {e}").into())+ })?;++ // Test the exact problematic query+ let sql =+ "SELECT p_size FROM part WHERE p_size IN (14, 6, 5, 31) and p_partkey > 1000";++ let logical_plan = ctx.sql(sql).await?.into_unoptimized_plan();+ let optimized_plan = ctx.state().optimize(&logical_plan)?;+ let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;++ // Serialize the physical plan - bug may happen here already but not necessarily manifests+ let codec = DefaultPhysicalExtensionCodec {};++ let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;++ // This will fail with the bug, but should succeed when fixed+ let _deserialized_plan = proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/udfs.rs b/datafusion/proto/tests/cases/plans/udfs.rsnew file mode 100644index 0000000000..08d00030e0--- /dev/null+++ b/datafusion/proto/tests/cases/plans/udfs.rs@@ -0,0 +1,573 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Plans carrying user defined functions, and the extension codec that+//! (de)serializes them.++use super::{roundtrip_test_and_return, roundtrip_test_with_context};+use crate::cases::{+ CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF,+ MyHigherOrderUdfNode, MyRegexUdf, MyRegexUdfNode,+};+use datafusion::arrow::array::ArrayRef;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::execution::TaskContext;+use datafusion::logical_expr::{Operator, Volatility, create_udf};+use datafusion::physical_expr::aggregate::AggregateExprBuilder;+use datafusion::physical_expr::expressions::Literal;+use datafusion::physical_expr::window::StandardWindowExpr;+use datafusion::physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};+use datafusion::physical_plan::aggregates::{+ AggregateExec, AggregateMode, PhysicalGroupBy,+};+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{BinaryExpr, PhysicalSortExpr, col, lit};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};+use datafusion::physical_plan::windows::{+ BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,+ create_udwf_window_expr,+};+use datafusion::physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr};+use datafusion::prelude::SessionContext;+use datafusion::scalar::ScalarValue;+use datafusion_common::config::ConfigOptions;+use datafusion_common::{Result, internal_datafusion_err, not_impl_err};+use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};+use datafusion_expr::{+ AggregateUDF, ColumnarValue, HigherOrderUDF, ScalarFunctionArgs, ScalarUDF,+ ScalarUDFImpl, Signature, WindowFrame, WindowFrameBound, WindowUDF,+};+use datafusion_functions_aggregate::min_max::max_udaf;+use datafusion_physical_expr::expressions::{LambdaVariable, is_not_null, lambda};+use datafusion_proto::physical_plan::{+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,+};+use prost::Message;+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_scalar_udf() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ let scalar_fn = Arc::new(|args: &[ColumnarValue]| {+ let ColumnarValue::Array(array) = &args[0] else {+ panic!("should be array")+ };+ Ok(ColumnarValue::from(Arc::new(array.clone()) as ArrayRef))+ });++ let udf = create_udf(+ "dummy",+ vec![DataType::Int64],+ DataType::Int64,+ Volatility::Immutable,+ scalar_fn.clone(),+ );++ let fun_def = Arc::new(udf.clone());++ let expr = ScalarFunctionExpr::new(+ "dummy",+ fun_def,+ vec![col("a", &schema)?],+ Field::new("f", DataType::Int64, true).into(),+ Arc::new(ConfigOptions::default()),+ );++ let project = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(expr),+ alias: "a".to_string(),+ }],+ input,+ )?;++ let ctx = SessionContext::new();++ ctx.register_udf(udf);++ roundtrip_test_with_context(Arc::new(project), &ctx)+}++#[derive(Debug)]+struct UDFExtensionCodec;++impl PhysicalExtensionCodec for UDFExtensionCodec {+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ not_impl_err!("No extension codec provided")+ }++ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ not_impl_err!("No extension codec provided")+ }++ fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {+ if name == "regex_udf" {+ let proto = MyRegexUdfNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode regex_udf: {err}")+ })?;++ Ok(Arc::new(ScalarUDF::from(MyRegexUdf::new(proto.pattern))))+ } else {+ not_impl_err!("unrecognized scalar UDF implementation, cannot decode")+ }+ }++ fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {+ let binding = node.inner();+ if let Some(udf) = binding.downcast_ref::<MyRegexUdf>() {+ let proto = MyRegexUdfNode {+ pattern: udf.pattern.clone(),+ };+ proto+ .encode(buf)+ .map_err(|err| internal_datafusion_err!("failed to encode udf: {err}"))?;+ }+ Ok(())+ }++ fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result<Arc<AggregateUDF>> {+ if name == "aggregate_udf" {+ let proto = MyAggregateUdfNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode aggregate_udf: {err}")+ })?;++ Ok(Arc::new(AggregateUDF::from(MyAggregateUDF::new(+ proto.result,+ ))))+ } else {+ not_impl_err!("unrecognized scalar UDF implementation, cannot decode")+ }+ }++ fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {+ let binding = node.inner();+ if let Some(udf) = binding.downcast_ref::<MyAggregateUDF>() {+ let proto = MyAggregateUdfNode {+ result: udf.result.clone(),+ };+ proto.encode(buf).map_err(|err| {+ internal_datafusion_err!("failed to encode udf: {err:?}")+ })?;+ }+ Ok(())+ }++ fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result<Arc<WindowUDF>> {+ if name == "custom_udwf" {+ let proto = CustomUDWFNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode custom_udwf: {err}")+ })?;++ Ok(Arc::new(WindowUDF::from(CustomUDWF::new(proto.payload))))+ } else {+ not_impl_err!(+ "unrecognized user-defined window function implementation, cannot decode"+ )+ }+ }++ fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec<u8>) -> Result<()> {+ let binding = node.inner();+ if let Some(udwf) = binding.downcast_ref::<CustomUDWF>() {+ let proto = CustomUDWFNode {+ payload: udwf.payload.clone(),+ };+ proto.encode(buf).map_err(|err| {+ internal_datafusion_err!("failed to encode udwf: {err:?}")+ })?;+ }+ Ok(())+ }++ fn try_decode_higher_order_function(+ &self,+ name: &str,+ buf: &[u8],+ ) -> Result<Arc<HigherOrderUDF>> {+ if name == "higher_order_udf" {+ let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {+ internal_datafusion_err!("failed to decode higher_order_udf: {err}")+ })?;++ Ok(Arc::new(HigherOrderUDF::new_from_impl(+ MyHigherOrderUDF::new(proto.payload),+ )))+ } else {+ not_impl_err!("unrecognized higher order UDF implementation, cannot decode")+ }+ }++ fn try_encode_higher_order_function(+ &self,+ node: &HigherOrderUDF,+ buf: &mut Vec<u8>,+ ) -> Result<()> {+ if let Some(hof) = (node.inner().as_ref() as &dyn std::any::Any)+ .downcast_ref::<MyHigherOrderUDF>()+ {+ let proto = MyHigherOrderUdfNode {+ payload: hof.payload.clone(),+ };+ proto.encode(buf).map_err(|err| {+ internal_datafusion_err!("failed to encode hof: {err:?}")+ })?;+ }+ Ok(())+ }+}++#[test]+fn roundtrip_scalar_udf_extension_codec() -> Result<()> {+ let field_text = Field::new("text", DataType::Utf8, true);+ let field_published = Field::new("published", DataType::Boolean, false);+ let field_author = Field::new("author", DataType::Utf8, false);+ let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));+ let input = Arc::new(EmptyExec::new(schema.clone()));++ let udf_expr = Arc::new(ScalarFunctionExpr::new(+ "regex_udf",+ Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),+ vec![col("text", &schema)?],+ Field::new("f", DataType::Int64, true).into(),+ Arc::new(ConfigOptions::default()),+ ));++ let filter = Arc::new(FilterExec::try_new(+ Arc::new(BinaryExpr::new(+ col("published", &schema)?,+ Operator::And,+ Arc::new(BinaryExpr::new(udf_expr.clone(), Operator::Gt, lit(0))),+ )),+ input,+ )?);+ let aggr_expr =+ AggregateExprBuilder::new(max_udaf(), vec![udf_expr as Arc<dyn PhysicalExpr>])+ .schema(schema.clone())+ .alias("max")+ .build()+ .map(Arc::new)?;++ let window = Arc::new(WindowAggExec::try_new(+ vec![Arc::new(PlainAggregateWindowExpr::new(+ aggr_expr.clone(),+ &[col("author", &schema)?],+ &[],+ Arc::new(WindowFrame::new(None)),+ None,+ ))],+ filter,+ true,+ )?);++ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new(vec![], vec![], vec![], false),+ vec![aggr_expr],+ vec![None],+ window,+ schema,+ )?);++ let ctx = SessionContext::new();+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;+ Ok(())+}++#[test]+fn roundtrip_higher_order_udf() -> Result<()> {+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));+ let list_field = Field::new(+ "list_col",+ DataType::List(Arc::clone(&element_field)),+ false,+ );+ let schema = Arc::new(Schema::new(vec![list_field]));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));++ let expr = HigherOrderFunctionExpr::try_new_with_schema(+ Arc::clone(&hof),+ vec![+ col("list_col", &schema)?,+ lambda(+ ["v"],+ is_not_null(Arc::new(LambdaVariable::new(1, element_field)))?,+ )?,+ ],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?;++ let project = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(expr),+ alias: "a".to_string(),+ }],+ input,+ )?;++ let ctx = SessionContext::new();+ ctx.register_higher_order_function(hof);++ roundtrip_test_with_context(Arc::new(project), &ctx)+}++#[test]+fn roundtrip_higher_order_udf_extension_codec() -> Result<()> {+ let element_field = Arc::new(Field::new("v", DataType::Int32, true));+ let list_field = Field::new(+ "list_col",+ DataType::List(Arc::clone(&element_field)),+ false,+ );+ let schema = Arc::new(Schema::new(vec![list_field]));+ let input = Arc::new(EmptyExec::new(schema.clone()));++ let lambda_body = Arc::new(LambdaVariable::new(1, Arc::clone(&element_field)));+ let lambda_expr = lambda(["v"], lambda_body)?;++ let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(+ "payload".to_string(),+ )));+ let hof_expr = Arc::new(HigherOrderFunctionExpr::try_new_with_schema(+ hof,+ vec![col("list_col", &schema)?, lambda_expr],+ &schema,+ Arc::new(ConfigOptions::default()),+ )?);++ let project = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: hof_expr,+ alias: "out".to_string(),+ }],+ input,+ )?;++ let ctx = SessionContext::new();+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(+ Arc::new(project),+ &ctx,+ &UDFExtensionCodec,+ &proto_converter,+ )?;+ Ok(())+}++#[test]+fn roundtrip_udwf_extension_codec() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let custom_udwf = Arc::new(WindowUDF::from(CustomUDWF::new("payload".to_string())));+ let udwf = create_udwf_window_expr(+ &custom_udwf,+ &[col("a", &schema)?],+ schema.as_ref(),+ "custom_udwf(a) PARTITION BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),+ false,+ )?;++ let window_frame = WindowFrame::new_bounds(+ datafusion_expr::WindowFrameUnits::Range,+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),+ WindowFrameBound::CurrentRow,+ );++ let udwf_expr = Arc::new(StandardWindowExpr::new(+ udwf,+ &[col("b", &schema)?],+ &[PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ }],+ Arc::new(window_frame),+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));+ let window = Arc::new(BoundedWindowAggExec::try_new(+ vec![udwf_expr],+ input,+ InputOrderMode::Sorted,+ true,+ )?);++ let ctx = SessionContext::new();+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;+ Ok(())+}++#[test]+fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {+ let field_text = Field::new("text", DataType::Utf8, true);+ let field_published = Field::new("published", DataType::Boolean, false);+ let field_author = Field::new("author", DataType::Utf8, false);+ let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));+ let input = Arc::new(EmptyExec::new(schema.clone()));++ let udf_expr = Arc::new(ScalarFunctionExpr::new(+ "regex_udf",+ Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),+ vec![col("text", &schema)?],+ Field::new("f", DataType::Int64, true).into(),+ Arc::new(ConfigOptions::default()),+ ));++ let udaf = Arc::new(AggregateUDF::from(MyAggregateUDF::new(+ "result".to_string(),+ )));+ let aggr_args: Vec<Arc<dyn PhysicalExpr>> =+ vec![Arc::new(Literal::new(ScalarValue::from(42)))];++ let aggr_expr = AggregateExprBuilder::new(Arc::clone(&udaf), aggr_args.clone())+ .schema(Arc::clone(&schema))+ .alias("aggregate_udf")+ .build()+ .map(Arc::new)?;++ let filter = Arc::new(FilterExec::try_new(+ Arc::new(BinaryExpr::new(+ col("published", &schema)?,+ Operator::And,+ Arc::new(BinaryExpr::new(udf_expr, Operator::Gt, lit(0))),+ )),+ input,+ )?);++ let window = Arc::new(WindowAggExec::try_new(+ vec![Arc::new(PlainAggregateWindowExpr::new(+ aggr_expr,+ &[col("author", &schema)?],+ &[],+ Arc::new(WindowFrame::new(None)),+ None,+ ))],+ filter,+ true,+ )?);++ let aggr_expr = AggregateExprBuilder::new(udaf, aggr_args.clone())+ .schema(Arc::clone(&schema))+ .alias("aggregate_udf")+ .distinct()+ .ignore_nulls()+ .build()+ .map(Arc::new)?;++ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new(vec![], vec![], vec![], false),+ vec![aggr_expr],+ vec![None],+ window,+ schema,+ )?);++ let ctx = SessionContext::new();+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;+ Ok(())+}++#[tokio::test]+async fn roundtrip_async_func_exec() -> Result<()> {+ #[derive(Debug, PartialEq, Eq, Hash)]+ struct TestAsyncUDF {+ signature: Signature,+ }++ impl TestAsyncUDF {+ fn new() -> Self {+ Self {+ signature: Signature::exact(vec![DataType::Int64], Volatility::Volatile),+ }+ }+ }++ impl ScalarUDFImpl for TestAsyncUDF {+ fn name(&self) -> &str {+ "test_async_udf"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Int64)+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ not_impl_err!("Must call from `invoke_async_with_args`")+ }+ }++ #[async_trait::async_trait]+ impl AsyncScalarUDFImpl for TestAsyncUDF {+ async fn invoke_async_with_args(+ &self,+ args: ScalarFunctionArgs,+ ) -> Result<ColumnarValue> {+ Ok(args.args[0].clone())+ }+ }++ let ctx = SessionContext::new();+ let async_udf = AsyncScalarUDF::new(Arc::new(TestAsyncUDF::new()));+ ctx.register_udf(async_udf.into_scalar_udf());++ let physical_plan = ctx+ .sql("select test_async_udf(1)")+ .await?+ .create_physical_plan()+ .await?;++ roundtrip_test_with_context(physical_plan, &ctx)?;++ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/windows.rs b/datafusion/proto/tests/cases/plans/windows.rsnew file mode 100644index 0000000000..cfd2532a88--- /dev/null+++ b/datafusion/proto/tests/cases/plans/windows.rs@@ -0,0 +1,308 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! The window execs and their window functions.++use super::roundtrip_test;+use datafusion::arrow::compute::kernels::sort::SortOptions;+use datafusion::arrow::datatypes::{DataType, Field, Schema};+use datafusion::functions_aggregate::count::count_udaf;+use datafusion::functions_aggregate::sum::sum_udaf;+use datafusion::functions_window::nth_value::nth_value_udwf;+use datafusion::functions_window::row_number::row_number_udwf;+use datafusion::physical_expr::aggregate::AggregateExprBuilder;+use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};+use datafusion::physical_plan::InputOrderMode;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{PhysicalSortExpr, cast, col, lit};+use datafusion::physical_plan::windows::{+ BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,+ create_udwf_window_expr,+};+use datafusion::scalar::ScalarValue;+use datafusion_common::Result;+use datafusion_expr::{WindowFrame, WindowFrameBound};+use datafusion_functions_aggregate::average::avg_udaf;+use std::sync::Arc;+use std::vec;++#[test]+fn roundtrip_udwf() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let udwf_expr = Arc::new(StandardWindowExpr::new(+ create_udwf_window_expr(+ &row_number_udwf(),+ &[],+ &schema,+ "row_number() PARTITION BY [a] ORDER BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),+ false,+ )?,+ &[+ col("a", &schema)?+ ],+ &[+ PhysicalSortExpr::new(col("b", &schema)?, SortOptions::new(true, true))+ ],+ Arc::new(WindowFrame::new(None)),+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(+ vec![udwf_expr],+ input,+ InputOrderMode::Sorted,+ true,+ )?))+}++#[test]+fn roundtrip_window() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ let window_frame = WindowFrame::new_bounds(+ datafusion_expr::WindowFrameUnits::Range,+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),+ WindowFrameBound::CurrentRow,+ );++ let nth_value_window =+ create_udwf_window_expr(+ &nth_value_udwf(),+ &[col("a", &schema)?,+ lit(2)], schema.as_ref(),+ "NTH_VALUE(a, 2) PARTITION BY [b] ORDER BY [a ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),+ false,+ )?;+ let udwf_expr = Arc::new(StandardWindowExpr::new(+ nth_value_window,+ &[col("b", &schema)?],+ &[PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ }],+ Arc::new(window_frame),+ ));++ let plain_aggr_window_expr = Arc::new(PlainAggregateWindowExpr::new(+ AggregateExprBuilder::new(+ avg_udaf(),+ vec![cast(col("b", &schema)?, &schema, DataType::Float64)?],+ )+ .schema(Arc::clone(&schema))+ .alias("avg(b)")+ .build()+ .map(Arc::new)?,+ &[],+ &[],+ Arc::new(WindowFrame::new(None)),+ None,+ ));++ let window_frame = WindowFrame::new_bounds(+ datafusion_expr::WindowFrameUnits::Range,+ WindowFrameBound::CurrentRow,+ WindowFrameBound::Preceding(ScalarValue::Int64(None)),+ );++ let args = vec![cast(col("a", &schema)?, &schema, DataType::Float64)?];+ let sum_expr = AggregateExprBuilder::new(sum_udaf(), args)+ .schema(Arc::clone(&schema))+ .alias("SUM(a) RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING")+ .build()+ .map(Arc::new)?;++ let sliding_aggr_window_expr = Arc::new(SlidingAggregateWindowExpr::new(+ sum_expr,+ &[],+ &[],+ Arc::new(window_frame),+ None,+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ roundtrip_test(Arc::new(WindowAggExec::try_new(+ vec![plain_aggr_window_expr, sliding_aggr_window_expr, udwf_expr],+ input,+ false,+ )?))+}++#[test]+fn roundtrip_window_distinct() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ // Create a distinct count window expression with unbounded frame (becomes PlainAggregateWindowExpr)+ let distinct_count_expr = Arc::new(PlainAggregateWindowExpr::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("count(DISTINCT a)")+ .distinct() // Enable distinct+ .build()+ .map(Arc::new)?,+ &[col("b", &schema)?], // partition by b+ &[], // no order by+ Arc::new(WindowFrame::new(None)), // unbounded frame+ None,+ ));++ // Create a distinct sum window expression with bounded frame (becomes SlidingAggregateWindowExpr)+ let bounded_frame = WindowFrame::new_bounds(+ datafusion_expr::WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(Some(1))),+ WindowFrameBound::CurrentRow,+ );++ let distinct_sum_expr = Arc::new(SlidingAggregateWindowExpr::new(+ AggregateExprBuilder::new(+ sum_udaf(),+ vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],+ )+ .schema(Arc::clone(&schema))+ .alias("sum(DISTINCT a)")+ .distinct() // Enable distinct+ .with_ignore_nulls(true) // Enable ignore nulls+ .build()+ .map(Arc::new)?,+ &[], // no partition by+ &[], // no order by+ Arc::new(bounded_frame), // bounded frame+ None,+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ roundtrip_test(Arc::new(WindowAggExec::try_new(+ vec![distinct_count_expr, distinct_sum_expr],+ input,+ false,+ )?))+}++#[test]+fn test_distinct_window_serialization_end_to_end() -> Result<()> {+ // Create a more comprehensive test that verifies distinct window functions+ // work properly through the entire serialization/deserialization pipeline+ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ // Test 1: DISTINCT COUNT with IGNORE NULLS+ let distinct_count_ignore_nulls = Arc::new(PlainAggregateWindowExpr::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("count_distinct_ignore_nulls")+ .distinct()+ .with_ignore_nulls(true)+ .build()+ .map(Arc::new)?,+ &[col("b", &schema)?],+ &[],+ Arc::new(WindowFrame::new(None)),+ None,+ ));++ // Test 2: DISTINCT SUM (without ignore nulls)+ let bounded_frame = WindowFrame::new_bounds(+ datafusion_expr::WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(Some(2))),+ WindowFrameBound::CurrentRow,+ );++ let distinct_sum = Arc::new(SlidingAggregateWindowExpr::new(+ AggregateExprBuilder::new(+ sum_udaf(),+ vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],+ )+ .schema(Arc::clone(&schema))+ .alias("sum_distinct")+ .distinct()+ .build()+ .map(Arc::new)?,+ &[],+ &[],+ Arc::new(bounded_frame),+ None,+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ let window_exec = Arc::new(WindowAggExec::try_new(+ vec![distinct_count_ignore_nulls, distinct_sum],+ input,+ false,+ )?);++ // Perform the roundtrip test+ roundtrip_test(window_exec)+}++/// Tests that `lead` window function with offset and default value args+/// survives a protobuf round-trip. This is a regression test for a bug+/// where `expressions()` (used during serialization) returns only the+/// column expression for lead/lag, silently dropping the offset and+/// default value literal args.+#[test]+fn roundtrip_lead_with_default_value() -> Result<()> {+ use datafusion::functions_window::lead_lag::lead_udwf;++ let field_a = Field::new("a", DataType::Int64, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));++ // lead(a, 2, 42) — column a, offset 2, default value 42+ let lead_window = create_udwf_window_expr(+ &lead_udwf(),+ &[col("a", &schema)?, lit(2i64), lit(42i64)],+ schema.as_ref(),+ "test lead with default".to_string(),+ false,+ )?;++ let udwf_expr = Arc::new(StandardWindowExpr::new(+ lead_window,+ &[col("b", &schema)?],+ &[PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ }],+ Arc::new(WindowFrame::new(None)),+ ));++ let input = Arc::new(EmptyExec::new(schema.clone()));++ roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(+ vec![udwf_expr],+ input,+ InputOrderMode::Sorted,+ true,+ )?))+}diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsdeleted file mode 100644index 4e33934c6b..0000000000--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ /dev/null@@ -1,5785 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use std::collections::HashMap;-use std::fmt::{Display, Formatter};-use std::sync::{Arc, RwLock};-use std::vec;--use arrow::array::RecordBatch;-use arrow::csv::WriterBuilder;-use arrow::datatypes::{Fields, TimeUnit};-use async_trait::async_trait;-use datafusion::arrow::array::ArrayRef;-use datafusion::arrow::compute::kernels::sort::SortOptions;-use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, SchemaRef};-use datafusion::datasource::empty::EmptyTable;-use datafusion::datasource::file_format::csv::CsvSink;-use datafusion::datasource::file_format::json::{JsonFormat, JsonSink};-use datafusion::datasource::file_format::parquet::ParquetSink;-use datafusion::datasource::listing::{- ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, PartitionedFile,-};-use datafusion::datasource::object_store::ObjectStoreUrl;-use datafusion::datasource::physical_plan::{- ArrowSource, CsvSource, FileGroup, FileOutputMode, FileScanConfig,- FileScanConfigBuilder, FileSinkConfig, JsonSource, ParquetSource,- wrap_partition_type_in_dict, wrap_partition_value_in_dict,-};-use datafusion::datasource::sink::{DataSink, DataSinkExec};-use datafusion::datasource::source::DataSourceExec;-use datafusion::execution::TaskContext;-use datafusion::functions_aggregate::count::count_udaf;-use datafusion::functions_aggregate::first_last::first_value_udaf;-use datafusion::functions_aggregate::sum::sum_udaf;-use datafusion::functions_window::nth_value::nth_value_udwf;-use datafusion::functions_window::row_number::row_number_udwf;-use datafusion::logical_expr::{JoinType, Operator, Volatility, create_udf};-use datafusion::physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};-use datafusion::physical_expr::expressions::Literal;-use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};-use datafusion::physical_expr::{- HigherOrderFunctionExpr, LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,-};-use datafusion::physical_optimizer::PhysicalOptimizerRule;-use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;-use datafusion::physical_optimizer::limit_pushdown::LimitPushdown;-use datafusion::physical_plan::aggregates::{- AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,-};-use datafusion::physical_plan::analyze::AnalyzeExec;-use datafusion::physical_plan::buffer::BufferExec;-#[expect(deprecated)]-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;-use datafusion::physical_plan::coop::CooperativeExec;-use datafusion::physical_plan::empty::EmptyExec;-use datafusion::physical_plan::explain::ExplainExec;-use datafusion::physical_plan::expressions::{- BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,- cast, col, in_list, like, lit,-};-use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};-use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};-use datafusion::physical_plan::joins::{- HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,- StreamJoinPartitionMode, SymmetricHashJoinExec,-};-use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};-use datafusion::physical_plan::metrics::MetricCategory;-use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;-use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};-use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;-use datafusion::physical_plan::repartition::RepartitionExec;-use datafusion::physical_plan::scalar_subquery::{- ScalarSubqueryExec, ScalarSubqueryLink,-};-use datafusion::physical_plan::sorts::sort::SortExec;-use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;-use datafusion::physical_plan::union::{InterleaveExec, UnionExec};-use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};-use datafusion::physical_plan::windows::{- BoundedWindowAggExec, PlainAggregateWindowExpr, WindowAggExec,- create_udwf_window_expr,-};-use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, InputOrderMode,- Partitioning, PhysicalExpr, PlanProperties, RangePartitioning,- SendableRecordBatchStream, SplitPoint, Statistics, displayable,-};-use datafusion::prelude::{ParquetReadOptions, SessionContext};-use datafusion::scalar::ScalarValue;-use datafusion_common::config::{ConfigOptions, TableParquetOptions};-use datafusion_common::display::{PlanType, StringifiedPlan};-use datafusion_common::file_options::csv_writer::CsvWriterOptions;-use datafusion_common::file_options::json_writer::JsonWriterOptions;-use datafusion_common::format::ExplainFormat;-use datafusion_common::parsers::CompressionTypeVariant;-use datafusion_common::stats::Precision;-use datafusion_common::tree_node::TreeNodeRecursion;-use datafusion_common::{- DataFusionError, JoinSide, NullEquality, Result, UnnestOptions, exec_datafusion_err,- internal_datafusion_err, internal_err, not_impl_err,-};-use datafusion_datasource::file::FileSource;-use datafusion_datasource::{TableSchema, TableSchemaBuilder};-use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};-use datafusion_expr::dml::InsertOp;-use datafusion_expr::{- Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, HigherOrderUDF,- ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,- WindowFrame, WindowFrameBound, WindowUDF,- physical_planning_context::{ScalarSubqueryResults, SubqueryIndex},-};-use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;-use datafusion_functions_aggregate::array_agg::array_agg_udaf;-use datafusion_functions_aggregate::average::avg_udaf;-use datafusion_functions_aggregate::min_max::max_udaf;-use datafusion_functions_aggregate::nth_value::nth_value_udaf;-use datafusion_functions_aggregate::string_agg::string_agg_udaf;-use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;-use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;-use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;-use datafusion_proto::bytes::{- physical_plan_from_bytes_with_proto_converter,- physical_plan_to_bytes_with_proto_converter,-};-use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;-use datafusion_proto::physical_plan::{- AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,- DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,- PhysicalPlanNodeExt, PhysicalProtoConverterExtension,-};-use datafusion_proto::protobuf;-use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};-use prost::Message;--use crate::cases::{- CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF,- MyHigherOrderUdfNode, MyRegexUdf, MyRegexUdfNode,-};-use datafusion_physical_expr::expressions::{LambdaVariable, is_not_null, lambda};-use datafusion_physical_expr::utils::reassign_expr_columns;--/// Perform a serde roundtrip and assert that the string representation of the before and after plans-/// are identical. Note that this often isn't sufficient to guarantee that no information is-/// lost during serde because the string representation of a plan often only shows a subset of state.-fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;- Ok(())-}--/// Perform a serde roundtrip and assert that the string representation of the before and after plans-/// are identical. Note that this often isn't sufficient to guarantee that no information is-/// lost during serde because the string representation of a plan often only shows a subset of state.-///-/// This version of the roundtrip_test method returns the final plan after serde so that it can be inspected-/// farther in tests.-fn roundtrip_test_and_return(- exec_plan: Arc<dyn ExecutionPlan>,- ctx: &SessionContext,- codec: &dyn PhysicalExtensionCodec,- proto_converter: &dyn PhysicalProtoConverterExtension,-) -> Result<Arc<dyn ExecutionPlan>> {- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan),- codec,- proto_converter,- )?;- let result_exec_plan = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- codec,- proto_converter,- )?;-- pretty_assertions::assert_eq!(- format!("{exec_plan:?}"),- format!("{result_exec_plan:?}")- );- Ok(result_exec_plan)-}--/// Perform a serde roundtrip and assert that the string representation of the before and after plans-/// are identical. Note that this often isn't sufficient to guarantee that no information is-/// lost during serde because the string representation of a plan often only shows a subset of state.-///-/// This version of the roundtrip_test function accepts a SessionContext, which is required when-/// performing serde on some plans.-fn roundtrip_test_with_context(- exec_plan: Arc<dyn ExecutionPlan>,- ctx: &SessionContext,-) -> Result<()> {- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;- Ok(())-}--/// Perform a serde roundtrip for the specified sql query, and assert that-/// query results are identical.-async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;-- roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;- Ok(())-}--/// returns a SessionContext with `alltypes_plain` registered-async fn all_types_context() -> Result<SessionContext> {- let ctx = SessionContext::new();-- let testdata = datafusion::test_util::parquet_test_data();- ctx.register_parquet(- "alltypes_plain",- &format!("{testdata}/alltypes_plain.parquet"),- ParquetReadOptions::default(),- )- .await?;-- Ok(ctx)-}--#[test]-fn roundtrip_empty() -> Result<()> {- roundtrip_test(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))-}--#[test]-fn roundtrip_empty_with_partitions() -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let plan = Arc::new(EmptyExec::new(Arc::new(Schema::empty())).with_partitions(4));- let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;- assert_eq!(plan.output_partitioning().partition_count(), 4);- Ok(())-}--#[test]-fn roundtrip_placeholder_row_with_partitions() -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let plan =- Arc::new(PlaceholderRowExec::new(Arc::new(Schema::empty())).with_partitions(4));- let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;- assert_eq!(plan.output_partitioning().partition_count(), 4);- Ok(())-}--/// Plans encoded before `partitions` was added carry no value for it, which-/// decodes as zero and must be treated as the previous default of one.-#[test]-fn decode_empty_and_placeholder_row_without_partitions() -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let schema: protobuf::Schema = (&Schema::empty()).try_into()?;-- for physical_plan_type in [- protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {- schema: Some(schema.clone()),- partitions: 0,- }),- protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(- protobuf::PlaceholderRowExecNode {- schema: Some(schema.clone()),- partitions: 0,- },- ),- ] {- let node = PhysicalPlanNode {- physical_plan_type: Some(physical_plan_type),- };- let plan = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;- assert_eq!(plan.output_partitioning().partition_count(), 1);- }- Ok(())-}--#[derive(Debug)]-struct DowncastDelegatingExec {- inner: Arc<dyn ExecutionPlan>,-}--impl DowncastDelegatingExec {- fn new(inner: Arc<dyn ExecutionPlan>) -> Self {- Self { inner }- }-}--impl DisplayAs for DowncastDelegatingExec {- fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {- self.inner.fmt_as(t, f)- }-}--impl ExecutionPlan for DowncastDelegatingExec {- fn name(&self) -> &str {- self.inner.name()- }-- fn properties(&self) -> &Arc<PlanProperties> {- self.inner.properties()- }-- fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {- self.inner.children()- }-- fn apply_expressions(- &self,- f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,- ) -> Result<TreeNodeRecursion> {- self.inner.apply_expressions(f)- }-- fn with_new_children(- self: Arc<Self>,- children: Vec<Arc<dyn ExecutionPlan>>,- ) -> Result<Arc<dyn ExecutionPlan>> {- let inner = Arc::clone(&self.inner).with_new_children(children)?;- Ok(Arc::new(Self::new(inner)))- }-- fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {- Some(self.inner.as_ref())- }-- fn execute(- &self,- partition: usize,- context: Arc<TaskContext>,- ) -> Result<SendableRecordBatchStream> {- self.inner.execute(partition, context)- }-}--#[test]-fn serialize_uses_downcast_delegate() -> Result<()> {- let inner: Arc<dyn ExecutionPlan> =- Arc::new(EmptyExec::new(Arc::new(Schema::empty())));- let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));- let codec = DefaultPhysicalExtensionCodec {};-- let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;-- assert!(matches!(- proto.physical_plan_type,- Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))- ));-- Ok(())-}--/// A wrapper delegating to a plan that serializes itself via the-/// `try_to_proto` hook must serialize as its delegate: the wrapper's default-/// hook returns `Ok(None)` and the delegate has no downcast-chain fallback.-#[test]-fn serialize_uses_downcast_delegate_for_self_serializing_plan() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));- let inner: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(- vec![ProjectionExpr {- expr: col("a", &schema)?,- alias: "a".to_string(),- }],- input,- )?);- let plan: Arc<dyn ExecutionPlan> = Arc::new(DowncastDelegatingExec::new(inner));- let codec = DefaultPhysicalExtensionCodec {};-- let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;-- assert!(matches!(- proto.physical_plan_type,- Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(- _- ))- ));-- Ok(())-}--#[test]-fn roundtrip_date_time_interval() -> Result<()> {- let schema = Schema::new(vec![- Field::new("some_date", DataType::Date32, false),- Field::new(- "some_interval",- DataType::Interval(IntervalUnit::DayTime),- false,- ),- ]);- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));- let date_expr = col("some_date", &schema)?;- let literal_expr = col("some_interval", &schema)?;- let date_time_interval_expr =- binary(date_expr, Operator::Plus, literal_expr, &schema)?;- let plan = Arc::new(ProjectionExec::try_new(- vec![ProjectionExpr {- expr: date_time_interval_expr,- alias: "result".to_string(),- }],- input,- )?);- roundtrip_test(plan)-}--#[test]-fn roundtrip_local_limit() -> Result<()> {- roundtrip_test(Arc::new(LocalLimitExec::new(- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),- 25,- )))-}--#[test]-fn roundtrip_global_limit() -> Result<()> {- roundtrip_test(Arc::new(GlobalLimitExec::new(- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),- 0,- Some(25),- )))-}--#[test]-fn roundtrip_global_skip_no_limit() -> Result<()> {- roundtrip_test(Arc::new(GlobalLimitExec::new(- Arc::new(EmptyExec::new(Arc::new(Schema::empty()))),- 10,- None, // no limit- )))-}--/// Sort key at index 1, so a decoder that misbinds column name vs index-/// cannot pass.-fn limit_test_schema() -> Arc<Schema> {- Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- ]))-}--/// Non-default sort options, so a decode that falls back to defaults cannot-/// pass.-fn limit_required_ordering(schema: &Schema) -> Result<Option<LexOrdering>> {- Ok(LexOrdering::new(vec![PhysicalSortExpr {- expr: col("b", schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- }]))-}--#[test]-fn roundtrip_limit_with_required_ordering() -> Result<()> {- let schema = limit_test_schema();- let required_ordering = limit_required_ordering(&schema)?;- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- let mut global =- GlobalLimitExec::new(Arc::new(EmptyExec::new(Arc::clone(&schema))), 3, Some(25));- global.set_required_ordering(required_ordering.clone());- let decoded =- roundtrip_test_and_return(Arc::new(global), &ctx, &codec, &proto_converter)?;- let decoded = decoded- .downcast_ref::<GlobalLimitExec>()- .expect("expected GlobalLimitExec");- assert_eq!(decoded.required_ordering(), &required_ordering);-- let mut local = LocalLimitExec::new(Arc::new(EmptyExec::new(schema)), 25);- local.set_required_ordering(required_ordering.clone());- let decoded =- roundtrip_test_and_return(Arc::new(local), &ctx, &codec, &proto_converter)?;- let decoded = decoded- .downcast_ref::<LocalLimitExec>()- .expect("expected LocalLimitExec");- assert_eq!(decoded.required_ordering(), &required_ordering);- Ok(())-}--/// A limit's `required_ordering` is the only record that an `ORDER BY ... LIMIT`-/// whose sort node was optimized away is order-sensitive, so it must survive-/// serde all the way into the scan's `preserve_order` flag.-#[test]-fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {- let file_schema = limit_test_schema();- let make_scan = || {- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .build();- DataSourceExec::from_data_source(scan_config)- };- let scan_after_limit_pushdown = |limit: GlobalLimitExec| -> Result<FileScanConfig> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let decoded =- roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;-- // Child replacement must not erase the decoded ordering before pushdown.- let rebuilt = decoded.with_new_children(vec![make_scan()])?;-- let optimized =- LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;- let scan = optimized- .downcast_ref::<DataSourceExec>()- .expect("limit should be absorbed into the scan");- Ok(scan- .data_source()- .downcast_ref::<FileScanConfig>()- .expect("expected FileScanConfig")- .clone())- };-- let mut limit = GlobalLimitExec::new(make_scan(), 0, Some(10));- limit.set_required_ordering(limit_required_ordering(&file_schema)?);- let scan_config = scan_after_limit_pushdown(limit)?;- assert_eq!(scan_config.limit, Some(10));- assert!(scan_config.preserve_order);-- let scan_config =- scan_after_limit_pushdown(GlobalLimitExec::new(make_scan(), 0, Some(10)))?;- assert_eq!(scan_config.limit, Some(10));- assert!(!scan_config.preserve_order);- Ok(())-}--#[test]-fn roundtrip_hash_join() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_a]);- let on = vec![(- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,- )];-- let schema_left = Arc::new(schema_left);- let schema_right = Arc::new(schema_right);- for join_type in &[- JoinType::Inner,- JoinType::Left,- JoinType::Right,- JoinType::Full,- JoinType::LeftAnti,- JoinType::RightAnti,- JoinType::LeftSemi,- JoinType::RightSemi,- ] {- for partition_mode in &[PartitionMode::Partitioned, PartitionMode::CollectLeft] {- roundtrip_test(Arc::new(HashJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- on.clone(),- None,- join_type,- None,- *partition_mode,- NullEquality::NullEqualsNothing,- false,- )?))?;- }- }- Ok(())-}--#[test]-fn roundtrip_nested_loop_join() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_a]);-- let schema_left = Arc::new(schema_left);- let schema_right = Arc::new(schema_right);- for join_type in &[- JoinType::Inner,- JoinType::Left,- JoinType::Right,- JoinType::Full,- JoinType::LeftAnti,- JoinType::RightAnti,- JoinType::LeftSemi,- JoinType::RightSemi,- ] {- roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- None,- join_type,- Some(vec![0]),- )?))?;- }- Ok(())-}--/// Regression: proto3 `repeated` fields cannot distinguish "absent" from "empty",-/// so a naive encoding collapses `Some(vec![])` and `None` into the same wire-/// representation. `try_embed_projection` (DataFusion 53+) produces-/// `HashJoinExec.projection = Some(vec![])` for `SELECT count(1) … JOIN …`,-/// which previously round-tripped to `None` and caused downstream consumers (e.g.-/// distributed Flight executors) to receive a different number of output-/// columns than the planner declared. Verify all three states preserve.-#[test]-fn roundtrip_hash_join_projection_states() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));- let schema_right = Arc::new(Schema::new(vec![field_a]));- let on = vec![(- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,- )];-- for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {- roundtrip_test(Arc::new(HashJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- on.clone(),- None,- &JoinType::Inner,- projection,- PartitionMode::Partitioned,- NullEquality::NullEqualsNothing,- false,- )?))?;- }- Ok(())-}--/// Regression: `HashJoinExecNode` had no `fetch` field, so the row limit that-/// the `limit_pushdown` physical optimizer rule pushes into the join via-/// `ExecutionPlan::with_fetch` was silently dropped by serde. Because that rule-/// also removes the enclosing `GlobalLimitExec` once the join absorbs the limit,-/// a round-tripped plan had no limit left at all and a distributed executor-/// returned more rows than the query asked for.-///-/// Note this cannot be covered by `roundtrip_test`: that helper compares-/// `format!("{plan:?}")`, and `HashJoinExec`'s `Debug` output does not include-/// `fetch`, so the before/after strings match even when the value is lost. The-/// assertions below therefore inspect `fetch()` directly.-#[test]-fn roundtrip_hash_join_fetch() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));- let schema_right = Arc::new(Schema::new(vec![field_a]));- let on = vec![(- Arc::new(Column::new("col", schema_left.index_of("col")?)) as _,- Arc::new(Column::new("col", schema_right.index_of("col")?)) as _,- )];-- // `usize::MAX` and `u32::MAX as usize` pin the decode-side `u64 -> usize`- // conversion: it is a checked `usize::try_from`, and a large fetch must- // survive the round trip exactly rather than being truncated or clamped.- // Both are representable on every target (on a 32-bit target `usize::MAX`- // is simply `u32::MAX`), so this stays portable. The truncating case- // itself -- a `u64` fetch above `usize::MAX` -- is only reachable on a- // 32-bit target and so is not exercised by this test on a 64-bit host.- for fetch in [None, Some(7), Some(u32::MAX as usize), Some(usize::MAX)] {- let join = HashJoinExec::try_new(- Arc::new(EmptyExec::new(Arc::clone(&schema_left))),- Arc::new(EmptyExec::new(Arc::clone(&schema_right))),- on.clone(),- None,- &JoinType::Inner,- None,- PartitionMode::Partitioned,- NullEquality::NullEqualsNothing,- false,- )?;-- let plan: Arc<dyn ExecutionPlan> = match fetch {- // This is how `limit_pushdown` installs the limit.- Some(fetch) => join- .with_fetch(Some(fetch))- .expect("HashJoinExec supports fetch"),- None => Arc::new(join),- };- assert_eq!(plan.fetch(), fetch);-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let deserialized =- roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;-- let deserialized_join = deserialized- .downcast_ref::<HashJoinExec>()- .expect("should be a HashJoinExec");- assert_eq!(deserialized_join.fetch(), fetch);- }- Ok(())-}--/// Same regression coverage for `NestedLoopJoinExec`, which shares the-/// `repeated uint32 projection` proto field shape with `HashJoinExec`.-#[test]-fn roundtrip_nested_loop_join_projection_states() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Arc::new(Schema::new(vec![field_a.clone()]));- let schema_right = Arc::new(Schema::new(vec![field_a]));-- for projection in [None, Some(vec![]), Some(vec![0]), Some(vec![1])] {- roundtrip_test(Arc::new(NestedLoopJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- None,- &JoinType::Inner,- projection,- )?))?;- }- Ok(())-}--#[test]-fn roundtrip_udwf() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let udwf_expr = Arc::new(StandardWindowExpr::new(- create_udwf_window_expr(- &row_number_udwf(),- &[],- &schema,- "row_number() PARTITION BY [a] ORDER BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),- false,- )?,- &[- col("a", &schema)?- ],- &[- PhysicalSortExpr::new(col("b", &schema)?, SortOptions::new(true, true))- ],- Arc::new(WindowFrame::new(None)),- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(- vec![udwf_expr],- input,- InputOrderMode::Sorted,- true,- )?))-}--#[test]-fn roundtrip_window() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let window_frame = WindowFrame::new_bounds(- datafusion_expr::WindowFrameUnits::Range,- WindowFrameBound::Preceding(ScalarValue::Int64(None)),- WindowFrameBound::CurrentRow,- );-- let nth_value_window =- create_udwf_window_expr(- &nth_value_udwf(),- &[col("a", &schema)?,- lit(2)], schema.as_ref(),- "NTH_VALUE(a, 2) PARTITION BY [b] ORDER BY [a ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),- false,- )?;- let udwf_expr = Arc::new(StandardWindowExpr::new(- nth_value_window,- &[col("b", &schema)?],- &[PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: false,- nulls_first: false,- },- }],- Arc::new(window_frame),- ));-- let plain_aggr_window_expr = Arc::new(PlainAggregateWindowExpr::new(- AggregateExprBuilder::new(- avg_udaf(),- vec![cast(col("b", &schema)?, &schema, DataType::Float64)?],- )- .schema(Arc::clone(&schema))- .alias("avg(b)")- .build()- .map(Arc::new)?,- &[],- &[],- Arc::new(WindowFrame::new(None)),- None,- ));-- let window_frame = WindowFrame::new_bounds(- datafusion_expr::WindowFrameUnits::Range,- WindowFrameBound::CurrentRow,- WindowFrameBound::Preceding(ScalarValue::Int64(None)),- );-- let args = vec![cast(col("a", &schema)?, &schema, DataType::Float64)?];- let sum_expr = AggregateExprBuilder::new(sum_udaf(), args)- .schema(Arc::clone(&schema))- .alias("SUM(a) RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING")- .build()- .map(Arc::new)?;-- let sliding_aggr_window_expr = Arc::new(SlidingAggregateWindowExpr::new(- sum_expr,- &[],- &[],- Arc::new(window_frame),- None,- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- roundtrip_test(Arc::new(WindowAggExec::try_new(- vec![plain_aggr_window_expr, sliding_aggr_window_expr, udwf_expr],- input,- false,- )?))-}--#[test]-fn roundtrip_window_distinct() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- // Create a distinct count window expression with unbounded frame (becomes PlainAggregateWindowExpr)- let distinct_count_expr = Arc::new(PlainAggregateWindowExpr::new(- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])- .schema(Arc::clone(&schema))- .alias("count(DISTINCT a)")- .distinct() // Enable distinct- .build()- .map(Arc::new)?,- &[col("b", &schema)?], // partition by b- &[], // no order by- Arc::new(WindowFrame::new(None)), // unbounded frame- None,- ));-- // Create a distinct sum window expression with bounded frame (becomes SlidingAggregateWindowExpr)- let bounded_frame = WindowFrame::new_bounds(- datafusion_expr::WindowFrameUnits::Rows,- WindowFrameBound::Preceding(ScalarValue::UInt64(Some(1))),- WindowFrameBound::CurrentRow,- );-- let distinct_sum_expr = Arc::new(SlidingAggregateWindowExpr::new(- AggregateExprBuilder::new(- sum_udaf(),- vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],- )- .schema(Arc::clone(&schema))- .alias("sum(DISTINCT a)")- .distinct() // Enable distinct- .with_ignore_nulls(true) // Enable ignore nulls- .build()- .map(Arc::new)?,- &[], // no partition by- &[], // no order by- Arc::new(bounded_frame), // bounded frame- None,- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- roundtrip_test(Arc::new(WindowAggExec::try_new(- vec![distinct_count_expr, distinct_sum_expr],- input,- false,- )?))-}--#[test]-fn test_distinct_window_serialization_end_to_end() -> Result<()> {- // Create a more comprehensive test that verifies distinct window functions- // work properly through the entire serialization/deserialization pipeline- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- // Test 1: DISTINCT COUNT with IGNORE NULLS- let distinct_count_ignore_nulls = Arc::new(PlainAggregateWindowExpr::new(- AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])- .schema(Arc::clone(&schema))- .alias("count_distinct_ignore_nulls")- .distinct()- .with_ignore_nulls(true)- .build()- .map(Arc::new)?,- &[col("b", &schema)?],- &[],- Arc::new(WindowFrame::new(None)),- None,- ));-- // Test 2: DISTINCT SUM (without ignore nulls)- let bounded_frame = WindowFrame::new_bounds(- datafusion_expr::WindowFrameUnits::Rows,- WindowFrameBound::Preceding(ScalarValue::UInt64(Some(2))),- WindowFrameBound::CurrentRow,- );-- let distinct_sum = Arc::new(SlidingAggregateWindowExpr::new(- AggregateExprBuilder::new(- sum_udaf(),- vec![cast(col("a", &schema)?, &schema, DataType::Float64)?],- )- .schema(Arc::clone(&schema))- .alias("sum_distinct")- .distinct()- .build()- .map(Arc::new)?,- &[],- &[],- Arc::new(bounded_frame),- None,- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- let window_exec = Arc::new(WindowAggExec::try_new(- vec![distinct_count_ignore_nulls, distinct_sum],- input,- false,- )?);-- // Perform the roundtrip test- roundtrip_test(window_exec)-}--#[test]-fn roundtrip_aggregate() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =- vec![(col("a", &schema)?, "unused".to_string())];-- let avg_expr = AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])- .schema(Arc::clone(&schema))- .alias("AVG(b)")- .build()?;- let nth_expr =- AggregateExprBuilder::new(nth_value_udaf(), vec![col("b", &schema)?, lit(1u64)])- .schema(Arc::clone(&schema))- .alias("NTH_VALUE(b, 1)")- .build()?;- let str_agg_expr =- AggregateExprBuilder::new(string_agg_udaf(), vec![col("b", &schema)?, lit(1u64)])- .schema(Arc::clone(&schema))- .alias("NTH_VALUE(b, 1)")- .build()?;-- let test_cases = vec![- // AVG- vec![Arc::new(avg_expr)],- // NTH_VALUE- vec![Arc::new(nth_expr)],- // STRING_AGG- vec![Arc::new(str_agg_expr)],- ];-- for aggregates in test_cases {- let schema = schema.clone();- roundtrip_test(Arc::new(AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new_single(groups.clone()),- aggregates,- vec![None],- Arc::new(EmptyExec::new(schema.clone())),- schema,- )?))?;- }-- Ok(())-}--#[test]-fn roundtrip_aggregate_with_limit() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =- vec![(col("a", &schema)?, "unused".to_string())];-- let aggregates = vec![- AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])- .schema(Arc::clone(&schema))- .alias("AVG(b)")- .build()- .map(Arc::new)?,- ];-- let agg = AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new_single(groups.clone()),- aggregates,- vec![None],- Arc::new(EmptyExec::new(schema.clone())),- schema,- )?;- let agg = agg.with_limit_options(Some(LimitOptions::new_with_order(12, false)));- roundtrip_test(Arc::new(agg))-}--#[test]-fn roundtrip_aggregate_with_approx_pencentile_cont() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =- vec![(col("a", &schema)?, "unused".to_string())];-- let aggregates = vec![- AggregateExprBuilder::new(- approx_percentile_cont_udaf(),- vec![col("b", &schema)?, lit(0.5)],- )- .schema(Arc::clone(&schema))- .alias("APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY b)")- .build()- .map(Arc::new)?,- ];-- let agg = AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new_single(groups.clone()),- aggregates,- vec![None],- Arc::new(EmptyExec::new(schema.clone())),- schema,- )?;- roundtrip_test(Arc::new(agg))-}--#[test]-fn roundtrip_aggregate_with_sort() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =- vec![(col("a", &schema)?, "unused".to_string())];- let sort_exprs = vec![PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- }];-- let aggregates = vec![- AggregateExprBuilder::new(array_agg_udaf(), vec![col("b", &schema)?])- .schema(Arc::clone(&schema))- .alias("ARRAY_AGG(b)")- .order_by(sort_exprs)- .build()- .map(Arc::new)?,- ];-- let agg = AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new_single(groups.clone()),- aggregates,- vec![None],- Arc::new(EmptyExec::new(schema.clone())),- schema,- )?;- roundtrip_test(Arc::new(agg))-}--#[test]-fn roundtrip_aggregate_udaf() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- #[derive(Debug)]- struct Example;- impl Accumulator for Example {- fn state(&mut self) -> Result<Vec<ScalarValue>> {- Ok(vec![ScalarValue::Int64(Some(0))])- }-- fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {- Ok(())- }-- fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {- Ok(())- }-- fn evaluate(&mut self) -> Result<ScalarValue> {- Ok(ScalarValue::Int64(Some(0)))- }-- fn size(&self) -> usize {- 0- }- }-- let return_type = DataType::Int64;- let accumulator: AccumulatorFactoryFunction = Arc::new(|_| Ok(Box::new(Example)));-- let udaf = AggregateUDF::from(SimpleAggregateUDF::new_with_signature(- "example",- Signature::exact(vec![DataType::Int64], Volatility::Immutable),- return_type,- accumulator,- vec![Field::new("value", DataType::Int64, true).into()],- ));-- let ctx = SessionContext::new();- ctx.register_udaf(udaf.clone());-- let groups: Vec<(Arc<dyn PhysicalExpr>, String)> =- vec![(col("a", &schema)?, "unused".to_string())];-- let aggregates = vec![- AggregateExprBuilder::new(Arc::new(udaf), vec![col("b", &schema)?])- .schema(Arc::clone(&schema))- .alias("example_agg")- .build()- .map(Arc::new)?,- ];-- roundtrip_test_with_context(- Arc::new(AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new_single(groups.clone()),- aggregates,- vec![None],- Arc::new(EmptyExec::new(schema.clone())),- schema,- )?),- &ctx,- )-}--#[test]-fn roundtrip_filter_with_not_and_in_list() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let field_c = Field::new("c", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));- let not = Arc::new(NotExpr::new(col("a", &schema)?));- let in_list = in_list(- col("b", &schema)?,- vec![- lit(ScalarValue::Int64(Some(1))),- lit(ScalarValue::Int64(Some(2))),- ],- &false,- schema.as_ref(),- )?;- let and = binary(not, Operator::And, in_list, &schema)?;- roundtrip_test(Arc::new(FilterExec::try_new(- and,- Arc::new(EmptyExec::new(schema.clone())),- )?))-}--#[test]-fn roundtrip_filter_with_fetch() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let predicate = col("a", &schema)?;- let filter = FilterExecBuilder::new(predicate, Arc::new(EmptyExec::new(schema)))- .with_fetch(Some(10))- .build()?;- assert_eq!(filter.fetch(), Some(10));- roundtrip_test(Arc::new(filter))-}--#[test]-fn roundtrip_filter_projection_states() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Boolean, false),- Field::new("b", DataType::Int64, false),- ]));- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- for projection in [None, Some(vec![]), Some(vec![0])] {- let filter = FilterExecBuilder::new(- col("a", &schema)?,- Arc::new(EmptyExec::new(Arc::clone(&schema))),- )- .apply_projection(projection.clone())?- .with_default_selectivity(37)- .with_batch_size(1024)- .with_fetch(Some(5))- .build()?;-- let result =- roundtrip_test_and_return(Arc::new(filter), &ctx, &codec, &proto_converter)?;- let result = result.downcast_ref::<FilterExec>().unwrap();- assert_eq!(result.projection().as_deref(), projection.as_deref());- assert_eq!(result.default_selectivity(), 37);- assert_eq!(result.batch_size(), 1024);- assert_eq!(result.fetch(), Some(5));- }-- Ok(())-}--#[test]-fn roundtrip_sort() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let sort_exprs = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- },- ]- .into();- roundtrip_test(Arc::new(SortExec::new(- sort_exprs,- Arc::new(EmptyExec::new(schema)),- )))-}--#[test]-fn roundtrip_sort_preserve_partitioning() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let sort_exprs: LexOrdering = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- },- ]- .into();-- roundtrip_test(Arc::new(SortExec::new(- sort_exprs.clone(),- Arc::new(EmptyExec::new(schema.clone())),- )))?;-- roundtrip_test(Arc::new(- SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)))- .with_preserve_partitioning(true),- ))-}--/// `SortExec::fetch` turns a sort into a top-k sort. Losing it during serde-/// would silently widen the result set, so exercise the `Some(..)` state-/// explicitly (`roundtrip_sort` only covers `None`).-///-/// `SortExec` currently derives `Debug`, so `roundtrip_test`'s-/// `format!("{plan:?}")` comparison does observe `fetch`. The assertions below-/// go through the accessor instead so that this coverage does not silently-/// disappear if `SortExec` ever grows a hand-written `Debug` impl.-#[test]-fn roundtrip_sort_with_fetch() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let sort_exprs: LexOrdering = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- },- ]- .into();-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- let roundtripped = roundtrip_test_and_return(- Arc::new(- SortExec::new(- sort_exprs.clone(),- Arc::new(EmptyExec::new(Arc::clone(&schema))),- )- .with_fetch(Some(7)),- ),- &ctx,- &codec,- &proto_converter,- )?;- let roundtripped = roundtripped- .downcast_ref::<SortExec>()- .expect("should decode back into a SortExec");- assert_eq!(roundtripped.fetch(), Some(7));- assert_eq!(roundtripped.expr(), &sort_exprs);-- // `fetch` combined with `preserve_partitioning`, since both share the same- // proto node.- let roundtripped = roundtrip_test_and_return(- Arc::new(- SortExec::new(sort_exprs.clone(), Arc::new(EmptyExec::new(schema)))- .with_fetch(Some(3))- .with_preserve_partitioning(true),- ),- &ctx,- &codec,- &proto_converter,- )?;- let roundtripped = roundtripped- .downcast_ref::<SortExec>()- .expect("should decode back into a SortExec");- assert_eq!(roundtripped.fetch(), Some(3));- assert!(roundtripped.preserve_partitioning());- Ok(())-}--/// Round trip a [`SortPreservingMergeExec`], which had no dedicated round trip-/// test at all.-///-/// Covers everything that is actually on the wire for this plan: the input, the-/// sort expressions and `fetch` in both its `None` and `Some(..)` states.-///-/// Note that `SortPreservingMergeExec::enable_round_robin_repartition` is-/// deliberately *not* asserted on here: it has no field in-/// `SortPreservingMergeExecNode`, so it is not serialized and decoding always-/// restores the `true` default from `SortPreservingMergeExec::new`. Asserting-/// round trip equality on it would give a false sense of coverage.-///-/// `SortPreservingMergeExec` derives `Debug`, so `roundtrip_test`'s-/// `format!("{plan:?}")` comparison does observe `expr` and `fetch`. The-/// assertions below use the accessors so the coverage survives a future-/// hand-written `Debug` impl.-#[test]-fn roundtrip_sort_preserving_merge() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let sort_exprs: LexOrdering = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- },- ]- .into();-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- // No fetch: `fetch` is encoded as -1 and must decode back to `None`.- let roundtripped = roundtrip_test_and_return(- Arc::new(SortPreservingMergeExec::new(- sort_exprs.clone(),- Arc::new(EmptyExec::new(Arc::clone(&schema))),- )),- &ctx,- &codec,- &proto_converter,- )?;- let roundtripped = roundtripped- .downcast_ref::<SortPreservingMergeExec>()- .expect("should decode back into a SortPreservingMergeExec");- assert_eq!(roundtripped.fetch(), None);- assert_eq!(roundtripped.expr(), &sort_exprs);- assert_eq!(roundtripped.input().schema(), schema);-- // With a fetch: dropping it would turn a bounded merge into an unbounded- // one and change the query result.- let roundtripped = roundtrip_test_and_return(- Arc::new(- SortPreservingMergeExec::new(- sort_exprs.clone(),- Arc::new(EmptyExec::new(Arc::clone(&schema))),- )- .with_fetch(Some(11)),- ),- &ctx,- &codec,- &proto_converter,- )?;- let roundtripped = roundtripped- .downcast_ref::<SortPreservingMergeExec>()- .expect("should decode back into a SortPreservingMergeExec");- assert_eq!(roundtripped.fetch(), Some(11));- assert_eq!(roundtripped.expr(), &sort_exprs);- Ok(())-}--#[test]-fn roundtrip_coalesce_batches_with_fetch() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- #[expect(deprecated)]- roundtrip_test(Arc::new(CoalesceBatchesExec::new(- Arc::new(EmptyExec::new(schema.clone())),- 8096,- )))?;-- #[expect(deprecated)]- roundtrip_test(Arc::new(- CoalesceBatchesExec::new(Arc::new(EmptyExec::new(schema)), 8096)- .with_fetch(Some(10)),- ))-}--#[test]-fn roundtrip_coalesce_partitions_with_fetch() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- roundtrip_test(Arc::new(CoalescePartitionsExec::new(Arc::new(- EmptyExec::new(schema.clone()),- ))))?;-- roundtrip_test(Arc::new(- CoalescePartitionsExec::new(Arc::new(EmptyExec::new(schema)))- .with_fetch(Some(10)),- ))-}--#[test]-fn roundtrip_cooperative() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));- roundtrip_test(Arc::new(CooperativeExec::new(Arc::new(EmptyExec::new(- schema,- )))))-}--#[test]-fn roundtrip_buffer() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let result = roundtrip_test_and_return(- Arc::new(BufferExec::new(Arc::new(EmptyExec::new(schema)), 4096)),- &ctx,- &codec,- &proto_converter,- )?;- let result = result.downcast_ref::<BufferExec>().unwrap();- assert_eq!(result.capacity(), 4096);- Ok(())-}--#[test]-fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));-- let predicate = Arc::new(BinaryExpr::new(- Arc::new(Column::new("col", 1)),- Operator::Eq,- lit("1"),- ));-- let mut options = TableParquetOptions::new();- options.global.pushdown_filters = true;-- let file_source = Arc::new(- ParquetSource::new(Arc::clone(&file_schema))- .with_table_parquet_options(options)- .with_predicate(predicate),- );-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(- vec![Field::new("col", DataType::Utf8, false)],- ))),- })- .build();-- roundtrip_test(DataSourceExec::from_data_source(scan_config))-}--#[test]-fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&file_schema),- })- .build();- let exec_plan = DataSourceExec::from_data_source(scan_config);-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let roundtripped =- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;-- let data_source = roundtripped- .downcast_ref::<DataSourceExec>()- .ok_or_else(|| {- internal_datafusion_err!("Expected DataSourceExec after roundtrip")- })?;- let file_scan = data_source- .data_source()- .downcast_ref::<FileScanConfig>()- .ok_or_else(|| {- internal_datafusion_err!("Expected FileScanConfig after roundtrip")- })?;- let parquet_source = file_scan- .file_source()- .downcast_ref::<ParquetSource>()- .ok_or_else(|| {- internal_datafusion_err!("Expected ParquetSource after roundtrip")- })?;-- assert!(- parquet_source.parquet_file_reader_factory().is_some(),- "Parquet reader factory should be attached after decoding from protobuf"- );- Ok(())-}--#[test]-fn roundtrip_arrow_scan() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));-- let table_schema = TableSchema::from(&file_schema);- let file_source = Arc::new(ArrowSource::new_file_source(table_schema));-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.arrow".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&file_schema),- })- .build();-- roundtrip_test(DataSourceExec::from_data_source(scan_config))-}--#[test]-fn roundtrip_json_scan() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.json".to_string(),- 1024,- )])])- .build();- roundtrip_test(DataSourceExec::from_data_source(scan_config))-}--#[cfg(feature = "avro")]-#[test]-fn roundtrip_avro_scan() -> Result<()> {- use datafusion_datasource_avro::source::AvroSource;-- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(AvroSource::new(TableSchema::from(&file_schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.avro".to_string(),- 1024,- )])])- .build();- roundtrip_test(DataSourceExec::from_data_source(scan_config))-}--#[test]-fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {- use datafusion::common::config::CsvOptions;-- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let table_schema = TableSchema::from(&file_schema);- let file_source =- Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {- has_header: Some(false),- delimiter: b'|',- quote: b'\'',- escape: Some(b'\\'),- comment: Some(b'#'),- newlines_in_values: Some(true),- truncated_rows: Some(true),- ..Default::default()- }));-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.csv".to_string(),- 1024,- )])])- .build();-- let ctx = SessionContext::new();- let roundtripped = roundtrip_test_and_return(- DataSourceExec::from_data_source(scan_config),- &ctx,- &DefaultPhysicalExtensionCodec {},- &DefaultPhysicalProtoConverter {},- )?;- let data_source = roundtripped- .downcast_ref::<DataSourceExec>()- .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;- let file_scan = data_source- .data_source()- .downcast_ref::<FileScanConfig>()- .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;- let csv_source = file_scan- .file_source()- .downcast_ref::<CsvSource>()- .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;-- assert!(!csv_source.has_header());- assert_eq!(csv_source.delimiter(), b'|');- assert_eq!(csv_source.quote(), b'\'');- assert_eq!(csv_source.escape(), Some(b'\\'));- assert_eq!(csv_source.comment(), Some(b'#'));- assert!(csv_source.newlines_in_values());- assert!(csv_source.truncate_rows());- Ok(())-}-#[tokio::test]-async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {- let mut file_group =- PartitionedFile::new("/path/to/part=0/file.parquet".to_string(), 1024);- file_group.partition_values =- vec![wrap_partition_value_in_dict(ScalarValue::Int64(Some(0)))];- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));-- let table_schema = TableSchemaBuilder::from(&schema)- .with_table_partition_cols(vec![Arc::new(Field::new(- "part".to_string(),- wrap_partition_type_in_dict(DataType::Int16),- false,- ))])- .build();-- let file_source = Arc::new(ParquetSource::new(table_schema.clone()));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_projection_indices(Some(vec![0, 1]))?- .with_file_group(FileGroup::new(vec![file_group]))- .build();-- roundtrip_test(DataSourceExec::from_data_source(scan_config))-}--#[test]-fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));-- let custom_predicate_expr = Arc::new(CustomPredicateExpr {- inner: Arc::new(Column::new("col", 1)),- });-- let file_source = Arc::new(- ParquetSource::new(Arc::clone(&file_schema))- .with_predicate(custom_predicate_expr),- );-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(- vec![Field::new("col", DataType::Utf8, false)],- ))),- })- .build();-- #[derive(Debug, Clone, Eq)]- struct CustomPredicateExpr {- inner: Arc<dyn PhysicalExpr>,- }-- // Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808- impl PartialEq for CustomPredicateExpr {- fn eq(&self, other: &Self) -> bool {- self.inner.eq(&other.inner)- }- }-- impl std::hash::Hash for CustomPredicateExpr {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- self.inner.hash(state);- }- }-- impl Display for CustomPredicateExpr {- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- write!(f, "CustomPredicateExpr")- }- }-- impl PhysicalExpr for CustomPredicateExpr {- fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {- unreachable!()- }-- fn nullable(&self, _input_schema: &Schema) -> Result<bool> {- unreachable!()- }-- fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {- unreachable!()- }-- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {- vec![&self.inner]- }-- fn with_new_children(- self: Arc<Self>,- _children: Vec<Arc<dyn PhysicalExpr>>,- ) -> Result<Arc<dyn PhysicalExpr>> {- Ok(self)- }-- fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- Display::fmt(self, f)- }- }-- #[derive(Debug)]- struct CustomPhysicalExtensionCodec;- impl PhysicalExtensionCodec for CustomPhysicalExtensionCodec {- fn try_decode(- &self,- _buf: &[u8],- _inputs: &[Arc<dyn ExecutionPlan>],- _ctx: &TaskContext,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Arc<dyn ExecutionPlan>> {- unreachable!()- }-- fn try_encode(- &self,- _node: Arc<dyn ExecutionPlan>,- _buf: &mut Vec<u8>,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<()> {- unreachable!()- }-- fn try_decode_expr(- &self,- buf: &[u8],- inputs: &[Arc<dyn PhysicalExpr>],- _ctx: &PhysicalExprDecodeCtx<'_>,- ) -> Result<Arc<dyn PhysicalExpr>> {- if buf == "CustomPredicateExpr".as_bytes() {- Ok(Arc::new(CustomPredicateExpr {- inner: inputs[0].clone(),- }))- } else {- internal_err!("Not supported")- }- }-- fn try_encode_expr(- &self,- node: &Arc<dyn PhysicalExpr>,- buf: &mut Vec<u8>,- _ctx: &PhysicalExprEncodeCtx<'_>,- ) -> Result<()> {- if node.downcast_ref::<CustomPredicateExpr>().is_some() {- buf.extend_from_slice("CustomPredicateExpr".as_bytes());- Ok(())- } else {- internal_err!("Not supported")- }- }- }-- let exec_plan = DataSourceExec::from_data_source(scan_config);-- let ctx = SessionContext::new();- roundtrip_test_and_return(- exec_plan,- &ctx,- &CustomPhysicalExtensionCodec {},- &DefaultPhysicalProtoConverter {},- )?;- Ok(())-}--#[test]-fn roundtrip_scalar_udf() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- let scalar_fn = Arc::new(|args: &[ColumnarValue]| {- let ColumnarValue::Array(array) = &args[0] else {- panic!("should be array")- };- Ok(ColumnarValue::from(Arc::new(array.clone()) as ArrayRef))- });-- let udf = create_udf(- "dummy",- vec![DataType::Int64],- DataType::Int64,- Volatility::Immutable,- scalar_fn.clone(),- );-- let fun_def = Arc::new(udf.clone());-- let expr = ScalarFunctionExpr::new(- "dummy",- fun_def,- vec![col("a", &schema)?],- Field::new("f", DataType::Int64, true).into(),- Arc::new(ConfigOptions::default()),- );-- let project = ProjectionExec::try_new(- vec![ProjectionExpr {- expr: Arc::new(expr),- alias: "a".to_string(),- }],- input,- )?;-- let ctx = SessionContext::new();-- ctx.register_udf(udf);-- roundtrip_test_with_context(Arc::new(project), &ctx)-}--#[derive(Debug)]-struct UDFExtensionCodec;--impl PhysicalExtensionCodec for UDFExtensionCodec {- fn try_decode(- &self,- _buf: &[u8],- _inputs: &[Arc<dyn ExecutionPlan>],- _ctx: &TaskContext,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Arc<dyn ExecutionPlan>> {- not_impl_err!("No extension codec provided")- }-- fn try_encode(- &self,- _node: Arc<dyn ExecutionPlan>,- _buf: &mut Vec<u8>,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<()> {- not_impl_err!("No extension codec provided")- }-- fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {- if name == "regex_udf" {- let proto = MyRegexUdfNode::decode(buf).map_err(|err| {- internal_datafusion_err!("failed to decode regex_udf: {err}")- })?;-- Ok(Arc::new(ScalarUDF::from(MyRegexUdf::new(proto.pattern))))- } else {- not_impl_err!("unrecognized scalar UDF implementation, cannot decode")- }- }-- fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {- let binding = node.inner();- if let Some(udf) = binding.downcast_ref::<MyRegexUdf>() {- let proto = MyRegexUdfNode {- pattern: udf.pattern.clone(),- };- proto- .encode(buf)- .map_err(|err| internal_datafusion_err!("failed to encode udf: {err}"))?;- }- Ok(())- }-- fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result<Arc<AggregateUDF>> {- if name == "aggregate_udf" {- let proto = MyAggregateUdfNode::decode(buf).map_err(|err| {- internal_datafusion_err!("failed to decode aggregate_udf: {err}")- })?;-- Ok(Arc::new(AggregateUDF::from(MyAggregateUDF::new(- proto.result,- ))))- } else {- not_impl_err!("unrecognized scalar UDF implementation, cannot decode")- }- }-- fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {- let binding = node.inner();- if let Some(udf) = binding.downcast_ref::<MyAggregateUDF>() {- let proto = MyAggregateUdfNode {- result: udf.result.clone(),- };- proto.encode(buf).map_err(|err| {- internal_datafusion_err!("failed to encode udf: {err:?}")- })?;- }- Ok(())- }-- fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result<Arc<WindowUDF>> {- if name == "custom_udwf" {- let proto = CustomUDWFNode::decode(buf).map_err(|err| {- internal_datafusion_err!("failed to decode custom_udwf: {err}")- })?;-- Ok(Arc::new(WindowUDF::from(CustomUDWF::new(proto.payload))))- } else {- not_impl_err!(- "unrecognized user-defined window function implementation, cannot decode"- )- }- }-- fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec<u8>) -> Result<()> {- let binding = node.inner();- if let Some(udwf) = binding.downcast_ref::<CustomUDWF>() {- let proto = CustomUDWFNode {- payload: udwf.payload.clone(),- };- proto.encode(buf).map_err(|err| {- internal_datafusion_err!("failed to encode udwf: {err:?}")- })?;- }- Ok(())- }-- fn try_decode_higher_order_function(- &self,- name: &str,- buf: &[u8],- ) -> Result<Arc<HigherOrderUDF>> {- if name == "higher_order_udf" {- let proto = MyHigherOrderUdfNode::decode(buf).map_err(|err| {- internal_datafusion_err!("failed to decode higher_order_udf: {err}")- })?;-- Ok(Arc::new(HigherOrderUDF::new_from_impl(- MyHigherOrderUDF::new(proto.payload),- )))- } else {- not_impl_err!("unrecognized higher order UDF implementation, cannot decode")- }- }-- fn try_encode_higher_order_function(- &self,- node: &HigherOrderUDF,- buf: &mut Vec<u8>,- ) -> Result<()> {- if let Some(hof) = (node.inner().as_ref() as &dyn std::any::Any)- .downcast_ref::<MyHigherOrderUDF>()- {- let proto = MyHigherOrderUdfNode {- payload: hof.payload.clone(),- };- proto.encode(buf).map_err(|err| {- internal_datafusion_err!("failed to encode hof: {err:?}")- })?;- }- Ok(())- }-}--#[test]-fn roundtrip_scalar_udf_extension_codec() -> Result<()> {- let field_text = Field::new("text", DataType::Utf8, true);- let field_published = Field::new("published", DataType::Boolean, false);- let field_author = Field::new("author", DataType::Utf8, false);- let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));- let input = Arc::new(EmptyExec::new(schema.clone()));-- let udf_expr = Arc::new(ScalarFunctionExpr::new(- "regex_udf",- Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),- vec![col("text", &schema)?],- Field::new("f", DataType::Int64, true).into(),- Arc::new(ConfigOptions::default()),- ));-- let filter = Arc::new(FilterExec::try_new(- Arc::new(BinaryExpr::new(- col("published", &schema)?,- Operator::And,- Arc::new(BinaryExpr::new(udf_expr.clone(), Operator::Gt, lit(0))),- )),- input,- )?);- let aggr_expr =- AggregateExprBuilder::new(max_udaf(), vec![udf_expr as Arc<dyn PhysicalExpr>])- .schema(schema.clone())- .alias("max")- .build()- .map(Arc::new)?;-- let window = Arc::new(WindowAggExec::try_new(- vec![Arc::new(PlainAggregateWindowExpr::new(- aggr_expr.clone(),- &[col("author", &schema)?],- &[],- Arc::new(WindowFrame::new(None)),- None,- ))],- filter,- true,- )?);-- let aggregate = Arc::new(AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new(vec![], vec![], vec![], false),- vec![aggr_expr],- vec![None],- window,- schema,- )?);-- let ctx = SessionContext::new();- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;- Ok(())-}--#[test]-fn roundtrip_higher_order_udf() -> Result<()> {- let element_field = Arc::new(Field::new("v", DataType::Int32, true));- let list_field = Field::new(- "list_col",- DataType::List(Arc::clone(&element_field)),- false,- );- let schema = Arc::new(Schema::new(vec![list_field]));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(- "payload".to_string(),- )));-- let expr = HigherOrderFunctionExpr::try_new_with_schema(- Arc::clone(&hof),- vec![- col("list_col", &schema)?,- lambda(- ["v"],- is_not_null(Arc::new(LambdaVariable::new(1, element_field)))?,- )?,- ],- &schema,- Arc::new(ConfigOptions::default()),- )?;-- let project = ProjectionExec::try_new(- vec![ProjectionExpr {- expr: Arc::new(expr),- alias: "a".to_string(),- }],- input,- )?;-- let ctx = SessionContext::new();- ctx.register_higher_order_function(hof);-- roundtrip_test_with_context(Arc::new(project), &ctx)-}--#[test]-fn roundtrip_higher_order_udf_extension_codec() -> Result<()> {- let element_field = Arc::new(Field::new("v", DataType::Int32, true));- let list_field = Field::new(- "list_col",- DataType::List(Arc::clone(&element_field)),- false,- );- let schema = Arc::new(Schema::new(vec![list_field]));- let input = Arc::new(EmptyExec::new(schema.clone()));-- let lambda_body = Arc::new(LambdaVariable::new(1, Arc::clone(&element_field)));- let lambda_expr = lambda(["v"], lambda_body)?;-- let hof = Arc::new(HigherOrderUDF::new_from_impl(MyHigherOrderUDF::new(- "payload".to_string(),- )));- let hof_expr = Arc::new(HigherOrderFunctionExpr::try_new_with_schema(- hof,- vec![col("list_col", &schema)?, lambda_expr],- &schema,- Arc::new(ConfigOptions::default()),- )?);-- let project = ProjectionExec::try_new(- vec![ProjectionExpr {- expr: hof_expr,- alias: "out".to_string(),- }],- input,- )?;-- let ctx = SessionContext::new();- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(- Arc::new(project),- &ctx,- &UDFExtensionCodec,- &proto_converter,- )?;- Ok(())-}--#[test]-fn roundtrip_udwf_extension_codec() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- let custom_udwf = Arc::new(WindowUDF::from(CustomUDWF::new("payload".to_string())));- let udwf = create_udwf_window_expr(- &custom_udwf,- &[col("a", &schema)?],- schema.as_ref(),- "custom_udwf(a) PARTITION BY [b] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW".to_string(),- false,- )?;-- let window_frame = WindowFrame::new_bounds(- datafusion_expr::WindowFrameUnits::Range,- WindowFrameBound::Preceding(ScalarValue::Int64(None)),- WindowFrameBound::CurrentRow,- );-- let udwf_expr = Arc::new(StandardWindowExpr::new(- udwf,- &[col("b", &schema)?],- &[PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: false,- nulls_first: false,- },- }],- Arc::new(window_frame),- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));- let window = Arc::new(BoundedWindowAggExec::try_new(- vec![udwf_expr],- input,- InputOrderMode::Sorted,- true,- )?);-- let ctx = SessionContext::new();- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;- Ok(())-}--#[test]-fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {- let field_text = Field::new("text", DataType::Utf8, true);- let field_published = Field::new("published", DataType::Boolean, false);- let field_author = Field::new("author", DataType::Utf8, false);- let schema = Arc::new(Schema::new(vec![field_text, field_published, field_author]));- let input = Arc::new(EmptyExec::new(schema.clone()));-- let udf_expr = Arc::new(ScalarFunctionExpr::new(- "regex_udf",- Arc::new(ScalarUDF::from(MyRegexUdf::new(".*".to_string()))),- vec![col("text", &schema)?],- Field::new("f", DataType::Int64, true).into(),- Arc::new(ConfigOptions::default()),- ));-- let udaf = Arc::new(AggregateUDF::from(MyAggregateUDF::new(- "result".to_string(),- )));- let aggr_args: Vec<Arc<dyn PhysicalExpr>> =- vec![Arc::new(Literal::new(ScalarValue::from(42)))];-- let aggr_expr = AggregateExprBuilder::new(Arc::clone(&udaf), aggr_args.clone())- .schema(Arc::clone(&schema))- .alias("aggregate_udf")- .build()- .map(Arc::new)?;-- let filter = Arc::new(FilterExec::try_new(- Arc::new(BinaryExpr::new(- col("published", &schema)?,- Operator::And,- Arc::new(BinaryExpr::new(udf_expr, Operator::Gt, lit(0))),- )),- input,- )?);-- let window = Arc::new(WindowAggExec::try_new(- vec![Arc::new(PlainAggregateWindowExpr::new(- aggr_expr,- &[col("author", &schema)?],- &[],- Arc::new(WindowFrame::new(None)),- None,- ))],- filter,- true,- )?);-- let aggr_expr = AggregateExprBuilder::new(udaf, aggr_args.clone())- .schema(Arc::clone(&schema))- .alias("aggregate_udf")- .distinct()- .ignore_nulls()- .build()- .map(Arc::new)?;-- let aggregate = Arc::new(AggregateExec::try_new(- AggregateMode::Final,- PhysicalGroupBy::new(vec![], vec![], vec![], false),- vec![aggr_expr],- vec![None],- window,- schema,- )?);-- let ctx = SessionContext::new();- let proto_converter = DefaultPhysicalProtoConverter {};- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;- Ok(())-}--#[test]-fn roundtrip_like() -> Result<()> {- let schema = Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- ]);- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));- let like_expr = like(- false,- false,- col("a", &schema)?,- col("b", &schema)?,- &schema,- )?;- let plan = Arc::new(ProjectionExec::try_new(- vec![ProjectionExpr {- expr: like_expr,- alias: "result".to_string(),- }],- input,- )?);- roundtrip_test(plan)-}--#[test]-fn roundtrip_analyze() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("plan_type", DataType::Utf8, false),- Field::new("plan", DataType::Utf8, false),- ]));- let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));- let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing];- let analyze = Arc::new(- AnalyzeExec::builder(true, true, input, Arc::clone(&schema))- .with_metric_categories(Some(metric_categories.clone()))- .with_format(ExplainFormat::Tree)- .build(),- );-- let ctx = SessionContext::new();- let roundtripped = roundtrip_test_and_return(- analyze,- &ctx,- &DefaultPhysicalExtensionCodec {},- &DefaultPhysicalProtoConverter {},- )?;- let roundtripped = roundtripped.downcast_ref::<AnalyzeExec>().unwrap();-- assert_eq!(roundtripped.schema(), schema);- assert!(roundtripped.verbose());- assert!(roundtripped.show_statistics());- assert_eq!(- roundtripped.metric_categories(),- Some(metric_categories.as_slice())- );- assert_eq!(roundtripped.format(), &ExplainFormat::Tree);- assert!(- roundtripped- .input()- .downcast_ref::<PlaceholderRowExec>()- .is_some()- );- Ok(())-}--#[test]-fn roundtrip_explain() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("plan_type", DataType::Utf8, false),- Field::new("plan", DataType::Utf8, false),- ]));- let stringified_plans = vec![- StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"),- StringifiedPlan::new(- PlanType::AnalyzedLogicalPlan {- analyzer_name: "analyzer".to_string(),- },- "analyzed logical",- ),- StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"),- StringifiedPlan::new(- PlanType::OptimizedLogicalPlan {- optimizer_name: "logical optimizer".to_string(),- },- "optimized logical",- ),- StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"),- StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"),- StringifiedPlan::new(- PlanType::InitialPhysicalPlanWithStats,- "initial physical with stats",- ),- StringifiedPlan::new(- PlanType::InitialPhysicalPlanWithSchema,- "initial physical with schema",- ),- StringifiedPlan::new(- PlanType::OptimizedPhysicalPlan {- optimizer_name: "physical optimizer".to_string(),- },- "optimized physical",- ),- StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"),- StringifiedPlan::new(- PlanType::FinalPhysicalPlanWithStats,- "final physical with stats",- ),- StringifiedPlan::new(- PlanType::FinalPhysicalPlanWithSchema,- "final physical with schema",- ),- StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"),- ];- let explain = Arc::new(ExplainExec::new(- Arc::clone(&schema),- stringified_plans.clone(),- true,- ));-- let ctx = SessionContext::new();- let roundtripped = roundtrip_test_and_return(- explain,- &ctx,- &DefaultPhysicalExtensionCodec {},- &DefaultPhysicalProtoConverter {},- )?;- let roundtripped = roundtripped.downcast_ref::<ExplainExec>().unwrap();-- assert_eq!(roundtripped.schema(), schema);- assert_eq!(roundtripped.stringified_plans(), stringified_plans);- assert!(roundtripped.verbose());- Ok(())-}--#[derive(Debug)]-struct ProtoHookSink {- schema: SchemaRef,-}--impl DisplayAs for ProtoHookSink {- fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {- write!(f, "ProtoHookSink")- }-}--#[async_trait]-impl DataSink for ProtoHookSink {- fn schema(&self) -> &SchemaRef {- &self.schema- }-- async fn write_all(- &self,- _data: SendableRecordBatchStream,- _context: &Arc<TaskContext>,- ) -> Result<u64> {- unreachable!("serialization test does not execute the sink")- }-- fn try_to_proto(- &self,- exec: &DataSinkExec,- ctx: &ExecutionPlanEncodeCtx<'_>,- ) -> Result<Option<PhysicalPlanNode>> {- let input = ctx.encode_child(exec.input())?;- let sort_order = exec.encode_sort_order(ctx)?;- assert!(matches!(- input.physical_plan_type,- Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))- ));- assert_eq!(- sort_order- .as_ref()- .map(|ordering| ordering.physical_sort_expr_nodes.len()),- Some(1)- );- assert_eq!(exec.schema().fields().len(), 1);-- Ok(Some(PhysicalPlanNode {- physical_plan_type: Some(- protobuf::physical_plan_node::PhysicalPlanType::Empty(- protobuf::EmptyExecNode {- schema: Some(exec.schema().as_ref().try_into()?),- partitions: 1,- },- ),- ),- }))- }-}--#[test]-fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {- let input_schema = Arc::new(Schema::new(vec![Field::new(- "value",- DataType::Int64,- false,- )]));- let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&input_schema)));- let sink = Arc::new(ProtoHookSink {- schema: Arc::clone(&input_schema),- });- let sort_order = [PhysicalSortRequirement::new(- Arc::new(Column::new("value", 0)),- Some(SortOptions::default()),- )]- .into();- let plan = Arc::new(DataSinkExec::new(input, sink, Some(sort_order)));-- let node = PhysicalPlanNode::try_from_physical_plan(- plan,- &DefaultPhysicalExtensionCodec {},- )?;-- assert!(matches!(- node.physical_plan_type,- Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))- ));- Ok(())-}--#[test]-fn file_sink_config_roundtrip_preserves_fields() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new(- "partition",- DataType::Utf8,- false,- )]));- let config = FileSinkConfig {- original_url: "file:///tmp/output".to_string(),- object_store_url: ObjectStoreUrl::local_filesystem(),- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp/output", 1)]),- table_paths: vec![ListingTableUrl::parse("file:///tmp/output")?],- output_schema: schema,- table_partition_cols: vec![("partition".to_string(), DataType::Utf8)],- insert_op: InsertOp::Overwrite,- keep_partition_by_columns: true,- file_extension: "parquet".to_string(),- file_output_mode: FileOutputMode::Directory,- };-- let encoded = protobuf::FileSinkConfig::try_from(&config)?;- assert_eq!(encoded.insert_op(), protobuf::InsertOp::Overwrite);- assert_eq!(- encoded.file_output_mode(),- protobuf::FileOutputMode::Directory- );-- let decoded = FileSinkConfig::try_from(&encoded)?;- assert_eq!(decoded.object_store_url, config.object_store_url);- assert_eq!(decoded.table_paths, config.table_paths);- assert_eq!(- decoded.output_schema.as_ref(),- config.output_schema.as_ref()- );- assert_eq!(decoded.table_partition_cols, config.table_partition_cols);- assert_eq!(decoded.insert_op, config.insert_op);- assert_eq!(- decoded.keep_partition_by_columns,- config.keep_partition_by_columns- );- assert_eq!(decoded.file_extension, config.file_extension);- assert_eq!(decoded.file_output_mode, config.file_output_mode);-- let [decoded_file] = decoded.file_group.files() else {- panic!("expected one decoded output file");- };- let [config_file] = config.file_group.files() else {- panic!("expected one configured output file");- };- assert_eq!(- decoded_file.object_meta.location,- config_file.object_meta.location- );- assert_eq!(decoded_file.object_meta.size, config_file.object_meta.size);- Ok(())-}--#[tokio::test]-async fn roundtrip_json_source() -> Result<()> {- let ctx = SessionContext::new();- ctx.register_json("t1", "../core/tests/data/1.json", Default::default())- .await?;- let plan = ctx.table("t1").await?.create_physical_plan().await?;- roundtrip_test(plan)-}--#[test]-fn roundtrip_json_sink() -> Result<()> {- let field_a = Field::new("plan_type", DataType::Utf8, false);- let field_b = Field::new("plan", DataType::Utf8, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));-- let file_sink_config = FileSinkConfig {- original_url: String::default(),- object_store_url: ObjectStoreUrl::local_filesystem(),- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),- table_paths: vec![ListingTableUrl::parse("file:///")?],- output_schema: schema.clone(),- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],- insert_op: InsertOp::Overwrite,- keep_partition_by_columns: true,- file_extension: "json".into(),- file_output_mode: FileOutputMode::SingleFile,- };- let data_sink = Arc::new(JsonSink::new(- file_sink_config,- JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED),- ));- let sort_order = [PhysicalSortRequirement::new(- Arc::new(Column::new("plan_type", 0)),- Some(SortOptions {- descending: true,- nulls_first: false,- }),- )]- .into();-- roundtrip_test(Arc::new(DataSinkExec::new(- input,- data_sink,- Some(sort_order),- )))-}--#[test]-fn roundtrip_csv_sink() -> Result<()> {- let field_a = Field::new("plan_type", DataType::Utf8, false);- let field_b = Field::new("plan", DataType::Utf8, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));-- let file_sink_config = FileSinkConfig {- original_url: String::default(),- object_store_url: ObjectStoreUrl::local_filesystem(),- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),- table_paths: vec![ListingTableUrl::parse("file:///")?],- output_schema: schema.clone(),- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],- insert_op: InsertOp::Overwrite,- keep_partition_by_columns: true,- file_extension: "csv".into(),- file_output_mode: FileOutputMode::Directory,- };- let data_sink = Arc::new(CsvSink::new(- file_sink_config,- CsvWriterOptions::new(WriterBuilder::default(), CompressionTypeVariant::ZSTD),- ));- let sort_order = [PhysicalSortRequirement::new(- Arc::new(Column::new("plan_type", 0)),- Some(SortOptions {- descending: true,- nulls_first: false,- }),- )]- .into();-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};-- let roundtrip_plan = roundtrip_test_and_return(- Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),- &ctx,- &codec,- &proto_converter,- )?;-- let roundtrip_plan = roundtrip_plan.downcast_ref::<DataSinkExec>().unwrap();- let csv_sink = roundtrip_plan.sink().downcast_ref::<CsvSink>().unwrap();- assert_eq!(- CompressionTypeVariant::ZSTD,- csv_sink.writer_options().compression- );-- Ok(())-}--#[test]-fn roundtrip_parquet_sink() -> Result<()> {- let field_a = Field::new("plan_type", DataType::Utf8, false);- let field_b = Field::new("plan", DataType::Utf8, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));-- let file_sink_config = FileSinkConfig {- original_url: String::default(),- object_store_url: ObjectStoreUrl::local_filesystem(),- file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),- table_paths: vec![ListingTableUrl::parse("file:///")?],- output_schema: schema.clone(),- table_partition_cols: vec![("plan_type".to_string(), DataType::Utf8)],- insert_op: InsertOp::Overwrite,- keep_partition_by_columns: true,- file_extension: "parquet".into(),- file_output_mode: FileOutputMode::Automatic,- };- let data_sink = Arc::new(ParquetSink::new(- file_sink_config,- TableParquetOptions::default(),- ));- let sort_order = [PhysicalSortRequirement::new(- Arc::new(Column::new("plan_type", 0)),- Some(SortOptions {- descending: true,- nulls_first: false,- }),- )]- .into();-- roundtrip_test(Arc::new(DataSinkExec::new(- input,- data_sink,- Some(sort_order),- )))-}--#[test]-fn roundtrip_sym_hash_join() -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let field_a = Field::new("col_a", DataType::Int64, false);- let field_b = Field::new("col_b", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_b.clone()]);- let on = vec![(- Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,- Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,- )];- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("col_a", 0)),- Operator::Gt,- Arc::new(Column::new("col_b", 1)),- )),- vec![- ColumnIndex {- index: 0,- side: JoinSide::Left,- },- ColumnIndex {- index: 0,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![field_a, field_b])),- );-- let schema_left = Arc::new(schema_left);- let schema_right = Arc::new(schema_right);- let left_order: LexOrdering = [PhysicalSortExpr {- expr: Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)),- options: SortOptions {- descending: true,- nulls_first: false,- },- }]- .into();- let right_order: LexOrdering = [PhysicalSortExpr {- expr: Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)),- options: SortOptions {- descending: false,- nulls_first: true,- },- }]- .into();- let ordering_cases = [- (None, None),- (Some(left_order.clone()), None),- (None, Some(right_order.clone())),- (Some(left_order), Some(right_order)),- ];- let ordering_options = |ordering: Option<&LexOrdering>| {- ordering- .map(|ordering| ordering.iter().map(|expr| expr.options).collect::<Vec<_>>())- };-- for join_type in [- JoinType::Inner,- JoinType::Left,- JoinType::Right,- JoinType::Full,- JoinType::LeftAnti,- JoinType::RightAnti,- JoinType::LeftSemi,- JoinType::RightSemi,- JoinType::LeftMark,- JoinType::RightMark,- ] {- for null_equality in [- NullEquality::NullEqualsNothing,- NullEquality::NullEqualsNull,- ] {- for filter in [None, Some(filter.clone())] {- for partition_mode in [- StreamJoinPartitionMode::Partitioned,- StreamJoinPartitionMode::SinglePartition,- ] {- for (left_order, right_order) in &ordering_cases {- let result = roundtrip_test_and_return(- Arc::new(SymmetricHashJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- on.clone(),- filter.clone(),- &join_type,- null_equality,- left_order.clone(),- right_order.clone(),- partition_mode,- )?),- &ctx,- &codec,- &proto_converter,- )?;- let result =- result.downcast_ref::<SymmetricHashJoinExec>().unwrap();- assert_eq!(result.join_type(), &join_type);- assert_eq!(result.null_equality(), null_equality);- assert_eq!(result.partition_mode(), partition_mode);- assert_eq!(- ordering_options(result.left_sort_exprs()),- ordering_options(left_order.as_ref())- );- assert_eq!(- ordering_options(result.right_sort_exprs()),- ordering_options(right_order.as_ref())- );- assert_eq!(- result.filter().map(JoinFilter::column_indices),- filter.as_ref().map(JoinFilter::column_indices)- );- }- }- }- }- }- Ok(())-}--#[test]-fn roundtrip_union() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_a]);- let left = EmptyExec::new(Arc::new(schema_left));- let right = EmptyExec::new(Arc::new(schema_right));- let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];- let union = UnionExec::try_new(inputs)?;- roundtrip_test(union)-}--#[test]-fn roundtrip_repartition_preserve_order() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));- let sort_exprs: LexOrdering = [PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions::default(),- }]- .into();-- // Create two sorted single-partition inputs, then union them to get- // a sorted input with 2 partitions.- let source1 = SortExec::new(- sort_exprs.clone(),- Arc::new(EmptyExec::new(Arc::clone(&schema))),- );- let source2 = SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)));- let union = UnionExec::try_new(vec![- Arc::new(source1) as Arc<dyn ExecutionPlan>,- Arc::new(source2) as Arc<dyn ExecutionPlan>,- ])?;-- let repartition = RepartitionExec::try_new(union, Partitioning::RoundRobinBatch(10))?- .with_preserve_order();- assert!(repartition.preserve_order());-- roundtrip_test(Arc::new(repartition))-}--#[test]-fn roundtrip_range_partitioning() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let range_partitioning = Partitioning::Range(RangePartitioning::new(- [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),- vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],- ));- // RepartitionExec is used only to carry the partitioning through proto.- // Executing range repartitioning is intentionally unsupported.- let repartition = RepartitionExec::try_new(input, range_partitioning)?;-- roundtrip_test(Arc::new(repartition))-}--/// `parse_protobuf_hash_partitioning` has no in-tree callers left; it delegates-/// to the shared `Partitioning::try_from_proto`, so pin that it still decodes-/// the hash message it is handed.-#[test]-fn parse_hash_partitioning_delegates_to_shared_decoder() -> Result<()> {- use datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;-- let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);- let ctx = SessionContext::new();- let task_ctx = ctx.task_ctx();- let codec = DefaultPhysicalExtensionCodec {};- let decode_ctx = PhysicalPlanDecodeContext::new(&task_ctx, &codec);- let proto_converter = DefaultPhysicalProtoConverter {};-- let hash_expr = serialize_physical_expr_with_converter(- &col("a", &schema)?,- &codec,- &proto_converter,- )?;- let hash = protobuf::PhysicalHashRepartition {- hash_expr: vec![hash_expr],- partition_count: 4,- };-- let partitioning = parse_protobuf_hash_partitioning(- Some(&hash),- &decode_ctx,- &schema,- &proto_converter,- )?;- let Some(Partitioning::Hash(exprs, count)) = partitioning else {- panic!("expected hash partitioning, got {partitioning:?}");- };- assert_eq!(count, 4);- assert_eq!(exprs.len(), 1);- assert_eq!(exprs[0].to_string(), col("a", &schema)?.to_string());-- // No message means no partitioning, as before.- assert!(- parse_protobuf_hash_partitioning(None, &decode_ctx, &schema, &proto_converter)?- .is_none()- );-- // The count is a `u64` on the wire and a `usize` in memory, so decoding- // narrows it. A count that does not fit is the case that motivated routing- // this through the shared decoder: it used to `unwrap()` and panic, and now- // reports an error. Only a target narrower than 64 bits can reach that arm- // -- on a 64-bit target every `u64` fits, and the assertion there is that- // the largest possible count survives whole rather than being truncated.- let oversized = protobuf::PhysicalHashRepartition {- hash_expr: vec![serialize_physical_expr_with_converter(- &col("a", &schema)?,- &codec,- &proto_converter,- )?],- partition_count: u64::MAX,- };- let decoded = parse_protobuf_hash_partitioning(- Some(&oversized),- &decode_ctx,- &schema,- &proto_converter,- );-- #[cfg(target_pointer_width = "64")]- {- let Some(Partitioning::Hash(_, count)) = decoded? else {- panic!("expected hash partitioning");- };- assert_eq!(count, usize::MAX);- }-- #[cfg(not(target_pointer_width = "64"))]- assert!(- decoded- .unwrap_err()- .to_string()- .contains("Partition count 18446744073709551615 exceeds usize::MAX")- );-- Ok(())-}--#[test]-fn roundtrip_interleave() -> Result<()> {- let field_a = Field::new("col", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_a]);- let partition = Partitioning::Hash(vec![], 3);- let left = RepartitionExec::try_new(- Arc::new(EmptyExec::new(Arc::new(schema_left))),- partition.clone(),- )?;- let right = RepartitionExec::try_new(- Arc::new(EmptyExec::new(Arc::new(schema_right))),- partition,- )?;- let inputs: Vec<Arc<dyn ExecutionPlan>> = vec![Arc::new(left), Arc::new(right)];- let interleave = InterleaveExec::try_new(inputs)?;- roundtrip_test(Arc::new(interleave))-}--#[test]-fn roundtrip_unnest() -> Result<()> {- let fa = Field::new("a", DataType::Int64, true);- let fb0 = Field::new_list_field(DataType::Utf8, true);- let fb = Field::new_list("b", fb0.clone(), false);- let fc1 = Field::new("c1", DataType::Boolean, false);- let fc2 = Field::new("c2", DataType::Date64, true);- let fc = Field::new_struct("c", Fields::from(vec![fc1.clone(), fc2.clone()]), true);- let fd0 = Field::new_list_field(DataType::Float32, false);- let fd = Field::new_list("d", fd0.clone(), true);- let fe1 = Field::new("e1", DataType::UInt16, false);- let fe2 = Field::new("e2", DataType::Duration(TimeUnit::Millisecond), true);- let fe3 = Field::new("e3", DataType::Timestamp(TimeUnit::Millisecond, None), true);- let fe_fields = Fields::from(vec![fe1.clone(), fe2.clone(), fe3.clone()]);- let fe = Field::new_struct("e", fe_fields, false);-- let fb0 = fb0.with_name("b");- let fd0 = fd0.with_name("d");- let input_schema = Arc::new(Schema::new(vec![fa.clone(), fb, fc, fd, fe]));- let output_schema =- Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));- let input = Arc::new(EmptyExec::new(input_schema));- let options = UnnestOptions {- null_handling: datafusion_common::NullHandling::Drop,- recursions: vec![datafusion_common::RecursionUnnestOption {- input_column: datafusion_common::Column::new_unqualified("b"),- output_column: datafusion_common::Column::new_unqualified("b"),- depth: 2,- }],- };- let unnest = UnnestExec::new(- input,- vec![- ListUnnest {- index_in_input_schema: 1,- depth: 1,- },- ListUnnest {- index_in_input_schema: 1,- depth: 2,- },- ListUnnest {- index_in_input_schema: 3,- depth: 2,- },- ],- vec![2, 4],- output_schema,- options.clone(),- )?;- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let result =- roundtrip_test_and_return(Arc::new(unnest), &ctx, &codec, &proto_converter)?;- let result = result.downcast_ref::<UnnestExec>().unwrap();- assert_eq!(result.options(), &options);-- Ok(())-}--#[tokio::test]-async fn roundtrip_coalesce() -> Result<()> {- let ctx = SessionContext::new();- ctx.register_table(- "t",- Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([- Arc::new(Field::new("f", DataType::Int64, false)),- ]))))),- )?;- let df = ctx.sql("select coalesce(f) as f from t").await?;- let plan = df.create_physical_plan().await?;-- let node = PhysicalPlanNode::try_from_physical_plan(- plan.clone(),- &DefaultPhysicalExtensionCodec {},- )?;- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())- .map_err(|e| DataFusionError::External(Box::new(e)))?;- let restored =- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;-- assert_eq!(- plan.schema(),- restored.schema(),- "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",- displayable(plan.as_ref())- .set_show_schema(true)- .indent(true),- displayable(restored.as_ref())- .set_show_schema(true)- .indent(true),- );-- Ok(())-}--#[tokio::test]-async fn roundtrip_generate_series() -> Result<()> {- let ctx = SessionContext::new();- ctx.register_table(- "t",- Arc::new(EmptyTable::new(Arc::new(Schema::new(Fields::from([- Arc::new(Field::new("f", DataType::Int64, false)),- ]))))),- )?;- let df = ctx.sql("select * from generate_series(1, 10000)").await?;- let plan = df.create_physical_plan().await?;-- let node = PhysicalPlanNode::try_from_physical_plan(- plan.clone(),- &DefaultPhysicalExtensionCodec {},- )?;- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())- .map_err(|e| DataFusionError::External(Box::new(e)))?;- let restored =- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;-- assert_eq!(- plan.schema(),- restored.schema(),- "Schema mismatch for plans:\n>> initial:\n{}>> final: \n{}",- displayable(plan.as_ref())- .set_show_schema(true)- .indent(true),- displayable(restored.as_ref())- .set_show_schema(true)- .indent(true),- );-- Ok(())-}--#[tokio::test]-async fn roundtrip_projection_source() -> Result<()> {- let schema = Arc::new(Schema::new(Fields::from([- Arc::new(Field::new("a", DataType::Utf8, false)),- Arc::new(Field::new("b", DataType::Utf8, false)),- Arc::new(Field::new("c", DataType::Int32, false)),- Arc::new(Field::new("d", DataType::Int32, false)),- ])));-- let statistics = Statistics::new_unknown(&schema);-- let file_source = Arc::new(ParquetSource::new(Arc::clone(&schema)));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(statistics)- .with_projection_indices(Some(vec![0, 1, 2]))?- .build();-- let filter = Arc::new(- FilterExecBuilder::new(- Arc::new(BinaryExpr::new(col("c", &schema)?, Operator::Eq, lit(1))),- DataSourceExec::from_data_source(scan_config),- )- .apply_projection(Some(vec![0, 1]))?- .build()?,- );-- roundtrip_test(filter)-}--#[tokio::test]-async fn roundtrip_parquet_select_star() -> Result<()> {- let ctx = all_types_context().await?;- let sql = "select * from alltypes_plain";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_parquet_select_projection() -> Result<()> {- let ctx = all_types_context().await?;- let sql = "select string_col, timestamp_col from alltypes_plain";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_parquet_select_star_predicate() -> Result<()> {- let ctx = all_types_context().await?;- let sql = "select * from alltypes_plain where id > 4";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_parquet_select_projection_predicate() -> Result<()> {- let ctx = all_types_context().await?;- let sql = "select string_col, timestamp_col from alltypes_plain where id > 4";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_empty_projection() -> Result<()> {- let ctx = all_types_context().await?;- let sql = "select 1 from alltypes_plain";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_memory_source_empty_projection() -> Result<()> {- // Memory scan: `Some(vec![])` must not decode back as `None`- let ctx = SessionContext::new();- let batch = RecordBatch::try_new(- Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Int64, false),- ])),- vec![- Arc::new(arrow::array::StringArray::from(vec!["Tom"])),- Arc::new(arrow::array::Int64Array::from(vec![18i64])),- ],- )?;- ctx.register_batch("tmem", batch)?;- let sql = "select 1 from tmem";- roundtrip_test_sql_with_context(sql, &ctx).await-}--#[tokio::test]-async fn roundtrip_physical_plan_node() {- use datafusion::prelude::*;- use datafusion_proto::physical_plan::{- AsExecutionPlan, DefaultPhysicalExtensionCodec,- };- use datafusion_proto::protobuf::PhysicalPlanNode;-- let ctx = SessionContext::new();-- ctx.register_parquet(- "pt",- &format!(- "{}/alltypes_plain.snappy.parquet",- datafusion_common::test_util::parquet_test_data()- ),- ParquetReadOptions::default(),- )- .await- .unwrap();-- let plan = ctx- .sql("select id, string_col, timestamp_col from pt where id > 4 order by string_col")- .await- .unwrap()- .create_physical_plan()- .await- .unwrap();-- let node: PhysicalPlanNode =- PhysicalPlanNode::try_from_physical_plan(plan, &DefaultPhysicalExtensionCodec {})- .unwrap();-- let plan = node- .try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})- .unwrap();-- let _ = plan.execute(0, ctx.task_ctx()).unwrap();-}--/// The deprecated `try_into_projection_physical_plan` shim now delegates to-/// [`ProjectionExec::try_from_proto`], which reads the enclosing-/// `PhysicalPlanNode` rather than a `ProjectionExecNode`. Assert the shim still-/// decodes the node passed as an argument, not `self`, so an out-of-tree caller-/// that passes a projection unrelated to `self` keeps the old behaviour.-#[test]-fn deprecated_projection_shim_decodes_argument_not_self() -> Result<()> {- use datafusion_proto::protobuf::PhysicalPlanNode;- use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;-- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);- let input = Arc::new(EmptyExec::new(Arc::new(schema.clone())));- let projection = Arc::new(ProjectionExec::try_new(- vec![ProjectionExpr::new(- col("a", &schema)?,- "renamed".to_string(),- )],- input,- )?);-- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let projection_node = PhysicalPlanNode::try_from_physical_plan_with_converter(- projection,- &codec,- &proto_converter,- )?;- let Some(PhysicalPlanType::Projection(projection_exec_node)) =- &projection_node.physical_plan_type- else {- panic!("expected a Projection node, got {projection_node:?}");- };-- // `self` is deliberately a different plan variant than the argument.- let unrelated_node = PhysicalPlanNode::try_from_physical_plan_with_converter(- Arc::new(EmptyExec::new(Arc::new(schema))),- &codec,- &proto_converter,- )?;-- let session_ctx = SessionContext::new();- let task_ctx = session_ctx.task_ctx();- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);- #[expect(deprecated)]- let decoded = unrelated_node.try_into_projection_physical_plan(- projection_exec_node,- &decode_ctx,- &proto_converter,- )?;-- let decoded = decoded- .downcast_ref::<ProjectionExec>()- .expect("decoded plan should be a ProjectionExec");- assert_eq!(decoded.expr().len(), 1);- assert_eq!(decoded.expr()[0].alias, "renamed");- Ok(())-}--/// Helper function to create a SessionContext with all TPC-H tables registered as external tables-async fn tpch_context() -> Result<SessionContext> {- use datafusion_common::test_util::datafusion_test_data;-- let ctx = SessionContext::new();- let test_data = datafusion_test_data();-- // TPC-H table names- let tables = [- "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation",- "region",- ];-- // Create external tables for all TPC-H tables- for table in &tables {- let table_sql = format!(- "CREATE EXTERNAL TABLE {table} STORED AS PARQUET LOCATION '{test_data}/tpch_{table}_small.parquet'"- );- ctx.sql(&table_sql).await.map_err(|e| {- DataFusionError::External(- format!("Failed to create {table} table: {e}").into(),- )- })?;- }-- Ok(ctx)-}--/// Helper function to get TPC-H query SQL-fn get_tpch_query_sql(query: usize) -> Result<Vec<String>> {- use std::fs;-- if !(1..=22).contains(&query) {- return Err(DataFusionError::External(- format!("Invalid TPC-H query number: {query}").into(),- ));- }-- let filename = format!("../../benchmarks/queries/q{query}.sql");- let contents = fs::read_to_string(&filename).map_err(|e| {- DataFusionError::External(- format!("Failed to read query file {filename}: {e}").into(),- )- })?;-- Ok(contents- .split(';')- .map(|s| s.trim())- .filter(|s| !s.is_empty())- .map(|s| s.to_string())- .collect())-}--#[tokio::test]-async fn test_serialize_deserialize_tpch_queries() -> Result<()> {- // Create context with TPC-H tables- let ctx = tpch_context().await?;-- // repeat to run all 22 queries- for query in 1..=22 {- // run all statements in the query- let sql = get_tpch_query_sql(query)?;- for stmt in sql {- let logical_plan = ctx.sql(&stmt).await?.into_unoptimized_plan();- let optimized_plan = ctx.state().optimize(&logical_plan)?;- let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;-- // serialize the physical plan- let codec = DefaultPhysicalExtensionCodec {};-- let proto =- PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;-- // deserialize the physical plan- let _deserialized_plan =- proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;- }- }-- Ok(())-}--// Bugs: https://github.com/apache/datafusion/issues/16772-#[tokio::test]-async fn test_round_trip_tpch_queries() -> Result<()> {- // Create context with TPC-H tables- let ctx = tpch_context().await?;-- // repeat to run all 22 queries- for query in 1..=22 {- // run all statements in the query- let sql = get_tpch_query_sql(query)?;- for stmt in sql {- roundtrip_test_sql_with_context(&stmt, &ctx).await?;- }- }-- Ok(())-}--// Bug 1 of https://github.com/apache/datafusion/issues/16772-/// Test that AggregateFunctionExpr human_display field is correctly preserved-/// during serialization/deserialization roundtrip.-///-/// Test for issue where the human_display field (used for EXPLAIN output)-/// was not being serialized to protobuf, causing it to be lost during roundtrip-/// and resulting in empty or incorrect display strings in query plans.-#[tokio::test]-async fn test_round_trip_human_display() -> Result<()> {- // Create context with TPC-H tables- let ctx = tpch_context().await?;-- let sql = "select r_name, count(1) from region group by r_name";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- let sql = "select r_name, count(*) from region group by r_name";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- let sql = "select r_name, count(r_name) from region group by r_name";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- let sql = "select count(*) as count_star from region";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- Ok(())-}--#[test]-fn test_round_trip_aliased_reverse_human_display() -> Result<()> {- let aggregate_expr = roundtrip_first_value_aggregate(- "agg",- "first_value(b) ORDER BY [b ASC NULLS LAST]",- Some("agg"),- )?;- let reversed = aggregate_expr- .reverse_expr()- .expect("expected reverse expr");-- assert_eq!(reversed.name(), "agg");- assert_eq!(reversed.human_display_alias(), Some("agg"));- assert_eq!(- reversed.human_display(),- Some("last_value(b) ORDER BY [b DESC NULLS FIRST]")- );-- Ok(())-}--#[test]-fn test_round_trip_human_display_alias_with_colon() -> Result<()> {- let aggregate_expr = roundtrip_first_value_aggregate(- "agg:one",- "first_value(b) ORDER BY [b ASC NULLS LAST]",- Some("agg:one"),- )?;-- assert_eq!(aggregate_expr.name(), "agg:one");- assert_eq!(aggregate_expr.human_display_alias(), Some("agg:one"));- assert_eq!(- aggregate_expr.human_display(),- Some("first_value(b) ORDER BY [b ASC NULLS LAST]")- );-- Ok(())-}--#[test]-fn test_round_trip_non_aliased_human_display_ending_like_alias() -> Result<()> {- let aggregate_expr =- roundtrip_first_value_aggregate("agg", "first_value(b) as agg", None)?;-- assert_eq!(aggregate_expr.name(), "agg");- assert_eq!(- aggregate_expr.human_display(),- Some("first_value(b) as agg")- );- assert_eq!(aggregate_expr.human_display_alias(), None);-- Ok(())-}--fn roundtrip_first_value_aggregate(- alias: &str,- human_display: &str,- human_display_alias: Option<&str>,-) -> Result<Arc<AggregateFunctionExpr>> {- let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));- let mut builder =- AggregateExprBuilder::new(first_value_udaf(), vec![col("b", &schema)?])- .order_by(vec![PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions::new(false, false),- }])- .schema(Arc::clone(&schema))- .alias(alias)- .human_display(human_display);- if let Some(human_display_alias) = human_display_alias {- builder = builder.human_display_alias(human_display_alias);- }- let agg_expr = builder.build().map(Arc::new)?;-- let plan = Arc::new(AggregateExec::try_new(- AggregateMode::Single,- PhysicalGroupBy::new(vec![], vec![], vec![], false),- vec![agg_expr],- vec![None],- Arc::new(EmptyExec::new(Arc::clone(&schema))),- schema,- )?);-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let roundtrip_plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;- let aggregate = roundtrip_plan- .as_ref()- .downcast_ref::<AggregateExec>()- .expect("expected AggregateExec after roundtrip");-- Ok(Arc::clone(&aggregate.aggr_expr()[0]))-}--// Bug 2 of https://github.com/apache/datafusion/issues/16772-/// Test that PhysicalGroupBy groups field is correctly serialized/deserialized-/// for simple aggregates (no GROUP BY clause).-///-/// Test for issue where simple aggregates like "SELECT SUM(col1 * col2) FROM table"-/// would incorrectly serialize groups as [[]] instead of [] during roundtrip serialization.-/// The groups field should be empty ([]) when there are no GROUP BY expressions.-#[tokio::test]-async fn test_round_trip_groups_display() -> Result<()> {- // Create context with TPC-H tables- let ctx = tpch_context().await?;-- let sql = "select sum(l_extendedprice * l_discount) as revenue from lineitem;";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- let sql = "select sum(l_extendedprice) as revenue from lineitem;";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- Ok(())-}--// Bug 3 of https://github.com/apache/datafusion/issues/16772-/// Test that ScalarFunctionExpr return_field name is correctly preserved-/// during serialization/deserialization roundtrip.-///-/// Test for issue where the return_field.name for scalar functions-/// was not being serialized to protobuf, causing it to be lost during roundtrip-/// and defaulting to a generic name like "f" instead of the proper function name.-#[tokio::test]-async fn test_round_trip_date_part_display() -> Result<()> {- // Create context with TPC-H tables- let ctx = tpch_context().await?;-- let sql = "select extract(year from l_shipdate) as l_year from lineitem ";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- let sql = "select extract(month from l_shipdate) as l_year from lineitem ";- roundtrip_test_sql_with_context(sql, &ctx).await?;-- Ok(())-}--#[tokio::test]-async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {- use datafusion_common::test_util::datafusion_test_data;-- let ctx = SessionContext::new();-- // Register the TPC-H part table using the local test data- let test_data = datafusion_test_data();- let table_sql = format!(- "CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '{test_data}/tpch_part_small.parquet'"- );- ctx.sql(&table_sql).await.map_err(|e| {- DataFusionError::External(format!("Failed to create part table: {e}").into())- })?;-- // Test the exact problematic query- let sql =- "SELECT p_size FROM part WHERE p_size IN (14, 6, 5, 31) and p_partkey > 1000";-- let logical_plan = ctx.sql(sql).await?.into_unoptimized_plan();- let optimized_plan = ctx.state().optimize(&logical_plan)?;- let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?;-- // Serialize the physical plan - bug may happen here already but not necessarily manifests- let codec = DefaultPhysicalExtensionCodec {};-- let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;-- // This will fail with the bug, but should succeed when fixed- let _deserialized_plan = proto.try_into_physical_plan(&ctx.task_ctx(), &codec)?;- Ok(())-}--#[tokio::test]-/// Tests that we can serialize an unoptimized "analyze" plan and it will work on the other end-async fn analyze_roundtrip_unoptimized() -> Result<()> {- let ctx = SessionContext::new();-- // No optimizations- let session_state =- datafusion::execution::SessionStateBuilder::new_from_existing(ctx.state())- .with_physical_optimizer_rules(vec![])- .build();-- let logical_plan = session_state- .create_logical_plan("explain analyze select 1")- .await?;- let plan = session_state.create_physical_plan(&logical_plan).await?;-- let node = PhysicalPlanNode::try_from_physical_plan(- plan.clone(),- &DefaultPhysicalExtensionCodec {},- )?;-- let node = PhysicalPlanNode::decode(node.encode_to_vec().as_slice())- .map_err(|e| DataFusionError::External(Box::new(e)))?;-- let unoptimized =- node.try_into_physical_plan(&ctx.task_ctx(), &DefaultPhysicalExtensionCodec {})?;-- let physical_planner =- datafusion::physical_planner::DefaultPhysicalPlanner::default();- physical_planner.optimize_physical_plan(unoptimized, &session_state, |_, _| {})?;- Ok(())-}--#[test]-fn roundtrip_sort_merge_join() -> Result<()> {- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let field_a = Field::new("col_a", DataType::Int64, false);- let field_b = Field::new("col_b", DataType::Int64, false);- let schema_left = Schema::new(vec![field_a.clone()]);- let schema_right = Schema::new(vec![field_b.clone()]);- let on = vec![(- Arc::new(Column::new("col_a", schema_left.index_of("col_a")?)) as _,- Arc::new(Column::new("col_b", schema_right.index_of("col_b")?)) as _,- )];-- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("col_a", 1)),- Operator::Gt,- Arc::new(Column::new("col_b", 0)),- )),- vec![- ColumnIndex {- index: 0,- side: JoinSide::Left,- },- ColumnIndex {- index: 0,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![field_a, field_b])),- );-- let schema_left = Arc::new(schema_left);- let schema_right = Arc::new(schema_right);- let sort_options = vec![SortOptions {- descending: true,- nulls_first: false,- }];- for null_equality in [- NullEquality::NullEqualsNothing,- NullEquality::NullEqualsNull,- ] {- for filter in [None, Some(filter.clone())] {- for join_type in [- JoinType::Inner,- JoinType::Left,- JoinType::Right,- JoinType::Full,- JoinType::LeftAnti,- JoinType::RightAnti,- JoinType::LeftSemi,- JoinType::RightSemi,- JoinType::LeftMark,- JoinType::RightMark,- ] {- let result = roundtrip_test_and_return(- Arc::new(SortMergeJoinExec::try_new(- Arc::new(EmptyExec::new(schema_left.clone())),- Arc::new(EmptyExec::new(schema_right.clone())),- on.clone(),- filter.clone(),- join_type,- sort_options.clone(),- null_equality,- )?),- &ctx,- &codec,- &proto_converter,- )?;- let result = result.downcast_ref::<SortMergeJoinExec>().unwrap();- assert_eq!(result.join_type(), join_type);- assert_eq!(result.null_equality(), null_equality);- assert_eq!(result.sort_options(), sort_options);- assert_eq!(- result.filter().as_ref().map(|f| f.column_indices()),- filter.as_ref().map(|f| f.column_indices())- );- }- }- }- Ok(())-}--#[tokio::test]-async fn roundtrip_logical_plan_sort_merge_join() -> Result<()> {- let ctx = SessionContext::new();- ctx.register_csv(- "t0",- "tests/testdata/test.csv",- datafusion::prelude::CsvReadOptions::default().has_header(true),- )- .await?;- ctx.register_csv(- "t1",- "tests/testdata/test.csv",- datafusion::prelude::CsvReadOptions::default().has_header(true),- )- .await?;-- ctx.sql("SET datafusion.optimizer.prefer_hash_join = false")- .await?- .show()- .await?;-- let query = "SELECT t1.* FROM t0 join t1 on t0.a = t1.a";- let plan = ctx.sql(query).await?.create_physical_plan().await?;- roundtrip_test(plan)-}--#[tokio::test]-async fn roundtrip_memory_source() -> Result<()> {- let ctx = SessionContext::new();- let plan = ctx- .sql("select * from values ('Tom', 18)")- .await?- .create_physical_plan()- .await?;- roundtrip_test(plan)-}--#[tokio::test]-async fn roundtrip_memory_source_sort_information_and_fetch() -> Result<()> {- use datafusion::datasource::memory::MemorySourceConfig;- use datafusion::datasource::source::DataSource as _;-- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Int64, false),- ]));- let batch = RecordBatch::try_new(- Arc::clone(&schema),- vec![- Arc::new(arrow::array::StringArray::from(vec!["Tom", "Bob"])),- Arc::new(arrow::array::Int64Array::from(vec![18i64, 21i64])),- ],- )?;- let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(- col("b", &schema)?,- SortOptions {- descending: true,- nulls_first: false,- },- )])- .unwrap();- let source = MemorySourceConfig::try_new(&[vec![batch]], Arc::clone(&schema), None)?- .with_limit(Some(1))- .with_show_sizes(false)- .try_with_sort_information(vec![ordering])?;- let exec_plan = DataSourceExec::from_data_source(source.clone());-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let decoded = roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;-- // The string representation does not include every field; check the- // decoded source directly.- let decoded = decoded- .downcast_ref::<DataSourceExec>()- .expect("expected DataSourceExec");- let decoded_source = decoded- .data_source()- .downcast_ref::<MemorySourceConfig>()- .expect("expected MemorySourceConfig");- assert_eq!(decoded_source.partitions(), source.partitions());- assert_eq!(decoded_source.original_schema(), source.original_schema());- assert_eq!(decoded_source.projection(), source.projection());- assert_eq!(decoded_source.sort_information(), source.sort_information());- assert_eq!(decoded_source.fetch(), Some(1));- assert!(!decoded_source.show_sizes());- Ok(())-}--#[tokio::test]-async fn roundtrip_listing_table_with_schema_metadata() -> Result<()> {- let ctx = SessionContext::new();- let file_format = JsonFormat::default();- let table_partition_cols = vec![("part".to_owned(), DataType::Int64)];- let data = "../core/tests/data/partitioned_table_json";- let listing_table_url = ListingTableUrl::parse(data)?;- let listing_options = ListingOptions::new(Arc::new(file_format))- .with_table_partition_cols(table_partition_cols);-- let config = ListingTableConfig::new(listing_table_url)- .with_listing_options(listing_options)- .infer_schema(&ctx.state())- .await?;-- // Decorate metadata onto the inferred ListingTable schema- let schema_with_meta = config- .file_schema- .clone()- .map(|s| {- let mut meta: HashMap<String, String> = HashMap::new();- meta.insert("foo.bar".to_string(), "baz".to_string());- s.as_ref().clone().with_metadata(meta)- })- .expect("Must decorate metadata");-- let config = config.with_schema(Arc::new(schema_with_meta));- ctx.register_table("hive_style", Arc::new(ListingTable::try_new(config)?))?;-- let plan = ctx- .sql("select * from hive_style limit 1")- .await?- .create_physical_plan()- .await?;-- roundtrip_test(plan)-}--#[tokio::test]-async fn roundtrip_async_func_exec() -> Result<()> {- #[derive(Debug, PartialEq, Eq, Hash)]- struct TestAsyncUDF {- signature: Signature,- }-- impl TestAsyncUDF {- fn new() -> Self {- Self {- signature: Signature::exact(vec![DataType::Int64], Volatility::Volatile),- }- }- }-- impl ScalarUDFImpl for TestAsyncUDF {- fn name(&self) -> &str {- "test_async_udf"- }-- fn signature(&self) -> &Signature {- &self.signature- }-- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(DataType::Int64)- }-- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {- not_impl_err!("Must call from `invoke_async_with_args`")- }- }-- #[async_trait::async_trait]- impl AsyncScalarUDFImpl for TestAsyncUDF {- async fn invoke_async_with_args(- &self,- args: ScalarFunctionArgs,- ) -> Result<ColumnarValue> {- Ok(args.args[0].clone())- }- }-- let ctx = SessionContext::new();- let async_udf = AsyncScalarUDF::new(Arc::new(TestAsyncUDF::new()));- ctx.register_udf(async_udf.into_scalar_udf());-- let physical_plan = ctx- .sql("select test_async_udf(1)")- .await?- .create_physical_plan()- .await?;-- roundtrip_test_with_context(physical_plan, &ctx)?;-- Ok(())-}--/// Test that HashTableLookupExpr serializes to lit(true)-///-/// HashTableLookupExpr contains a runtime hash table that cannot be serialized.-/// The serialization code replaces it with lit(true) which is safe because-/// it's a performance optimization filter, not a correctness requirement.-#[test]-fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {- use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;- use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};-- // Create a simple schema and input plan- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));- let input = Arc::new(EmptyExec::new(schema.clone()));-- // Create a HashTableLookupExpr - it will be replaced with lit(true) during serialization- let hash_map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0))));- let on_columns = vec![col("col", &schema)?];- let lookup_expr: Arc<dyn PhysicalExpr> = Arc::new(HashTableLookupExpr::new(- on_columns,- datafusion::physical_plan::joins::SeededRandomState::with_seed(0),- hash_map,- "test_lookup".to_string(),- ));-- // Create a filter with the lookup expression- let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?);-- // Serialize- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};-- let proto: PhysicalPlanNode =- PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)- .expect("serialization should succeed");-- // Deserialize- let result: Arc<dyn ExecutionPlan> = proto- .try_into_physical_plan(&ctx.task_ctx(), &codec)- .expect("deserialization should succeed");-- // The deserialized plan should have lit(true) instead of HashTableLookupExpr- // Verify the filter predicate is a Literal(true)- let result_filter = result.downcast_ref::<FilterExec>().unwrap();- let predicate = result_filter.predicate();- let literal = predicate.downcast_ref::<Literal>().unwrap();- assert_eq!(*literal.value(), ScalarValue::Boolean(Some(true)));-- Ok(())-}--#[test]-fn roundtrip_hash_expr() -> Result<()> {- use datafusion::physical_plan::joins::{HashExpr, SeededRandomState};-- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Utf8, false),- ]));-- // Create a HashExpr with test columns and seeds- let on_columns = vec![col("a", &schema)?, col("b", &schema)?];- let hash_expr: Arc<dyn PhysicalExpr> = Arc::new(HashExpr::new(- on_columns,- SeededRandomState::with_seed(0), // arbitrary random seed for testing- "test_hash".to_string(),- ));-- // Wrap in a filter by comparing hash value to a literal- // hash_expr > 0 is always boolean- let filter_expr = binary(hash_expr, Operator::Gt, lit(0u64), &schema)?;- let filter = Arc::new(FilterExec::try_new(- filter_expr,- Arc::new(EmptyExec::new(schema)),- )?);-- // Confirm that the debug string contains the random state seeds- assert!(- format!("{filter:?}").contains("test_hash(a@0, b@1, [0])"),- "Debug string missing seeds: {filter:?}"- );- roundtrip_test(filter)-}--#[test]-fn custom_proto_converter_intercepts() -> Result<()> {- #[derive(Default)]- struct CustomConverterInterceptor {- num_proto_plans: RwLock<usize>,- num_physical_plans: RwLock<usize>,- num_proto_exprs: RwLock<usize>,- num_physical_exprs: RwLock<usize>,- }-- impl PhysicalProtoConverterExtension for CustomConverterInterceptor {- fn proto_to_execution_plan(- &self,- proto: &PhysicalPlanNode,- ctx: &PhysicalPlanDecodeContext<'_>,- ) -> Result<Arc<dyn ExecutionPlan>> {- {- let mut counter = self- .num_proto_plans- .write()- .map_err(|err| exec_datafusion_err!("{err}"))?;- *counter += 1;- }- self.default_proto_to_execution_plan(proto, ctx)- }-- fn execution_plan_to_proto(- &self,- plan: &Arc<dyn ExecutionPlan>,- codec: &dyn PhysicalExtensionCodec,- ) -> Result<PhysicalPlanNode>- where- Self: Sized,- {- {- let mut counter = self- .num_physical_plans- .write()- .map_err(|err| exec_datafusion_err!("{err}"))?;- *counter += 1;- }- PhysicalPlanNode::try_from_physical_plan_with_converter(- Arc::clone(plan),- codec,- self,- )- }-- fn proto_to_physical_expr(- &self,- proto: &PhysicalExprNode,- input_schema: &Schema,- ctx: &PhysicalPlanDecodeContext<'_>,- ) -> Result<Arc<dyn PhysicalExpr>>- where- Self: Sized,- {- {- let mut counter = self- .num_proto_exprs- .write()- .map_err(|err| exec_datafusion_err!("{err}"))?;- *counter += 1;- }- self.default_proto_to_physical_expr(proto, input_schema, ctx)- }-- fn physical_expr_to_proto(- &self,- expr: &Arc<dyn PhysicalExpr>,- codec: &dyn PhysicalExtensionCodec,- ) -> Result<PhysicalExprNode> {- {- let mut counter = self- .num_physical_exprs- .write()- .map_err(|err| exec_datafusion_err!("{err}"))?;- *counter += 1;- }- serialize_physical_expr_with_converter(expr, codec, self)- }- }-- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let sort_exprs = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: true,- nulls_first: false,- },- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: SortOptions {- descending: false,- nulls_first: true,- },- },- ]- .into();-- let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = CustomConverterInterceptor::default();- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;-- assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);- assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);- assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);- assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);-- Ok(())-}--#[test]-fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {- let data_type = DataType::Struct(Fields::from(vec![Field::new(- "item",- DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),- true,- )]));-- let schema = Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), true)]));- let scan = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let scalar = lit(ScalarValue::try_from(data_type)?);- let filter = Arc::new(FilterExec::try_new(- Arc::new(BinaryExpr::new(scalar, Operator::Eq, col("a", &schema)?)),- scan,- )?);-- roundtrip_test(filter)-}--/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.-fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {- Arc::new(DynamicFilterPhysicalExpr::new(- vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],- lit(true),- )) as Arc<dyn PhysicalExpr>-}--/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.-fn make_reassigned_dynamic_filter(- filter: Arc<dyn PhysicalExpr>,-) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {- let schema = Arc::new(Schema::new(vec![- Field::new("b", DataType::Int64, false),- Field::new("a", DataType::Int64, false),- ]));- let reassigned = reassign_expr_columns(filter, &schema)?;- Ok((schema, reassigned))-}--/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the-/// default serializer from `PhysicalExpr::expression_id`.-fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {- expr.expr_id- .expect("expected PhysicalExprNode.expr_id to be populated")-}--/// Roundtrip a single physical expression shaped like so:-///-/// ```text-/// BinaryExpr(AND)-/// / \-/// filter_expr_1 filter_expr_2-/// ```-///-/// Returns filter_expr_1 and filter_expr_2 after deserialization.-fn roundtrip_dynamic_filter_expr_pair(- filter_expr_1: Arc<dyn PhysicalExpr>,- filter_expr_2: Arc<dyn PhysicalExpr>,- schema: Arc<Schema>,-) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {- let pair_expr = Arc::new(BinaryExpr::new(- Arc::clone(&filter_expr_1),- Operator::And,- Arc::clone(&filter_expr_2),- )) as Arc<dyn PhysicalExpr>;-- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;- let ctx = SessionContext::new();- let task_ctx = ctx.task_ctx();- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);- let deserialized_expr =- converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;-- let binary = deserialized_expr- .downcast_ref::<BinaryExpr>()- .expect("Expected BinaryExpr");-- Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))-}--/// Roundtrip an execution plan shaped like so:-///-/// ```text-/// FilterExec(dynamic_filter_1 on a@0)-/// ProjectionExec(a := Column("a", source_index))-/// DataSourceExec-/// ParquetSource(predicate = dynamic_filter_2)-/// ```-///-/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with-/// different children.-///-/// Returns-/// - `dynamic_filter_1` before serialization-/// - `dynamic_filter_2` before serialization-/// - `dynamic_filter_1` after serialization-/// - `dynamic_filter_2` after serialization-#[expect(clippy::type_complexity)]-fn roundtrip_dynamic_filter_plan_pair() -> Result<(- Arc<dyn PhysicalExpr>,- Arc<dyn PhysicalExpr>,- Arc<dyn PhysicalExpr>,- Arc<dyn PhysicalExpr>,-)> {- let filter_expr_1 = make_dynamic_filter();- let (data_source_schema, filter_expr_2) =- make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;- let left_before = Arc::clone(&filter_expr_1);- let right_before = Arc::clone(&filter_expr_2);- let file_source = Arc::new(- ParquetSource::new(Arc::clone(&data_source_schema))- .with_predicate(Arc::clone(&filter_expr_2)),- );- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .build();- let data_source_exec =- DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;-- let projection_exec = Arc::new(ProjectionExec::try_new(- vec![ProjectionExpr {- expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,- alias: "a".to_string(),- }],- data_source_exec,- )?) as Arc<dyn ExecutionPlan>;- let filter_exec = Arc::new(FilterExec::try_new(- Arc::clone(&filter_expr_1),- projection_exec,- )?) as Arc<dyn ExecutionPlan>;-- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;-- let ctx = SessionContext::new();- let task_ctx = ctx.task_ctx();- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);- let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;-- let outer_filter = deserialized_plan- .downcast_ref::<FilterExec>()- .expect("Expected outer FilterExec");- let left_filter = Arc::clone(outer_filter.predicate());- let projection = outer_filter.children()[0]- .downcast_ref::<ProjectionExec>()- .expect("Expected ProjectionExec");- let data_source = projection- .input()- .downcast_ref::<DataSourceExec>()- .expect("Expected DataSourceExec");- let scan_config = data_source- .data_source()- .downcast_ref::<FileScanConfig>()- .expect("Expected FileScanConfig");- let right_filter = scan_config- .file_source()- .filter()- .expect("Expected pushed-down predicate");-- Ok((left_before, right_before, left_filter, right_filter))-}--/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible-/// via the other. This helps assert that referential integrity is maintained after-/// deserializing.-fn assert_dynamic_filter_update_is_visible(- left_filter: &Arc<dyn PhysicalExpr>,- right_filter: &Arc<dyn PhysicalExpr>,-) -> Result<()> {- let left_filter = left_filter- .downcast_ref::<DynamicFilterPhysicalExpr>()- .expect("Expected dynamic filter");- let right_filter = right_filter- .downcast_ref::<DynamicFilterPhysicalExpr>()- .expect("Expected dynamic filter");-- // Sanity check that the filters have the same generation.- let original_generation = left_filter.snapshot_generation();- assert_eq!(original_generation, right_filter.snapshot_generation(),);-- left_filter.update(lit(123_i64))?;-- // Assert that both generations updated.- assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);- assert_eq!(- left_filter.snapshot_generation(),- right_filter.snapshot_generation(),- );-- // Ensure both filters have the updated expr.- let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;- assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);- assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);-- Ok(())-}--/// Extract the dynamic-filter predicate that was pushed down to the parquet-/// scan at the bottom of the plan tree.-fn parquet_source_predicate(child: &Arc<dyn ExecutionPlan>) -> Arc<dyn PhysicalExpr> {- let data_source = child- .downcast_ref::<DataSourceExec>()- .expect("Child should be DataSourceExec");- let (_, parquet_source) = data_source- .downcast_to_file_source::<ParquetSource>()- .expect("Should be ParquetSource");- parquet_source- .filter()- .expect("ParquetSource should have a predicate after roundtrip")-}--/// Assert that two dynamic filters are equal both structurally (Debug output)-/// and by identity (`expression_id`).-fn assert_dynamic_filters_equal(- expected: &Arc<dyn PhysicalExpr>,- actual: &Arc<dyn PhysicalExpr>,-) {- // Structural.- let expected_dbg = format!("{expected:?}");- let actual_dbg = format!("{actual:?}");- if expected_dbg == actual_dbg {- return;- }-- // Note that the `DeduplicatingDeserializer` routes every cache hit through- // `with_new_children`. This produces an equivalent expression, but with- // remapped children that are equal to the original. Handle that case here.- let rewritten = Arc::clone(expected)- .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())- .expect("with_new_children on a dynamic filter should not fail");- assert_eq!(format!("{rewritten:?}"), actual_dbg);-}--// Two clones of a dynamic filter expression should be deduped to the exact same expression.-#[test]-fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let filter_expr_1 = make_dynamic_filter();- let filter_expr_2 = Arc::clone(&filter_expr_1);-- let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =- roundtrip_dynamic_filter_expr_pair(- Arc::clone(&filter_expr_1),- Arc::clone(&filter_expr_2),- schema,- )?;-- // Assert the filters are not modified during roundtrip.- assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);- assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);- assert_dynamic_filters_equal(- &filter_expr_1_after_roundtrip,- &filter_expr_2_after_roundtrip,- );-- // Assert referential integrity.- assert_dynamic_filter_update_is_visible(- &filter_expr_1_after_roundtrip,- &filter_expr_2_after_roundtrip,- )?;-- Ok(())-}--/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter-/// with different children.-#[test]-fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {- let (- filter_expr_1,- filter_expr_2,- filter_expr_1_after_roundtrip,- filter_expr_2_after_roundtrip,- ) = roundtrip_dynamic_filter_plan_pair()?;-- // Assert the filters are not modified during roundtrip.- assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);- assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);-- // Assert referential integrity.- assert_dynamic_filter_update_is_visible(- &filter_expr_1_after_roundtrip,- &filter_expr_2_after_roundtrip,- )?;-- Ok(())-}--#[test]-fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {- let filter_expr = make_dynamic_filter();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};-- let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;- let expr_id1 = proto_expression_id(&proto1);-- let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;- let expr_id2 = proto_expression_id(&proto2);-- assert_eq!(- expr_id1, expr_id2,- "Expected the same dynamic filter expression id across serializations"- );-- Ok(())-}--/// Tests that `lead` window function with offset and default value args-/// survives a protobuf round-trip. This is a regression test for a bug-/// where `expressions()` (used during serialization) returns only the-/// column expression for lead/lag, silently dropping the offset and-/// default value literal args.-#[test]-fn roundtrip_lead_with_default_value() -> Result<()> {- use datafusion::functions_window::lead_lag::lead_udwf;-- let field_a = Field::new("a", DataType::Int64, false);- let field_b = Field::new("b", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b]));-- // lead(a, 2, 42) — column a, offset 2, default value 42- let lead_window = create_udwf_window_expr(- &lead_udwf(),- &[col("a", &schema)?, lit(2i64), lit(42i64)],- schema.as_ref(),- "test lead with default".to_string(),- false,- )?;-- let udwf_expr = Arc::new(StandardWindowExpr::new(- lead_window,- &[col("b", &schema)?],- &[PhysicalSortExpr {- expr: col("a", &schema)?,- options: SortOptions {- descending: false,- nulls_first: false,- },- }],- Arc::new(WindowFrame::new(None)),- ));-- let input = Arc::new(EmptyExec::new(schema.clone()));-- roundtrip_test(Arc::new(BoundedWindowAggExec::try_new(- vec![udwf_expr],- input,- InputOrderMode::Sorted,- true,- )?))-}--/// Verify that ScalarSubqueryExpr nodes in the input plan are connected to the-/// same shared results container as ScalarSubqueryExec after a proto round-trip.-#[test]-fn roundtrip_scalar_subquery_exec() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let results = ScalarSubqueryResults::new(1);-- // Build the input plan: a filter whose predicate references the- // scalar subquery result via ScalarSubqueryExpr.- let sq_expr = Arc::new(ScalarSubqueryExpr::new(- DataType::Int64,- true,- SubqueryIndex::new(0),- results.clone(),- ));- let predicate = binary(col("a", &schema)?, Operator::Eq, sq_expr, &schema)?;- let filter =- FilterExec::try_new(predicate, Arc::new(EmptyExec::new(schema.clone())))?;-- // Build a trivial subquery plan.- let subquery_plan =- Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(- "x",- DataType::Int64,- true,- )]))));-- let exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(- Arc::new(filter),- vec![ScalarSubqueryLink {- plan: subquery_plan,- index: SubqueryIndex::new(0),- }],- results,- ));-- // Perform the round-trip using DeduplicatingProtoConverter, which- // creates a DeduplicatingDeserializer that threads scalar subquery- // results through expression deserialization.- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec),- &codec,- &converter,- )?;- let ctx = SessionContext::new();- let deserialized = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &converter,- )?;-- // Verify the deserialized ScalarSubqueryExec's results container is- // shared with the ScalarSubqueryExpr in the input plan.- let sq_exec = deserialized- .downcast_ref::<ScalarSubqueryExec>()- .expect("expected ScalarSubqueryExec");- let exec_results = sq_exec.results();-- // Walk the input plan to find the ScalarSubqueryExpr and verify it- // points to the same results container.- let filter_exec = sq_exec- .input()- .downcast_ref::<FilterExec>()- .expect("expected FilterExec");- let binary_expr = filter_exec- .predicate()- .downcast_ref::<BinaryExpr>()- .expect("expected BinaryExpr");- let deserialized_sq_expr = binary_expr- .right()- .downcast_ref::<ScalarSubqueryExpr>()- .expect("expected ScalarSubqueryExpr");-- assert!(- ScalarSubqueryResults::ptr_eq(exec_results, deserialized_sq_expr.results()),- "ScalarSubqueryExpr should share the same results container as ScalarSubqueryExec"- );- Ok(())-}--/// Verify that nested ScalarSubqueryExec nodes deserialize with distinct-/// scoped results containers, and that each ScalarSubqueryExpr is wired to the-/// container for its own surrounding ScalarSubqueryExec.-#[test]-fn roundtrip_nested_scalar_subquery_exec_scopes_results() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let subquery_schema =- Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));-- let inner_results = ScalarSubqueryResults::new(1);- let inner_sq_expr = Arc::new(ScalarSubqueryExpr::new(- DataType::Int64,- true,- SubqueryIndex::new(0),- inner_results.clone(),- ));- let inner_predicate =- binary(col("a", &schema)?, Operator::Eq, inner_sq_expr, &schema)?;- let inner_filter = Arc::new(FilterExec::try_new(- inner_predicate,- Arc::new(EmptyExec::new(schema.clone())),- )?);- let inner_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(- inner_filter,- vec![ScalarSubqueryLink {- plan: Arc::new(EmptyExec::new(subquery_schema.clone())),- index: SubqueryIndex::new(0),- }],- inner_results,- ));-- let outer_results = ScalarSubqueryResults::new(1);- let outer_sq_expr = Arc::new(ScalarSubqueryExpr::new(- DataType::Int64,- true,- SubqueryIndex::new(0),- outer_results.clone(),- ));- let outer_predicate =- binary(col("a", &schema)?, Operator::Eq, outer_sq_expr, &schema)?;- let outer_filter = Arc::new(FilterExec::try_new(outer_predicate, inner_exec)?);- let outer_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(- outer_filter,- vec![ScalarSubqueryLink {- plan: Arc::new(EmptyExec::new(subquery_schema)),- index: SubqueryIndex::new(0),- }],- outer_results,- ));-- let bytes = datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&outer_exec))?;- let ctx = SessionContext::new();- let deserialized = datafusion_proto::bytes::physical_plan_from_bytes(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- )?;-- let outer_exec = deserialized- .downcast_ref::<ScalarSubqueryExec>()- .expect("expected outer ScalarSubqueryExec");- let outer_results = outer_exec.results();- let outer_filter = outer_exec- .input()- .downcast_ref::<FilterExec>()- .expect("expected outer FilterExec");- let outer_binary = outer_filter- .predicate()- .downcast_ref::<BinaryExpr>()- .expect("expected outer BinaryExpr");- let outer_sq_expr = outer_binary- .right()- .downcast_ref::<ScalarSubqueryExpr>()- .expect("expected outer ScalarSubqueryExpr");-- let inner_exec = outer_filter- .input()- .downcast_ref::<ScalarSubqueryExec>()- .expect("expected inner ScalarSubqueryExec");- let inner_results = inner_exec.results();- let inner_filter = inner_exec- .input()- .downcast_ref::<FilterExec>()- .expect("expected inner FilterExec");- let inner_binary = inner_filter- .predicate()- .downcast_ref::<BinaryExpr>()- .expect("expected inner BinaryExpr");- let inner_sq_expr = inner_binary- .right()- .downcast_ref::<ScalarSubqueryExpr>()- .expect("expected inner ScalarSubqueryExpr");-- assert!(- ScalarSubqueryResults::ptr_eq(outer_results, outer_sq_expr.results()),- "outer ScalarSubqueryExpr should use outer ScalarSubqueryExec results"- );- assert!(- ScalarSubqueryResults::ptr_eq(inner_results, inner_sq_expr.results()),- "inner ScalarSubqueryExpr should use inner ScalarSubqueryExec results"- );- assert!(- !ScalarSubqueryResults::ptr_eq(outer_results, inner_results),- "nested ScalarSubqueryExec nodes should not share results containers"- );- assert!(- !ScalarSubqueryResults::ptr_eq(outer_results, inner_sq_expr.results()),- "inner ScalarSubqueryExpr must not read from outer results"- );- assert!(- !ScalarSubqueryResults::ptr_eq(inner_results, outer_sq_expr.results()),- "outer ScalarSubqueryExpr must not read from inner results"- );-- Ok(())-}--/// Verify that the default physical plan bytes round-trip preserves executable-/// scalar subquery plans.-#[tokio::test]-async fn roundtrip_scalar_subquery_exec_with_default_converter_executes() -> Result<()> {- let ctx = SessionContext::new();- let sql = "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) AS s \- FROM (VALUES (2), (1)) AS t(x) \- ORDER BY s";-- let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;- assert!(- format!("{initial_plan:?}").contains("ScalarSubqueryExec"),- "expected ScalarSubqueryExec in plan:\n{initial_plan:?}"- );-- let bytes =- datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&initial_plan))?;- let roundtripped = datafusion_proto::bytes::physical_plan_from_bytes(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- )?;- assert!(- format!("{roundtripped:?}").contains("ScalarSubqueryExec"),- "expected ScalarSubqueryExec after roundtrip:\n{roundtripped:?}"- );-- let batches = datafusion::physical_plan::common::collect(- roundtripped.execute(0, ctx.task_ctx())?,- )- .await?;- datafusion::assert_batches_eq!(- &["+----+", "| s |", "+----+", "| 21 |", "| 22 |", "+----+",],- &batches- );-- Ok(())-}--/// Test that a chain of the same operator (a AND b AND c) is linearized-/// and roundtrips correctly.-#[test]-fn roundtrip_binary_expr_chain_same_op() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Boolean, false);- let field_c = Field::new("c", DataType::Boolean, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));- let ab = binary(- col("a", &schema)?,- Operator::And,- col("b", &schema)?,- &schema,- )?;- let abc = binary(ab, Operator::And, col("c", &schema)?, &schema)?;- roundtrip_test(Arc::new(FilterExec::try_new(- abc,- Arc::new(EmptyExec::new(schema)),- )?))-}--/// Test that mixed operators (a AND b OR c) are NOT linearized together —-/// only chains of the same operator are flattened.-#[test]-fn roundtrip_binary_expr_mixed_ops() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Boolean, false);- let field_c = Field::new("c", DataType::Boolean, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c]));- // (a AND b) OR c — AND and OR are different operators, so linearization stops- let a_and_b = binary(- col("a", &schema)?,- Operator::And,- col("b", &schema)?,- &schema,- )?;- let expr = binary(a_and_b, Operator::Or, col("c", &schema)?, &schema)?;- roundtrip_test(Arc::new(FilterExec::try_new(- expr,- Arc::new(EmptyExec::new(schema)),- )?))-}--/// Test that a deeply nested chain of AND expressions (like many WHERE conditions)-/// roundtrips correctly. This is the scenario from issue #18602.-#[test]-fn roundtrip_binary_expr_deeply_nested_and_chain() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let schema = Arc::new(Schema::new(vec![field_a]));-- // Build a chain: a AND a AND a AND ... (100 times)- let col_a = col("a", &schema)?;- let mut expr = Arc::clone(&col_a);- for _ in 0..99 {- expr = binary(expr, Operator::And, Arc::clone(&col_a), &schema)?;- }-- roundtrip_test(Arc::new(FilterExec::try_new(- expr,- Arc::new(EmptyExec::new(schema)),- )?))-}--/// Test that a deeply nested chain of OR expressions roundtrips correctly.-#[test]-fn roundtrip_binary_expr_deeply_nested_or_chain() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let schema = Arc::new(Schema::new(vec![field_a]));-- let col_a = col("a", &schema)?;- let mut expr = Arc::clone(&col_a);- for _ in 0..99 {- expr = binary(expr, Operator::Or, Arc::clone(&col_a), &schema)?;- }-- roundtrip_test(Arc::new(FilterExec::try_new(- expr,- Arc::new(EmptyExec::new(schema)),- )?))-}--/// Test that alternating AND/OR operators produce correct results —-/// each sub-chain gets linearized independently.-#[test]-fn roundtrip_binary_expr_alternating_and_or() -> Result<()> {- let field_a = Field::new("a", DataType::Boolean, false);- let field_b = Field::new("b", DataType::Boolean, false);- let field_c = Field::new("c", DataType::Boolean, false);- let field_d = Field::new("d", DataType::Boolean, false);- let schema = Arc::new(Schema::new(vec![field_a, field_b, field_c, field_d]));-- // (a AND b) OR (c AND d)- let a_and_b = binary(- col("a", &schema)?,- Operator::And,- col("b", &schema)?,- &schema,- )?;- let c_and_d = binary(- col("c", &schema)?,- Operator::And,- col("d", &schema)?,- &schema,- )?;- let expr = binary(a_and_b, Operator::Or, c_and_d, &schema)?;-- roundtrip_test(Arc::new(FilterExec::try_new(- expr,- Arc::new(EmptyExec::new(schema)),- )?))-}--/// Verify that the linearized proto format has a flat operands list-/// rather than deeply nested l/r fields.-#[test]-fn test_linearization_produces_flat_operands() -> Result<()> {- // Build: a AND a AND a AND a (4 operands, 3 levels of nesting)- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::new(BinaryExpr::new(- Arc::new(BinaryExpr::new(- Arc::clone(&col_a),- Operator::And,- Arc::clone(&col_a),- )),- Operator::And,- Arc::clone(&col_a),- )),- Operator::And,- Arc::clone(&col_a),- ));-- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;-- // The top-level should use the operands field with 4 entries- match &proto.expr_type {- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {- assert!(- b.l.is_none(),- "l should be None when using linearized operands"- );- assert!(- b.r.is_none(),- "r should be None when using linearized operands"- );- assert_eq!(- b.operands.len(),- 4,- "Expected 4 linearized operands for a AND a AND a AND a"- );- assert_eq!(b.op, "And");- }- other => panic!("Expected BinaryExpr, got {other:?}"),- }-- Ok(())-}--/// Test that linearization stops when encountering a different operator.-/// For (a AND b) OR c, only the top-level OR should be represented, and-/// the left-hand AND subtree should be a separate nested BinaryExpr.-#[test]-fn test_linearization_stops_at_different_op() -> Result<()> {- // (a AND b) OR c- let a_and_b: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::new(Column::new("a", 0)),- Operator::And,- Arc::new(Column::new("b", 1)),- ));- let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- a_and_b,- Operator::Or,- Arc::new(Column::new("c", 2)),- ));-- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let proto = proto_converter.physical_expr_to_proto(&expr, &codec)?;-- // The top-level OR should have only 2 operands (can't linearize through AND)- match &proto.expr_type {- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => {- assert_eq!(- b.operands.len(),- 2,- "Expected 2 operands for (a AND b) OR c"- );- assert_eq!(b.op, "Or");- // The first operand should be a nested AND BinaryExpr- match &b.operands[0].expr_type {- Some(protobuf::physical_expr_node::ExprType::BinaryExpr(inner)) => {- assert_eq!(inner.op, "And");- assert_eq!(inner.operands.len(), 2);- }- other => panic!("Expected inner BinaryExpr(AND), got {other:?}"),- }- }- other => panic!("Expected BinaryExpr, got {other:?}"),- }-- Ok(())-}--/// Create a DataSourceExec backed by a ParquetSource that accepts filter pushdown,-/// along with a ConfigOptions that enables all dynamic filter pushdown options.-fn datasource_for_dynamic_filter_pushdown(- schema: &Arc<Schema>,-) -> (Arc<dyn ExecutionPlan>, ConfigOptions) {- let mut parquet_options = TableParquetOptions::new();- parquet_options.global.pushdown_filters = true;- let source = Arc::new(- ParquetSource::new(Arc::clone(schema))- .with_table_parquet_options(parquet_options),- );- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)- .with_file(PartitionedFile::new("/path/to/file.parquet", 1024))- .build();-- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_join_dynamic_filter_pushdown = true;- config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;- config.optimizer.enable_topk_dynamic_filter_pushdown = true;-- (DataSourceExec::from_data_source(scan_config), config)-}--/// Test that plan containing a HashJoinExec with dynamic filter pushdown-/// can be serialized and deserialized while preserving references to the dynamic filter.-#[test]-fn test_hash_join_with_dynamic_filter_roundtrip() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));-- let left_child = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let (right_child, config) = datasource_for_dynamic_filter_pushdown(&schema);-- let on: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> = vec![(- Arc::new(Column::new("col", 0)),- Arc::new(Column::new("col", 0)),- )];-- let hash_join = Arc::new(HashJoinExec::try_new(- left_child,- right_child,- on,- None,- &JoinType::Inner,- None,- PartitionMode::CollectLeft,- NullEquality::NullEqualsNothing,- false,- )?) as Arc<dyn ExecutionPlan>;-- // Run the optimizer rule for filter pushdown.- let optimizer = FilterPushdown::new_post_optimization();- let plan = optimizer.optimize(hash_join, &config)?;-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let deserialized = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;-- // Extract the deserialized HashJoinExec and its dynamic filter.- let deserialized_join = deserialized- .downcast_ref::<HashJoinExec>()- .expect("Should be HashJoinExec");- let deserialized_hash_join_df = deserialized_join- .dynamic_expressions_produced()- .into_iter()- .next()- .expect("HashJoinExec should have a dynamic filter after roundtrip");-- // Extract the dynamic filter pushed down to the probe side's ParquetSource.- let deserialized_predicate = parquet_source_predicate(deserialized_join.right());-- // The HashJoinExec's dynamic filter and the probe side's predicate should- // refer to the same underlying expression.- let plan_df = deserialized_hash_join_df;- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;-- Ok(())-}--/// returns a SessionContext with an empty `netflow` table registered-fn netflow_context() -> Result<SessionContext> {- let ctx = SessionContext::new();- let schema = Arc::new(Schema::new(vec![- Field::new("dst_geo_country_name", DataType::Utf8, true),- Field::new("dst_geo_city_name", DataType::Utf8, true),- Field::new("packets", DataType::UInt64, true),- Field::new("src_addr", DataType::Utf8, true),- Field::new("dst_addr", DataType::Utf8, true),- ]));-- ctx.register_table("netflow", Arc::new(EmptyTable::new(schema)))?;-- Ok(ctx)-}--/// Regression test for issue #18602:-/// https://github.com/apache/datafusion/issues/18602-///-/// The physical filter expression here contains a long chain of `AND` predicates.-/// Before linearizing `PhysicalBinaryExprNode`, encoding then decoding the protobuf-/// could fail with `DecodeError: recursion limit reached`.-#[tokio::test]-async fn roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {- let ctx = netflow_context()?;- let sql = "SELECT \- dst_geo_country_name AS x_axis_1, \- dst_geo_city_name AS x_axis_2, \- sum(packets) AS y_axis_1 \- FROM netflow \- WHERE dst_geo_country_name IS NOT NULL \- AND src_addr NOT LIKE '10.201.%' \- AND dst_addr NOT LIKE '10.201.%' \- AND src_addr NOT LIKE '10.202.%' \- AND dst_addr NOT LIKE '10.202.%' \- AND src_addr NOT LIKE '10.203.%' \- AND dst_addr NOT LIKE '10.203.%' \- AND src_addr NOT LIKE '10.204.%' \- AND dst_addr NOT LIKE '10.204.%' \- AND src_addr NOT LIKE '172.16.186.%' \- AND dst_addr NOT LIKE '172.16.186.%' \- AND src_addr NOT LIKE '172.16.187.%' \- AND dst_addr NOT LIKE '172.16.187.%' \- AND src_addr NOT LIKE '172.16.188.%' \- AND dst_addr NOT LIKE '172.16.188.%' \- AND src_addr NOT LIKE '10.102.45.%' \- AND dst_addr NOT LIKE '10.102.45.%' \- AND src_addr NOT LIKE '172.25.210.%' \- AND dst_addr NOT LIKE '172.25.210.%' \- AND src_addr NOT LIKE '172.25.211.%' \- AND dst_addr NOT LIKE '172.25.211.%' \- AND src_addr NOT LIKE '141.226.101.%' \- AND dst_addr NOT LIKE '141.226.101.%' \- AND src_addr NOT LIKE '167.86.40.%' \- AND dst_addr NOT LIKE '167.86.40.%' \- AND src_addr NOT LIKE '66.22.38.%' \- AND dst_addr NOT LIKE '66.22.38.%' \- AND src_addr != '168.143.191.55' \- AND dst_addr != '168.143.191.55' \- AND src_addr != '82.112.107.142' \- AND dst_addr != '82.112.107.142' \- AND src_addr != '20.76.39.176' \- AND dst_addr != '20.76.39.176' \- AND src_addr != '162.159.129.83' \- AND dst_addr != '162.159.129.83' \- AND src_addr != '34.201.223.155' \- AND dst_addr != '34.201.223.155' \- AND src_addr != '34.201.223.156' \- AND dst_addr != '34.201.223.156' \- AND src_addr != '34.201.223.157' \- AND dst_addr != '34.201.223.157' \- AND src_addr != '134.201.223.157' \- AND dst_addr != '134.201.223.157' \- AND src_addr != '341.201.223.157' \- AND dst_addr != '341.201.223.157' \- GROUP BY x_axis_1, x_axis_2 \- ORDER BY y_axis_1 DESC \- LIMIT 20";-- roundtrip_test_sql_with_context(sql, &ctx).await-}--/// Test that plan containing a AggregateExec with dynamic filter pushdown-/// can be serialized and deserialized while preserving references to the dynamic filter.-#[test]-fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));-- let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);-- let agg = Arc::new(AggregateExec::try_new(- AggregateMode::Partial,- PhysicalGroupBy::new_single(vec![]),- vec![- AggregateExprBuilder::new(- datafusion::functions_aggregate::min_max::min_udaf(),- vec![Arc::clone(&col_a)],- )- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .map(Arc::new)?,- ],- vec![None],- child,- Arc::clone(&schema),- )?) as Arc<dyn ExecutionPlan>;-- // Run the optimizer rule for filter pushdown.- let optimizer = FilterPushdown::new_post_optimization();- let plan = optimizer.optimize(agg, &config)?;-- // Roundtrip with deduplication.- //- // Note: We don't use `roundtrip_test_and_return` here because there's a- // pre-existing issue with PhysicalGroupBy serialization where empty groups- // `[[]]` become `[]` after roundtrip. This behavior is unrelated to this test.- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&plan),- &codec,- &converter,- )?;- let deserialized = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &converter,- )?;-- // Extract the deserialized AggregateExec and its dynamic filter.- let deserialized_agg = deserialized- .downcast_ref::<AggregateExec>()- .expect("Should be AggregateExec");- let deserialized_agg_df = deserialized_agg- .dynamic_expressions_produced()- .into_iter()- .next()- .expect("AggregateExec should have a dynamic filter after roundtrip");-- // Extract the dynamic filter pushed down to the child ParquetSource.- let deserialized_predicate = parquet_source_predicate(deserialized_agg.input());-- // The AggregateExec's dynamic filter and the child's predicate should- // refer to the same underlying expression.- let plan_df = deserialized_agg_df;- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;-- Ok(())-}--#[test]-fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let aggregate = Arc::new(AggregateExec::try_new(- AggregateMode::Partial,- PhysicalGroupBy::new_single(vec![]),- vec![- AggregateExprBuilder::new(- datafusion::functions_aggregate::min_max::min_udaf(),- vec![col_a],- )- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .map(Arc::new)?,- ],- vec![None],- child,- Arc::clone(&schema),- )?) as Arc<dyn ExecutionPlan>;-- let mut config = ConfigOptions::default();- config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;- let optimizer = FilterPushdown::new_post_optimization();- let plan = optimizer.optimize(aggregate, &config)?;- assert!(- plan.downcast_ref::<AggregateExec>()- .expect("Should be AggregateExec")- .dynamic_expressions_produced()- .is_empty()- );-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let converter = DefaultPhysicalProtoConverter {};- let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;- let deserialized = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &converter,- )?;-- assert!(- deserialized- .downcast_ref::<AggregateExec>()- .expect("Should be AggregateExec")- .dynamic_expressions_produced()- .is_empty()- );- Ok(())-}--/// Test that plan containing a SortExec with dynamic filter pushdown-/// can be serialized and deserialized while preserving references to the dynamic filter.-#[test]-fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));-- let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);-- let sort = Arc::new(- SortExec::new(- LexOrdering::new(vec![PhysicalSortExpr {- expr: Arc::clone(&col_a),- options: SortOptions::default(),- }])- .unwrap(),- child,- )- .with_fetch(Some(10)),- ) as Arc<dyn ExecutionPlan>;-- // Verify the optimizer kept the dynamic filter on the SortExec.- let optimizer = FilterPushdown::new_post_optimization();- let plan = optimizer.optimize(sort, &config)?;-- // Roundtrip with deduplication.- //- // Note: We don't use `roundtrip_test_and_return` here because- // `DeduplicatingDeserializer` rewrites cache hits via `with_new_children`,- // which sets `remapped_children: Some(...)` on the second encounter of a- // shared `DynamicFilterPhysicalExpr`. SortExec's `Debug` includes its- // dynamic filter, so the original-vs-deserialized structural equality check- // would fail purely on this artifact.- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let converter = DeduplicatingProtoConverter {};- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&plan),- &codec,- &converter,- )?;- let deserialized = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &converter,- )?;-- // Extract the deserialized SortExec and its dynamic filter.- let deserialized_sort = deserialized- .downcast_ref::<SortExec>()- .expect("Should be SortExec");- let deserialized_sort_df = deserialized_sort- .dynamic_expressions_produced()- .into_iter()- .next()- .expect("SortExec should have a dynamic filter after roundtrip");-- // Extract the dynamic filter pushed down to the child ParquetSource.- let deserialized_predicate = parquet_source_predicate(deserialized_sort.input());-- // The SortExec's dynamic filter and the child's predicate should- // refer to the same underlying expression.- let plan_df = deserialized_sort_df;- assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);- assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;-- Ok(())-}--/// A custom [`ExecutionPlan`] which stores [`PhysicalExpr`]s.-struct CustomExecWithExprs {- exprs: Vec<Arc<dyn PhysicalExpr>>,- child: Arc<dyn ExecutionPlan>,-}--#[derive(Clone, PartialEq, Message)]-struct CustomExecWithExprsProto {- #[prost(message, repeated, tag = "1")]- exprs: Vec<PhysicalExprNode>,-}--impl std::fmt::Debug for CustomExecWithExprs {- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- f.debug_struct("CustomExecWithExprs")- .field("exprs", &self.exprs)- .field("child", &self.child)- .finish()- }-}--impl CustomExecWithExprs {- fn new(exprs: Vec<Arc<dyn PhysicalExpr>>, child: Arc<dyn ExecutionPlan>) -> Self {- Self { exprs, child }- }-}--impl DisplayAs for CustomExecWithExprs {- fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {- write!(f, "CustomExecWithExprs")- }-}--impl ExecutionPlan for CustomExecWithExprs {- fn name(&self) -> &str {- "CustomExecWithExprs"- }-- fn schema(&self) -> SchemaRef {- self.child.schema()- }-- fn properties(&self) -> &Arc<PlanProperties> {- self.child.properties()- }-- fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {- vec![&self.child]- }-- fn apply_expressions(- &self,- f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,- ) -> Result<TreeNodeRecursion> {- datafusion_physical_plan::apply_expression_roots(&self.exprs, f)- }-- fn with_new_children(- self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,- ) -> Result<Arc<dyn ExecutionPlan>> {- unreachable!()- }-- fn execute(- &self,- _partition: usize,- _context: Arc<TaskContext>,- ) -> Result<SendableRecordBatchStream> {- unreachable!()- }-}--/// A [`PhysicalExtensionCodec`] for [`CustomExecWithExprs`].-#[derive(Debug)]-struct CustomExecWithExprsCodec {}--impl PhysicalExtensionCodec for CustomExecWithExprsCodec {- fn try_decode(- &self,- buf: &[u8],- inputs: &[Arc<dyn ExecutionPlan>],- ctx: &TaskContext,- proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Arc<dyn ExecutionPlan>> {- let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);- let input_schema = inputs[0].schema();- let proto = CustomExecWithExprsProto::decode(buf)- .map_err(|e| internal_datafusion_err!("Failed to decode custom exec: {e}"))?;- let exprs = proto- .exprs- .iter()- .map(|expr_proto| {- proto_converter.proto_to_physical_expr(- expr_proto,- input_schema.as_ref(),- &decode_ctx,- )- })- .collect::<Result<Vec<_>>>()?;-- Ok(Arc::new(CustomExecWithExprs::new(exprs, inputs[0].clone())))- }-- fn try_encode(- &self,- node: Arc<dyn ExecutionPlan>,- buf: &mut Vec<u8>,- proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<()> {- let custom = node- .downcast_ref::<CustomExecWithExprs>()- .ok_or_else(|| internal_datafusion_err!("Expected CustomExecWithExprs"))?;- let proto = CustomExecWithExprsProto {- exprs: custom- .exprs- .iter()- .map(|expr| proto_converter.physical_expr_to_proto(expr, self))- .collect::<Result<Vec<_>>>()?,- };- proto- .encode(buf)- .map_err(|e| internal_datafusion_err!("Failed to encode custom exec: {e}"))?;-- Ok(())- }-}--/// Tests that a custom [`ExecutionPlan`] with [`PhysicalExpr`] can-/// dedupe dynamic filters by using the proto converter in its-/// [`PhysicalExtensionCodec`] implementation.-#[test]-fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {- // Create the plan:- //- // FilterExec(dynamic_filter)- // -> CustomExecWithExprs(exprs: [dynamic_filter])- // -> EmptyExec- //- // The same dynamic filter expression is saved in both the FilterExec and CustomExecWithExprs.- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(- vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],- lit(true),- ));- let dynamic_filter_expr: Arc<dyn PhysicalExpr> = dynamic_filter;-- let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let custom_exec = Arc::new(CustomExecWithExprs::new(- vec![Arc::clone(&dynamic_filter_expr)],- empty,- ));- let filter_exec = Arc::new(FilterExec::try_new(- Arc::clone(&dynamic_filter_expr),- custom_exec,- )?) as Arc<dyn ExecutionPlan>;-- // Roundtrip with DeduplicatingProtoConverter- let codec = CustomExecWithExprsCodec {};- let converter = DeduplicatingProtoConverter {};-- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&filter_exec),- &codec,- &converter,- )?;-- let ctx = SessionContext::new();- let deser_converter = DeduplicatingProtoConverter {};- let deserialized = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &deser_converter,- )?;-- // Extract the deserialized FilterExec's dynamic filter- let deser_filter = deserialized- .downcast_ref::<FilterExec>()- .expect("Top-level should be FilterExec");- let deser_filter_df = deser_filter.predicate();-- // Extract the deserialized custom node's dynamic filter- let deser_custom = deser_filter- .input()- .downcast_ref::<CustomExecWithExprs>()- .expect("FilterExec child should be CustomExecWithExprs");- assert_eq!(deser_custom.exprs.len(), 1, "Should have one expression");- let [deser_custom_df] = deser_custom.exprs.as_slice() else {- return internal_err!("Custom node should have one expression");- };-- // Pass the un-remapped filter first so the helper's `with_new_children`- // rewrite can reconstruct the remapped form on the other side.- assert_dynamic_filters_equal(deser_custom_df, deser_filter_df);- assert_dynamic_filter_update_is_visible(deser_custom_df, deser_filter_df)?;-- Ok(())-}--fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanConfig> {- let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let result_plan =- roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;-- let data_source_exec = result_plan- .downcast_ref::<DataSourceExec>()- .expect("Expected DataSourceExec");- let file_scan_config = data_source_exec- .data_source()- .downcast_ref::<FileScanConfig>()- .expect("Expected FileScanConfig");- Ok(file_scan_config.clone())-}--#[test]-fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));- let output_partitioning =- Partitioning::Hash(vec![Arc::new(Column::new("col", 0))], 1);- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_output_partitioning(Some(output_partitioning.clone()))- .build();-- assert_eq!(- roundtrip_file_scan_config(scan_config)?.output_partitioning,- Some(output_partitioning)- );-- Ok(())-}--#[test]-fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {- let file_schema =- Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)]));- let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));- let output_partitioning = Partitioning::Range(RangePartitioning::new(- LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(- "col", 0,- )))])- .unwrap(),- vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],- ));- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![- FileGroup::new(vec![PartitionedFile::new(- "/path/to/file-1.parquet".to_string(),- 1024,- )]),- FileGroup::new(vec![PartitionedFile::new(- "/path/to/file-2.parquet".to_string(),- 1024,- )]),- ])- .with_output_partitioning(Some(output_partitioning.clone()))- .build();-- assert_eq!(- roundtrip_file_scan_config(scan_config)?.output_partitioning,- Some(output_partitioning)- );-- Ok(())-}--/// A custom `PhysicalExpr` whose extension codec embeds a nested-/// `PhysicalExprNode` *inside its own blob* (rather than the standard-/// `PhysicalExtensionExprNode.inputs` field). This is the case that only-/// works if the expr-level codec methods receive the encode/decode context.-#[derive(Debug)]-struct WrapperExpr {- inner: Arc<dyn PhysicalExpr>,-}--impl Display for WrapperExpr {- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- write!(f, "WrapperExpr({})", self.inner)- }-}--impl PartialEq for WrapperExpr {- fn eq(&self, other: &Self) -> bool {- self.inner.eq(&other.inner)- }-}-impl Eq for WrapperExpr {}--impl std::hash::Hash for WrapperExpr {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- self.inner.hash(state);- }-}--impl PhysicalExpr for WrapperExpr {- fn data_type(&self, input_schema: &Schema) -> Result<DataType> {- self.inner.data_type(input_schema)- }- fn nullable(&self, input_schema: &Schema) -> Result<bool> {- self.inner.nullable(input_schema)- }- fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {- internal_err!("WrapperExpr is not executable in this test")- }- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {- vec![&self.inner]- }- fn with_new_children(- self: Arc<Self>,- children: Vec<Arc<dyn PhysicalExpr>>,- ) -> Result<Arc<dyn PhysicalExpr>> {- Ok(Arc::new(WrapperExpr {- inner: Arc::clone(&children[0]),- }))- }- fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- Display::fmt(self, f)- }-}--/// Wire layout for [`WrapperExpr`]: a single nested `PhysicalExprNode`.-#[derive(Clone, PartialEq, prost::Message)]-struct WrapperExprProto {- #[prost(message, optional, boxed, tag = "1")]- inner: Option<Box<PhysicalExprNode>>,-}--#[derive(Debug)]-struct WrapperCodec;--impl PhysicalExtensionCodec for WrapperCodec {- fn try_decode(- &self,- _buf: &[u8],- _inputs: &[Arc<dyn ExecutionPlan>],- _ctx: &TaskContext,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Arc<dyn ExecutionPlan>> {- internal_err!("not used")- }- fn try_encode(- &self,- _node: Arc<dyn ExecutionPlan>,- _buf: &mut Vec<u8>,- _proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<()> {- internal_err!("not used")- }- fn try_decode_expr(- &self,- buf: &[u8],- _inputs: &[Arc<dyn PhysicalExpr>],- ctx: &PhysicalExprDecodeCtx<'_>,- ) -> Result<Arc<dyn PhysicalExpr>> {- let proto = WrapperExprProto::decode(buf)- .map_err(|e| internal_datafusion_err!("decode WrapperExprProto: {e}"))?;- let inner_proto = proto- .inner- .ok_or_else(|| internal_datafusion_err!("missing inner"))?;- // Decode the nested expr through the context so it resolves against- // the real schema/registry AND participates in dedup — no fabricated- // `SessionContext` or hard-coded schema required.- let inner = ctx.decode(&inner_proto)?;- Ok(Arc::new(WrapperExpr { inner }))- }- fn try_encode_expr(- &self,- node: &Arc<dyn PhysicalExpr>,- buf: &mut Vec<u8>,- ctx: &PhysicalExprEncodeCtx<'_>,- ) -> Result<()> {- let wrapper = node- .downcast_ref::<WrapperExpr>()- .ok_or_else(|| internal_datafusion_err!("not WrapperExpr"))?;- // Encode the nested expr through the context so an active- // `DeduplicatingProtoConverter` stamps a matching `expr_id`.- let inner_proto = ctx.encode_child(&wrapper.inner)?;- let proto = WrapperExprProto {- inner: Some(Box::new(inner_proto)),- };- proto- .encode(buf)- .map_err(|e| internal_datafusion_err!("encode WrapperExprProto: {e}"))?;- Ok(())- }-}--/// A `DynamicFilterPhysicalExpr` referenced both as a bare expression and-/// nested inside a custom expression's codec blob must reconstruct to a-/// single shared `Inner` after roundtrip.-///-/// This exercises the expr-level codec hooks receiving the encode/decode-/// context: `try_encode_expr` routes its nested `PhysicalExprNode` through-/// `ctx.encode_child` and `try_decode_expr` through `ctx.decode`, so the-/// nested filter picks up the same `DeduplicatingProtoConverter` /-/// `DeduplicatingDeserializer` cache as the bare reference. Without the-/// context the nested expr would serialize with `expr_id: None` and decode-/// into a distinct `Inner`, breaking heap-max propagation across the-/// extension boundary in distributed execution.-#[test]-fn extension_codec_expr_participates_in_deduplication() -> Result<()> {- use prost::Message;-- // A single composite expression holding TWO references to the same- // dynamic filter: bare on the left of an AND, wrapped on the right.- let dyn_filter = make_dynamic_filter();- let wrapper: Arc<dyn PhysicalExpr> = Arc::new(WrapperExpr {- inner: Arc::clone(&dyn_filter),- });- let composite: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::clone(&dyn_filter),- Operator::And,- Arc::clone(&wrapper),- ));-- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));- let codec = WrapperCodec;- let converter = DeduplicatingProtoConverter {};-- // Encode, then round-trip through prost bytes to mimic the wire.- let proto = converter.physical_expr_to_proto(&composite, &codec)?;- let bytes = proto.encode_to_vec();- let decoded_proto = PhysicalExprNode::decode(bytes.as_slice()).unwrap();-- let ctx = SessionContext::new();- let task_ctx = ctx.task_ctx();- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);- let decoded =- converter.proto_to_physical_expr(&decoded_proto, &schema, &decode_ctx)?;-- let binary = decoded- .downcast_ref::<BinaryExpr>()- .expect("must decode back to BinaryExpr");- let decoded_left = Arc::clone(binary.left());- let decoded_right = Arc::clone(binary.right());- let decoded_wrapper = decoded_right- .downcast_ref::<WrapperExpr>()- .expect("right side must decode back to WrapperExpr");-- // The load-bearing check: an `update()` on the bare-side filter must be- // observable from the wrapped-side filter, proving both refs back the- // same `Inner`.- assert_dynamic_filter_update_is_visible(&decoded_left, &decoded_wrapper.inner)?;-- Ok(())-}