Apache DataFusion SQL Query Engine
refactor(proto): destructure lambda expression serde hooks (#24775)
Burak Şenf96892a9b5df
diff --git a/datafusion/physical-expr/src/expressions/lambda.rs b/datafusion/physical-expr/src/expressions/lambda.rsindex be1539e208..03e827339b 100644--- a/datafusion/physical-expr/src/expressions/lambda.rs+++ b/datafusion/physical-expr/src/expressions/lambda.rs@@ -168,10 +168,11 @@ impl LambdaExpr {protobuf::physical_expr_node::ExprType::Lambda,"LambdaExpr",);+ let protobuf::PhysicalLambdaExprNode { params, body } = &**lambda;Ok(Arc::new(LambdaExpr::try_new(- lambda.params.clone(),- ctx.decode_required_expression(lambda.body.as_deref(), "LambdaExpr", "body")?,+ params.clone(),+ ctx.decode_required_expression(body.as_deref(), "LambdaExpr", "body")?,)?))}@@ -303,12 +304,24 @@ impl PhysicalExpr for LambdaExpr {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {use datafusion_proto_models::protobuf;+ // Exhaustive destructure: adding a field to `LambdaExpr` without+ // deciding how it is serialized is a compile error, not a silent+ // round-trip gap.+ let Self {+ params,+ body,+ // Derived from `params` and `body` by `try_new` on decode.+ projected_body: _,+ projection: _,+ used_param_indices: _,+ } = self;+Ok(Some(protobuf::PhysicalExprNode {expr_id: None,expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(protobuf::PhysicalLambdaExprNode {- params: self.params().to_vec(),- body: Some(Box::new(ctx.encode_child(self.body())?)),+ params: params.clone(),+ body: Some(Box::new(ctx.encode_child(body)?)),},))),}))@@ -496,3 +509,148 @@ mod tests {);}}++/// Tests for the `try_to_proto` / `try_from_proto` hooks.+#[cfg(all(test, feature = "proto"))]+mod proto_tests {+ use super::*;+ use crate::proto_test_util::{+ StubDecoder, StubEncoder, UnreachableDecoder, column_node,+ };+ use arrow::datatypes::Field;+ use datafusion_common::DataFusionError;+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_proto_models::protobuf::{self, PhysicalExprNode};++ /// A one-parameter lambda whose body references that parameter.+ fn proto_lambda_fixture() -> LambdaExpr {+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));+ LambdaExpr::try_new(+ vec!["v".to_string()],+ Arc::new(LambdaVariable::new(0, v_field)),+ )+ .unwrap()+ }++ fn proto_lambda_node(+ params: Vec<String>,+ body: Option<Box<PhysicalExprNode>>,+ ) -> PhysicalExprNode {+ PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(+ protobuf::PhysicalLambdaExprNode { params, body },+ ))),+ }+ }++ #[test]+ fn try_to_proto_encodes_lambda() {+ let lambda = proto_lambda_fixture();+ let encoder = StubEncoder::ok();+ let ctx = PhysicalExprEncodeCtx::new(&encoder);++ let node = lambda+ .try_to_proto(&ctx)+ .unwrap()+ .expect("LambdaExpr should encode to Some(node)");++ assert!(node.expr_id.is_none());+ let lambda_node = match node.expr_type {+ Some(protobuf::physical_expr_node::ExprType::Lambda(boxed)) => *boxed,+ other => panic!("expected a LambdaExpr node, got {other:?}"),+ };+ assert_eq!(lambda_node.params, vec!["v".to_string()]);+ assert!(lambda_node.body.is_some());+ }++ #[test]+ fn try_to_proto_propagates_body_encode_error() {+ let lambda = proto_lambda_fixture();+ let encoder = StubEncoder::failing_on(1);+ let ctx = PhysicalExprEncodeCtx::new(&encoder);++ let err = lambda.try_to_proto(&ctx).unwrap_err();+ assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));+ }++ /// Decoding goes through `try_new`, which recomputes `projection`,+ /// `projected_body`, and `used_param_indices` from the decoded body, so+ /// they need no wire representation.+ #[test]+ fn try_from_proto_recomputes_derived_state() {+ let node =+ proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));+ let schema = Schema::empty();+ let decoder = StubDecoder::ok();+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let decoded = LambdaExpr::try_from_proto(&node, &ctx).unwrap();+ let lambda = decoded+ .downcast_ref::<LambdaExpr>()+ .expect("decoded expr should be a LambdaExpr");++ assert_eq!(lambda.params(), &["v".to_string()]);+ // The stub decodes the body to a `Column` at index 0: the projection+ // covers exactly that index, no declared param is referenced, and an+ // identity projection leaves the body untouched.+ assert_eq!(lambda.projection(), &[0]);+ assert!(lambda.used_param_indices().is_empty());+ assert!(lambda.projected_body().eq(lambda.body()));+ }++ #[test]+ fn try_from_proto_rejects_non_lambda_node() {+ let node = column_node("a");+ let schema = Schema::empty();+ let decoder = UnreachableDecoder;+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();+ assert!(+ matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaExpr"))+ );+ }++ #[test]+ fn try_from_proto_rejects_missing_body() {+ let node = proto_lambda_node(vec!["v".to_string()], None);+ let schema = Schema::empty();+ let decoder = UnreachableDecoder;+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();+ assert!(+ matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaExpr is missing required field 'body'"))+ );+ }++ #[test]+ fn try_from_proto_rejects_duplicate_params() {+ let node = proto_lambda_node(+ vec!["v".to_string(), "v".to_string()],+ Some(Box::new(column_node("body"))),+ );+ let schema = Schema::empty();+ let decoder = StubDecoder::ok();+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();+ assert!(+ matches!(err, DataFusionError::Plan(msg) if msg.contains("lambda params must be unique"))+ );+ }++ #[test]+ fn try_from_proto_propagates_body_decode_error() {+ let node =+ proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));+ let schema = Schema::empty();+ let decoder = StubDecoder::failing_on(1);+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();+ assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));+ }+}diff --git a/datafusion/physical-expr/src/expressions/lambda_variable.rs b/datafusion/physical-expr/src/expressions/lambda_variable.rsindex f7e6910020..195c1837e1 100644--- a/datafusion/physical-expr/src/expressions/lambda_variable.rs+++ b/datafusion/physical-expr/src/expressions/lambda_variable.rs@@ -89,11 +89,12 @@ impl LambdaVariable {protobuf::physical_expr_node::ExprType::LambdaVariable,"LambdaVariable",);+ let protobuf::PhysicalLambdaVariableExprNode { index, field } = var;Ok(Arc::new(LambdaVariable::new(- var.index as usize,+ *index as usize,Arc::new(- require_proto_field(var.field.as_ref(), "LambdaVariable", "field")?+ require_proto_field(field.as_ref(), "LambdaVariable", "field")?.try_into()?,),)))@@ -169,12 +170,17 @@ impl PhysicalExpr for LambdaVariable {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {use datafusion_proto_models::protobuf;+ // Exhaustive destructure: adding a field to `LambdaVariable` without+ // deciding how it is serialized is a compile error, not a silent+ // round-trip gap.+ let Self { index, field } = self;+Ok(Some(protobuf::PhysicalExprNode {expr_id: None,expr_type: Some(protobuf::physical_expr_node::ExprType::LambdaVariable(protobuf::PhysicalLambdaVariableExprNode {- index: self.index() as u32,- field: Some(self.field().as_ref().try_into()?),+ index: *index as u32,+ field: Some(field.as_ref().try_into()?),},)),}))@@ -188,3 +194,102 @@ pub fn lambda_variable(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalExOk(Arc::new(LambdaVariable::new(index, field)))}++/// Tests for the `try_to_proto` / `try_from_proto` hooks.+#[cfg(all(test, feature = "proto"))]+mod proto_tests {+ use super::*;+ use crate::proto_test_util::{StubEncoder, UnreachableDecoder, column_node};+ use arrow::datatypes::Field;+ use datafusion_common::DataFusionError;+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_proto_models::protobuf::{self, physical_expr_node};++ fn v_field() -> FieldRef {+ Arc::new(Field::new("v", DataType::Int32, true))+ }++ #[test]+ fn try_to_proto_encodes_lambda_variable() {+ let var = LambdaVariable::new(3, v_field());+ // LambdaVariable has no child exprs so the encoder is never called.+ let encoder = StubEncoder::ok();+ let ctx = PhysicalExprEncodeCtx::new(&encoder);++ let node = var+ .try_to_proto(&ctx)+ .unwrap()+ .expect("LambdaVariable should encode to Some(node)");++ assert!(node.expr_id.is_none());+ let protobuf::PhysicalLambdaVariableExprNode { index, field } =+ match node.expr_type {+ Some(physical_expr_node::ExprType::LambdaVariable(v)) => v,+ other => panic!("expected a LambdaVariable node, got {other:?}"),+ };+ assert_eq!(index, 3);+ let field = field.expect("field should be encoded");+ assert_eq!(field.name, "v");+ assert!(field.nullable);+ }++ #[test]+ fn try_from_proto_rejects_non_lambda_variable_node() {+ let node = column_node("a");+ let schema = Schema::empty();+ let decoder = UnreachableDecoder;+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();+ assert!(+ matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaVariable"))+ );+ }++ #[test]+ fn try_from_proto_rejects_missing_field() {+ let node = protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(physical_expr_node::ExprType::LambdaVariable(+ protobuf::PhysicalLambdaVariableExprNode {+ index: 0,+ field: None,+ },+ )),+ };+ let schema = Schema::empty();+ let decoder = UnreachableDecoder;+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();+ assert!(+ matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaVariable is missing required field 'field'"))+ );+ }++ /// `PartialEq` covers every field of `LambdaVariable`, so equality after+ /// the roundtrip proves the encoding is lossless.+ #[test]+ fn lambda_variable_proto_roundtrip() {+ let var = LambdaVariable::new(5, v_field());+ let encoder = StubEncoder::ok();+ let enc_ctx = PhysicalExprEncodeCtx::new(&encoder);++ let node = var+ .try_to_proto(&enc_ctx)+ .unwrap()+ .expect("LambdaVariable should encode to Some(node)");++ let schema = Schema::empty();+ let decoder = UnreachableDecoder;+ let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let decoded = LambdaVariable::try_from_proto(&node, &dec_ctx).unwrap();+ let decoded = decoded+ .downcast_ref::<LambdaVariable>()+ .expect("decoded expr should be a LambdaVariable");++ assert_eq!(decoded, &var);+ }+}diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rsindex 9142e44bee..5fce0231a6 100644--- a/datafusion/proto/tests/cases/plans/exprs.rs+++ b/datafusion/proto/tests/cases/plans/exprs.rs@@ -23,7 +23,7 @@ use arrow::datatypes::Fields;use datafusion::arrow::compute::SortOptions;use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};use datafusion::logical_expr::Operator;-use datafusion::physical_expr::expressions::Literal;+use datafusion::physical_expr::expressions::{LambdaVariable, Literal, lambda};use datafusion::physical_plan::empty::EmptyExec;use datafusion::physical_plan::expressions::{BinaryExpr, Column, PhysicalSortExpr, SqlSimilarToPattern, binary, col, like, lit,@@ -248,6 +248,24 @@ fn roundtrip_sql_similar_to_pattern() -> Result<()> {Ok(())}+#[test]+fn roundtrip_lambda_with_variable_dispatch() -> Result<()> {+ let field = Arc::new(Field::new("v", DataType::Int32, true));+ let expr = lambda(["v"], Arc::new(LambdaVariable::new(0, field)))?;++ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DefaultPhysicalProtoConverter {};+ let proto = converter.physical_expr_to_proto(&expr, &codec)?;+ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let decoded =+ converter.proto_to_physical_expr(&proto, &Schema::empty(), &decode_ctx)?;++ assert!(decoded.as_ref().eq(expr.as_ref()));+ Ok(())+}+#[test]fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {let data_type = DataType::Struct(Fields::from(vec![Field::new(