Skip to content

Apache DataFusion SQL Query Engine

refactor(proto): destructure lambda expression serde hooks (#24775)

Burak Şenf96892a9b5df
diff --git a/datafusion/physical-expr/src/expressions/lambda.rs b/datafusion/physical-expr/src/expressions/lambda.rs
index be1539e208..03e827339b 100644
--- a/datafusion/physical-expr/src/expressions/lambda.rs
+++ b/datafusion/physical-expr/src/expressions/lambda.rs
@@ -168,10 +168,11 @@ impl LambdaExpr {
protobuf::physical_expr_node::ExprType::Lambda,
"LambdaExpr",
);
+ let protobuf::PhysicalLambdaExprNode { params, body } = &**lambda;
Ok(Arc::new(LambdaExpr::try_new(
- lambda.params.clone(),
- ctx.decode_required_expression(lambda.body.as_deref(), "LambdaExpr", "body")?,
+ params.clone(),
+ ctx.decode_required_expression(body.as_deref(), "LambdaExpr", "body")?,
)?))
}
@@ -303,12 +304,24 @@ impl PhysicalExpr for LambdaExpr {
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
use datafusion_proto_models::protobuf;
+ // Exhaustive destructure: adding a field to `LambdaExpr` without
+ // deciding how it is serialized is a compile error, not a silent
+ // round-trip gap.
+ let Self {
+ params,
+ body,
+ // Derived from `params` and `body` by `try_new` on decode.
+ projected_body: _,
+ projection: _,
+ used_param_indices: _,
+ } = self;
+
Ok(Some(protobuf::PhysicalExprNode {
expr_id: None,
expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(
protobuf::PhysicalLambdaExprNode {
- params: self.params().to_vec(),
- body: Some(Box::new(ctx.encode_child(self.body())?)),
+ params: params.clone(),
+ body: Some(Box::new(ctx.encode_child(body)?)),
},
))),
}))
@@ -496,3 +509,148 @@ mod tests {
);
}
}
+
+/// Tests for the `try_to_proto` / `try_from_proto` hooks.
+#[cfg(all(test, feature = "proto"))]
+mod proto_tests {
+ use super::*;
+ use crate::proto_test_util::{
+ StubDecoder, StubEncoder, UnreachableDecoder, column_node,
+ };
+ use arrow::datatypes::Field;
+ use datafusion_common::DataFusionError;
+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+ use datafusion_proto_models::protobuf::{self, PhysicalExprNode};
+
+ /// A one-parameter lambda whose body references that parameter.
+ fn proto_lambda_fixture() -> LambdaExpr {
+ let v_field = Arc::new(Field::new("v", DataType::Int32, true));
+ LambdaExpr::try_new(
+ vec!["v".to_string()],
+ Arc::new(LambdaVariable::new(0, v_field)),
+ )
+ .unwrap()
+ }
+
+ fn proto_lambda_node(
+ params: Vec<String>,
+ body: Option<Box<PhysicalExprNode>>,
+ ) -> PhysicalExprNode {
+ PhysicalExprNode {
+ expr_id: None,
+ expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(
+ protobuf::PhysicalLambdaExprNode { params, body },
+ ))),
+ }
+ }
+
+ #[test]
+ fn try_to_proto_encodes_lambda() {
+ let lambda = proto_lambda_fixture();
+ let encoder = StubEncoder::ok();
+ let ctx = PhysicalExprEncodeCtx::new(&encoder);
+
+ let node = lambda
+ .try_to_proto(&ctx)
+ .unwrap()
+ .expect("LambdaExpr should encode to Some(node)");
+
+ assert!(node.expr_id.is_none());
+ let lambda_node = match node.expr_type {
+ Some(protobuf::physical_expr_node::ExprType::Lambda(boxed)) => *boxed,
+ other => panic!("expected a LambdaExpr node, got {other:?}"),
+ };
+ assert_eq!(lambda_node.params, vec!["v".to_string()]);
+ assert!(lambda_node.body.is_some());
+ }
+
+ #[test]
+ fn try_to_proto_propagates_body_encode_error() {
+ let lambda = proto_lambda_fixture();
+ let encoder = StubEncoder::failing_on(1);
+ let ctx = PhysicalExprEncodeCtx::new(&encoder);
+
+ let err = lambda.try_to_proto(&ctx).unwrap_err();
+ assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
+ }
+
+ /// Decoding goes through `try_new`, which recomputes `projection`,
+ /// `projected_body`, and `used_param_indices` from the decoded body, so
+ /// they need no wire representation.
+ #[test]
+ fn try_from_proto_recomputes_derived_state() {
+ let node =
+ proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));
+ let schema = Schema::empty();
+ let decoder = StubDecoder::ok();
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let decoded = LambdaExpr::try_from_proto(&node, &ctx).unwrap();
+ let lambda = decoded
+ .downcast_ref::<LambdaExpr>()
+ .expect("decoded expr should be a LambdaExpr");
+
+ assert_eq!(lambda.params(), &["v".to_string()]);
+ // The stub decodes the body to a `Column` at index 0: the projection
+ // covers exactly that index, no declared param is referenced, and an
+ // identity projection leaves the body untouched.
+ assert_eq!(lambda.projection(), &[0]);
+ assert!(lambda.used_param_indices().is_empty());
+ assert!(lambda.projected_body().eq(lambda.body()));
+ }
+
+ #[test]
+ fn try_from_proto_rejects_non_lambda_node() {
+ let node = column_node("a");
+ let schema = Schema::empty();
+ let decoder = UnreachableDecoder;
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(
+ matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaExpr"))
+ );
+ }
+
+ #[test]
+ fn try_from_proto_rejects_missing_body() {
+ let node = proto_lambda_node(vec!["v".to_string()], None);
+ let schema = Schema::empty();
+ let decoder = UnreachableDecoder;
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(
+ matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaExpr is missing required field 'body'"))
+ );
+ }
+
+ #[test]
+ fn try_from_proto_rejects_duplicate_params() {
+ let node = proto_lambda_node(
+ vec!["v".to_string(), "v".to_string()],
+ Some(Box::new(column_node("body"))),
+ );
+ let schema = Schema::empty();
+ let decoder = StubDecoder::ok();
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(
+ matches!(err, DataFusionError::Plan(msg) if msg.contains("lambda params must be unique"))
+ );
+ }
+
+ #[test]
+ fn try_from_proto_propagates_body_decode_error() {
+ let node =
+ proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));
+ let schema = Schema::empty();
+ let decoder = StubDecoder::failing_on(1);
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
+ }
+}
diff --git a/datafusion/physical-expr/src/expressions/lambda_variable.rs b/datafusion/physical-expr/src/expressions/lambda_variable.rs
index f7e6910020..195c1837e1 100644
--- a/datafusion/physical-expr/src/expressions/lambda_variable.rs
+++ b/datafusion/physical-expr/src/expressions/lambda_variable.rs
@@ -89,11 +89,12 @@ impl LambdaVariable {
protobuf::physical_expr_node::ExprType::LambdaVariable,
"LambdaVariable",
);
+ let protobuf::PhysicalLambdaVariableExprNode { index, field } = var;
Ok(Arc::new(LambdaVariable::new(
- var.index as usize,
+ *index as usize,
Arc::new(
- require_proto_field(var.field.as_ref(), "LambdaVariable", "field")?
+ require_proto_field(field.as_ref(), "LambdaVariable", "field")?
.try_into()?,
),
)))
@@ -169,12 +170,17 @@ impl PhysicalExpr for LambdaVariable {
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
use datafusion_proto_models::protobuf;
+ // Exhaustive destructure: adding a field to `LambdaVariable` without
+ // deciding how it is serialized is a compile error, not a silent
+ // round-trip gap.
+ let Self { index, field } = self;
+
Ok(Some(protobuf::PhysicalExprNode {
expr_id: None,
expr_type: Some(protobuf::physical_expr_node::ExprType::LambdaVariable(
protobuf::PhysicalLambdaVariableExprNode {
- index: self.index() as u32,
- field: Some(self.field().as_ref().try_into()?),
+ index: *index as u32,
+ field: Some(field.as_ref().try_into()?),
},
)),
}))
@@ -188,3 +194,102 @@ pub fn lambda_variable(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalEx
Ok(Arc::new(LambdaVariable::new(index, field)))
}
+
+/// Tests for the `try_to_proto` / `try_from_proto` hooks.
+#[cfg(all(test, feature = "proto"))]
+mod proto_tests {
+ use super::*;
+ use crate::proto_test_util::{StubEncoder, UnreachableDecoder, column_node};
+ use arrow::datatypes::Field;
+ use datafusion_common::DataFusionError;
+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+ use datafusion_proto_models::protobuf::{self, physical_expr_node};
+
+ fn v_field() -> FieldRef {
+ Arc::new(Field::new("v", DataType::Int32, true))
+ }
+
+ #[test]
+ fn try_to_proto_encodes_lambda_variable() {
+ let var = LambdaVariable::new(3, v_field());
+ // LambdaVariable has no child exprs so the encoder is never called.
+ let encoder = StubEncoder::ok();
+ let ctx = PhysicalExprEncodeCtx::new(&encoder);
+
+ let node = var
+ .try_to_proto(&ctx)
+ .unwrap()
+ .expect("LambdaVariable should encode to Some(node)");
+
+ assert!(node.expr_id.is_none());
+ let protobuf::PhysicalLambdaVariableExprNode { index, field } =
+ match node.expr_type {
+ Some(physical_expr_node::ExprType::LambdaVariable(v)) => v,
+ other => panic!("expected a LambdaVariable node, got {other:?}"),
+ };
+ assert_eq!(index, 3);
+ let field = field.expect("field should be encoded");
+ assert_eq!(field.name, "v");
+ assert!(field.nullable);
+ }
+
+ #[test]
+ fn try_from_proto_rejects_non_lambda_variable_node() {
+ let node = column_node("a");
+ let schema = Schema::empty();
+ let decoder = UnreachableDecoder;
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(
+ matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaVariable"))
+ );
+ }
+
+ #[test]
+ fn try_from_proto_rejects_missing_field() {
+ let node = protobuf::PhysicalExprNode {
+ expr_id: None,
+ expr_type: Some(physical_expr_node::ExprType::LambdaVariable(
+ protobuf::PhysicalLambdaVariableExprNode {
+ index: 0,
+ field: None,
+ },
+ )),
+ };
+ let schema = Schema::empty();
+ let decoder = UnreachableDecoder;
+ let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();
+ assert!(
+ matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaVariable is missing required field 'field'"))
+ );
+ }
+
+ /// `PartialEq` covers every field of `LambdaVariable`, so equality after
+ /// the roundtrip proves the encoding is lossless.
+ #[test]
+ fn lambda_variable_proto_roundtrip() {
+ let var = LambdaVariable::new(5, v_field());
+ let encoder = StubEncoder::ok();
+ let enc_ctx = PhysicalExprEncodeCtx::new(&encoder);
+
+ let node = var
+ .try_to_proto(&enc_ctx)
+ .unwrap()
+ .expect("LambdaVariable should encode to Some(node)");
+
+ let schema = Schema::empty();
+ let decoder = UnreachableDecoder;
+ let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+
+ let decoded = LambdaVariable::try_from_proto(&node, &dec_ctx).unwrap();
+ let decoded = decoded
+ .downcast_ref::<LambdaVariable>()
+ .expect("decoded expr should be a LambdaVariable");
+
+ assert_eq!(decoded, &var);
+ }
+}
diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rs
index 9142e44bee..5fce0231a6 100644
--- a/datafusion/proto/tests/cases/plans/exprs.rs
+++ b/datafusion/proto/tests/cases/plans/exprs.rs
@@ -23,7 +23,7 @@ use arrow::datatypes::Fields;
use datafusion::arrow::compute::SortOptions;
use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};
use datafusion::logical_expr::Operator;
-use datafusion::physical_expr::expressions::Literal;
+use datafusion::physical_expr::expressions::{LambdaVariable, Literal, lambda};
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::expressions::{
BinaryExpr, Column, PhysicalSortExpr, SqlSimilarToPattern, binary, col, like, lit,
@@ -248,6 +248,24 @@ fn roundtrip_sql_similar_to_pattern() -> Result<()> {
Ok(())
}
+#[test]
+fn roundtrip_lambda_with_variable_dispatch() -> Result<()> {
+ let field = Arc::new(Field::new("v", DataType::Int32, true));
+ let expr = lambda(["v"], Arc::new(LambdaVariable::new(0, field)))?;
+
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DefaultPhysicalProtoConverter {};
+ let proto = converter.physical_expr_to_proto(&expr, &codec)?;
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let decoded =
+ converter.proto_to_physical_expr(&proto, &Schema::empty(), &decode_ctx)?;
+
+ assert!(decoded.as_ref().eq(expr.as_ref()));
+ Ok(())
+}
+
#[test]
fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {
let data_type = DataType::Struct(Fields::from(vec![Field::new(