Apache DataFusion SQL Query Engine
refactor: Migrate ScalarSubqueryExpr to self-serialization proto pattern (#23130)
Matthew Patton360a56dfe48e
diff --git a/datafusion/physical-expr/src/scalar_subquery.rs b/datafusion/physical-expr/src/scalar_subquery.rsindex ea00847151..f7270f8054 100644--- a/datafusion/physical-expr/src/scalar_subquery.rs+++ b/datafusion/physical-expr/src/scalar_subquery.rs@@ -59,22 +59,34 @@ impl ScalarSubqueryExpr {}}+ pub fn results(&self) -> &ScalarSubqueryResults {+ &self.results+ }++ #[deprecated(+ since = "55.0.0",+ note = "was only used for proto serialization, which no longer needs it; use `return_field` for type/nullability. It will be removed in 61.0.0 or 6 months after 55.0.0 is released, whichever is longer."+ )]pub fn data_type(&self) -> &DataType {&self.data_type}+ #[deprecated(+ since = "55.0.0",+ note = "was only used for proto serialization, which no longer needs it; use `return_field` for type/nullability. It will be removed in 61.0.0 or 6 months after 55.0.0 is released, whichever is longer."+ )]pub fn nullable(&self) -> bool {self.nullable}/// Returns the index of this subquery in the shared results container.+ #[deprecated(+ since = "55.0.0",+ note = "was only used for proto serialization, which no longer needs it. It will be removed in 61.0.0 or 6 months after 55.0.0 is released, whichever is longer."+ )]pub fn index(&self) -> SubqueryIndex {self.index}-- pub fn results(&self) -> &ScalarSubqueryResults {- &self.results- }}impl fmt::Display for ScalarSubqueryExpr {@@ -139,6 +151,69 @@ impl PhysicalExpr for ScalarSubqueryExpr {fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {write!(f, "(scalar subquery)")}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;+ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(+ protobuf::PhysicalScalarSubqueryExprNode {+ data_type: Some((&self.data_type).try_into()?),+ nullable: self.nullable,+ index: u32::try_from(self.index.as_usize()).map_err(|_| {+ internal_datafusion_err!(+ "scalar subquery index {} does not fit in u32",+ self.index.as_usize()+ )+ })?,+ },+ )),+ }))+ }+}++#[cfg(feature = "proto")]+impl ScalarSubqueryExpr {+ /// Reconstruct a [`ScalarSubqueryExpr`] from its protobuf representation.+ ///+ /// Unlike other expressions, this takes a third argument: the shared+ /// [`ScalarSubqueryResults`] container. That container is a runtime-only+ /// `Arc` shared with the surrounding `ScalarSubqueryExec` and is not part of+ /// the wire format, so it cannot be reconstructed here or carried on the+ /// decode context (which lives in a crate that cannot depend on+ /// `datafusion-expr`). The match arm in `from_proto.rs` fetches it from the+ /// plan-level decode context and passes it in.+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalExprNode,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ results: &ScalarSubqueryResults,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_physical_expr_common::expect_expr_variant;+ use datafusion_physical_expr_common::physical_expr::proto_decode::require_proto_field;+ use datafusion_proto_models::protobuf;++ let sq = expect_expr_variant!(+ node,+ protobuf::physical_expr_node::ExprType::ScalarSubquery,+ "ScalarSubqueryExpr",+ );+ let data_type = require_proto_field(+ sq.data_type.as_ref(),+ "ScalarSubqueryExpr",+ "data_type",+ )?+ .try_into()?;+ Ok(Arc::new(ScalarSubqueryExpr::new(+ data_type,+ sq.nullable,+ SubqueryIndex::new(sq.index as usize),+ results.clone(),+ )))+ }}#[cfg(test)]@@ -238,3 +313,123 @@ mod tests {assert_ne!(e1a, e3);}}++/// Tests for the `try_to_proto` / `try_from_proto` hooks.+#[cfg(all(test, feature = "proto"))]+mod proto_tests {+ use super::*;+ use crate::proto_test_util::{StubEncoder, UnreachableDecoder, column_node};+ use datafusion_common::DataFusionError;+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_proto_models::protobuf::{+ PhysicalExprNode, PhysicalScalarSubqueryExprNode, physical_expr_node,+ };++ /// Build a `ScalarSubquery` proto node directly, with control over each+ /// field, so the decode error paths can be exercised independently.+ fn proto_scalar_subquery_node(+ data_type: Option<datafusion_proto_models::datafusion_common::ArrowType>,+ nullable: bool,+ index: u32,+ ) -> PhysicalExprNode {+ PhysicalExprNode {+ expr_id: None,+ expr_type: Some(physical_expr_node::ExprType::ScalarSubquery(+ PhysicalScalarSubqueryExprNode {+ data_type,+ nullable,+ index,+ },+ )),+ }+ }++ #[test]+ fn round_trips_through_proto() {+ // A three-slot results container so index 2 is meaningful.+ let results = ScalarSubqueryResults::new(3);+ let expr = ScalarSubqueryExpr::new(+ DataType::Int32,+ true,+ SubqueryIndex::new(2),+ results.clone(),+ );++ // Encode: the expression serializes itself via try_to_proto.+ let encoder = StubEncoder::ok();+ let enc_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let node = expr+ .try_to_proto(&enc_ctx)+ .unwrap()+ .expect("ScalarSubqueryExpr should encode to Some(node)");++ assert!(node.expr_id.is_none());+ let sq = match &node.expr_type {+ Some(physical_expr_node::ExprType::ScalarSubquery(sq)) => sq,+ other => panic!("expected a ScalarSubquery node, got {other:?}"),+ };+ assert!(sq.nullable);+ assert_eq!(sq.index, 2);+ let encoded_type: DataType = sq+ .data_type+ .as_ref()+ .expect("data_type encoded")+ .try_into()+ .unwrap();+ assert_eq!(encoded_type, DataType::Int32);++ // Decode: reconstruct from the proto node, threading in the shared+ // results container the surrounding exec would provide.+ let decoder = UnreachableDecoder;+ let schema = Schema::empty();+ let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);+ let decoded =+ ScalarSubqueryExpr::try_from_proto(&node, &dec_ctx, &results).unwrap();+ let decoded = decoded+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("decoded expr should be a ScalarSubqueryExpr");++ // data_type + nullable survive the round-trip (observed via return_field).+ let field = decoded.return_field(&Schema::empty()).unwrap();+ assert_eq!(field.data_type(), &DataType::Int32);+ assert!(field.is_nullable());++ // Same shared container + same index → equal to the original.+ assert_eq!(decoded, &expr);+ }++ #[test]+ fn rejects_non_scalar_subquery_node() {+ let node = column_node("a");+ let results = ScalarSubqueryResults::new(1);+ let decoder = UnreachableDecoder;+ let schema = Schema::empty();+ let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err =+ ScalarSubqueryExpr::try_from_proto(&node, &dec_ctx, &results).unwrap_err();+ assert!(matches!(+ err,+ DataFusionError::Internal(msg)+ if msg.contains("PhysicalExprNode is not a ScalarSubqueryExpr")+ ));+ }++ #[test]+ fn rejects_missing_data_type() {+ let node = proto_scalar_subquery_node(None, false, 0);+ let results = ScalarSubqueryResults::new(1);+ let decoder = UnreachableDecoder;+ let schema = Schema::empty();+ let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ let err =+ ScalarSubqueryExpr::try_from_proto(&node, &dec_ctx, &results).unwrap_err();+ assert!(matches!(+ err,+ DataFusionError::Internal(msg)+ if msg.contains("ScalarSubqueryExpr is missing required field 'data_type'")+ ));+ }+}diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex b908b504bb..d003d6db12 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -42,7 +42,6 @@ use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::WindowFunctionDefinition;use datafusion_expr::dml::InsertOp;-use datafusion_expr::execution_props::SubqueryIndex;use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;@@ -366,26 +365,14 @@ pub fn parse_physical_expr_with_converter(}ExprType::LikeExpr(_) => LikeExpr::try_from_proto(proto, &decode_ctx)?,ExprType::HashExpr(_) => HashExpr::try_from_proto(proto, &decode_ctx)?,- ExprType::ScalarSubquery(sq) => {- let data_type: arrow::datatypes::DataType = sq- .data_type- .as_ref()- .ok_or_else(|| {- proto_error("Missing data_type in PhysicalScalarSubqueryExprNode")- })?- .try_into()?;+ ExprType::ScalarSubquery(_) => {let results = ctx.scalar_subquery_results().ok_or_else(|| {proto_error("ScalarSubqueryExpr can only be deserialized as part \of a surrounding ScalarSubqueryExec",)})?;- Arc::new(ScalarSubqueryExpr::new(- data_type,- sq.nullable,- SubqueryIndex::new(sq.index as usize),- results.clone(),- ))+ ScalarSubqueryExpr::try_from_proto(proto, &decode_ctx, results)?}ExprType::DynamicFilter(_) => {DynamicFilterPhysicalExpr::try_from_proto(proto, &decode_ctx)?diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 4025b580e8..cade397c3b 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -31,7 +31,6 @@ use datafusion_datasource_json::file_format::JsonSink;#[cfg(feature = "parquet")]use datafusion_datasource_parquet::file_format::ParquetSink;use datafusion_expr::WindowFrame;-use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion_physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;@@ -331,17 +330,6 @@ pub fn serialize_physical_expr_with_converter(},)),})- } else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {- Ok(protobuf::PhysicalExprNode {- expr_id,- expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(- protobuf::PhysicalScalarSubqueryExprNode {- data_type: Some(expr.data_type().try_into()?),- nullable: expr.nullable(),- index: expr.index().as_usize() as u32,- },- )),- })} else {let mut buf: Vec<u8> = vec![];match codec.try_encode_expr(value, &mut buf) {