Apache DataFusion SQL Query Engine
fix(proto): prevent logical plan serialization stack overflow (#24124)
Mithun Chicklore Yogendrab225ded73384
diff --git a/Cargo.lock b/Cargo.lockindex 3ddb32f60f..4b4c40791e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2546,6 +2546,7 @@ dependencies = ["object_store","pretty_assertions","prost",+ "recursive","serde_json","tokio",]diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 3144809379..fb3308ae52 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -42,6 +42,7 @@ name = "datafusion_proto"[features]default = ["parquet"]+recursive_protection = ["dep:recursive"]json = ["serde_json","datafusion-proto-common/json",@@ -76,6 +77,7 @@ datafusion-proto-common = { workspace = true }datafusion-proto-models = { workspace = true }object_store = { workspace = true }prost = { workspace = true }+recursive = { workspace = true, optional = true }serde_json = { workspace = true, optional = true }[dev-dependencies]diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex f273b33431..021645036d 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -115,6 +115,29 @@ pub trait AsLogicalPlan: Debug + Send + Sync + Clone {Self: Sized;}+// In debug builds, keep each serializer arm's local temporaries out of the+// recursive dispatcher frame. Without this call boundary, they inflate the+// frame of every recursive invocation.+#[cfg_attr(debug_assertions, inline(never))]+fn serialize_logical_plan_arm<F>(serializer: F) -> Result<LogicalPlanNode>+where+ F: FnOnce() -> Result<LogicalPlanNode>,+{+ serializer()+}++macro_rules! dispatch_logical_plan {+ ($plan:expr, { $($pattern:pat => $body:expr $(,)?)+ }) => {+ match $plan {+ $(+ $pattern => serialize_logical_plan_arm(|| -> Result<LogicalPlanNode> {+ $body+ }),+ )++ }+ };+}+pub trait LogicalExtensionCodec: Debug + Send + Sync + std::any::Any {fn try_decode(&self,@@ -1285,6 +1308,7 @@ impl AsLogicalPlan for LogicalPlanNode {}}+ #[cfg_attr(feature = "recursive_protection", recursive::recursive)]fn try_from_logical_plan(plan: &LogicalPlan,extension_codec: &dyn LogicalExtensionCodec,@@ -1292,7 +1316,7 @@ impl AsLogicalPlan for LogicalPlanNode {whereSelf: Sized,{- match plan {+ dispatch_logical_plan!(plan, {LogicalPlan::Values(Values { values, .. }) => {let n_cols = if values.is_empty() {0@@ -2214,6 +2238,6 @@ impl AsLogicalPlan for LogicalPlanNode {))),})}- }+ })}}diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rsindex 7a95ee0c29..430b1405b1 100644--- a/datafusion/proto/tests/cases/mod.rs+++ b/datafusion/proto/tests/cases/mod.rs@@ -35,6 +35,7 @@ use std::sync::Arc;mod roundtrip_logical_plan;mod roundtrip_physical_plan;mod serialize;+mod stack_safety;#[derive(Debug, PartialEq, Eq, Hash)]struct MyRegexUdf {diff --git a/datafusion/proto/tests/cases/stack_safety.rs b/datafusion/proto/tests/cases/stack_safety.rsnew file mode 100644index 0000000000..5caf4119a7--- /dev/null+++ b/datafusion/proto/tests/cases/stack_safety.rs@@ -0,0 +1,99 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::process::Command;+use std::sync::Arc;++use datafusion_common::DFSchema;+use datafusion_expr::logical_plan::{EmptyRelation, LogicalPlan, LogicalPlanBuilder};+use datafusion_proto::bytes::logical_plan_to_bytes;++const CHILD_ENV: &str = "DATAFUSION_PROTO_ISSUE_23823_CHILD";+const ALIAS_DEPTH_ENV: &str = "DATAFUSION_PROTO_ISSUE_23823_ALIAS_DEPTH";+const TWO_MIB_TEST_NAME: &str =+ "cases::stack_safety::logical_plan_serialization_fits_a_two_mib_stack";+#[cfg(feature = "recursive_protection")]+const GROWABLE_STACK_TEST_NAME: &str =+ "cases::stack_safety::deeply_nested_logical_plan_serialization_uses_a_growable_stack";++fn deeply_aliased_plan(alias_depth: usize) -> LogicalPlan {+ let mut plan = LogicalPlan::EmptyRelation(EmptyRelation {+ produce_one_row: false,+ schema: Arc::new(DFSchema::empty()),+ });++ for level in 0..alias_depth {+ plan = LogicalPlanBuilder::from(plan)+ .alias(format!("level_{level}"))+ .unwrap()+ .build()+ .unwrap();+ }++ plan+}++fn serialize_on_two_mib_stack(alias_depth: usize) {+ let plan = deeply_aliased_plan(alias_depth);+ std::thread::Builder::new()+ .name("two-megabyte-stack".into())+ .stack_size(2 * 1024 * 1024)+ .spawn(move || logical_plan_to_bytes(&plan).unwrap())+ .unwrap()+ .join()+ .unwrap();+}++fn run_in_child(test_name: &str, alias_depth: usize) {+ if std::env::var_os(CHILD_ENV).is_some() {+ let alias_depth = std::env::var(ALIAS_DEPTH_ENV).unwrap().parse().unwrap();+ serialize_on_two_mib_stack(alias_depth);+ return;+ }++ // A native stack overflow aborts the process. Re-run this exact test in a+ // child process so a regression produces a normal test failure.+ let output = Command::new(std::env::current_exe().unwrap())+ .args(["--exact", test_name, "--nocapture"])+ .env(CHILD_ENV, "1")+ .env(ALIAS_DEPTH_ENV, alias_depth.to_string())+ .output()+ .unwrap();++ assert!(+ output.status.success(),+ "child process failed with status {}\nstdout:\n{}\nstderr:\n{}",+ output.status,+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr),+ );+}++#[test]+fn logical_plan_serialization_fits_a_two_mib_stack() {+ // Ten aliases reproduce #23823. Use 100 to provide a safety margin while+ // verifying the dispatcher reduction without runtime stack growth.+ run_in_child(TWO_MIB_TEST_NAME, 100);+}++#[cfg(feature = "recursive_protection")]+#[test]+fn deeply_nested_logical_plan_serialization_uses_a_growable_stack() {+ // This depth exceeds the 2 MiB thread stack without recursive protection,+ // exercising the `recursive` stack-growth checkpoint.+ run_in_child(GROWABLE_STACK_TEST_NAME, 2_000);+}