Skip to content

Apache DataFusion SQL Query Engine

fix(proto): prevent logical plan serialization stack overflow (#24124)

Mithun Chicklore Yogendrab225ded73384
diff --git a/Cargo.lock b/Cargo.lock
index 3ddb32f60f..4b4c40791e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2546,6 +2546,7 @@ dependencies = [
"object_store",
"pretty_assertions",
"prost",
+ "recursive",
"serde_json",
"tokio",
]
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index 3144809379..fb3308ae52 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -42,6 +42,7 @@ name = "datafusion_proto"
[features]
default = ["parquet"]
+recursive_protection = ["dep:recursive"]
json = [
"serde_json",
"datafusion-proto-common/json",
@@ -76,6 +77,7 @@ datafusion-proto-common = { workspace = true }
datafusion-proto-models = { workspace = true }
object_store = { workspace = true }
prost = { workspace = true }
+recursive = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
[dev-dependencies]
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index f273b33431..021645036d 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -115,6 +115,29 @@ pub trait AsLogicalPlan: Debug + Send + Sync + Clone {
Self: Sized;
}
+// In debug builds, keep each serializer arm's local temporaries out of the
+// recursive dispatcher frame. Without this call boundary, they inflate the
+// frame of every recursive invocation.
+#[cfg_attr(debug_assertions, inline(never))]
+fn serialize_logical_plan_arm<F>(serializer: F) -> Result<LogicalPlanNode>
+where
+ F: FnOnce() -> Result<LogicalPlanNode>,
+{
+ serializer()
+}
+
+macro_rules! dispatch_logical_plan {
+ ($plan:expr, { $($pattern:pat => $body:expr $(,)?)+ }) => {
+ match $plan {
+ $(
+ $pattern => serialize_logical_plan_arm(|| -> Result<LogicalPlanNode> {
+ $body
+ }),
+ )+
+ }
+ };
+}
+
pub trait LogicalExtensionCodec: Debug + Send + Sync + std::any::Any {
fn try_decode(
&self,
@@ -1285,6 +1308,7 @@ impl AsLogicalPlan for LogicalPlanNode {
}
}
+ #[cfg_attr(feature = "recursive_protection", recursive::recursive)]
fn try_from_logical_plan(
plan: &LogicalPlan,
extension_codec: &dyn LogicalExtensionCodec,
@@ -1292,7 +1316,7 @@ impl AsLogicalPlan for LogicalPlanNode {
where
Self: Sized,
{
- match plan {
+ dispatch_logical_plan!(plan, {
LogicalPlan::Values(Values { values, .. }) => {
let n_cols = if values.is_empty() {
0
@@ -2214,6 +2238,6 @@ impl AsLogicalPlan for LogicalPlanNode {
))),
})
}
- }
+ })
}
}
diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rs
index 7a95ee0c29..430b1405b1 100644
--- a/datafusion/proto/tests/cases/mod.rs
+++ b/datafusion/proto/tests/cases/mod.rs
@@ -35,6 +35,7 @@ use std::sync::Arc;
mod roundtrip_logical_plan;
mod roundtrip_physical_plan;
mod serialize;
+mod stack_safety;
#[derive(Debug, PartialEq, Eq, Hash)]
struct MyRegexUdf {
diff --git a/datafusion/proto/tests/cases/stack_safety.rs b/datafusion/proto/tests/cases/stack_safety.rs
new file mode 100644
index 0000000000..5caf4119a7
--- /dev/null
+++ b/datafusion/proto/tests/cases/stack_safety.rs
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::process::Command;
+use std::sync::Arc;
+
+use datafusion_common::DFSchema;
+use datafusion_expr::logical_plan::{EmptyRelation, LogicalPlan, LogicalPlanBuilder};
+use datafusion_proto::bytes::logical_plan_to_bytes;
+
+const CHILD_ENV: &str = "DATAFUSION_PROTO_ISSUE_23823_CHILD";
+const ALIAS_DEPTH_ENV: &str = "DATAFUSION_PROTO_ISSUE_23823_ALIAS_DEPTH";
+const TWO_MIB_TEST_NAME: &str =
+ "cases::stack_safety::logical_plan_serialization_fits_a_two_mib_stack";
+#[cfg(feature = "recursive_protection")]
+const GROWABLE_STACK_TEST_NAME: &str =
+ "cases::stack_safety::deeply_nested_logical_plan_serialization_uses_a_growable_stack";
+
+fn deeply_aliased_plan(alias_depth: usize) -> LogicalPlan {
+ let mut plan = LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row: false,
+ schema: Arc::new(DFSchema::empty()),
+ });
+
+ for level in 0..alias_depth {
+ plan = LogicalPlanBuilder::from(plan)
+ .alias(format!("level_{level}"))
+ .unwrap()
+ .build()
+ .unwrap();
+ }
+
+ plan
+}
+
+fn serialize_on_two_mib_stack(alias_depth: usize) {
+ let plan = deeply_aliased_plan(alias_depth);
+ std::thread::Builder::new()
+ .name("two-megabyte-stack".into())
+ .stack_size(2 * 1024 * 1024)
+ .spawn(move || logical_plan_to_bytes(&plan).unwrap())
+ .unwrap()
+ .join()
+ .unwrap();
+}
+
+fn run_in_child(test_name: &str, alias_depth: usize) {
+ if std::env::var_os(CHILD_ENV).is_some() {
+ let alias_depth = std::env::var(ALIAS_DEPTH_ENV).unwrap().parse().unwrap();
+ serialize_on_two_mib_stack(alias_depth);
+ return;
+ }
+
+ // A native stack overflow aborts the process. Re-run this exact test in a
+ // child process so a regression produces a normal test failure.
+ let output = Command::new(std::env::current_exe().unwrap())
+ .args(["--exact", test_name, "--nocapture"])
+ .env(CHILD_ENV, "1")
+ .env(ALIAS_DEPTH_ENV, alias_depth.to_string())
+ .output()
+ .unwrap();
+
+ assert!(
+ output.status.success(),
+ "child process failed with status {}\nstdout:\n{}\nstderr:\n{}",
+ output.status,
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr),
+ );
+}
+
+#[test]
+fn logical_plan_serialization_fits_a_two_mib_stack() {
+ // Ten aliases reproduce #23823. Use 100 to provide a safety margin while
+ // verifying the dispatcher reduction without runtime stack growth.
+ run_in_child(TWO_MIB_TEST_NAME, 100);
+}
+
+#[cfg(feature = "recursive_protection")]
+#[test]
+fn deeply_nested_logical_plan_serialization_uses_a_growable_stack() {
+ // This depth exceeds the 2 MiB thread stack without recursive protection,
+ // exercising the `recursive` stack-growth checkpoint.
+ run_in_child(GROWABLE_STACK_TEST_NAME, 2_000);
+}