Skip to content

Apache DataFusion SQL Query Engine

Reduce datafusion-spark dependency footprint (#24351)

dentiny84a8e6db91d1
diff --git a/Cargo.lock b/Cargo.lock
index 4cb6117da3..5365be12a1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2623,14 +2623,13 @@ dependencies = [
"crc32fast",
"criterion",
"datafusion",
- "datafusion-catalog",
"datafusion-common",
- "datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
"datafusion-functions-nested",
+ "datafusion-session",
"log",
"num-traits",
"percent-encoding",
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 93987b553f..01fd9ec9b3 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -49,14 +49,13 @@ chrono = { workspace = true }
crc32fast = "1.4"
# Optional dependency for SessionStateBuilderSpark extension trait
datafusion = { workspace = true, optional = true, default-features = false }
-datafusion-catalog = { workspace = true }
datafusion-common = { workspace = true }
-datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
-datafusion-functions = { workspace = true, features = ["crypto_expressions"] }
+datafusion-functions = { workspace = true }
datafusion-functions-aggregate = { workspace = true }
datafusion-functions-aggregate-common = { workspace = true }
datafusion-functions-nested = { workspace = true }
+datafusion-session = { workspace = true }
log = { workspace = true }
num-traits = { workspace = true }
percent-encoding = "2.3.2"
diff --git a/datafusion/spark/src/function/string/quote.rs b/datafusion/spark/src/function/string/quote.rs
index 39ad8bf841..55d827fb7a 100644
--- a/datafusion/spark/src/function/string/quote.rs
+++ b/datafusion/spark/src/function/string/quote.rs
@@ -17,12 +17,14 @@
use arrow::array::{ArrayRef, OffsetSizeTrait, StringArray};
use arrow::datatypes::DataType;
-use datafusion::logical_expr::{Coercion, ColumnarValue, Signature, TypeSignatureClass};
use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
use datafusion_common::types::{NativeType, logical_string};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, exec_err};
-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Volatility};
+use datafusion_expr::{
+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ TypeSignatureClass, Volatility,
+};
use datafusion_functions::utils::make_scalar_function;
use std::sync::Arc;
diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs
index aba7b7ceb7..5ccf8d91df 100644
--- a/datafusion/spark/src/function/table/mod.rs
+++ b/datafusion/spark/src/function/table/mod.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use datafusion_catalog::TableFunction;
+use datafusion_session::TableFunction;
use std::sync::Arc;
pub mod expr_fn {}
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index 6cd4678da7..c75f94142f 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -42,7 +42,7 @@
//! Spark functions taking priority.
//!
//! ```
-//! # use datafusion_execution::FunctionRegistry;
+//! # use datafusion_expr::registry::FunctionRegistry;
//! # use datafusion_expr::{ScalarUDF, AggregateUDF, WindowUDF, HigherOrderUDF};
//! # use datafusion_expr::planner::ExprPlanner;
//! # use datafusion_common::Result;
@@ -137,10 +137,10 @@ mod session_state;
#[cfg(feature = "core")]
pub use session_state::SessionStateBuilderSpark;
-use datafusion_catalog::TableFunction;
use datafusion_common::Result;
-use datafusion_execution::FunctionRegistry;
+use datafusion_expr::registry::FunctionRegistry;
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use datafusion_session::TableFunction;
use log::debug;
use std::sync::Arc;