Apache DataFusion SQL Query Engine
feat(cli): add opt-in Spark function registration (#24179)
Goutam Adwant571477c4a7a5
diff --git a/Cargo.lock b/Cargo.lockindex 5365be12a1..d7576659f0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1870,6 +1870,7 @@ dependencies = ["ctor","datafusion","datafusion-common",+ "datafusion-spark","dirs","env_logger","futures",diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.tomlindex 62eedafe79..50b4002541 100644--- a/datafusion-cli/Cargo.toml+++ b/datafusion-cli/Cargo.toml@@ -56,6 +56,7 @@ datafusion = { workspace = true, features = ["unicode_expressions",] }datafusion-common = { workspace = true }+datafusion-spark = { workspace = true, features = ["core"] }dirs = "6.0.0"env_logger = { workspace = true }futures = { workspace = true }diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex 20a2537d7c..4be6058aef 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -23,6 +23,7 @@ use std::process::ExitCode;use std::sync::{Arc, LazyLock};use datafusion::error::{DataFusionError, Result};+use datafusion::execution::SessionStateBuilder;use datafusion::execution::context::SessionConfig;use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool,@@ -46,9 +47,11 @@ use datafusion_cli::{};use clap::Parser;+use datafusion::common::config::Dialect;use datafusion::common::config_err;use datafusion::config::ConfigOptions;use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};+use datafusion_spark::SessionStateBuilderSpark;use mimalloc::MiMalloc;#[global_allocator]@@ -143,6 +146,12 @@ struct Args {#[clap(long, help = "Enables console syntax highlighting")]color: bool,+ #[clap(+ long,+ help = "Enable Apache Spark-compatible SQL dialect, functions, and expression planning"+ )]+ spark: bool,+#[clap(short = 'd',long,@@ -243,9 +252,16 @@ async fn main_inner() -> Result<()> {let runtime_env = rt_builder.build_arc()?;+ let mut state_builder = SessionStateBuilder::new()+ .with_config(session_config)+ .with_runtime_env(runtime_env)+ .with_default_features();+ if args.spark {+ state_builder = state_builder.with_spark_features();+ }+// enable dynamic file query- let ctx = SessionContext::new_with_config_rt(session_config, runtime_env)- .enable_url_table();+ let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table();ctx.refresh_catalogs().await?;// install dynamic catalog provider that can register required object storesctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(@@ -350,6 +366,10 @@ fn get_session_config(args: &Args) -> Result<SessionConfig> {config_options.format.null = String::from("NULL");}+ if args.spark {+ config_options.sql_parser.dialect = Dialect::Spark;+ }+let mut session_config =SessionConfig::from(config_options).with_information_schema(true);diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rsindex 4dc244445a..9bfeb65278 100644--- a/datafusion-cli/tests/cli_integration.rs+++ b/datafusion-cli/tests/cli_integration.rs@@ -173,6 +173,37 @@ fn cli_quick_test<'a>(assert_cmd_snapshot!(cmd);}+#[test]+fn spark_features_require_opt_in() {+ let dialect_query = "SELECT CAST(1 AS LONG);";+ let spark_query = "SELECT concat_ws(',', array(1, 2)), CAST(1 AS LONG);";++ let output = cli()+ .args(["-q", "--command", dialect_query])+ .output()+ .expect("failed to run datafusion-cli without --spark");+ let stdout = String::from_utf8_lossy(&output.stdout);+ assert!(!output.status.success(), "query unexpectedly succeeded");+ assert!(+ stdout.contains("Unsupported SQL type LONG"),+ "expected Spark SQL syntax to be unavailable without --spark, got:\n{stdout}"+ );++ let output = cli()+ .args(["-q", "--spark", "--format", "csv", "--command", spark_query])+ .output()+ .expect("failed to run datafusion-cli with --spark");+ let stdout = String::from_utf8_lossy(&output.stdout);+ assert!(+ output.status.success(),+ "query failed with --spark, got:\n{stdout}"+ );+ assert!(+ stdout.lines().any(|line| line == "\"1,2\",1"),+ "expected Spark planner and dialect result, got:\n{stdout}"+ );+}+/// Read data piped into the CLI via the `/dev/stdin` pseudo-path.////// Unix-only: `/dev/stdin` does not exist on Windows. This drives the realdiff --git a/datafusion/spark/README.md b/datafusion/spark/README.mdindex 7cb24084cd..0ad21604ce 100644--- a/datafusion/spark/README.md+++ b/datafusion/spark/README.md@@ -23,6 +23,18 @@ under the License.This crate is a submodule of DataFusion that provides [Apache Spark] compatible expressions for use with DataFusion.+## Using the DataFusion CLI++Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register+the Spark-compatible scalar, aggregate, and window functions for the session:++```shell+datafusion-cli --spark+```++The option is disabled by default because Spark-compatible functions can+override DataFusion functions with the same name.+[apache arrow]: https://arrow.apache.org/[apache datafusion]: https://datafusion.apache.org/[apache spark]: https://spark.apache.org/diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.mdindex e61001c6e4..a4550213fa 100644--- a/datafusion/sqllogictest/test_files/spark/README.md+++ b/datafusion/sqllogictest/test_files/spark/README.md@@ -52,6 +52,10 @@ statement okset datafusion.execution.enable_ansi_mode = false;```+Run `datafusion-cli --spark` to explore Spark-compatible functions+interactively. This enables the Spark SQL dialect and registers the+Spark-compatible function library for the CLI session.+### Finding Test CasesTo verify and compare function behavior at a minimum, you can refer to the following documentation sources:diff --git a/docs/source/user-guide/cli/usage.md b/docs/source/user-guide/cli/usage.mdindex 75c6698f00..24811a3ce9 100644--- a/docs/source/user-guide/cli/usage.md+++ b/docs/source/user-guide/cli/usage.md@@ -52,6 +52,8 @@ Options:[possible values: numbers(0/10/...), inf(no limit)] [default: 40]--colorEnables console syntax highlighting+ --spark+ Enable Apache Spark-compatible SQL dialect, functions, and expression planning-d, --disk-limit <DISK_LIMIT>Available disk space for spilling queries (e.g. '10g'), default to None (uses DataFusion's default value of '100g')--object-store-profiling <OBJECT_STORE_PROFILING>@@ -63,6 +65,18 @@ Options:Print version```+## Apache Spark compatibility++Use `--spark` to enable the Apache Spark SQL dialect, expression planner, and+function library for the CLI session:++```bash+datafusion-cli --spark -c "SELECT next_day('2015-07-27'::DATE, 'Sun'::STRING);"+```++Spark compatibility is disabled by default. Enabling it overrides DataFusion+functions with the same name for the duration of the session.+## CommandsAvailable commands inside DataFusion CLI are: