Apache DataFusion SQL Query Engine
Expand scalar async UDF results to the batch row count (#25828)
Haresh Khanna1aed7dcf1547
diff --git a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rsindex 150801e38f..167792adae 100644--- a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs@@ -89,6 +89,35 @@ async fn test_async_udf_with_non_modular_batch_size() -> Result<()> {Ok(())}+// An async UDF may return a scalar when all of its arguments are scalars. The+// scalar applies to every row, so it must be expanded to the number of rows in+// each chunk rather than to a single row.+#[tokio::test]+async fn test_async_udf_with_scalar_result() -> Result<()> {+ let ctx = register_table_and_udf()?;++ let df = ctx+ .sql("SELECT id, test_async_udf('constant') as result FROM test_table")+ .await?;++ let result = df.collect().await?;++ assert_batches_eq!(+ &[+ "+----+----------+",+ "| id | result |",+ "+----+----------+",+ "| 0 | constant |",+ "| 1 | constant |",+ "| 2 | constant |",+ "+----+----------+"+ ],+ &result+ );++ Ok(())+}+// This test checks if metrics are printed for `AsyncFuncExec`#[tokio::test]async fn test_async_udf_metrics() -> Result<()> {diff --git a/datafusion/physical-expr/src/async_scalar_function.rs b/datafusion/physical-expr/src/async_scalar_function.rsindex e8ee9a6948..b2fa2861e4 100644--- a/datafusion/physical-expr/src/async_scalar_function.rs+++ b/datafusion/physical-expr/src/async_scalar_function.rs@@ -162,7 +162,8 @@ impl AsyncFuncExpr {return_field: Arc::clone(&self.return_field),config_options: Arc::clone(&config_options),})- .await?,+ .await?+ .into_array_of_size(current_batch.num_rows())?,);}} else {@@ -181,20 +182,13 @@ impl AsyncFuncExpr {return_field: Arc::clone(&self.return_field),config_options: Arc::clone(&config_options),})- .await?,+ .await?+ .into_array_of_size(batch.num_rows())?,);}- let datas = result_batches- .into_iter()- .map(|cv| match cv {- ColumnarValue::Array(arr) => Ok(arr),- ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?),- })- .collect::<Result<Vec<_>>>()?;-// Get references to the arrays as dyn Array to call concat- let dyn_arrays = datas+ let dyn_arrays = result_batches.iter().map(|arr| arr as &dyn arrow::array::Array).collect::<Vec<_>>();