Skip to content

Apache DataFusion SQL Query Engine

Expand scalar async UDF results to the batch row count (#25828)

Haresh Khanna1aed7dcf1547
diff --git a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs
index 150801e38f..167792adae 100644
--- a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs
+++ b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs
@@ -89,6 +89,35 @@ async fn test_async_udf_with_non_modular_batch_size() -> Result<()> {
Ok(())
}
+// An async UDF may return a scalar when all of its arguments are scalars. The
+// scalar applies to every row, so it must be expanded to the number of rows in
+// each chunk rather than to a single row.
+#[tokio::test]
+async fn test_async_udf_with_scalar_result() -> Result<()> {
+ let ctx = register_table_and_udf()?;
+
+ let df = ctx
+ .sql("SELECT id, test_async_udf('constant') as result FROM test_table")
+ .await?;
+
+ let result = df.collect().await?;
+
+ assert_batches_eq!(
+ &[
+ "+----+----------+",
+ "| id | result |",
+ "+----+----------+",
+ "| 0 | constant |",
+ "| 1 | constant |",
+ "| 2 | constant |",
+ "+----+----------+"
+ ],
+ &result
+ );
+
+ Ok(())
+}
+
// This test checks if metrics are printed for `AsyncFuncExec`
#[tokio::test]
async fn test_async_udf_metrics() -> Result<()> {
diff --git a/datafusion/physical-expr/src/async_scalar_function.rs b/datafusion/physical-expr/src/async_scalar_function.rs
index e8ee9a6948..b2fa2861e4 100644
--- a/datafusion/physical-expr/src/async_scalar_function.rs
+++ b/datafusion/physical-expr/src/async_scalar_function.rs
@@ -162,7 +162,8 @@ impl AsyncFuncExpr {
return_field: Arc::clone(&self.return_field),
config_options: Arc::clone(&config_options),
})
- .await?,
+ .await?
+ .into_array_of_size(current_batch.num_rows())?,
);
}
} else {
@@ -181,20 +182,13 @@ impl AsyncFuncExpr {
return_field: Arc::clone(&self.return_field),
config_options: Arc::clone(&config_options),
})
- .await?,
+ .await?
+ .into_array_of_size(batch.num_rows())?,
);
}
- let datas = result_batches
- .into_iter()
- .map(|cv| match cv {
- ColumnarValue::Array(arr) => Ok(arr),
- ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?),
- })
- .collect::<Result<Vec<_>>>()?;
-
// Get references to the arrays as dyn Array to call concat
- let dyn_arrays = datas
+ let dyn_arrays = result_batches
.iter()
.map(|arr| arr as &dyn arrow::array::Array)
.collect::<Vec<_>>();