Skip to content

Apache DataFusion SQL Query Engine

feat(dataframe): improve DataFrame::from_columns input types (#24630) (#24633)

Sergey Zhukove36c5f842aae
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
index 1299e5fb7b..ed3dc5ea83 100644
--- a/datafusion/core/src/dataframe/mod.rs
+++ b/datafusion/core/src/dataframe/mod.rs
@@ -2614,7 +2614,7 @@ impl DataFrame {
/// # async fn main() -> Result<()> {
/// let id: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
/// let name: ArrayRef = Arc::new(StringArray::from(vec!["foo", "bar", "baz"]));
- /// let df = DataFrame::from_columns(vec![("id", id), ("name", name)])?;
+ /// let df = DataFrame::from_columns([("id", id), ("name", name)])?;
/// let expected = vec![
/// "+----+------+",
/// "| id | name |",
@@ -2628,17 +2628,16 @@ impl DataFrame {
/// # Ok(())
/// # }
/// ```
- pub fn from_columns(columns: Vec<(&str, ArrayRef)>) -> Result<Self> {
- let fields = columns
- .iter()
- .map(|(name, array)| Field::new(*name, array.data_type().clone(), true))
- .collect::<Vec<_>>();
-
- let arrays = columns
+ pub fn from_columns<'a, I>(columns: I) -> Result<Self>
+ where
+ I: IntoIterator<Item = (&'a str, ArrayRef)>,
+ {
+ let (fields, arrays): (Vec<_>, Vec<_>) = columns
.into_iter()
- .map(|(_, array)| array)
- .collect::<Vec<_>>();
-
+ .map(|(name, array)| {
+ (Field::new(name, array.data_type().clone(), true), array)
+ })
+ .unzip();
let schema = Arc::new(Schema::new(fields));
let batch = RecordBatch::try_new(schema, arrays)?;
let ctx = SessionContext::new();
@@ -2695,7 +2694,7 @@ macro_rules! dataframe {
use datafusion::prelude::DataFrame;
use datafusion::common::test_util::IntoArrayRef;
- let columns = vec![
+ let columns = [
$(
($name, $data.into_array_ref()),
)+
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 1676a69de8..527c956749 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -6968,7 +6968,7 @@ async fn test_dataframe_from_columns() -> Result<()> {
let strings: ArrayRef =
Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));
- let df = DataFrame::from_columns(vec![
+ let columns = [
("bool", bools),
("i8", i8s),
("i16", i16s),
@@ -6982,10 +6982,10 @@ async fn test_dataframe_from_columns() -> Result<()> {
("f32", f32s),
("f64", f64s),
("str", strings),
- ])?;
+ ];
- assert_eq!(df.schema().fields().len(), 13);
- assert_eq!(df.clone().count().await?, 3);
+ let df1 = DataFrame::from_columns(columns.clone())?;
+ let df2 = DataFrame::from_columns(columns.to_vec())?;
let expected_types = [
("bool", DataType::Boolean),
@@ -7003,14 +7003,89 @@ async fn test_dataframe_from_columns() -> Result<()> {
("str", DataType::Utf8),
];
- let schema = df.schema();
+ for df in [df1, df2] {
+ assert_eq!(df.schema().fields().len(), expected_types.len());
+ assert_eq!(df.clone().count().await?, 3);
- for (name, data_type) in expected_types {
- assert_eq!(schema.field_with_name(None, name)?.data_type(), &data_type);
+ let schema = df.schema();
+
+ for (name, data_type) in &expected_types {
+ assert_eq!(schema.field_with_name(None, name)?.data_type(), data_type);
+ }
+
+ let rows = df.sort(vec![col("i32").sort(true, true)])?;
+
+ assert_batches_eq!(
+ &[
+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
+ "| bool | i8 | i16 | i32 | i64 | u8 | u16 | u32 | u64 | f16 | f32 | f64 | str |",
+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
+ "| true | -1 | -1 | -1 | -1 | 0 | 0 | 0 | 0 | 1 | 1.0 | 1.0 | foo |",
+ "| false | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 2 | 2.0 | 2.0 | bar |",
+ "| true | 1 | 1 | 1 | 1 | 2 | 2 | 2 | 2 | 3 | 3.0 | 3.0 | |",
+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
+ ],
+ &rows.collect().await?
+ );
}
- let rows = df.sort(vec![col("i32").sort(true, true)])?;
+ Ok(())
+}
+
+#[test]
+fn test_dataframe_from_columns_empty() {
+ let result = DataFrame::from_columns(vec![]);
+ assert!(result.is_err());
+
+ let result = DataFrame::from_columns([]);
+ assert!(result.is_err());
+}
+
+#[tokio::test]
+async fn test_dataframe_from_columns_with_iterator() -> Result<()> {
+ let bools: ArrayRef = Arc::new(BooleanArray::from(vec![true, false, true]));
+ let i8s: ArrayRef = Arc::new(Int8Array::from(vec![-1, 0, 1]));
+ let i16s: ArrayRef = Arc::new(Int16Array::from(vec![-1, 0, 1]));
+ let i32s: ArrayRef = Arc::new(Int32Array::from(vec![-1, 0, 1]));
+ let i64s: ArrayRef = Arc::new(Int64Array::from(vec![-1, 0, 1]));
+
+ let u8s: ArrayRef = Arc::new(UInt8Array::from(vec![0, 1, 2]));
+ let u16s: ArrayRef = Arc::new(UInt16Array::from(vec![0, 1, 2]));
+ let u32s: ArrayRef = Arc::new(UInt32Array::from(vec![0, 1, 2]));
+ let u64s: ArrayRef = Arc::new(UInt64Array::from(vec![0, 1, 2]));
+
+ let f16s: ArrayRef = Arc::new(Float16Array::from(vec![
+ half::f16::from_f64(1.0),
+ half::f16::from_f64(2.0),
+ half::f16::from_f64(3.0),
+ ]));
+ let f32s: ArrayRef = Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0]));
+ let f64s: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));
+
+ let strings: ArrayRef =
+ Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));
+
+ let columns = [
+ ("bool", bools),
+ ("i8", i8s),
+ ("i16", i16s),
+ ("i32", i32s),
+ ("i64", i64s),
+ ("u8", u8s),
+ ("u16", u16s),
+ ("u32", u32s),
+ ("u64", u64s),
+ ("f16", f16s),
+ ("f32", f32s),
+ ("f64", f64s),
+ ("str", strings),
+ ];
+ let df = DataFrame::from_columns(columns.into_iter())?;
+
+ assert_eq!(df.schema().fields().len(), 13);
+ assert_eq!(df.clone().count().await?, 3);
+ let rows = df.sort(vec![col("i32").sort(true, true)])?;
assert_batches_eq!(
&[
"+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 97ac11a016..881aee0c63 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -97,4 +97,25 @@ The output type of the `floor` and `ceil` UDFs has been changed from the exact i
Change the expected type or wrap the expression in `CAST`. It's recommended to avoid relying on decimal's exact precision and scale.
-[#24703]: https://github.com/apache/datafusion/pull/24703
+### `DataFrame::from_columns` accepts `IntoIterator`
+
+`DataFrame::from_columns` now accepts any `IntoIterator<Item = (&str, ArrayRef)>`
+instead of specifically accepting a `Vec<(&str, ArrayRef)>`.
+
+```rust,ignore
+// Existing Vec usage continues to work
+let df = DataFrame::from_columns(vec![
+ ("id", id),
+ ("name", name),
+])?;
+
+// Arrays can now be used directly
+let df = DataFrame::from_columns([
+ ("id", id),
+ ("name", name),
+])?;
+```
+
+Most existing call sites using `Vec` require no changes. Code that relies on
+the exact non-generic function signature of `DataFrame::from_columns` may need
+to be updated to account for the new generic API.