Apache DataFusion SQL Query Engine
feat(dataframe): improve DataFrame::from_columns input types (#24630) (#24633)
Sergey Zhukove36c5f842aae
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 1299e5fb7b..ed3dc5ea83 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -2614,7 +2614,7 @@ impl DataFrame {/// # async fn main() -> Result<()> {/// let id: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));/// let name: ArrayRef = Arc::new(StringArray::from(vec!["foo", "bar", "baz"]));- /// let df = DataFrame::from_columns(vec![("id", id), ("name", name)])?;+ /// let df = DataFrame::from_columns([("id", id), ("name", name)])?;/// let expected = vec![/// "+----+------+",/// "| id | name |",@@ -2628,17 +2628,16 @@ impl DataFrame {/// # Ok(())/// # }/// ```- pub fn from_columns(columns: Vec<(&str, ArrayRef)>) -> Result<Self> {- let fields = columns- .iter()- .map(|(name, array)| Field::new(*name, array.data_type().clone(), true))- .collect::<Vec<_>>();-- let arrays = columns+ pub fn from_columns<'a, I>(columns: I) -> Result<Self>+ where+ I: IntoIterator<Item = (&'a str, ArrayRef)>,+ {+ let (fields, arrays): (Vec<_>, Vec<_>) = columns.into_iter()- .map(|(_, array)| array)- .collect::<Vec<_>>();-+ .map(|(name, array)| {+ (Field::new(name, array.data_type().clone(), true), array)+ })+ .unzip();let schema = Arc::new(Schema::new(fields));let batch = RecordBatch::try_new(schema, arrays)?;let ctx = SessionContext::new();@@ -2695,7 +2694,7 @@ macro_rules! dataframe {use datafusion::prelude::DataFrame;use datafusion::common::test_util::IntoArrayRef;- let columns = vec![+ let columns = [$(($name, $data.into_array_ref()),)+diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 1676a69de8..527c956749 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -6968,7 +6968,7 @@ async fn test_dataframe_from_columns() -> Result<()> {let strings: ArrayRef =Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));- let df = DataFrame::from_columns(vec![+ let columns = [("bool", bools),("i8", i8s),("i16", i16s),@@ -6982,10 +6982,10 @@ async fn test_dataframe_from_columns() -> Result<()> {("f32", f32s),("f64", f64s),("str", strings),- ])?;+ ];- assert_eq!(df.schema().fields().len(), 13);- assert_eq!(df.clone().count().await?, 3);+ let df1 = DataFrame::from_columns(columns.clone())?;+ let df2 = DataFrame::from_columns(columns.to_vec())?;let expected_types = [("bool", DataType::Boolean),@@ -7003,14 +7003,89 @@ async fn test_dataframe_from_columns() -> Result<()> {("str", DataType::Utf8),];- let schema = df.schema();+ for df in [df1, df2] {+ assert_eq!(df.schema().fields().len(), expected_types.len());+ assert_eq!(df.clone().count().await?, 3);- for (name, data_type) in expected_types {- assert_eq!(schema.field_with_name(None, name)?.data_type(), &data_type);+ let schema = df.schema();++ for (name, data_type) in &expected_types {+ assert_eq!(schema.field_with_name(None, name)?.data_type(), data_type);+ }++ let rows = df.sort(vec![col("i32").sort(true, true)])?;++ assert_batches_eq!(+ &[+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",+ "| bool | i8 | i16 | i32 | i64 | u8 | u16 | u32 | u64 | f16 | f32 | f64 | str |",+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",+ "| true | -1 | -1 | -1 | -1 | 0 | 0 | 0 | 0 | 1 | 1.0 | 1.0 | foo |",+ "| false | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 2 | 2.0 | 2.0 | bar |",+ "| true | 1 | 1 | 1 | 1 | 2 | 2 | 2 | 2 | 3 | 3.0 | 3.0 | |",+ "+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",+ ],+ &rows.collect().await?+ );}- let rows = df.sort(vec![col("i32").sort(true, true)])?;+ Ok(())+}++#[test]+fn test_dataframe_from_columns_empty() {+ let result = DataFrame::from_columns(vec![]);+ assert!(result.is_err());++ let result = DataFrame::from_columns([]);+ assert!(result.is_err());+}++#[tokio::test]+async fn test_dataframe_from_columns_with_iterator() -> Result<()> {+ let bools: ArrayRef = Arc::new(BooleanArray::from(vec![true, false, true]));+ let i8s: ArrayRef = Arc::new(Int8Array::from(vec![-1, 0, 1]));+ let i16s: ArrayRef = Arc::new(Int16Array::from(vec![-1, 0, 1]));+ let i32s: ArrayRef = Arc::new(Int32Array::from(vec![-1, 0, 1]));+ let i64s: ArrayRef = Arc::new(Int64Array::from(vec![-1, 0, 1]));++ let u8s: ArrayRef = Arc::new(UInt8Array::from(vec![0, 1, 2]));+ let u16s: ArrayRef = Arc::new(UInt16Array::from(vec![0, 1, 2]));+ let u32s: ArrayRef = Arc::new(UInt32Array::from(vec![0, 1, 2]));+ let u64s: ArrayRef = Arc::new(UInt64Array::from(vec![0, 1, 2]));++ let f16s: ArrayRef = Arc::new(Float16Array::from(vec![+ half::f16::from_f64(1.0),+ half::f16::from_f64(2.0),+ half::f16::from_f64(3.0),+ ]));+ let f32s: ArrayRef = Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0]));+ let f64s: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));++ let strings: ArrayRef =+ Arc::new(StringArray::from(vec![Some("foo"), Some("bar"), None]));++ let columns = [+ ("bool", bools),+ ("i8", i8s),+ ("i16", i16s),+ ("i32", i32s),+ ("i64", i64s),+ ("u8", u8s),+ ("u16", u16s),+ ("u32", u32s),+ ("u64", u64s),+ ("f16", f16s),+ ("f32", f32s),+ ("f64", f64s),+ ("str", strings),+ ];+ let df = DataFrame::from_columns(columns.into_iter())?;++ assert_eq!(df.schema().fields().len(), 13);+ assert_eq!(df.clone().count().await?, 3);+ let rows = df.sort(vec![col("i32").sort(true, true)])?;assert_batches_eq!(&["+-------+----+-----+-----+-----+----+-----+-----+-----+-----+-----+-----+-----+",diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 97ac11a016..881aee0c63 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -97,4 +97,25 @@ The output type of the `floor` and `ceil` UDFs has been changed from the exact iChange the expected type or wrap the expression in `CAST`. It's recommended to avoid relying on decimal's exact precision and scale.-[#24703]: https://github.com/apache/datafusion/pull/24703+### `DataFrame::from_columns` accepts `IntoIterator`++`DataFrame::from_columns` now accepts any `IntoIterator<Item = (&str, ArrayRef)>`+instead of specifically accepting a `Vec<(&str, ArrayRef)>`.++```rust,ignore+// Existing Vec usage continues to work+let df = DataFrame::from_columns(vec![+ ("id", id),+ ("name", name),+])?;++// Arrays can now be used directly+let df = DataFrame::from_columns([+ ("id", id),+ ("name", name),+])?;+```++Most existing call sites using `Vec` require no changes. Code that relies on+the exact non-generic function signature of `DataFrame::from_columns` may need+to be updated to account for the new generic API.