Apache DataFusion SQL Query Engine
fix: render binary columns as hex in DataFrame::describe() (#21728)
Diego Perez Giordán6644732a8a7a
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 0f38988c69..3d6b832aa6 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -48,6 +48,7 @@ use std::sync::Arc;use arrow::array::{Array, ArrayRef, Int64Array, StringArray};use arrow::compute::{cast, concat};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::util::display::{ArrayFormatter, FormatOptions};use arrow_schema::FieldRef;use datafusion_common::config::{CsvOptions, JsonOptions};use datafusion_common::{@@ -979,8 +980,13 @@ impl DataFrame {/// Return a new `DataFrame` that has statistics for a DataFrame.///- /// Only summarizes numeric datatypes at the moment and returns nulls for- /// non numeric datatypes. The output format is modeled after pandas+ /// The summary contains the `count`, `null_count`, `mean`, `std`, `min`,+ /// `max`, and `median` of each column. `count` and `null_count` are+ /// computed for every column; `min` and `max` for every column except+ /// `Boolean`; and `mean`, `std`, and `median` only for numeric columns+ /// (other columns report `null` for these). `min`/`max` of binary columns+ /// (`Binary`, `LargeBinary`, `BinaryView`, `FixedSizeBinary`) are rendered+ /// as lowercase hex. The output format is modeled after pandas////// # Example/// ```@@ -1074,9 +1080,7 @@ impl DataFrame {vec![],original_schema_fields.clone()- .filter(|f| {- !matches!(f.data_type(), DataType::Binary | DataType::Boolean)- })+ .filter(|f| !matches!(f.data_type(), DataType::Boolean)).map(|f| min(ident(f.name())).alias(f.name())).collect::<Vec<_>>(),),@@ -1085,9 +1089,7 @@ impl DataFrame {vec![],original_schema_fields.clone()- .filter(|f| {- !matches!(f.data_type(), DataType::Binary | DataType::Boolean)- })+ .filter(|f| !matches!(f.data_type(), DataType::Boolean)).map(|f| max(ident(f.name())).alias(f.name())).collect::<Vec<_>>(),),@@ -1126,6 +1128,22 @@ impl DataFrame {Arc::new(StringArray::from(vec!["null"]))} else if field.data_type().is_numeric() {cast(column, &DataType::Float64)?+ } else if field.data_type().is_binary() {+ let formatter = ArrayFormatter::try_new(+ column.as_ref(),+ &FormatOptions::default(),+ )?;+ let values: Vec<Option<String>> = (0..column.len())+ .map(|i| {+ if column.is_null(i) {+ None+ } else {+ let value = formatter.value(i);+ Some(value.to_string())+ }+ })+ .collect();+ Arc::new(StringArray::from(values))} else {cast(column, &DataType::Utf8)?}@@ -1133,7 +1151,8 @@ impl DataFrame {_ => Arc::new(StringArray::from(vec!["null"])),}}- //Handling error when only boolean/binary column, and in other cases+ // Handles the case where all columns were filtered out+ // (e.g. only boolean columns for mean/std/min/max/median)Err(err)if err.to_string().contains("Error during planning: \@@ -1517,7 +1536,7 @@ impl DataFrame {/// # }pub async fn to_string(self) -> Result<String> {let options = self.session_state.config().options().format.clone();- let arrow_options: arrow::util::display::FormatOptions = (&options).try_into()?;+ let arrow_options: FormatOptions = (&options).try_into()?;let registry = self.session_state.extension_type_registry();let formatter_factory = DFArrayFormatterFactory::new(Arc::clone(registry));diff --git a/datafusion/core/tests/dataframe/describe.rs b/datafusion/core/tests/dataframe/describe.rsindex 9aa8a49c97..056bc21a69 100644--- a/datafusion/core/tests/dataframe/describe.rs+++ b/datafusion/core/tests/dataframe/describe.rs@@ -15,6 +15,12 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;++use arrow::array::{+ BinaryArray, BinaryViewArray, FixedSizeBinaryArray, LargeBinaryArray, RecordBatch,+};+use arrow::datatypes::{DataType, Field, Schema};use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_common::test_util::batches_to_string;use datafusion_common::{Result, test_util::parquet_test_data};@@ -112,6 +118,59 @@ async fn describe_null() -> Result<()> {Ok(())}+#[tokio::test]+async fn describe_binary_columns() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("bin", DataType::Binary, true),+ Field::new("lbin", DataType::LargeBinary, true),+ Field::new("vbin", DataType::BinaryView, true),+ Field::new("fbin", DataType::FixedSizeBinary(2), true),+ ]));++ let bin: BinaryArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]+ .into_iter()+ .collect();+ let lbin: LargeBinaryArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]+ .into_iter()+ .collect();+ let vbin: BinaryViewArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]+ .into_iter()+ .collect();+ let fbin = FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ [Some([0x00u8, 0x01]), Some([0xff, 0xee]), None].into_iter(),+ 2,+ )?;+ let batch = RecordBatch::try_new(+ schema,+ vec![+ Arc::new(bin),+ Arc::new(lbin),+ Arc::new(vbin),+ Arc::new(fbin),+ ],+ )?;+ let ctx = SessionContext::new();+ ctx.register_batch("t", batch)?;+ let result = ctx.table("t").await?.describe().await?.collect().await?;++ assert_snapshot!(batches_to_string(&result),+ @r"+ +------------+------+------+------+------++ | describe | bin | lbin | vbin | fbin |+ +------------+------+------+------+------++ | count | 2 | 2 | 2 | 2 |+ | null_count | 1 | 1 | 1 | 1 |+ | mean | null | null | null | null |+ | std | null | null | null | null |+ | min | 0001 | 0001 | 0001 | 0001 |+ | max | ffee | ffee | ffee | ffee |+ | median | null | null | null | null |+ +------------+------+------+------+------++");++ Ok(())+}+/// Return a SessionContext with parquet file registeredasync fn parquet_context() -> SessionContext {let ctx = SessionContext::new();