Skip to content

Apache DataFusion SQL Query Engine

minor: refactoring of some `ScalarValue` code (#19439)

Jeffrey Vo33ac70dd6d63
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index fe0c92e0ec..859b324430 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -78,14 +78,13 @@ use arrow::compute::kernels::numeric::{
add, add_wrapping, div, mul, mul_wrapping, rem, sub, sub_wrapping,
};
use arrow::datatypes::{
- ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType,
- DECIMAL128_MAX_PRECISION, DataType, Date32Type, Decimal32Type, Decimal64Type,
- Decimal128Type, Decimal256Type, DecimalType, Field, Float32Type, Int8Type, Int16Type,
- Int32Type, Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,
- IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit,
- TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
- TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields,
- UnionMode, i256, validate_decimal_precision_and_scale,
+ ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType, DataType, Date32Type,
+ Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, Field,
+ Float32Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTime,
+ IntervalDayTimeType, IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit,
+ IntervalYearMonthType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
+ TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,
+ UInt64Type, UnionFields, UnionMode, i256, validate_decimal_precision_and_scale,
};
use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string};
use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array};
@@ -1152,13 +1151,8 @@ impl ScalarValue {
/// Create a decimal Scalar from value/precision and scale.
pub fn try_new_decimal128(value: i128, precision: u8, scale: i8) -> Result<Self> {
- // make sure the precision and scale is valid
- if precision <= DECIMAL128_MAX_PRECISION && scale.unsigned_abs() <= precision {
- return Ok(ScalarValue::Decimal128(Some(value), precision, scale));
- }
- _internal_err!(
- "Can not new a decimal type ScalarValue for precision {precision} and scale {scale}"
- )
+ Self::validate_decimal_or_internal_err::<Decimal128Type>(precision, scale)?;
+ Ok(ScalarValue::Decimal128(Some(value), precision, scale))
}
/// Create a Null instance of ScalarValue for this datatype
@@ -1250,7 +1244,7 @@ impl ScalarValue {
index_type.clone(),
Box::new(value_type.as_ref().try_into()?),
),
- // `ScalaValue::List` contains single element `ListArray`.
+ // `ScalarValue::List` contains single element `ListArray`.
DataType::List(field_ref) => ScalarValue::List(Arc::new(
GenericListArray::new_null(Arc::clone(field_ref), 1),
)),
@@ -1258,7 +1252,7 @@ impl ScalarValue {
DataType::LargeList(field_ref) => ScalarValue::LargeList(Arc::new(
GenericListArray::new_null(Arc::clone(field_ref), 1),
)),
- // `ScalaValue::FixedSizeList` contains single element `FixedSizeList`.
+ // `ScalarValue::FixedSizeList` contains single element `FixedSizeList`.
DataType::FixedSizeList(field_ref, fixed_length) => {
ScalarValue::FixedSizeList(Arc::new(FixedSizeListArray::new_null(
Arc::clone(field_ref),
@@ -1338,6 +1332,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing PI
pub fn new_pi(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(f16::PI)),
DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::PI)),
DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::PI)),
_ => _internal_err!("PI is not supported for data type: {}", datatype),
@@ -1347,6 +1342,8 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing PI's upper bound
pub fn new_pi_upper(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ // TODO: half::f16 doesn't seem to have equivalent
+ // https://github.com/apache/datafusion/issues/19465
DataType::Float32 => Ok(ScalarValue::from(consts::PI_UPPER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::PI_UPPER_F64)),
_ => {
@@ -1358,6 +1355,8 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing -PI's lower bound
pub fn new_negative_pi_lower(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ // TODO: half::f16 doesn't seem to have equivalent
+ // https://github.com/apache/datafusion/issues/19465
DataType::Float32 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F64)),
_ => {
@@ -1369,6 +1368,8 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing FRAC_PI_2's upper bound
pub fn new_frac_pi_2_upper(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ // TODO: half::f16 doesn't seem to have equivalent
+ // https://github.com/apache/datafusion/issues/19465
DataType::Float32 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F64)),
_ => {
@@ -1380,6 +1381,8 @@ impl ScalarValue {
// Returns a [`ScalarValue`] representing FRAC_PI_2's lower bound
pub fn new_neg_frac_pi_2_lower(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ // TODO: half::f16 doesn't seem to have equivalent
+ // https://github.com/apache/datafusion/issues/19465
DataType::Float32 => {
Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F32))
}
@@ -1395,6 +1398,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing -PI
pub fn new_negative_pi(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(-f16::PI)),
DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::PI)),
DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::PI)),
_ => _internal_err!("-PI is not supported for data type: {}", datatype),
@@ -1404,6 +1408,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing PI/2
pub fn new_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(f16::FRAC_PI_2)),
DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::FRAC_PI_2)),
DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::FRAC_PI_2)),
_ => _internal_err!("PI/2 is not supported for data type: {}", datatype),
@@ -1413,6 +1418,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing -PI/2
pub fn new_neg_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(-f16::FRAC_PI_2)),
DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::FRAC_PI_2)),
DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::FRAC_PI_2)),
_ => _internal_err!("-PI/2 is not supported for data type: {}", datatype),
@@ -1422,6 +1428,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing infinity
pub fn new_infinity(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(f16::INFINITY)),
DataType::Float32 => Ok(ScalarValue::from(f32::INFINITY)),
DataType::Float64 => Ok(ScalarValue::from(f64::INFINITY)),
_ => {
@@ -1433,6 +1440,7 @@ impl ScalarValue {
/// Returns a [`ScalarValue`] representing negative infinity
pub fn new_neg_infinity(datatype: &DataType) -> Result<ScalarValue> {
match datatype {
+ DataType::Float16 => Ok(ScalarValue::from(f16::NEG_INFINITY)),
DataType::Float32 => Ok(ScalarValue::from(f32::NEG_INFINITY)),
DataType::Float64 => Ok(ScalarValue::from(f64::NEG_INFINITY)),
_ => {
@@ -1456,7 +1464,7 @@ impl ScalarValue {
DataType::UInt16 => ScalarValue::UInt16(Some(0)),
DataType::UInt32 => ScalarValue::UInt32(Some(0)),
DataType::UInt64 => ScalarValue::UInt64(Some(0)),
- DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(0.0))),
+ DataType::Float16 => ScalarValue::Float16(Some(f16::ZERO)),
DataType::Float32 => ScalarValue::Float32(Some(0.0)),
DataType::Float64 => ScalarValue::Float64(Some(0.0)),
DataType::Decimal32(precision, scale) => {
@@ -1671,7 +1679,7 @@ impl ScalarValue {
DataType::UInt16 => ScalarValue::UInt16(Some(1)),
DataType::UInt32 => ScalarValue::UInt32(Some(1)),
DataType::UInt64 => ScalarValue::UInt64(Some(1)),
- DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(1.0))),
+ DataType::Float16 => ScalarValue::Float16(Some(f16::ONE)),
DataType::Float32 => ScalarValue::Float32(Some(1.0)),
DataType::Float64 => ScalarValue::Float64(Some(1.0)),
DataType::Decimal32(precision, scale) => {
@@ -1737,7 +1745,7 @@ impl ScalarValue {
DataType::Int16 | DataType::UInt16 => ScalarValue::Int16(Some(-1)),
DataType::Int32 | DataType::UInt32 => ScalarValue::Int32(Some(-1)),
DataType::Int64 | DataType::UInt64 => ScalarValue::Int64(Some(-1)),
- DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(-1.0))),
+ DataType::Float16 => ScalarValue::Float16(Some(f16::NEG_ONE)),
DataType::Float32 => ScalarValue::Float32(Some(-1.0)),
DataType::Float64 => ScalarValue::Float64(Some(-1.0)),
DataType::Decimal32(precision, scale) => {
@@ -1964,9 +1972,7 @@ impl ScalarValue {
| ScalarValue::Float16(None)
| ScalarValue::Float32(None)
| ScalarValue::Float64(None) => Ok(self.clone()),
- ScalarValue::Float16(Some(v)) => {
- Ok(ScalarValue::Float16(Some(f16::from_f32(-v.to_f32()))))
- }
+ ScalarValue::Float16(Some(v)) => Ok(ScalarValue::Float16(Some(-v))),
ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))),
ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))),
ScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int8(Some(v.neg_checked()?))),
@@ -2087,6 +2093,7 @@ impl ScalarValue {
let r = add_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
Self::try_from_array(r.as_ref(), 0)
}
+
/// Checked addition of `ScalarValue`
///
/// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
@@ -2719,71 +2726,6 @@ impl ScalarValue {
Ok(array)
}
- fn build_decimal32_array(
- value: Option<i32>,
- precision: u8,
- scale: i8,
- size: usize,
- ) -> Result<Decimal32Array> {
- Ok(match value {
- Some(val) => Decimal32Array::from(vec![val; size])
- .with_precision_and_scale(precision, scale)?,
- None => {
- let mut builder = Decimal32Array::builder(size)
- .with_precision_and_scale(precision, scale)?;
- builder.append_nulls(size);
- builder.finish()
- }
- })
- }
-
- fn build_decimal64_array(
- value: Option<i64>,
- precision: u8,
- scale: i8,
- size: usize,
- ) -> Result<Decimal64Array> {
- Ok(match value {
- Some(val) => Decimal64Array::from(vec![val; size])
- .with_precision_and_scale(precision, scale)?,
- None => {
- let mut builder = Decimal64Array::builder(size)
- .with_precision_and_scale(precision, scale)?;
- builder.append_nulls(size);
- builder.finish()
- }
- })
- }
-
- fn build_decimal128_array(
- value: Option<i128>,
- precision: u8,
- scale: i8,
- size: usize,
- ) -> Result<Decimal128Array> {
- Ok(match value {
- Some(val) => Decimal128Array::from(vec![val; size])
- .with_precision_and_scale(precision, scale)?,
- None => {
- let mut builder = Decimal128Array::builder(size)
- .with_precision_and_scale(precision, scale)?;
- builder.append_nulls(size);
- builder.finish()
- }
- })
- }
-
- fn build_decimal256_array(
- value: Option<i256>,
- precision: u8,
- scale: i8,
- size: usize,
- ) -> Result<Decimal256Array> {
- Ok(repeat_n(value, size)
- .collect::<Decimal256Array>()
- .with_precision_and_scale(precision, scale)?)
- }
-
/// Converts `Vec<ScalarValue>` where each element has type corresponding to
/// `data_type`, to a single element [`ListArray`].
///
@@ -2939,18 +2881,35 @@ impl ScalarValue {
/// - a `Dictionary` that fails be converted to a dictionary array of size
pub fn to_array_of_size(&self, size: usize) -> Result<ArrayRef> {
Ok(match self {
- ScalarValue::Decimal32(e, precision, scale) => Arc::new(
- ScalarValue::build_decimal32_array(*e, *precision, *scale, size)?,
+ ScalarValue::Decimal32(Some(e), precision, scale) => Arc::new(
+ Decimal32Array::from_value(*e, size)
+ .with_precision_and_scale(*precision, *scale)?,
),
- ScalarValue::Decimal64(e, precision, scale) => Arc::new(
- ScalarValue::build_decimal64_array(*e, *precision, *scale, size)?,
+ ScalarValue::Decimal32(None, precision, scale) => {
+ new_null_array(&DataType::Decimal32(*precision, *scale), size)
+ }
+ ScalarValue::Decimal64(Some(e), precision, scale) => Arc::new(
+ Decimal64Array::from_value(*e, size)
+ .with_precision_and_scale(*precision, *scale)?,
),
- ScalarValue::Decimal128(e, precision, scale) => Arc::new(
- ScalarValue::build_decimal128_array(*e, *precision, *scale, size)?,
+ ScalarValue::Decimal64(None, precision, scale) => {
+ new_null_array(&DataType::Decimal64(*precision, *scale), size)
+ }
+ ScalarValue::Decimal128(Some(e), precision, scale) => Arc::new(
+ Decimal128Array::from_value(*e, size)
+ .with_precision_and_scale(*precision, *scale)?,
),
- ScalarValue::Decimal256(e, precision, scale) => Arc::new(
- ScalarValue::build_decimal256_array(*e, *precision, *scale, size)?,
+ ScalarValue::Decimal128(None, precision, scale) => {
+ new_null_array(&DataType::Decimal128(*precision, *scale), size)
+ }
+ ScalarValue::Decimal256(Some(e), precision, scale) => Arc::new(
+ Decimal256Array::from_value(*e, size)
+ .with_precision_and_scale(*precision, *scale)?,
),
+ ScalarValue::Decimal256(None, precision, scale) => {
+ new_null_array(&DataType::Decimal256(*precision, *scale), size)
+ }
+
ScalarValue::Boolean(e) => match e {
None => new_null_array(&DataType::Boolean, size),
Some(true) => {
@@ -3239,10 +3198,7 @@ impl ScalarValue {
.map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
Arc::new(ar)
}
- None => {
- let dt = self.data_type();
- new_null_array(&dt, size)
- }
+ None => new_null_array(&DataType::Union(fields.clone(), *mode), size),
},
ScalarValue::Dictionary(key_type, v) => {
// values array is one element long (the value)
@@ -5123,7 +5079,8 @@ mod tests {
use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};
use arrow::compute::{is_null, kernels};
use arrow::datatypes::{
- ArrowNumericType, DECIMAL256_MAX_PRECISION, Fields, Float64Type, TimeUnit,
+ ArrowNumericType, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, Fields,
+ Float64Type, TimeUnit,
};
use arrow::error::ArrowError;
use arrow::util::pretty::pretty_format_columns;