Skip to content

Apache DataFusion SQL Query Engine

perf: optimize CASE WHEN lookup table (2.5-22.5 times faster) (#18183)

Raz Luvatond85595e0322a
diff --git a/Cargo.lock b/Cargo.lock
index 85872c4fa7..b449a5ba9b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2491,6 +2491,7 @@ dependencies = [
"datafusion-functions",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
+ "half",
"hashbrown 0.14.5",
"indexmap 2.12.1",
"insta",
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index cc92ce5bac..70cf88b5af 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -55,6 +55,7 @@ parking_lot = { workspace = true }
paste = { workspace = true }
petgraph = "0.8.3"
tokio = { workspace = true }
+half = { workspace = true }
[dev-dependencies]
arrow = { workspace = true, features = ["test_utils"] }
diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs
index dbd138b761..f8bc26c43e 100644
--- a/datafusion/physical-expr/src/expressions/case.rs
+++ b/datafusion/physical-expr/src/expressions/case.rs
@@ -15,9 +15,10 @@
// specific language governing permissions and limitations
// under the License.
+mod literal_lookup_table;
+
use super::{Column, Literal};
use crate::PhysicalExpr;
-use crate::expressions::case::ResultState::{Complete, Empty, Partial};
use crate::expressions::{lit, try_cast};
use arrow::array::*;
use arrow::compute::kernels::zip::zip;
@@ -28,7 +29,6 @@ use arrow::compute::{
use arrow::datatypes::{DataType, Schema, UInt32Type, UnionMode};
use arrow::error::ArrowError;
use datafusion_common::cast::as_boolean_array;
-use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::{
DataFusionError, HashMap, HashSet, Result, ScalarValue, assert_or_internal_err,
exec_err, internal_datafusion_err, internal_err,
@@ -38,11 +38,13 @@ use std::borrow::Cow;
use std::hash::Hash;
use std::{any::Any, sync::Arc};
+use crate::expressions::case::literal_lookup_table::LiteralLookupTable;
+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_physical_expr_common::datum::compare_with_eq;
use itertools::Itertools;
use std::fmt::{Debug, Formatter};
-type WhenThen = (Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>);
+pub(super) type WhenThen = (Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>);
#[derive(Debug, Hash, PartialEq, Eq)]
enum EvalMethod {
@@ -73,8 +75,37 @@ enum EvalMethod {
///
/// CASE WHEN condition THEN expression ELSE expression END
ExpressionOrExpression(ProjectedCaseBody),
+
+ /// This is a specialization for [`EvalMethod::WithExpression`] when the value and results are literals
+ ///
+ /// See [`LiteralLookupTable`] for more details
+ WithExprScalarLookupTable(LiteralLookupTable),
+}
+
+/// Implementing hash so we can use `derive` on [`EvalMethod`].
+///
+/// not implementing actual [`Hash`] as it is not dyn compatible so we cannot implement it for
+/// `dyn` [`literal_lookup_table::WhenLiteralIndexMap`].
+///
+/// So implementing empty hash is still valid as the data is derived from `PhysicalExpr` s which are already hashed
+impl Hash for LiteralLookupTable {
+ fn hash<H: std::hash::Hasher>(&self, _state: &mut H) {}
+}
+
+/// Implementing Equal so we can use `derive` on [`EvalMethod`].
+///
+/// not implementing actual [`PartialEq`] as it is not dyn compatible so we cannot implement it for
+/// `dyn` [`literal_lookup_table::WhenLiteralIndexMap`].
+///
+/// So we always return true as the data is derived from `PhysicalExpr` s which are already compared
+impl PartialEq for LiteralLookupTable {
+ fn eq(&self, _other: &Self) -> bool {
+ true
+ }
}
+impl Eq for LiteralLookupTable {}
+
/// The body of a CASE expression which consists of an optional base expression, the "when/then"
/// branches and an optional "else" branch.
#[derive(Debug, Hash, PartialEq, Eq)]
@@ -591,7 +622,7 @@ impl ResultBuilder {
Self {
data_type: data_type.clone(),
row_count,
- state: Empty,
+ state: ResultState::Empty,
}
}
@@ -671,21 +702,21 @@ impl ResultBuilder {
);
match &mut self.state {
- Empty => {
+ ResultState::Empty => {
let array_index = PartialResultIndex::zero();
let mut indices = vec![PartialResultIndex::none(); self.row_count];
for row_ix in row_indices.as_primitive::<UInt32Type>().values().iter() {
indices[*row_ix as usize] = array_index;
}
- self.state = Partial {
+ self.state = ResultState::Partial {
arrays: vec![row_values],
indices,
};
Ok(())
}
- Partial { arrays, indices } => {
+ ResultState::Partial { arrays, indices } => {
let array_index = PartialResultIndex::try_new(arrays.len())?;
arrays.push(row_values);
@@ -705,7 +736,7 @@ impl ResultBuilder {
}
Ok(())
}
- Complete(_) => internal_err!(
+ ResultState::Complete(_) => internal_err!(
"Cannot add a partial result when complete result is already set"
),
}
@@ -718,23 +749,23 @@ impl ResultBuilder {
/// without any merging overhead.
fn set_complete_result(&mut self, value: ColumnarValue) -> Result<()> {
match &self.state {
- Empty => {
- self.state = Complete(value);
+ ResultState::Empty => {
+ self.state = ResultState::Complete(value);
Ok(())
}
- Partial { .. } => {
+ ResultState::Partial { .. } => {
internal_err!(
"Cannot set a complete result when there are already partial results"
)
}
- Complete(_) => internal_err!("Complete result already set"),
+ ResultState::Complete(_) => internal_err!("Complete result already set"),
}
}
/// Finishes building the result and returns the final array.
fn finish(self) -> Result<ColumnarValue> {
match self.state {
- Empty => {
+ ResultState::Empty => {
// No complete result and no partial results.
// This can happen for case expressions with no else branch where no rows
// matched.
@@ -742,11 +773,11 @@ impl ResultBuilder {
&self.data_type,
)?))
}
- Partial { arrays, indices } => {
+ ResultState::Partial { arrays, indices } => {
// Merge partial results into a single array.
Ok(ColumnarValue::Array(merge_n(&arrays, &indices)?))
}
- Complete(v) => {
+ ResultState::Complete(v) => {
// If we have a complete result, we can just return it.
Ok(v)
}
@@ -781,28 +812,40 @@ impl CaseExpr {
else_expr,
};
- let eval_method = if body.expr.is_some() {
- EvalMethod::WithExpression(body.project()?)
- } else if body.when_then_expr.len() == 1
- && is_cheap_and_infallible(&(body.when_then_expr[0].1))
- && body.else_expr.is_none()
- {
- EvalMethod::InfallibleExprOrNull
- } else if body.when_then_expr.len() == 1
- && body.when_then_expr[0].1.as_any().is::<Literal>()
- && body.else_expr.is_some()
- && body.else_expr.as_ref().unwrap().as_any().is::<Literal>()
- {
- EvalMethod::ScalarOrScalar
- } else if body.when_then_expr.len() == 1 && body.else_expr.is_some() {
- EvalMethod::ExpressionOrExpression(body.project()?)
- } else {
- EvalMethod::NoExpression(body.project()?)
- };
+ let eval_method = Self::find_best_eval_method(&body)?;
Ok(Self { body, eval_method })
}
+ fn find_best_eval_method(body: &CaseBody) -> Result<EvalMethod> {
+ if body.expr.is_some() {
+ if let Some(mapping) = LiteralLookupTable::maybe_new(body) {
+ return Ok(EvalMethod::WithExprScalarLookupTable(mapping));
+ }
+
+ return Ok(EvalMethod::WithExpression(body.project()?));
+ }
+
+ Ok(
+ if body.when_then_expr.len() == 1
+ && is_cheap_and_infallible(&(body.when_then_expr[0].1))
+ && body.else_expr.is_none()
+ {
+ EvalMethod::InfallibleExprOrNull
+ } else if body.when_then_expr.len() == 1
+ && body.when_then_expr[0].1.as_any().is::<Literal>()
+ && body.else_expr.is_some()
+ && body.else_expr.as_ref().unwrap().as_any().is::<Literal>()
+ {
+ EvalMethod::ScalarOrScalar
+ } else if body.when_then_expr.len() == 1 && body.else_expr.is_some() {
+ EvalMethod::ExpressionOrExpression(body.project()?)
+ } else {
+ EvalMethod::NoExpression(body.project()?)
+ },
+ )
+ }
+
/// Optional base expression that can be compared to literal values in the "when" expressions
pub fn expr(&self) -> Option<&Arc<dyn PhysicalExpr>> {
self.body.expr.as_ref()
@@ -1275,6 +1318,28 @@ impl CaseExpr {
self.body.expr_or_expr(batch, when_value)
}
}
+
+ fn with_lookup_table(
+ &self,
+ batch: &RecordBatch,
+ lookup_table: &LiteralLookupTable,
+ ) -> Result<ColumnarValue> {
+ let expr = self.body.expr.as_ref().unwrap();
+ let evaluated_expression = expr.evaluate(batch)?;
+
+ let is_scalar = matches!(evaluated_expression, ColumnarValue::Scalar(_));
+ let evaluated_expression = evaluated_expression.to_array(1)?;
+
+ let values = lookup_table.map_keys_to_values(&evaluated_expression)?;
+
+ let result = if is_scalar {
+ ColumnarValue::Scalar(ScalarValue::try_from_array(values.as_ref(), 0)?)
+ } else {
+ ColumnarValue::Array(values)
+ };
+
+ Ok(result)
+ }
}
impl PhysicalExpr for CaseExpr {
@@ -1370,6 +1435,9 @@ impl PhysicalExpr for CaseExpr {
}
EvalMethod::ScalarOrScalar => self.scalar_or_scalar(batch),
EvalMethod::ExpressionOrExpression(p) => self.expr_or_expr(batch, p),
+ EvalMethod::WithExprScalarLookupTable(lookup_table) => {
+ self.with_lookup_table(batch, lookup_table)
+ }
}
}
@@ -1515,6 +1583,7 @@ mod tests {
use datafusion_expr::type_coercion::binary::comparison_coercion;
use datafusion_expr_common::operator::Operator;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
+ use half::f16;
#[test]
fn case_with_expr() -> Result<()> {
@@ -1585,6 +1654,86 @@ mod tests {
Ok(())
}
+ // Make sure we are not failing when got literal in case when but input is dictionary encoded
+ #[test]
+ fn case_with_expr_primitive_dictionary() -> Result<()> {
+ let schema = Schema::new(vec![Field::new(
+ "a",
+ DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::UInt64)),
+ true,
+ )]);
+ let keys = UInt8Array::from(vec![0u8, 1u8, 2u8, 3u8]);
+ let values = UInt64Array::from(vec![Some(10), Some(20), None, Some(30)]);
+ let dictionary = DictionaryArray::new(keys, Arc::new(values));
+ let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(dictionary)])?;
+
+ let schema = batch.schema();
+
+ // CASE a WHEN 10 THEN 123 WHEN 30 THEN 456 END
+ let when1 = lit(10_u64);
+ let then1 = lit(123_i32);
+ let when2 = lit(30_u64);
+ let then2 = lit(456_i32);
+
+ let expr = generate_case_when_with_type_coercion(
+ Some(col("a", &schema)?),
+ vec![(when1, then1), (when2, then2)],
+ None,
+ schema.as_ref(),
+ )?;
+ let result = expr
+ .evaluate(&batch)?
+ .into_array(batch.num_rows())
+ .expect("Failed to convert to array");
+ let result = as_int32_array(&result)?;
+
+ let expected = &Int32Array::from(vec![Some(123), None, None, Some(456)]);
+
+ assert_eq!(expected, result);
+
+ Ok(())
+ }
+
+ // Make sure we are not failing when got literal in case when but input is dictionary encoded
+ #[test]
+ fn case_with_expr_boolean_dictionary() -> Result<()> {
+ let schema = Schema::new(vec![Field::new(
+ "a",
+ DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Boolean)),
+ true,
+ )]);
+ let keys = UInt8Array::from(vec![0u8, 1u8, 2u8, 3u8]);
+ let values = BooleanArray::from(vec![Some(true), Some(false), None, Some(true)]);
+ let dictionary = DictionaryArray::new(keys, Arc::new(values));
+ let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(dictionary)])?;
+
+ let schema = batch.schema();
+
+ // CASE a WHEN true THEN 123 WHEN false THEN 456 END
+ let when1 = lit(true);
+ let then1 = lit(123i32);
+ let when2 = lit(false);
+ let then2 = lit(456i32);
+
+ let expr = generate_case_when_with_type_coercion(
+ Some(col("a", &schema)?),
+ vec![(when1, then1), (when2, then2)],
+ None,
+ schema.as_ref(),
+ )?;
+ let result = expr
+ .evaluate(&batch)?
+ .into_array(batch.num_rows())
+ .expect("Failed to convert to array");
+ let result = as_int32_array(&result)?;
+
+ let expected = &Int32Array::from(vec![Some(123), Some(456), None, Some(123)]);
+
+ assert_eq!(expected, result);
+
+ Ok(())
+ }
+
#[test]
fn case_with_expr_all_null_dictionary() -> Result<()> {
let schema = Schema::new(vec![Field::new(
@@ -2646,4 +2795,501 @@ mod tests {
assert_not_nullable(expr, schema);
}
}
+
+ // Test Lookup evaluation
+
+ fn test_case_when_literal_lookup(
+ values: ArrayRef,
+ lookup_map: &[(ScalarValue, ScalarValue)],
+ else_value: Option<ScalarValue>,
+ expected: ArrayRef,
+ ) {
+ // Create lookup
+ // CASE <expr>
+ // WHEN <when_constant_1> THEN <then_constant_1>
+ // WHEN <when_constant_2> THEN <then_constant_2>
+ // [ ELSE <else_constant> ]
+
+ let schema = Schema::new(vec![Field::new(
+ "a",
+ values.data_type().clone(),
+ values.is_nullable(),
+ )]);
+ let schema = Arc::new(schema);
+
+ let batch = RecordBatch::try_new(schema, vec![values])
+ .expect("failed to create RecordBatch");
+
+ let schema = batch.schema_ref();
+ let case = col("a", schema).expect("failed to create col");
+
+ let when_then = lookup_map
+ .iter()
+ .map(|(when, then)| {
+ (
+ Arc::new(Literal::new(when.clone())) as _,
+ Arc::new(Literal::new(then.clone())) as _,
+ )
+ })
+ .collect::<Vec<WhenThen>>();
+
+ let else_expr = else_value.map(|else_value| {
+ Arc::new(Literal::new(else_value)) as Arc<dyn PhysicalExpr>
+ });
+ let expr = CaseExpr::try_new(Some(case), when_then, else_expr)
+ .expect("failed to create case");
+
+ // Assert that we are testing what we intend to assert
+ assert!(
+ matches!(
+ expr.eval_method,
+ EvalMethod::WithExprScalarLookupTable { .. }
+ ),
+ "we should use the expected eval method"
+ );
+
+ let actual = expr
+ .evaluate(&batch)
+ .expect("failed to evaluate case")
+ .into_array(batch.num_rows())
+ .expect("Failed to convert to array");
+
+ assert_eq!(
+ actual.data_type(),
+ expected.data_type(),
+ "Data type mismatch"
+ );
+
+ assert_eq!(
+ actual.as_ref(),
+ expected.as_ref(),
+ "actual (left) does not match expected (right)"
+ );
+ }
+
+ fn create_lookup<When, Then>(
+ when_then_pairs: impl IntoIterator<Item = (When, Then)>,
+ ) -> Vec<(ScalarValue, ScalarValue)>
+ where
+ ScalarValue: From<When>,
+ ScalarValue: From<Then>,
+ {
+ when_then_pairs
+ .into_iter()
+ .map(|(when, then)| (ScalarValue::from(when), ScalarValue::from(then)))
+ .collect()
+ }
+
+ fn create_input_and_expected<Input, Expected, InputFromItem, ExpectedFromItem>(
+ input_and_expected_pairs: impl IntoIterator<Item = (InputFromItem, ExpectedFromItem)>,
+ ) -> (Input, Expected)
+ where
+ Input: Array + From<Vec<InputFromItem>>,
+ Expected: Array + From<Vec<ExpectedFromItem>>,
+ {
+ let (input_items, expected_items): (Vec<InputFromItem>, Vec<ExpectedFromItem>) =
+ input_and_expected_pairs.into_iter().unzip();
+
+ (Input::from(input_items), Expected::from(expected_items))
+ }
+
+ fn test_lookup_eval_with_and_without_else(
+ lookup_map: &[(ScalarValue, ScalarValue)],
+ input_values: ArrayRef,
+ expected: StringArray,
+ ) {
+ // Testing without ELSE should fallback to None
+ test_case_when_literal_lookup(
+ Arc::clone(&input_values),
+ lookup_map,
+ None,
+ Arc::new(expected.clone()),
+ );
+
+ // Testing with Else
+ let else_value = "___fallback___";
+
+ // Changing each expected None to be fallback
+ let expected_with_else = expected
+ .iter()
+ .map(|item| item.unwrap_or(else_value))
+ .map(Some)
+ .collect::<StringArray>();
+
+ // Test case
+ test_case_when_literal_lookup(
+ input_values,
+ lookup_map,
+ Some(ScalarValue::Utf8(Some(else_value.to_string()))),
+ Arc::new(expected_with_else),
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_int32_to_string() {
+ let lookup_map = create_lookup([
+ (Some(4), Some("four")),
+ (Some(2), Some("two")),
+ (Some(3), Some("three")),
+ (Some(1), Some("one")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Int32Array, StringArray, _, _>([
+ (1, Some("one")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (3, Some("three")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ (5, None), // No match in WHEN
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_none_case_should_never_match() {
+ let lookup_map = create_lookup([
+ (Some(4), Some("four")),
+ (None, Some("none")),
+ (Some(2), Some("two")),
+ (Some(1), Some("one")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Int32Array, StringArray, _, _>([
+ (Some(1), Some("one")),
+ (Some(5), None), // No match in WHEN
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (Some(2), Some("two")),
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (Some(2), Some("two")),
+ (Some(5), None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_int32_to_string_with_duplicate_cases() {
+ let lookup_map = create_lookup([
+ (Some(4), Some("four")),
+ (Some(4), Some("no 4")),
+ (Some(2), Some("two")),
+ (Some(2), Some("no 2")),
+ (Some(3), Some("three")),
+ (Some(3), Some("no 3")),
+ (Some(2), Some("no 2")),
+ (Some(4), Some("no 4")),
+ (Some(2), Some("no 2")),
+ (Some(3), Some("no 3")),
+ (Some(4), Some("no 4")),
+ (Some(2), Some("no 2")),
+ (Some(3), Some("no 3")),
+ (Some(3), Some("no 3")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Int32Array, StringArray, _, _>([
+ (1, None), // No match in WHEN
+ (2, Some("two")),
+ (3, Some("three")),
+ (3, Some("three")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ (5, None), // No match in WHEN
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_f32_to_string_with_special_values_and_duplicate_cases()
+ {
+ let lookup_map = create_lookup([
+ (Some(4.0), Some("four point zero")),
+ (Some(f32::NAN), Some("NaN")),
+ (Some(3.2), Some("three point two")),
+ // Duplicate case to make sure it is not used
+ (Some(f32::NAN), Some("should not use this NaN branch")),
+ (Some(f32::INFINITY), Some("Infinity")),
+ (Some(0.0), Some("zero")),
+ // Duplicate case to make sure it is not used
+ (
+ Some(f32::INFINITY),
+ Some("should not use this Infinity branch"),
+ ),
+ (Some(1.1), Some("one point one")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Float32Array, StringArray, _, _>([
+ (1.1, Some("one point one")),
+ (f32::NAN, Some("NaN")),
+ (3.2, Some("three point two")),
+ (3.2, Some("three point two")),
+ (0.0, Some("zero")),
+ (f32::INFINITY, Some("Infinity")),
+ (3.2, Some("three point two")),
+ (f32::NEG_INFINITY, None), // No match in WHEN
+ (f32::NEG_INFINITY, None), // No match in WHEN
+ (3.2, Some("three point two")),
+ (-0.0, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_f16_to_string_with_special_values() {
+ let lookup_map = create_lookup([
+ (
+ ScalarValue::Float16(Some(f16::from_f32(3.2))),
+ Some("3 dot 2"),
+ ),
+ (ScalarValue::Float16(Some(f16::NAN)), Some("NaN")),
+ (
+ ScalarValue::Float16(Some(f16::from_f32(17.4))),
+ Some("17 dot 4"),
+ ),
+ (ScalarValue::Float16(Some(f16::INFINITY)), Some("Infinity")),
+ (ScalarValue::Float16(Some(f16::ZERO)), Some("zero")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Float16Array, StringArray, _, _>([
+ (f16::from_f32(3.2), Some("3 dot 2")),
+ (f16::NAN, Some("NaN")),
+ (f16::from_f32(17.4), Some("17 dot 4")),
+ (f16::from_f32(17.4), Some("17 dot 4")),
+ (f16::INFINITY, Some("Infinity")),
+ (f16::from_f32(17.4), Some("17 dot 4")),
+ (f16::NEG_INFINITY, None), // No match in WHEN
+ (f16::NEG_INFINITY, None), // No match in WHEN
+ (f16::from_f32(17.4), Some("17 dot 4")),
+ (f16::NEG_ZERO, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_f32_to_string_with_special_values() {
+ let lookup_map = create_lookup([
+ (3.2, Some("3 dot 2")),
+ (f32::NAN, Some("NaN")),
+ (17.4, Some("17 dot 4")),
+ (f32::INFINITY, Some("Infinity")),
+ (f32::ZERO, Some("zero")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Float32Array, StringArray, _, _>([
+ (3.2, Some("3 dot 2")),
+ (f32::NAN, Some("NaN")),
+ (17.4, Some("17 dot 4")),
+ (17.4, Some("17 dot 4")),
+ (f32::INFINITY, Some("Infinity")),
+ (17.4, Some("17 dot 4")),
+ (f32::NEG_INFINITY, None), // No match in WHEN
+ (f32::NEG_INFINITY, None), // No match in WHEN
+ (17.4, Some("17 dot 4")),
+ (-0.0, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_case_when_literal_lookup_f64_to_string_with_special_values() {
+ let lookup_map = create_lookup([
+ (3.2, Some("3 dot 2")),
+ (f64::NAN, Some("NaN")),
+ (17.4, Some("17 dot 4")),
+ (f64::INFINITY, Some("Infinity")),
+ (f64::ZERO, Some("zero")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Float64Array, StringArray, _, _>([
+ (3.2, Some("3 dot 2")),
+ (f64::NAN, Some("NaN")),
+ (17.4, Some("17 dot 4")),
+ (17.4, Some("17 dot 4")),
+ (f64::INFINITY, Some("Infinity")),
+ (17.4, Some("17 dot 4")),
+ (f64::NEG_INFINITY, None), // No match in WHEN
+ (f64::NEG_INFINITY, None), // No match in WHEN
+ (17.4, Some("17 dot 4")),
+ (-0.0, None), // No match in WHEN
+ ]);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ // Test that we don't lose the decimal precision and scale info
+ #[test]
+ fn test_decimal_with_non_default_precision_and_scale() {
+ let lookup_map = create_lookup([
+ (ScalarValue::Decimal32(Some(4), 3, 2), Some("four")),
+ (ScalarValue::Decimal32(Some(2), 3, 2), Some("two")),
+ (ScalarValue::Decimal32(Some(3), 3, 2), Some("three")),
+ (ScalarValue::Decimal32(Some(1), 3, 2), Some("one")),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<Decimal32Array, StringArray, _, _>([
+ (1, Some("one")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (3, Some("three")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ (5, None), // No match in WHEN
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ ]);
+
+ let input_values = input_values
+ .with_precision_and_scale(3, 2)
+ .expect("must be able to set precision and scale");
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ // Test that we don't lose the timezone info
+ #[test]
+ fn test_timestamp_with_non_default_timezone() {
+ let timezone: Option<Arc<str>> = Some("-10:00".into());
+ let lookup_map = create_lookup([
+ (
+ ScalarValue::TimestampMillisecond(Some(4), timezone.clone()),
+ Some("four"),
+ ),
+ (
+ ScalarValue::TimestampMillisecond(Some(2), timezone.clone()),
+ Some("two"),
+ ),
+ (
+ ScalarValue::TimestampMillisecond(Some(3), timezone.clone()),
+ Some("three"),
+ ),
+ (
+ ScalarValue::TimestampMillisecond(Some(1), timezone.clone()),
+ Some("one"),
+ ),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<TimestampMillisecondArray, StringArray, _, _>([
+ (1, Some("one")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (3, Some("three")),
+ (2, Some("two")),
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ (5, None), // No match in WHEN
+ (3, Some("three")),
+ (5, None), // No match in WHEN
+ ]);
+
+ let input_values = input_values.with_timezone_opt(timezone);
+
+ test_lookup_eval_with_and_without_else(
+ &lookup_map,
+ Arc::new(input_values),
+ expected,
+ );
+ }
+
+ #[test]
+ fn test_with_strings_to_int32() {
+ let lookup_map = create_lookup([
+ (Some("why"), Some(42)),
+ (Some("what"), Some(22)),
+ (Some("when"), Some(17)),
+ ]);
+
+ let (input_values, expected) =
+ create_input_and_expected::<StringArray, Int32Array, _, _>([
+ (Some("why"), Some(42)),
+ (Some("5"), None), // No match in WHEN
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (Some("what"), Some(22)),
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (None, None), // None cases are never match in CASE <expr> WHEN <value> syntax
+ (Some("what"), Some(22)),
+ (Some("5"), None), // No match in WHEN
+ ]);
+
+ let input_values = Arc::new(input_values) as ArrayRef;
+
+ // Testing without ELSE should fallback to None
+ test_case_when_literal_lookup(
+ Arc::clone(&input_values),
+ &lookup_map,
+ None,
+ Arc::new(expected.clone()),
+ );
+
+ // Testing with Else
+ let else_value = 101;
+
+ // Changing each expected None to be fallback
+ let expected_with_else = expected
+ .iter()
+ .map(|item| item.unwrap_or(else_value))
+ .map(Some)
+ .collect::<Int32Array>();
+
+ // Test case
+ test_case_when_literal_lookup(
+ input_values,
+ &lookup_map,
+ Some(ScalarValue::Int32(Some(else_value))),
+ Arc::new(expected_with_else),
+ );
+ }
}
diff --git a/datafusion/physical-expr/src/expressions/case/literal_lookup_table/boolean_lookup_table.rs b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/boolean_lookup_table.rs
new file mode 100644
index 0000000000..15b3d04955
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/boolean_lookup_table.rs
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::expressions::case::literal_lookup_table::WhenLiteralIndexMap;
+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray};
+use arrow::datatypes::DataType;
+use datafusion_common::{ScalarValue, internal_err};
+
+#[derive(Clone, Debug)]
+pub(super) struct BooleanIndexMap {
+ true_index: Option<u32>,
+ false_index: Option<u32>,
+}
+
+impl BooleanIndexMap {
+ /// Try creating a new lookup table from the given literals and else index
+ /// The index of each literal in the vector is used as the mapped value in the lookup table.
+ ///
+ /// `literals` are guaranteed to be unique and non-nullable
+ pub(super) fn try_new(
+ unique_non_null_literals: Vec<ScalarValue>,
+ ) -> datafusion_common::Result<Self> {
+ let mut true_index: Option<u32> = None;
+ let mut false_index: Option<u32> = None;
+
+ for (index, literal) in unique_non_null_literals.into_iter().enumerate() {
+ match literal {
+ ScalarValue::Boolean(Some(true)) => {
+ if true_index.is_some() {
+ return internal_err!(
+ "Duplicate true literal found in literals for BooleanIndexMap"
+ );
+ }
+ true_index = Some(index as u32);
+ }
+ ScalarValue::Boolean(Some(false)) => {
+ if false_index.is_some() {
+ return internal_err!(
+ "Duplicate false literal found in literals for BooleanIndexMap"
+ );
+ }
+ false_index = Some(index as u32);
+ }
+ ScalarValue::Boolean(None) => {
+ return internal_err!(
+ "Null literal found in non-null literals for BooleanIndexMap"
+ );
+ }
+ _ => {
+ return internal_err!(
+ "Non-boolean literal found in literals for BooleanIndexMap"
+ );
+ }
+ }
+ }
+
+ Ok(Self {
+ true_index,
+ false_index,
+ })
+ }
+
+ fn map_boolean_array_to_when_indices(
+ &self,
+ array: &BooleanArray,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>> {
+ let true_index = self.true_index.unwrap_or(else_index);
+ let false_index = self.false_index.unwrap_or(else_index);
+
+ Ok(array
+ .into_iter()
+ .map(|value| match value {
+ Some(true) => true_index,
+ Some(false) => false_index,
+ None => else_index,
+ })
+ .collect::<Vec<u32>>())
+ }
+}
+
+impl WhenLiteralIndexMap for BooleanIndexMap {
+ fn map_to_when_indices(
+ &self,
+ array: &ArrayRef,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>> {
+ match array.data_type() {
+ DataType::Boolean => {
+ self.map_boolean_array_to_when_indices(array.as_boolean(), else_index)
+ }
+ // We support dictionary boolean array as we create the lookup table in `CaseWhen` expression
+ // creation when we don't know the schema, so we may receive dictionary encoded boolean arrays at execution time.
+ DataType::Dictionary(_, value_type)
+ if value_type.as_ref() == &DataType::Boolean =>
+ {
+ // Since it is not common to have dictionary encoded boolean arrays
+ // at all than it is ok to do the cast here to simplify the implementation.
+ let converted = arrow::compute::cast(array.as_ref(), &DataType::Boolean)?;
+ self.map_boolean_array_to_when_indices(converted.as_boolean(), else_index)
+ }
+ _ => internal_err!(
+ "Expected boolean array for BooleanIndexMap, got {:?}",
+ array.data_type()
+ ),
+ }
+ }
+}
diff --git a/datafusion/physical-expr/src/expressions/case/literal_lookup_table/bytes_like_lookup_table.rs b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/bytes_like_lookup_table.rs
new file mode 100644
index 0000000000..e5cf3f84fd
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/bytes_like_lookup_table.rs
@@ -0,0 +1,223 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::expressions::case::literal_lookup_table::WhenLiteralIndexMap;
+use arrow::array::{
+ Array, ArrayRef, AsArray, BinaryArray, BinaryViewArray, DictionaryArray,
+ FixedSizeBinaryArray, LargeBinaryArray, LargeStringArray, StringArray,
+ StringViewArray, downcast_integer,
+};
+use arrow::datatypes::{
+ ArrowDictionaryKeyType, BinaryViewType, DataType, StringViewType,
+};
+use datafusion_common::{HashMap, ScalarValue, internal_err, plan_datafusion_err};
+use std::fmt::Debug;
+
+/// Map from byte-like literal values to their first occurrence index
+///
+/// This is a wrapper for handling different kinds of literal maps
+#[derive(Clone, Debug)]
+pub(super) struct BytesLikeIndexMap {
+ /// Map from non-null literal value the first occurrence index in the literals
+ map: HashMap<Vec<u8>, u32>,
+}
+
+impl BytesLikeIndexMap {
+ /// Try creating a new lookup table from the given literals and else index
+ /// The index of each literal in the vector is used as the mapped value in the lookup table.
+ ///
+ /// `literals` are guaranteed to be unique and non-nullable
+ pub(super) fn try_new(
+ unique_non_null_literals: Vec<ScalarValue>,
+ ) -> datafusion_common::Result<Self> {
+ let input = ScalarValue::iter_to_array(unique_non_null_literals)?;
+
+ // Literals are guaranteed to not contain nulls
+ if input.logical_null_count() > 0 {
+ return internal_err!("Literal values for WHEN clauses cannot contain nulls");
+ }
+
+ let map: HashMap<Vec<u8>, u32> = try_get_bytes_iterator(&input)?
+ // Flattening Option<&[u8]> to &[u8] as literals cannot contain nulls
+ .flatten()
+ .enumerate()
+ .map(|(map_index, value)| (value.to_vec(), map_index as u32))
+ // Because literals are unique we can collect directly, and we can avoid only inserting the first occurrence
+ .collect();
+
+ Ok(Self { map })
+ }
+}
+
+impl WhenLiteralIndexMap for BytesLikeIndexMap {
+ fn map_to_when_indices(
+ &self,
+ array: &ArrayRef,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>> {
+ let indices = try_get_bytes_iterator(array)?
+ .map(|value| match value {
+ Some(value) => self.map.get(value).copied().unwrap_or(else_index),
+ None => else_index,
+ })
+ .collect::<Vec<u32>>();
+
+ Ok(indices)
+ }
+}
+
+fn try_get_bytes_iterator(
+ array: &ArrayRef,
+) -> datafusion_common::Result<Box<dyn Iterator<Item = Option<&[u8]>> + '_>> {
+ Ok(match array.data_type() {
+ DataType::Utf8 => Box::new(array.as_string::<i32>().into_iter().map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ })),
+
+ DataType::LargeUtf8 => {
+ Box::new(array.as_string::<i64>().into_iter().map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ }))
+ }
+
+ DataType::Binary => Box::new(array.as_binary::<i32>().into_iter()),
+
+ DataType::LargeBinary => Box::new(array.as_binary::<i64>().into_iter()),
+
+ DataType::FixedSizeBinary(_) => Box::new(array.as_binary::<i64>().into_iter()),
+
+ DataType::Utf8View => Box::new(
+ array
+ .as_byte_view::<StringViewType>()
+ .into_iter()
+ .map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ }),
+ ),
+ DataType::BinaryView => {
+ Box::new(array.as_byte_view::<BinaryViewType>().into_iter())
+ }
+
+ DataType::Dictionary(key, _) => {
+ macro_rules! downcast_dictionary_array_helper {
+ ($t:ty) => {{ get_bytes_iterator_for_dictionary(array.as_dictionary::<$t>())? }};
+ }
+
+ downcast_integer! {
+ key.as_ref() => (downcast_dictionary_array_helper),
+ k => unreachable!("unsupported dictionary key type: {}", k)
+ }
+ }
+ t => {
+ return Err(plan_datafusion_err!(
+ "Unsupported data type for bytes lookup table: {}",
+ t
+ ));
+ }
+ })
+}
+
+fn get_bytes_iterator_for_dictionary<K: ArrowDictionaryKeyType + Send + Sync>(
+ array: &DictionaryArray<K>,
+) -> datafusion_common::Result<Box<dyn Iterator<Item = Option<&[u8]>> + '_>> {
+ Ok(match array.values().data_type() {
+ DataType::Utf8 => Box::new(
+ array
+ .downcast_dict::<StringArray>()
+ .unwrap()
+ .into_iter()
+ .map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ }),
+ ),
+
+ DataType::LargeUtf8 => Box::new(
+ array
+ .downcast_dict::<LargeStringArray>()
+ .unwrap()
+ .into_iter()
+ .map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ }),
+ ),
+
+ DataType::Binary => {
+ Box::new(array.downcast_dict::<BinaryArray>().unwrap().into_iter())
+ }
+
+ DataType::LargeBinary => Box::new(
+ array
+ .downcast_dict::<LargeBinaryArray>()
+ .unwrap()
+ .into_iter(),
+ ),
+
+ DataType::FixedSizeBinary(_) => Box::new(
+ array
+ .downcast_dict::<FixedSizeBinaryArray>()
+ .unwrap()
+ .into_iter(),
+ ),
+
+ DataType::Utf8View => Box::new(
+ array
+ .downcast_dict::<StringViewArray>()
+ .unwrap()
+ .into_iter()
+ .map(|item| {
+ item.map(|v| {
+ let bytes: &[u8] = v.as_ref();
+
+ bytes
+ })
+ }),
+ ),
+ DataType::BinaryView => Box::new(
+ array
+ .downcast_dict::<BinaryViewArray>()
+ .unwrap()
+ .into_iter(),
+ ),
+
+ t => {
+ return Err(plan_datafusion_err!(
+ "Unsupported data type for lookup table dictionary value: {}",
+ t
+ ));
+ }
+ })
+}
diff --git a/datafusion/physical-expr/src/expressions/case/literal_lookup_table/mod.rs b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/mod.rs
new file mode 100644
index 0000000000..67b045f998
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/mod.rs
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod boolean_lookup_table;
+mod bytes_like_lookup_table;
+mod primitive_lookup_table;
+
+use crate::expressions::Literal;
+use crate::expressions::case::CaseBody;
+use crate::expressions::case::literal_lookup_table::boolean_lookup_table::BooleanIndexMap;
+use crate::expressions::case::literal_lookup_table::bytes_like_lookup_table::BytesLikeIndexMap;
+use crate::expressions::case::literal_lookup_table::primitive_lookup_table::PrimitiveIndexMap;
+use arrow::array::{Array, ArrayRef, UInt32Array, downcast_primitive};
+use arrow::datatypes::DataType;
+use datafusion_common::{ScalarValue, arrow_datafusion_err, plan_datafusion_err};
+use indexmap::IndexMap;
+use std::fmt::Debug;
+
+/// Optimization for CASE expressions with literal WHEN and THEN clauses
+///
+/// for this form:
+/// ```sql
+/// CASE <expr_a>
+/// WHEN <literal_a> THEN <literal_e>
+/// WHEN <literal_b> THEN <literal_f>
+/// WHEN <literal_c> THEN <literal_g>
+/// WHEN <literal_d> THEN <literal_h>
+/// ELSE <optional-fallback_literal>
+/// END
+/// ```
+///
+/// # Improvement idea
+/// TODO - we should think of unwrapping the `IN` expressions into multiple equality comparisons
+/// so it will use this optimization as well, e.g.
+/// ```sql
+/// -- Before
+/// CASE
+/// WHEN (<expr_a> = <literal_a>) THEN <literal_e>
+/// WHEN (<expr_a> in (<literal_b>, <literal_c>) THEN <literal_f>
+/// WHEN (<expr_a> = <literal_d>) THEN <literal_g>
+/// ELSE <optional-fallback_literal>
+///
+/// -- After
+/// CASE
+/// WHEN (<expr_a> = <literal_a>) THEN <literal_e>
+/// WHEN (<expr_a> = <literal_b>) THEN <literal_f>
+/// WHEN (<expr_a> = <literal_c>) THEN <literal_g>
+/// WHEN (<expr_a> = <literal_d>) THEN <literal_h>
+/// ELSE <optional-fallback_literal>
+/// END
+/// ```
+///
+#[derive(Debug)]
+pub(in super::super) struct LiteralLookupTable {
+ /// The lookup table to use for evaluating the CASE expression
+ lookup: Box<dyn WhenLiteralIndexMap>,
+
+ else_index: u32,
+
+ /// [`ArrayRef`] where `array[i] = then_literals[i]`
+ /// the last value in the array is the else_expr
+ ///
+ /// This will be used to take from based on the indices returned by the lookup table to build the final output
+ then_and_else_values: ArrayRef,
+}
+
+impl LiteralLookupTable {
+ pub(in super::super) fn maybe_new(body: &CaseBody) -> Option<Self> {
+ // We can't use the optimization if we don't have any when then pairs
+ if body.when_then_expr.is_empty() {
+ return None;
+ }
+
+ // If we only have 1 than this optimization is not useful
+ if body.when_then_expr.len() == 1 {
+ return None;
+ }
+
+ // Try to downcast all the WHEN/THEN expressions to literals
+ let when_then_exprs_maybe_literals = body
+ .when_then_expr
+ .iter()
+ .map(|(when, then)| {
+ let when_maybe_literal = when.as_any().downcast_ref::<Literal>();
+ let then_maybe_literal = then.as_any().downcast_ref::<Literal>();
+
+ when_maybe_literal.zip(then_maybe_literal)
+ })
+ .collect::<Vec<_>>();
+
+ // If not all the WHEN/THEN expressions are literals we cannot use this optimization
+ if when_then_exprs_maybe_literals.contains(&None) {
+ return None;
+ }
+
+ let when_then_exprs_scalars = when_then_exprs_maybe_literals
+ .into_iter()
+ // Unwrap the options as we have already checked there is no None
+ .flatten()
+ .map(|(when_lit, then_lit)| {
+ (when_lit.value().clone(), then_lit.value().clone())
+ })
+ // Only keep non-null WHEN literals
+ // as they cannot be matched - case NULL WHEN NULL THEN ... ELSE ... END always goes to ELSE
+ .filter(|(when_lit, _)| !when_lit.is_null())
+ .collect::<Vec<_>>();
+
+ if when_then_exprs_scalars.is_empty() {
+ // All WHEN literals were nulls, so cannot use optimization
+ //
+ // instead, another optimization would be to go straight to the ELSE clause
+ return None;
+ }
+
+ // Keep only the first occurrence of each when literal (as the first match is used)
+ // and remove nulls (as they cannot be matched - case NULL WHEN NULL THEN ... ELSE ... END always goes to ELSE)
+ let (when, then): (Vec<ScalarValue>, Vec<ScalarValue>) = {
+ let mut map = IndexMap::with_capacity(body.when_then_expr.len());
+
+ for (when, then) in when_then_exprs_scalars.into_iter() {
+ // Don't overwrite existing entries as we want to keep the first occurrence
+ if !map.contains_key(&when) {
+ map.insert(when, then);
+ }
+ }
+
+ map.into_iter().unzip()
+ };
+
+ let else_value: ScalarValue = if let Some(else_expr) = &body.else_expr {
+ let literal = else_expr.as_any().downcast_ref::<Literal>()?;
+
+ literal.value().clone()
+ } else {
+ let Ok(null_scalar) = ScalarValue::try_new_null(&then[0].data_type()) else {
+ return None;
+ };
+
+ null_scalar
+ };
+
+ {
+ let when_data_type = when[0].data_type();
+
+ // If not all the WHEN literals are the same data type we cannot use this optimization
+ if when.iter().any(|l| l.data_type() != when_data_type) {
+ return None;
+ }
+ }
+
+ {
+ let data_type = then[0].data_type();
+
+ // If not all the then and the else literals are the same data type we cannot use this optimization
+ if then.iter().any(|l| l.data_type() != data_type) {
+ return None;
+ }
+
+ if else_value.data_type() != data_type {
+ return None;
+ }
+ }
+
+ let then_and_else_values = ScalarValue::iter_to_array(
+ then.iter()
+ // The else is in the end
+ .chain(std::iter::once(&else_value))
+ .cloned(),
+ )
+ .ok()?;
+ // The else expression is in the end
+ let else_index = then_and_else_values.len() as u32 - 1;
+
+ let lookup = try_creating_lookup_table(when).ok()?;
+
+ Some(Self {
+ lookup,
+ then_and_else_values,
+ else_index,
+ })
+ }
+
+ pub(in super::super) fn map_keys_to_values(
+ &self,
+ keys_array: &ArrayRef,
+ ) -> datafusion_common::Result<ArrayRef> {
+ let take_indices = self
+ .lookup
+ .map_to_when_indices(keys_array, self.else_index)?;
+
+ // Zero-copy conversion
+ let take_indices = UInt32Array::from(take_indices);
+
+ // An optimize version would depend on the type of the values_to_take_from
+ // For example, if the type is view we can just keep pointing to the same value (similar to dictionary)
+ // if the type is dictionary we can just use the indices as is (or cast them to the key type) and create a new dictionary array
+ let output =
+ arrow::compute::take(&self.then_and_else_values, &take_indices, None)
+ .map_err(|e| arrow_datafusion_err!(e))?;
+
+ Ok(output)
+ }
+}
+
+/// Map values that match the WHEN literal to the index of their corresponding WHEN clause
+///
+/// For example, for this CASE expression:
+///
+/// ```sql
+/// CASE <expr_a>
+/// WHEN <literal_a> THEN <result_e>
+/// WHEN <literal_b> THEN <result_f>
+/// WHEN <literal_c> THEN <result_g>
+/// WHEN <literal_d> THEN <result_h>
+/// ELSE <fallback_result>
+/// END
+/// ```
+///
+/// this will map <literal_a> to 0, <literal_b> to 1, <literal_c> to 2, <literal_d> to 3
+pub(super) trait WhenLiteralIndexMap: Debug + Send + Sync {
+ /// Given an array of values, returns a vector of WHEN clause indices corresponding to each value in the provided array.
+ ///
+ /// For example, for this CASE expression:
+ ///
+ /// ```sql
+ /// CASE <expr_a>
+ /// WHEN <literal_a> THEN <result_e>
+ /// WHEN <literal_b> THEN <result_f>
+ /// WHEN <literal_c> THEN <result_g>
+ /// WHEN <literal_d> THEN <result_h>
+ /// ELSE <fallback_result>
+ /// END
+ /// ```
+ ///
+ /// the array will be the evaluated values of `<expr_a>`
+ /// and if that array is:
+ /// - `[<literal_a>, <literal_c>, <literal_x>, <literal_b>, <literal_a>]`
+ ///
+ /// the returned vector will be:
+ /// - `[0, 2, else_index, 1, 0]`
+ ///
+ fn map_to_when_indices(
+ &self,
+ array: &ArrayRef,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>>;
+}
+
+fn try_creating_lookup_table(
+ unique_non_null_literals: Vec<ScalarValue>,
+) -> datafusion_common::Result<Box<dyn WhenLiteralIndexMap>> {
+ assert_ne!(
+ unique_non_null_literals.len(),
+ 0,
+ "Must have at least one literal"
+ );
+ match unique_non_null_literals[0].data_type() {
+ DataType::Boolean => {
+ let lookup_table = BooleanIndexMap::try_new(unique_non_null_literals)?;
+ Ok(Box::new(lookup_table))
+ }
+
+ data_type if data_type.is_primitive() => {
+ macro_rules! create_matching_map {
+ ($t:ty) => {{
+ let lookup_table =
+ PrimitiveIndexMap::<$t>::try_new(unique_non_null_literals)?;
+ Ok(Box::new(lookup_table))
+ }};
+ }
+
+ downcast_primitive! {
+ data_type => (create_matching_map),
+ _ => Err(plan_datafusion_err!(
+ "Unsupported field type for primitive: {:?}",
+ data_type
+ )),
+ }
+ }
+
+ DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Binary
+ | DataType::LargeBinary
+ | DataType::FixedSizeBinary(_)
+ | DataType::Utf8View
+ | DataType::BinaryView => {
+ let lookup_table = BytesLikeIndexMap::try_new(unique_non_null_literals)?;
+ Ok(Box::new(lookup_table))
+ }
+
+ DataType::Dictionary(_key, value)
+ if matches!(
+ value.as_ref(),
+ DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Binary
+ | DataType::LargeBinary
+ | DataType::FixedSizeBinary(_)
+ | DataType::Utf8View
+ | DataType::BinaryView
+ ) =>
+ {
+ let lookup_table = BytesLikeIndexMap::try_new(unique_non_null_literals)?;
+ Ok(Box::new(lookup_table))
+ }
+
+ _ => Err(plan_datafusion_err!(
+ "Unsupported data type for lookup table: {}",
+ unique_non_null_literals[0].data_type()
+ )),
+ }
+}
diff --git a/datafusion/physical-expr/src/expressions/case/literal_lookup_table/primitive_lookup_table.rs b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/primitive_lookup_table.rs
new file mode 100644
index 0000000000..36d282c2a4
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/case/literal_lookup_table/primitive_lookup_table.rs
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::expressions::case::literal_lookup_table::WhenLiteralIndexMap;
+use arrow::array::{
+ Array, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, AsArray, PrimitiveArray,
+};
+use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, i256};
+use datafusion_common::{HashMap, ScalarValue, internal_err};
+use half::f16;
+use std::fmt::Debug;
+use std::hash::Hash;
+
+#[derive(Clone)]
+pub(super) struct PrimitiveIndexMap<T>
+where
+ T: ArrowPrimitiveType,
+ T::Native: ToHashableKey,
+{
+ data_type: DataType,
+ /// Literal value to map index
+ ///
+ /// If searching this map becomes a bottleneck consider using linear map implementations for small hashmaps
+ map: HashMap<<T::Native as ToHashableKey>::HashableKey, u32>,
+}
+
+impl<T> Debug for PrimitiveIndexMap<T>
+where
+ T: ArrowPrimitiveType,
+ T::Native: ToHashableKey,
+{
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("PrimitiveIndexMap")
+ .field("map", &self.map)
+ .finish()
+ }
+}
+
+impl<T> PrimitiveIndexMap<T>
+where
+ T: ArrowPrimitiveType,
+ T::Native: ToHashableKey,
+{
+ /// Try creating a new lookup table from the given literals and else index.
+ /// The index of each literal in the vector is used as the mapped value in the lookup table.
+ ///
+ /// `literals` are guaranteed to be unique and non-nullable
+ pub(super) fn try_new(
+ unique_non_null_literals: Vec<ScalarValue>,
+ ) -> datafusion_common::Result<Self> {
+ let input = ScalarValue::iter_to_array(unique_non_null_literals)?;
+
+ // Literals are guaranteed to not contain nulls
+ if input.null_count() > 0 {
+ return internal_err!("Literal values for WHEN clauses cannot contain nulls");
+ }
+
+ let map = input
+ .as_primitive::<T>()
+ .values()
+ .iter()
+ .enumerate()
+ // Because literals are unique we can collect directly, and we can avoid only inserting the first occurrence
+ .map(|(map_index, value)| (value.into_hashable_key(), map_index as u32))
+ .collect();
+
+ Ok(Self {
+ map,
+ data_type: input.data_type().clone(),
+ })
+ }
+
+ fn map_primitive_array_to_when_indices(
+ &self,
+ array: &PrimitiveArray<T>,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>> {
+ let indices = array
+ .into_iter()
+ .map(|value| match value {
+ Some(value) => self
+ .map
+ .get(&value.into_hashable_key())
+ .copied()
+ .unwrap_or(else_index),
+
+ None => else_index,
+ })
+ .collect::<Vec<u32>>();
+
+ Ok(indices)
+ }
+}
+
+impl<T> WhenLiteralIndexMap for PrimitiveIndexMap<T>
+where
+ T: ArrowPrimitiveType,
+ T::Native: ToHashableKey,
+{
+ fn map_to_when_indices(
+ &self,
+ array: &ArrayRef,
+ else_index: u32,
+ ) -> datafusion_common::Result<Vec<u32>> {
+ match array.data_type() {
+ dt if dt == &self.data_type => {
+ let primitive_array = array.as_primitive::<T>();
+
+ self.map_primitive_array_to_when_indices(primitive_array, else_index)
+ }
+ // We support dictionary primitive array as we create the lookup table in `CaseWhen` expression
+ // creation when we don't know the schema, so we may receive dictionary encoded primitive arrays at execution time.
+ DataType::Dictionary(_, value_type)
+ if value_type.as_ref() == &self.data_type =>
+ {
+ // Cast here to simplify the implementation.
+ let converted = arrow::compute::cast(array.as_ref(), &self.data_type)?;
+ self.map_primitive_array_to_when_indices(
+ converted.as_primitive::<T>(),
+ else_index,
+ )
+ }
+ _ => internal_err!(
+ "PrimitiveIndexMap expected array of type {:?} but got {:?}",
+ self.data_type,
+ array.data_type()
+ ),
+ }
+ }
+}
+
+// TODO - We need to port it to arrow so that it can be reused in other places
+
+/// Trait that help convert a value to a key that is hashable and equatable
+/// This is needed as some types like f16/f32/f64 do not implement Hash/Eq directly
+pub(super) trait ToHashableKey: ArrowNativeTypeOp {
+ /// The type that is hashable and equatable
+ /// It must be an Arrow native type but it NOT GUARANTEED to be the same as Self
+ /// this is just a helper trait so you can reuse the same code for all arrow native types
+ type HashableKey: Hash + Eq + Debug + Clone + Copy + Send + Sync;
+
+ /// Converts self to a hashable key
+ /// the result of this value can be used as the key in hash maps/sets
+ fn into_hashable_key(self) -> Self::HashableKey;
+}
+
+macro_rules! impl_to_hashable_key {
+ (@single_already_hashable | $t:ty) => {
+ impl ToHashableKey for $t {
+ type HashableKey = $t;
+
+ #[inline]
+ fn into_hashable_key(self) -> Self::HashableKey {
+ self
+ }
+ }
+ };
+ (@already_hashable | $($t:ty),+ $(,)?) => {
+ $(
+ impl_to_hashable_key!(@single_already_hashable | $t);
+ )+
+ };
+ (@float | $t:ty => $hashable:ty) => {
+ impl ToHashableKey for $t {
+ type HashableKey = $hashable;
+
+ #[inline]
+ fn into_hashable_key(self) -> Self::HashableKey {
+ self.to_bits()
+ }
+ }
+ };
+}
+
+impl_to_hashable_key!(@already_hashable | i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, IntervalDayTime, IntervalMonthDayNano);
+impl_to_hashable_key!(@float | f16 => u16);
+impl_to_hashable_key!(@float | f32 => u32);
+impl_to_hashable_key!(@float | f64 => u64);
+
+#[cfg(test)]
+mod tests {
+ use super::ToHashableKey;
+ use arrow::array::downcast_primitive;
+
+ // This test ensure that all arrow primitive types implement ToHashableKey
+ // otherwise the code will not compile
+ #[test]
+ fn should_implement_to_hashable_key_for_all_primitives() {
+ #[derive(Debug, Default)]
+ struct ExampleSet<T>
+ where
+ T: arrow::datatypes::ArrowPrimitiveType,
+ T::Native: ToHashableKey,
+ {
+ _map: std::collections::HashSet<<T::Native as ToHashableKey>::HashableKey>,
+ }
+
+ macro_rules! create_matching_set {
+ ($t:ty) => {{
+ let _lookup_table = ExampleSet::<$t> {
+ _map: Default::default(),
+ };
+
+ return;
+ }};
+ }
+
+ let data_type = arrow::datatypes::DataType::Float16;
+
+ downcast_primitive! {
+ data_type => (create_matching_set),
+ _ => panic!("not implemented for {data_type}"),
+ }
+ }
+}