Apache DataFusion SQL Query Engine
Add additional tests for InListExpr (#19050)
Adrian Garcia Badaracco21a16e44bc7e
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 9f2204b66a..fba8f4fbe4 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -226,31 +226,62 @@ impl StaticFilter for Int32StaticFilter {}fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {+ // Handle dictionary arrays by recursing on the values+ downcast_dictionary_array! {+ v => {+ let values_contains = self.contains(v.values().as_ref(), negated)?;+ let result = take(&values_contains, v.keys(), None)?;+ return Ok(downcast_array(result.as_ref()))+ }+ _ => {}+ }+let v = v.as_primitive_opt::<Int32Type>().ok_or_else(|| exec_datafusion_err!("Failed to downcast array"))?;- let result = match (v.null_count() > 0, negated) {+ let haystack_has_nulls = self.null_count > 0;+ let has_nulls = v.null_count() > 0 || haystack_has_nulls;++ let result = match (has_nulls, negated) {(true, false) => {- // has nulls, not negated"- BooleanArray::from_iter(- v.iter().map(|value| Some(self.values.contains(&value?))),- )+ // needle has nulls, not negated+ BooleanArray::from_iter(v.iter().map(|value| match value {+ None => None,+ Some(v) => {+ if self.values.contains(&v) {+ Some(true)+ } else if haystack_has_nulls {+ None+ } else {+ Some(false)+ }+ }+ }))}(true, true) => {- // has nulls, negated- BooleanArray::from_iter(- v.iter().map(|value| Some(!self.values.contains(&value?))),- )+ // needle has nulls, negated+ BooleanArray::from_iter(v.iter().map(|value| match value {+ None => None,+ Some(v) => {+ if self.values.contains(&v) {+ Some(false)+ } else if haystack_has_nulls {+ None+ } else {+ Some(true)+ }+ }+ }))}(false, false) => {- //no null, not negated+ // No nulls anywhere, not negatedBooleanArray::from_iter(v.values().iter().map(|value| self.values.contains(value)),)}(false, true) => {- // no null, negated+ // No nulls anywhere, negatedBooleanArray::from_iter(v.values().iter().map(|value| !self.values.contains(value)),)@@ -284,15 +315,26 @@ fn evaluate_list(/// Try to evaluate a list of expressions as constants.///-/// Returns an ArrayRef if all expressions are constants (can be evaluated on an-/// empty RecordBatch), otherwise returns an error. This is used to detect when-/// a list contains only literals, casts of literals, or other constant expressions.+/// Returns:+/// - `Ok(Some(ArrayRef))` if all expressions are constants (can be evaluated on an empty RecordBatch)+/// - `Ok(None)` if the list contains non-constant expressions+/// - `Err(...)` only for actual errors (not for non-constant expressions)+///+/// This is used to detect when a list contains only literals, casts of literals,+/// or other constant expressions.fn try_evaluate_constant_list(list: &[Arc<dyn PhysicalExpr>],schema: &Schema,-) -> Result<ArrayRef> {+) -> Result<Option<ArrayRef>> {let batch = RecordBatch::new_empty(Arc::new(schema.clone()));- evaluate_list(list, &batch)+ match evaluate_list(list, &batch) {+ Ok(array) => Ok(Some(array)),+ Err(_) => {+ // Non-constant expressions can't be evaluated on an empty batch+ // This is not an error, just means we can't use a static filter+ Ok(None)+ }+ }}impl InListExpr {@@ -363,6 +405,42 @@ impl InListExpr {Some(instantiate_static_filter(array)?),))}++ /// Create a new InList expression, using a static filter when possible.+ ///+ /// This validates data types and attempts to create a static filter for constant+ /// list expressions. Uses specialized StaticFilter implementations for better+ /// performance (e.g., Int32StaticFilter for Int32).+ ///+ /// Returns an error if data types don't match. If the list contains non-constant+ /// expressions, falls back to dynamic evaluation at runtime.+ pub fn try_new(+ expr: Arc<dyn PhysicalExpr>,+ list: Vec<Arc<dyn PhysicalExpr>>,+ negated: bool,+ schema: &Schema,+ ) -> Result<Self> {+ // Check the data types match+ let expr_data_type = expr.data_type(schema)?;+ for list_expr in list.iter() {+ let list_expr_data_type = list_expr.data_type(schema)?;+ assert_or_internal_err!(+ DFSchema::datatype_is_logically_equal(+ &expr_data_type,+ &list_expr_data_type+ ),+ "The data type inlist should be same, the value type is {expr_data_type}, one of list expr type is {list_expr_data_type}"+ );+ }++ // Try to create a static filter if all list expressions are constants+ let static_filter = match try_evaluate_constant_list(&list, schema)? {+ Some(in_array) => Some(instantiate_static_filter(in_array)?),+ None => None, // Non-constant expressions, fall back to dynamic evaluation+ };++ Ok(Self::new(expr, list, negated, static_filter))+ }}impl std::fmt::Display for InListExpr {fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {@@ -564,36 +642,12 @@ pub fn in_list(negated: &bool,schema: &Schema,) -> Result<Arc<dyn PhysicalExpr>> {- // check the data type- let expr_data_type = expr.data_type(schema)?;- for list_expr in list.iter() {- let list_expr_data_type = list_expr.data_type(schema)?;- assert_or_internal_err!(- DFSchema::datatype_is_logically_equal(&expr_data_type, &list_expr_data_type),- "The data type inlist should be same, the value type is {expr_data_type}, one of list expr type is {list_expr_data_type}"- );- }-- // Try to create a static filter for constant expressions- let static_filter = try_evaluate_constant_list(&list, schema)- .and_then(ArrayStaticFilter::try_new)- .ok()- .map(|static_filter| {- Arc::new(static_filter) as Arc<dyn StaticFilter + Send + Sync>- });-- Ok(Arc::new(InListExpr::new(- expr,- list,- *negated,- static_filter,- )))+ Ok(Arc::new(InListExpr::try_new(expr, list, *negated, schema)?))}#[cfg(test)]mod tests {use super::*;- use crate::expressions;use crate::expressions::{col, lit, try_cast};use arrow::buffer::NullBuffer;use datafusion_common::plan_err;@@ -636,14 +690,6 @@ mod tests {}}- fn try_cast_static_filter_to_set(- list: &[Arc<dyn PhysicalExpr>],- schema: &Schema,- ) -> Result<ArrayStaticFilter> {- let array = try_evaluate_constant_list(list, schema)?;- ArrayStaticFilter::try_new(array)- }-// Attempts to coerce the types of `list_type` to be comparable with the// `expr_type`fn get_coerce_type(expr_type: &DataType, list_type: &[DataType]) -> Option<DataType> {@@ -695,573 +741,461 @@ mod tests {/// and list expressions are already the correct types and don't require casting.macro_rules! in_list_raw {($BATCH:expr, $LIST:expr, $NEGATED:expr, $EXPECTED:expr, $COL:expr, $SCHEMA:expr) => {{- let expr = in_list($COL, $LIST, $NEGATED, $SCHEMA).unwrap();+ let col_expr = $COL;+ let expr = in_list(Arc::clone(&col_expr), $LIST, $NEGATED, $SCHEMA).unwrap();let result = expr.evaluate(&$BATCH)?.into_array($BATCH.num_rows()).expect("Failed to convert to array");let result = as_boolean_array(&result);let expected = &BooleanArray::from($EXPECTED);- assert_eq!(expected, result);+ assert_eq!(+ expected,+ result,+ "Failed for: {}\n{}: {:?}",+ fmt_sql(expr.as_ref()),+ fmt_sql(col_expr.as_ref()),+ col_expr+ .evaluate(&$BATCH)?+ .into_array($BATCH.num_rows())+ .unwrap()+ );}};}- #[test]- fn in_list_utf8() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);- let a = StringArray::from(vec![Some("a"), Some("d"), None]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in ("a", "b")"- let list = vec![lit("a"), lit("b")];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in ("a", "b")"- let list = vec![lit("a"), lit("b")];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in ("a", "b", null)"- let list = vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))];- in_list!(- batch,- list,- &false,- vec![Some(true), None, None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in ("a", "b", null)"- let list = vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))];- in_list!(- batch,- list,- &true,- vec![Some(false), None, None],- Arc::clone(&col_a),- &schema- );-- Ok(())- }-- #[test]- fn in_list_binary() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Binary, true)]);- let a = BinaryArray::from(vec![- Some([1, 2, 3].as_slice()),- Some([1, 2, 2].as_slice()),- None,- ]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in ([1, 2, 3], [4, 5, 6])"- let list = vec![lit([1, 2, 3].as_slice()), lit([4, 5, 6].as_slice())];- in_list!(- batch,- list.clone(),- &false,- vec![Some(true), Some(false), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in ([1, 2, 3], [4, 5, 6])"- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in ([1, 2, 3], [4, 5, 6], null)"- let list = vec![- lit([1, 2, 3].as_slice()),- lit([4, 5, 6].as_slice()),- lit(ScalarValue::Binary(None)),- ];- in_list!(- batch,- list.clone(),- &false,- vec![Some(true), None, None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in ([1, 2, 3], [4, 5, 6], null)"- in_list!(- batch,- list,- &true,- vec![Some(false), None, None],- Arc::clone(&col_a),- &schema- );-- Ok(())+ /// Test case for primitive types following the standard IN LIST pattern.+ ///+ /// Each test case represents a data type with:+ /// - `value_in`: A value that appears in both the test array and the IN list (matches → true)+ /// - `value_not_in`: A value that appears in the test array but NOT in the IN list (doesn't match → false)+ /// - `other_list_values`: Additional values in the IN list besides `value_in`+ /// - `null_value`: Optional null scalar value for NULL handling tests. When None, tests+ /// without nulls are run, exercising the `(false, false)` and `(false, true)` branches.+ struct InListPrimitiveTestCase {+ name: &'static str,+ value_in: ScalarValue,+ value_not_in: ScalarValue,+ other_list_values: Vec<ScalarValue>,+ null_value: Option<ScalarValue>,+ }++ /// Generic test data struct for primitive types.+ ///+ /// Holds test values needed for IN LIST tests, allowing the data+ /// to be declared explicitly and reused across multiple types.+ #[derive(Clone)]+ struct PrimitiveTestCaseData<T> {+ value_in: T,+ value_not_in: T,+ other_list_values: Vec<T>,}- #[test]- fn in_list_int64() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]);- let a = Int64Array::from(vec![Some(0), Some(2), None]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in (0, 1)"- let list = vec![lit(0i64), lit(1i64)];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0, 1)"- let list = vec![lit(0i64), lit(1i64)];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (0, 1, NULL)"- let list = vec![lit(0i64), lit(1i64), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &false,- vec![Some(true), None, None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0, 1, NULL)"- let list = vec![lit(0i64), lit(1i64), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &true,- vec![Some(false), None, None],- Arc::clone(&col_a),- &schema- );-- Ok(())+ /// Helper to create test cases for any primitive type using generic data.+ ///+ /// Uses TryInto for flexible type conversion, allowing test data to be+ /// declared in any convertible type (e.g., i32 for all integer types).+ /// Creates a test case WITH null support (for null handling tests).+ fn primitive_test_case<T, D, F>(+ name: &'static str,+ constructor: F,+ data: PrimitiveTestCaseData<D>,+ ) -> InListPrimitiveTestCase+ where+ D: TryInto<T> + Clone,+ <D as TryInto<T>>::Error: Debug,+ F: Fn(Option<T>) -> ScalarValue,+ T: Clone,+ {+ InListPrimitiveTestCase {+ name,+ value_in: constructor(Some(data.value_in.try_into().unwrap())),+ value_not_in: constructor(Some(data.value_not_in.try_into().unwrap())),+ other_list_values: data+ .other_list_values+ .into_iter()+ .map(|v| constructor(Some(v.try_into().unwrap())))+ .collect(),+ null_value: Some(constructor(None)),+ }}- #[test]- fn in_list_float64() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Float64, true)]);- let a = Float64Array::from(vec![- Some(0.0),- Some(0.2),- None,- Some(f64::NAN),- Some(-f64::NAN),- ]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in (0.0, 0.1)"- let list = vec![lit(0.0f64), lit(0.1f64)];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None, Some(false), Some(false)],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0.0, 0.1)"- let list = vec![lit(0.0f64), lit(0.1f64)];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None, Some(true), Some(true)],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (0.0, 0.1, NULL)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &false,- vec![Some(true), None, None, None, None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0.0, 0.1, NULL)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &true,- vec![Some(false), None, None, None, None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (0.0, 0.1, NaN)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(f64::NAN)];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None, Some(true), Some(false)],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0.0, 0.1, NaN)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(f64::NAN)];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None, Some(false), Some(true)],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (0.0, 0.1, -NaN)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(-f64::NAN)];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None, Some(false), Some(true)],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0.0, 0.1, -NaN)"- let list = vec![lit(0.0f64), lit(0.1f64), lit(-f64::NAN)];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None, Some(true), Some(false)],- Arc::clone(&col_a),- &schema- );-- Ok(())+ /// Helper to create test cases WITHOUT null support.+ /// These test cases exercise the `(false, true)` branch (no nulls, negated).+ fn primitive_test_case_no_nulls<T, D, F>(+ name: &'static str,+ constructor: F,+ data: PrimitiveTestCaseData<D>,+ ) -> InListPrimitiveTestCase+ where+ D: TryInto<T> + Clone,+ <D as TryInto<T>>::Error: Debug,+ F: Fn(Option<T>) -> ScalarValue,+ T: Clone,+ {+ InListPrimitiveTestCase {+ name,+ value_in: constructor(Some(data.value_in.try_into().unwrap())),+ value_not_in: constructor(Some(data.value_not_in.try_into().unwrap())),+ other_list_values: data+ .other_list_values+ .into_iter()+ .map(|v| constructor(Some(v.try_into().unwrap())))+ .collect(),+ null_value: None,+ }}- #[test]- fn in_list_bool() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);- let a = BooleanArray::from(vec![Some(true), None]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in (true)"- let list = vec![lit(true)];- in_list!(- batch,- list,- &false,- vec![Some(true), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (true)"- let list = vec![lit(true)];- in_list!(- batch,- list,- &true,- vec![Some(false), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (true, NULL)"- let list = vec![lit(true), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &false,- vec![Some(true), None],- Arc::clone(&col_a),- &schema- );+ /// Runs test cases for multiple types, providing detailed SQL error messages on failure.+ ///+ /// For each test case, runs IN LIST scenarios based on whether null_value is Some or None:+ /// - With null_value (Some): 4 tests including null handling+ /// - Without null_value (None): 2 tests exercising the no-nulls paths+ fn run_test_cases(test_cases: Vec<InListPrimitiveTestCase>) -> Result<()> {+ for test_case in test_cases {+ let test_name = test_case.name;++ // Get the data type from the scalar value+ let data_type = test_case.value_in.data_type();++ // Build the base list: [value_in, ...other_list_values]+ let build_base_list = || -> Vec<Arc<dyn PhysicalExpr>> {+ let mut list = vec![lit(test_case.value_in.clone())];+ list.extend(test_case.other_list_values.iter().map(|v| lit(v.clone())));+ list+ };- // expression: "a not in (true, NULL)"- let list = vec![lit(true), lit(ScalarValue::Null)];- in_list!(- batch,- list,- &true,- vec![Some(false), None],- Arc::clone(&col_a),- &schema- );+ match &test_case.null_value {+ Some(null_val) => {+ // Tests WITH nulls in the needle array+ let schema =+ Schema::new(vec![Field::new("a", data_type.clone(), true)]);++ // Create array from scalar values: [value_in, value_not_in, None]+ let array = ScalarValue::iter_to_array(vec![+ test_case.value_in.clone(),+ test_case.value_not_in.clone(),+ null_val.clone(),+ ])?;++ let col_a = col("a", &schema)?;+ let batch = RecordBatch::try_new(+ Arc::new(schema.clone()),+ vec![Arc::clone(&array)],+ )?;++ // Test 1: a IN (list) → [true, false, null]+ let list = build_base_list();+ in_list!(+ batch,+ list,+ &false,+ vec![Some(true), Some(false), None],+ Arc::clone(&col_a),+ &schema+ );++ // Test 2: a NOT IN (list) → [false, true, null]+ let list = build_base_list();+ in_list!(+ batch,+ list,+ &true,+ vec![Some(false), Some(true), None],+ Arc::clone(&col_a),+ &schema+ );++ // Test 3: a IN (list, NULL) → [true, null, null]+ let mut list = build_base_list();+ list.push(lit(null_val.clone()));+ in_list!(+ batch,+ list,+ &false,+ vec![Some(true), None, None],+ Arc::clone(&col_a),+ &schema+ );++ // Test 4: a NOT IN (list, NULL) → [false, null, null]+ let mut list = build_base_list();+ list.push(lit(null_val.clone()));+ in_list!(+ batch,+ list,+ &true,+ vec![Some(false), None, None],+ Arc::clone(&col_a),+ &schema+ );+ }+ None => {+ // Tests WITHOUT nulls - exercises the (false, false) and (false, true) branches+ let schema =+ Schema::new(vec![Field::new("a", data_type.clone(), false)]);++ // Create array from scalar values: [value_in, value_not_in] (no NULL)+ let array = ScalarValue::iter_to_array(vec![+ test_case.value_in.clone(),+ test_case.value_not_in.clone(),+ ])?;++ let col_a = col("a", &schema)?;+ let batch = RecordBatch::try_new(+ Arc::new(schema.clone()),+ vec![Arc::clone(&array)],+ )?;++ // Test 1: a IN (list) → [true, false] - exercises (false, false) branch+ let list = build_base_list();+ in_list!(+ batch,+ list,+ &false,+ vec![Some(true), Some(false)],+ Arc::clone(&col_a),+ &schema+ );++ // Test 2: a NOT IN (list) → [false, true] - exercises (false, true) branch+ let list = build_base_list();+ in_list!(+ batch,+ list,+ &true,+ vec![Some(false), Some(true)],+ Arc::clone(&col_a),+ &schema+ );++ eprintln!(+ "Test '{test_name}': exercised (false, true) branch (no nulls, negated)",+ );+ }+ }+ }Ok(())}+ /// Test IN LIST for all integer types (Int8/16/32/64, UInt8/16/32/64).+ ///+ /// Test data: 0 (in list), 2 (not in list), [1, 3, 5] (other list values)#[test]- fn in_list_date64() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Date64, true)]);- let a = Date64Array::from(vec![Some(0), Some(2), None]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;-- // expression: "a in (0, 1)"- let list = vec![- lit(ScalarValue::Date64(Some(0))),- lit(ScalarValue::Date64(Some(1))),- ];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a not in (0, 1)"- let list = vec![- lit(ScalarValue::Date64(Some(0))),- lit(ScalarValue::Date64(Some(1))),- ];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None],- Arc::clone(&col_a),- &schema- );-- // expression: "a in (0, 1, NULL)"- let list = vec![- lit(ScalarValue::Date64(Some(0))),- lit(ScalarValue::Date64(Some(1))),- lit(ScalarValue::Null),- ];- in_list!(- batch,- list,- &false,- vec![Some(true), None, None],- Arc::clone(&col_a),- &schema- );+ fn in_list_int_types() -> Result<()> {+ let int_data = PrimitiveTestCaseData {+ value_in: 0,+ value_not_in: 2,+ other_list_values: vec![1, 3, 5],+ };- // expression: "a not in (0, 1, NULL)"- let list = vec![- lit(ScalarValue::Date64(Some(0))),- lit(ScalarValue::Date64(Some(1))),- lit(ScalarValue::Null),- ];- in_list!(- batch,- list,- &true,- vec![Some(false), None, None],- Arc::clone(&col_a),- &schema- );+ run_test_cases(vec![+ // Tests WITH nulls+ primitive_test_case("int8", ScalarValue::Int8, int_data.clone()),+ primitive_test_case("int16", ScalarValue::Int16, int_data.clone()),+ primitive_test_case("int32", ScalarValue::Int32, int_data.clone()),+ primitive_test_case("int64", ScalarValue::Int64, int_data.clone()),+ primitive_test_case("uint8", ScalarValue::UInt8, int_data.clone()),+ primitive_test_case("uint16", ScalarValue::UInt16, int_data.clone()),+ primitive_test_case("uint32", ScalarValue::UInt32, int_data.clone()),+ primitive_test_case("uint64", ScalarValue::UInt64, int_data.clone()),+ // Tests WITHOUT nulls - exercises (false, true) branch+ primitive_test_case_no_nulls("int32_no_nulls", ScalarValue::Int32, int_data),+ ])+ }++ /// Test IN LIST for all string types (Utf8, LargeUtf8, Utf8View).+ ///+ /// Test data: "a" (in list), "d" (not in list), ["b", "c"] (other list values)+ #[test]+ fn in_list_string_types() -> Result<()> {+ let string_data = PrimitiveTestCaseData {+ value_in: "a",+ value_not_in: "d",+ other_list_values: vec!["b", "c"],+ };- Ok(())+ run_test_cases(vec![+ primitive_test_case("utf8", ScalarValue::Utf8, string_data.clone()),+ primitive_test_case(+ "large_utf8",+ ScalarValue::LargeUtf8,+ string_data.clone(),+ ),+ primitive_test_case("utf8_view", ScalarValue::Utf8View, string_data),+ ])}+ /// Test IN LIST for all binary types (Binary, LargeBinary, BinaryView).+ ///+ /// Test data: [1,2,3] (in list), [1,2,2] (not in list), [[4,5,6], [7,8,9]] (other list values)#[test]- fn in_list_date32() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", DataType::Date32, true)]);- let a = Date32Array::from(vec![Some(0), Some(2), None]);- let col_a = col("a", &schema)?;- let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;+ fn in_list_binary_types() -> Result<()> {+ let binary_data = PrimitiveTestCaseData {+ value_in: vec![1_u8, 2, 3],+ value_not_in: vec![1_u8, 2, 2],+ other_list_values: vec![vec![4_u8, 5, 6], vec![7_u8, 8, 9]],+ };- // expression: "a in (0, 1)"- let list = vec![- lit(ScalarValue::Date32(Some(0))),- lit(ScalarValue::Date32(Some(1))),- ];- in_list!(- batch,- list,- &false,- vec![Some(true), Some(false), None],- Arc::clone(&col_a),- &schema- );+ run_test_cases(vec![+ primitive_test_case("binary", ScalarValue::Binary, binary_data.clone()),+ primitive_test_case(+ "large_binary",+ ScalarValue::LargeBinary,+ binary_data.clone(),+ ),+ primitive_test_case("binary_view", ScalarValue::BinaryView, binary_data),+ ])+ }- // expression: "a not in (0, 1)"- let list = vec![- lit(ScalarValue::Date32(Some(0))),- lit(ScalarValue::Date32(Some(1))),- ];- in_list!(- batch,- list,- &true,- vec![Some(false), Some(true), None],- Arc::clone(&col_a),- &schema- );+ /// Test IN LIST for date types (Date32, Date64).+ ///+ /// Test data: 0 (in list), 2 (not in list), [1, 3] (other list values)+ #[test]+ fn in_list_date_types() -> Result<()> {+ let date_data = PrimitiveTestCaseData {+ value_in: 0,+ value_not_in: 2,+ other_list_values: vec![1, 3],+ };- // expression: "a in (0, 1, NULL)"- let list = vec![- lit(ScalarValue::Date32(Some(0))),- lit(ScalarValue::Date32(Some(1))),- lit(ScalarValue::Null),- ];- in_list!(- batch,- list,- &false,- vec![Some(true), None, None],- Arc::clone(&col_a),- &schema- );+ run_test_cases(vec![+ primitive_test_case("date32", ScalarValue::Date32, date_data.clone()),+ primitive_test_case("date64", ScalarValue::Date64, date_data),+ ])+ }- // expression: "a not in (0, 1, NULL)"- let list = vec![- lit(ScalarValue::Date32(Some(0))),- lit(ScalarValue::Date32(Some(1))),- lit(ScalarValue::Null),- ];- in_list!(- batch,- list,- &true,- vec![Some(false), None, None],- Arc::clone(&col_a),- &schema- );+ /// Test IN LIST for Decimal128 type.+ ///+ /// Test data: 0 (in list), 200 (not in list), [100, 300] (other list values) with precision=10, scale=2+ #[test]+ fn in_list_decimal() -> Result<()> {+ run_test_cases(vec![InListPrimitiveTestCase {+ name: "decimal128",+ value_in: ScalarValue::Decimal128(Some(0), 10, 2),+ value_not_in: ScalarValue::Decimal128(Some(200), 10, 2),+ other_list_values: vec![+ ScalarValue::Decimal128(Some(100), 10, 2),+ ScalarValue::Decimal128(Some(300), 10, 2),+ ],+ null_value: Some(ScalarValue::Decimal128(None, 10, 2)),+ }])+ }- Ok(())+ /// Test IN LIST for timestamp types.+ ///+ /// Test data: 0 (in list), 2000 (not in list), [1000, 3000] (other list values)+ #[test]+ fn in_list_timestamp_types() -> Result<()> {+ run_test_cases(vec![InListPrimitiveTestCase {+ name: "timestamp_nanosecond",+ value_in: ScalarValue::TimestampNanosecond(Some(0), None),+ value_not_in: ScalarValue::TimestampNanosecond(Some(2000), None),+ other_list_values: vec![+ ScalarValue::TimestampNanosecond(Some(1000), None),+ ScalarValue::TimestampNanosecond(Some(3000), None),+ ],+ null_value: Some(ScalarValue::TimestampNanosecond(None, None)),+ }])}#[test]- fn in_list_decimal() -> Result<()> {- // Now, we can check the NULL type- let schema =- Schema::new(vec![Field::new("a", DataType::Decimal128(13, 4), true)]);- let array = vec![Some(100_0000_i128), None, Some(200_5000_i128)]- .into_iter()- .collect::<Decimal128Array>();- let array = array.with_precision_and_scale(13, 4).unwrap();+ fn in_list_float64() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Float64, true)]);+ let a = Float64Array::from(vec![+ Some(0.0),+ Some(0.2),+ None,+ Some(f64::NAN),+ Some(-f64::NAN),+ ]);let col_a = col("a", &schema)?;- let batch =- RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)])?;+ let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;- // expression: "a in (100,200), the data type of list is INT32- let list = vec![lit(100i32), lit(200i32)];+ // expression: "a in (0.0, 0.1)"+ let list = vec![lit(0.0f64), lit(0.1f64)];in_list!(batch,list,&false,- vec![Some(true), None, Some(false)],+ vec![Some(true), Some(false), None, Some(false), Some(false)],Arc::clone(&col_a),&schema);- // expression: "a not in (100,200)- let list = vec![lit(100i32), lit(200i32)];++ // expression: "a not in (0.0, 0.1)"+ let list = vec![lit(0.0f64), lit(0.1f64)];in_list!(batch,list,&true,- vec![Some(false), None, Some(true)],+ vec![Some(false), Some(true), None, Some(true), Some(true)],Arc::clone(&col_a),&schema);- // expression: "a in (200,NULL), the data type of list is INT32 AND NULL- let list = vec![lit(ScalarValue::Int32(Some(100))), lit(ScalarValue::Null)];+ // expression: "a in (0.0, 0.1, NULL)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(ScalarValue::Null)];in_list!(batch,- list.clone(),+ list,&false,- vec![Some(true), None, None],+ vec![Some(true), None, None, None, None],Arc::clone(&col_a),&schema);- // expression: "a not in (200,NULL), the data type of list is INT32 AND NULL++ // expression: "a not in (0.0, 0.1, NULL)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(ScalarValue::Null)];in_list!(batch,list,&true,- vec![Some(false), None, None],+ vec![Some(false), None, None, None, None],Arc::clone(&col_a),&schema);- // expression: "a in (200.5, 100), the data type of list is FLOAT32 and INT32- let list = vec![lit(200.50f32), lit(100i32)];+ // expression: "a in (0.0, 0.1, NaN)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(f64::NAN)];in_list!(batch,list,&false,- vec![Some(true), None, Some(true)],+ vec![Some(true), Some(false), None, Some(true), Some(false)],Arc::clone(&col_a),&schema);- // expression: "a not in (200.5, 100), the data type of list is FLOAT32 and INT32- let list = vec![lit(200.50f32), lit(101i32)];+ // expression: "a not in (0.0, 0.1, NaN)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(f64::NAN)];in_list!(batch,list,&true,- vec![Some(true), None, Some(false)],+ vec![Some(false), Some(true), None, Some(false), Some(true)],Arc::clone(&col_a),&schema);- // test the optimization: set- // expression: "a in (99..300), the data type of list is INT32- let list = (99i32..300).map(lit).collect::<Vec<_>>();-+ // expression: "a in (0.0, 0.1, -NaN)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(-f64::NAN)];in_list!(batch,- list.clone(),+ list,&false,- vec![Some(true), None, Some(false)],+ vec![Some(true), Some(false), None, Some(false), Some(true)],Arc::clone(&col_a),&schema);+ // expression: "a not in (0.0, 0.1, -NaN)"+ let list = vec![lit(0.0f64), lit(0.1f64), lit(-f64::NAN)];in_list!(batch,list,&true,- vec![Some(false), None, Some(true)],+ vec![Some(false), Some(true), None, Some(true), Some(false)],Arc::clone(&col_a),&schema);@@ -1270,150 +1204,52 @@ mod tests {}#[test]- fn test_cast_static_filter_to_set() -> Result<()> {- // random schema- let schema =- Schema::new(vec![Field::new("a", DataType::Decimal128(13, 4), true)]);-- // list of phy expr- let mut phy_exprs = vec![- lit(1i64),- expressions::cast(lit(2i32), &schema, DataType::Int64)?,- try_cast(lit(3.13f32), &schema, DataType::Int64)?,- ];- let static_filter = try_cast_static_filter_to_set(&phy_exprs, &schema).unwrap();-- let array = Int64Array::from(vec![1, 2, 3, 4]);- let r = static_filter.contains(&array, false).unwrap();- assert_eq!(r, BooleanArray::from(vec![true, true, true, false]));-- try_cast_static_filter_to_set(&phy_exprs, &schema).unwrap();- // cast(cast(lit())), but the cast to the same data type, one case will be ignored- phy_exprs.push(expressions::cast(- expressions::cast(lit(2i32), &schema, DataType::Int64)?,- &schema,- DataType::Int64,- )?);- try_cast_static_filter_to_set(&phy_exprs, &schema).unwrap();-- phy_exprs.clear();-- // case(cast(lit())), the cast to the diff data type- phy_exprs.push(expressions::cast(- expressions::cast(lit(2i32), &schema, DataType::Int64)?,- &schema,- DataType::Int32,- )?);- try_cast_static_filter_to_set(&phy_exprs, &schema).unwrap();-- // column- phy_exprs.push(col("a", &schema)?);- assert!(try_cast_static_filter_to_set(&phy_exprs, &schema).is_err());-- Ok(())- }-- #[test]- fn in_list_timestamp() -> Result<()> {- let schema = Schema::new(vec![Field::new(- "a",- DataType::Timestamp(TimeUnit::Microsecond, None),- true,- )]);- let a = TimestampMicrosecondArray::from(vec![- Some(1388588401000000000),- Some(1288588501000000000),- None,- ]);+ fn in_list_bool() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);+ let a = BooleanArray::from(vec![Some(true), None]);let col_a = col("a", &schema)?;let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;- let list = vec![- lit(ScalarValue::TimestampMicrosecond(- Some(1388588401000000000),- None,- )),- lit(ScalarValue::TimestampMicrosecond(- Some(1388588401000000001),- None,- )),- lit(ScalarValue::TimestampMicrosecond(- Some(1388588401000000002),- None,- )),- ];-+ // expression: "a in (true)"+ let list = vec![lit(true)];in_list!(batch,- list.clone(),+ list,&false,- vec![Some(true), Some(false), None],+ vec![Some(true), None],Arc::clone(&col_a),&schema);+ // expression: "a not in (true)"+ let list = vec![lit(true)];in_list!(batch,- list.clone(),+ list,&true,- vec![Some(false), Some(true), None],+ vec![Some(false), None],Arc::clone(&col_a),&schema);- Ok(())- }-- #[test]- fn in_expr_with_multiple_element_in_list() -> Result<()> {- let schema = Schema::new(vec![- Field::new("a", DataType::Float64, true),- Field::new("b", DataType::Float64, true),- Field::new("c", DataType::Float64, true),- ]);- let a = Float64Array::from(vec![- Some(0.0),- Some(1.0),- Some(2.0),- Some(f64::NAN),- Some(-f64::NAN),- ]);- let b = Float64Array::from(vec![- Some(8.0),- Some(1.0),- Some(5.0),- Some(f64::NAN),- Some(3.0),- ]);- let c = Float64Array::from(vec![- Some(6.0),- Some(7.0),- None,- Some(5.0),- Some(-f64::NAN),- ]);- let col_a = col("a", &schema)?;- let col_b = col("b", &schema)?;- let col_c = col("c", &schema)?;- let batch = RecordBatch::try_new(- Arc::new(schema.clone()),- vec![Arc::new(a), Arc::new(b), Arc::new(c)],- )?;- let list = vec![Arc::clone(&col_b), Arc::clone(&col_c)];+ // expression: "a in (true, NULL)"+ let list = vec![lit(true), lit(ScalarValue::Null)];in_list!(batch,- list.clone(),+ list,&false,- vec![Some(false), Some(true), None, Some(true), Some(true)],+ vec![Some(true), None],Arc::clone(&col_a),&schema);+ // expression: "a not in (true, NULL)"+ let list = vec![lit(true), lit(ScalarValue::Null)];in_list!(batch,list,&true,- vec![Some(true), Some(false), None, Some(false), Some(false)],+ vec![Some(false), None],Arc::clone(&col_a),&schema);@@ -2823,4 +2659,322 @@ mod tests {Ok(())}++ #[test]+ fn test_in_list_dictionary_int32() -> Result<()> {+ // Create schema with dictionary-encoded Int32 column+ let dict_type =+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32));+ let schema = Schema::new(vec![Field::new("a", dict_type.clone(), false)]);+ let col_a = col("a", &schema)?;++ // Create IN list with Int32 literals: (100, 200, 300)+ let list = vec![lit(100i32), lit(200i32), lit(300i32)];++ // Create InListExpr via in_list() - this uses Int32StaticFilter for Int32 lists+ let expr = in_list(col_a, list, &false, &schema)?;++ // Create dictionary-encoded batch with values [100, 200, 500]+ // Dictionary: keys [0, 1, 2] -> values [100, 200, 500]+ // Using values clearly distinct from keys to avoid confusion+ let keys = Int8Array::from(vec![0, 1, 2]);+ let values = Int32Array::from(vec![100, 200, 500]);+ let dict_array: ArrayRef =+ Arc::new(DictionaryArray::try_new(keys, Arc::new(values))?);+ let batch = RecordBatch::try_new(Arc::new(schema), vec![dict_array])?;++ // Expected: [100 IN (100,200,300), 200 IN (100,200,300), 500 IN (100,200,300)] = [true, true, false]+ let result = expr.evaluate(&batch)?.into_array(3)?;+ let result = as_boolean_array(&result);+ assert_eq!(result, &BooleanArray::from(vec![true, true, false]));+ Ok(())+ }++ #[test]+ fn test_in_list_dictionary_types() -> Result<()> {+ // Helper functions for creating dictionary literals+ fn dict_lit_int64(key_type: DataType, value: i64) -> Arc<dyn PhysicalExpr> {+ lit(ScalarValue::Dictionary(+ Box::new(key_type),+ Box::new(ScalarValue::Int64(Some(value))),+ ))+ }++ fn dict_lit_float64(key_type: DataType, value: f64) -> Arc<dyn PhysicalExpr> {+ lit(ScalarValue::Dictionary(+ Box::new(key_type),+ Box::new(ScalarValue::Float64(Some(value))),+ ))+ }++ // Test case structures+ struct DictNeedleTest {+ list_values: Vec<Arc<dyn PhysicalExpr>>,+ expected: Vec<Option<bool>>,+ }++ struct DictionaryInListTestCase {+ name: &'static str,+ dict_type: DataType,+ dict_keys: Vec<Option<i8>>,+ dict_values: ArrayRef,+ list_values_no_null: Vec<Arc<dyn PhysicalExpr>>,+ list_values_with_null: Vec<Arc<dyn PhysicalExpr>>,+ expected_1: Vec<Option<bool>>,+ expected_2: Vec<Option<bool>>,+ expected_3: Vec<Option<bool>>,+ expected_4: Vec<Option<bool>>,+ dict_needle_test: Option<DictNeedleTest>,+ }++ // Test harness function+ fn run_dictionary_in_list_test(+ test_case: DictionaryInListTestCase,+ ) -> Result<()> {+ // Create schema with dictionary type+ let schema =+ Schema::new(vec![Field::new("a", test_case.dict_type.clone(), true)]);+ let col_a = col("a", &schema)?;++ // Create dictionary array from keys and values+ let keys = Int8Array::from(test_case.dict_keys.clone());+ let dict_array: ArrayRef =+ Arc::new(DictionaryArray::try_new(keys, test_case.dict_values)?);+ let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![dict_array])?;++ let exp1 = test_case.expected_1.clone();+ let exp2 = test_case.expected_2.clone();+ let exp3 = test_case.expected_3.clone();+ let exp4 = test_case.expected_4;++ // Test 1: a IN (values_no_null)+ in_list!(+ batch,+ test_case.list_values_no_null.clone(),+ &false,+ exp1,+ Arc::clone(&col_a),+ &schema+ );++ // Test 2: a NOT IN (values_no_null)+ in_list!(+ batch,+ test_case.list_values_no_null.clone(),+ &true,+ exp2,+ Arc::clone(&col_a),+ &schema+ );++ // Test 3: a IN (values_with_null)+ in_list!(+ batch,+ test_case.list_values_with_null.clone(),+ &false,+ exp3,+ Arc::clone(&col_a),+ &schema+ );++ // Test 4: a NOT IN (values_with_null)+ in_list!(+ batch,+ test_case.list_values_with_null,+ &true,+ exp4,+ Arc::clone(&col_a),+ &schema+ );++ // Optional: Dictionary needle test (if provided)+ if let Some(needle_test) = test_case.dict_needle_test {+ in_list_raw!(+ batch,+ needle_test.list_values,+ &false,+ needle_test.expected,+ Arc::clone(&col_a),+ &schema+ );+ }++ Ok(())+ }++ // Test case 1: UTF8+ // Dictionary: keys [0, 1, null] → values ["a", "d", -]+ // Rows: ["a", "d", null]+ let utf8_case = DictionaryInListTestCase {+ name: "dictionary_utf8",+ dict_type: DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Utf8),+ ),+ dict_keys: vec![Some(0), Some(1), None],+ dict_values: Arc::new(StringArray::from(vec![Some("a"), Some("d")])),+ list_values_no_null: vec![lit("a"), lit("b")],+ list_values_with_null: vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))],+ expected_1: vec![Some(true), Some(false), None],+ expected_2: vec![Some(false), Some(true), None],+ expected_3: vec![Some(true), None, None],+ expected_4: vec![Some(false), None, None],+ dict_needle_test: None,+ };++ // Test case 2: Int64 with dictionary needles+ // Dictionary: keys [0, 1, null] → values [10, 20, -]+ // Rows: [10, 20, null]+ let int64_case = DictionaryInListTestCase {+ name: "dictionary_int64",+ dict_type: DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Int64),+ ),+ dict_keys: vec![Some(0), Some(1), None],+ dict_values: Arc::new(Int64Array::from(vec![Some(10), Some(20)])),+ list_values_no_null: vec![lit(10i64), lit(15i64)],+ list_values_with_null: vec![+ lit(10i64),+ lit(15i64),+ lit(ScalarValue::Int64(None)),+ ],+ expected_1: vec![Some(true), Some(false), None],+ expected_2: vec![Some(false), Some(true), None],+ expected_3: vec![Some(true), None, None],+ expected_4: vec![Some(false), None, None],+ dict_needle_test: Some(DictNeedleTest {+ list_values: vec![+ dict_lit_int64(DataType::Int16, 10),+ dict_lit_int64(DataType::Int16, 15),+ ],+ expected: vec![Some(true), Some(false), None],+ }),+ };++ // Test case 3: Float64 with NaN and dictionary needles+ // Dictionary: keys [0, 1, null, 2] → values [1.5, 3.7, NaN, -]+ // Rows: [1.5, 3.7, null, NaN]+ // Note: NaN is a value (not null), so it goes in the values array+ let float64_case = DictionaryInListTestCase {+ name: "dictionary_float64",+ dict_type: DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Float64),+ ),+ dict_keys: vec![Some(0), Some(1), None, Some(2)],+ dict_values: Arc::new(Float64Array::from(vec![+ Some(1.5), // index 0+ Some(3.7), // index 1+ Some(f64::NAN), // index 2+ ])),+ list_values_no_null: vec![lit(1.5f64), lit(2.0f64)],+ list_values_with_null: vec![+ lit(1.5f64),+ lit(2.0f64),+ lit(ScalarValue::Float64(None)),+ ],+ // Test 1: a IN (1.5, 2.0) → [true, false, null, false]+ // NaN is false because NaN not in list and no NULL in list+ expected_1: vec![Some(true), Some(false), None, Some(false)],+ // Test 2: a NOT IN (1.5, 2.0) → [false, true, null, true]+ // NaN is true because NaN not in list+ expected_2: vec![Some(false), Some(true), None, Some(true)],+ // Test 3: a IN (1.5, 2.0, NULL) → [true, null, null, null]+ // 3.7 and NaN become null due to NULL in list (three-valued logic)+ expected_3: vec![Some(true), None, None, None],+ // Test 4: a NOT IN (1.5, 2.0, NULL) → [false, null, null, null]+ // 3.7 and NaN become null due to NULL in list+ expected_4: vec![Some(false), None, None, None],+ dict_needle_test: Some(DictNeedleTest {+ list_values: vec![+ dict_lit_float64(DataType::UInt16, 1.5),+ dict_lit_float64(DataType::UInt16, 2.0),+ ],+ expected: vec![Some(true), Some(false), None, Some(false)],+ }),+ };++ // Execute all test cases+ let test_name = utf8_case.name;+ run_dictionary_in_list_test(utf8_case).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Dictionary test '{test_name}' failed: {e}"+ ))+ })?;++ let test_name = int64_case.name;+ run_dictionary_in_list_test(int64_case).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Dictionary test '{test_name}' failed: {e}"+ ))+ })?;++ let test_name = float64_case.name;+ run_dictionary_in_list_test(float64_case).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Dictionary test '{test_name}' failed: {e}"+ ))+ })?;++ // Additional test: Dictionary deduplication with repeated keys+ // This tests that multiple rows with the same key (pointing to the same value)+ // are evaluated correctly+ let dedup_case = DictionaryInListTestCase {+ name: "dictionary_deduplication",+ dict_type: DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Utf8),+ ),+ // Keys: [0, 1, 0, 1, null] - keys 0 and 1 are repeated+ // This creates data: ["a", "d", "a", "d", null]+ dict_keys: vec![Some(0), Some(1), Some(0), Some(1), None],+ dict_values: Arc::new(StringArray::from(vec![Some("a"), Some("d")])),+ list_values_no_null: vec![lit("a"), lit("b")],+ list_values_with_null: vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))],+ // Test 1: a IN ("a", "b") → [true, false, true, false, null]+ // Rows 0 and 2 both have key 0 → "a", so both are true+ expected_1: vec![Some(true), Some(false), Some(true), Some(false), None],+ // Test 2: a NOT IN ("a", "b") → [false, true, false, true, null]+ expected_2: vec![Some(false), Some(true), Some(false), Some(true), None],+ // Test 3: a IN ("a", "b", NULL) → [true, null, true, null, null]+ // "d" becomes null due to NULL in list+ expected_3: vec![Some(true), None, Some(true), None, None],+ // Test 4: a NOT IN ("a", "b", NULL) → [false, null, false, null, null]+ expected_4: vec![Some(false), None, Some(false), None, None],+ dict_needle_test: None,+ };++ let test_name = dedup_case.name;+ run_dictionary_in_list_test(dedup_case).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Dictionary test '{test_name}' failed: {e}"+ ))+ })?;++ // Additional test for Float64 NaN in IN list+ let dict_type =+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Float64));+ let schema = Schema::new(vec![Field::new("a", dict_type.clone(), true)]);+ let col_a = col("a", &schema)?;++ let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);+ let values = Float64Array::from(vec![Some(1.5), Some(3.7), Some(f64::NAN)]);+ let dict_array: ArrayRef =+ Arc::new(DictionaryArray::try_new(keys, Arc::new(values))?);+ let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![dict_array])?;++ // Test: a IN (1.5, 2.0, NaN)+ let list_with_nan = vec![lit(1.5f64), lit(2.0f64), lit(f64::NAN)];+ in_list!(+ batch,+ list_with_nan,+ &false,+ vec![Some(true), Some(false), None, Some(true)],+ col_a,+ &schema+ );++ Ok(())+ }}