Skip to content

Apache DataFusion SQL Query Engine

perf: short-circuit and collect_bool for IN list with column references (#20694)

Zhang Xiaofengc9190542413c
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 44a6572f53..e30f256352 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -793,63 +793,71 @@ impl PhysicalExpr for InListExpr {
// comparator for unsupported types (nested, RunEndEncoded, etc.).
let value = value.into_array(num_rows)?;
let lhs_supports_arrow_eq = supports_arrow_eq(value.data_type());
- let found = self.list.iter().map(|expr| expr.evaluate(batch)).try_fold(
- BooleanArray::new(BooleanBuffer::new_unset(num_rows), None),
- |result, expr| -> Result<BooleanArray> {
- let rhs = match expr? {
- ColumnarValue::Array(array) => {
- if lhs_supports_arrow_eq
- && supports_arrow_eq(array.data_type())
- {
- arrow_eq(&value, &array)?
- } else {
- let cmp = make_comparator(
- value.as_ref(),
- array.as_ref(),
- SortOptions::default(),
- )?;
- (0..num_rows)
- .map(|i| {
- if value.is_null(i) || array.is_null(i) {
- return None;
- }
- Some(cmp(i, i).is_eq())
- })
- .collect::<BooleanArray>()
- }
+
+ // Helper: compare value against a single list expression
+ let compare_one = |expr: &Arc<dyn PhysicalExpr>| -> Result<BooleanArray> {
+ match expr.evaluate(batch)? {
+ ColumnarValue::Array(array) => {
+ if lhs_supports_arrow_eq
+ && supports_arrow_eq(array.data_type())
+ {
+ Ok(arrow_eq(&value, &array)?)
+ } else {
+ let cmp = make_comparator(
+ value.as_ref(),
+ array.as_ref(),
+ SortOptions::default(),
+ )?;
+ let buffer = BooleanBuffer::collect_bool(num_rows, |i| {
+ cmp(i, i).is_eq()
+ });
+ let nulls =
+ NullBuffer::union(value.nulls(), array.nulls());
+ Ok(BooleanArray::new(buffer, nulls))
}
- ColumnarValue::Scalar(scalar) => {
- // Check if scalar is null once, before the loop
- if scalar.is_null() {
- // If scalar is null, all comparisons return null
- BooleanArray::from(vec![None; num_rows])
- } else if lhs_supports_arrow_eq {
- let scalar_datum = scalar.to_scalar()?;
- arrow_eq(&value, &scalar_datum)?
- } else {
- // Convert scalar to 1-element array
- let array = scalar.to_array()?;
- let cmp = make_comparator(
- value.as_ref(),
- array.as_ref(),
- SortOptions::default(),
- )?;
- // Compare each row of value with the single scalar element
- (0..num_rows)
- .map(|i| {
- if value.is_null(i) {
- None
- } else {
- Some(cmp(i, 0).is_eq())
- }
- })
- .collect::<BooleanArray>()
- }
+ }
+ ColumnarValue::Scalar(scalar) => {
+ // Check if scalar is null once, before the loop
+ if scalar.is_null() {
+ // If scalar is null, all comparisons return null
+ Ok(BooleanArray::from(vec![None; num_rows]))
+ } else if lhs_supports_arrow_eq {
+ let scalar_datum = scalar.to_scalar()?;
+ Ok(arrow_eq(&value, &scalar_datum)?)
+ } else {
+ // Convert scalar to 1-element array
+ let array = scalar.to_array()?;
+ let cmp = make_comparator(
+ value.as_ref(),
+ array.as_ref(),
+ SortOptions::default(),
+ )?;
+ // Compare each row of value with the single scalar element
+ let buffer = BooleanBuffer::collect_bool(num_rows, |i| {
+ cmp(i, 0).is_eq()
+ });
+ Ok(BooleanArray::new(buffer, value.nulls().cloned()))
}
- };
- Ok(or_kleene(&result, &rhs)?)
- },
- )?;
+ }
+ }
+ };
+
+ // Evaluate first expression directly to avoid a redundant
+ // or_kleene with an all-false accumulator.
+ let mut found = if let Some(first) = self.list.first() {
+ compare_one(first)?
+ } else {
+ BooleanArray::new(BooleanBuffer::new_unset(num_rows), None)
+ };
+
+ for expr in self.list.iter().skip(1) {
+ // Short-circuit: if every non-null row is already true,
+ // no further list items can change the result.
+ if found.null_count() == 0 && found.true_count() == num_rows {
+ break;
+ }
+ found = or_kleene(&found, &compare_one(expr)?)?;
+ }
if self.negated { not(&found)? } else { found }
}
@@ -3724,4 +3732,150 @@ mod tests {
assert_eq!(result, &BooleanArray::from(vec![true, false, false]));
Ok(())
}
+
+ /// Tests that short-circuit evaluation produces correct results.
+ /// When all rows match after the first list item, remaining items
+ /// should be skipped without affecting correctness.
+ #[test]
+ fn test_in_list_with_columns_short_circuit() -> Result<()> {
+ // a IN (b, c) where b already matches every row of a
+ // The short-circuit should skip evaluating c
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ Field::new("c", DataType::Int32, false),
+ ]);
+ let batch = RecordBatch::try_new(
+ Arc::new(schema.clone()),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3])),
+ Arc::new(Int32Array::from(vec![1, 2, 3])), // b == a for all rows
+ Arc::new(Int32Array::from(vec![99, 99, 99])),
+ ],
+ )?;
+
+ let col_a = col("a", &schema)?;
+ let list = vec![col("b", &schema)?, col("c", &schema)?];
+ let expr = make_in_list_with_columns(col_a, list, false);
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_boolean_array(&result);
+ assert_eq!(result, &BooleanArray::from(vec![true, true, true]));
+ Ok(())
+ }
+
+ /// Short-circuit must NOT skip when nulls are present (three-valued logic).
+ /// Even if all non-null values are true, null rows keep the result as null.
+ #[test]
+ fn test_in_list_with_columns_short_circuit_with_nulls() -> Result<()> {
+ // a IN (b, c) where a has nulls
+ // Even if b matches all non-null rows, result should preserve nulls
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, false),
+ Field::new("c", DataType::Int32, false),
+ ]);
+ let batch = RecordBatch::try_new(
+ Arc::new(schema.clone()),
+ vec![
+ Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
+ Arc::new(Int32Array::from(vec![1, 2, 3])), // matches non-null rows
+ Arc::new(Int32Array::from(vec![99, 99, 99])),
+ ],
+ )?;
+
+ let col_a = col("a", &schema)?;
+ let list = vec![col("b", &schema)?, col("c", &schema)?];
+ let expr = make_in_list_with_columns(col_a, list, false);
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_boolean_array(&result);
+ // row 0: 1 IN (1, 99) → true
+ // row 1: NULL IN (2, 99) → NULL
+ // row 2: 3 IN (3, 99) → true
+ assert_eq!(
+ result,
+ &BooleanArray::from(vec![Some(true), None, Some(true)])
+ );
+ Ok(())
+ }
+
+ /// Tests the make_comparator + collect_bool fallback path using
+ /// struct column references (nested types don't support arrow_eq).
+ #[test]
+ fn test_in_list_with_columns_struct() -> Result<()> {
+ let struct_fields = Fields::from(vec![
+ Field::new("x", DataType::Int32, false),
+ Field::new("y", DataType::Utf8, false),
+ ]);
+ let struct_dt = DataType::Struct(struct_fields.clone());
+
+ let schema = Schema::new(vec![
+ Field::new("a", struct_dt.clone(), true),
+ Field::new("b", struct_dt.clone(), false),
+ Field::new("c", struct_dt.clone(), false),
+ ]);
+
+ // a: [{1,"a"}, {2,"b"}, NULL, {4,"d"}]
+ // b: [{1,"a"}, {9,"z"}, {3,"c"}, {4,"d"}]
+ // c: [{9,"z"}, {2,"b"}, {9,"z"}, {9,"z"}]
+ let a = Arc::new(StructArray::new(
+ struct_fields.clone(),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
+ Arc::new(StringArray::from(vec!["a", "b", "c", "d"])),
+ ],
+ Some(vec![true, true, false, true].into()),
+ ));
+ let b = Arc::new(StructArray::new(
+ struct_fields.clone(),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 9, 3, 4])),
+ Arc::new(StringArray::from(vec!["a", "z", "c", "d"])),
+ ],
+ None,
+ ));
+ let c = Arc::new(StructArray::new(
+ struct_fields.clone(),
+ vec![
+ Arc::new(Int32Array::from(vec![9, 2, 9, 9])),
+ Arc::new(StringArray::from(vec!["z", "b", "z", "z"])),
+ ],
+ None,
+ ));
+
+ let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b, c])?;
+
+ let col_a = col("a", &schema)?;
+ let list = vec![col("b", &schema)?, col("c", &schema)?];
+ let expr = make_in_list_with_columns(col_a, list, false);
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_boolean_array(&result);
+ // row 0: {1,"a"} IN ({1,"a"}, {9,"z"}) → true (matches b)
+ // row 1: {2,"b"} IN ({9,"z"}, {2,"b"}) → true (matches c)
+ // row 2: NULL IN ({3,"c"}, {9,"z"}) → NULL
+ // row 3: {4,"d"} IN ({4,"d"}, {9,"z"}) → true (matches b)
+ assert_eq!(
+ result,
+ &BooleanArray::from(vec![Some(true), Some(true), None, Some(true)])
+ );
+
+ // Also test NOT IN
+ let col_a = col("a", &schema)?;
+ let list = vec![col("b", &schema)?, col("c", &schema)?];
+ let expr = make_in_list_with_columns(col_a, list, true);
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ let result = as_boolean_array(&result);
+ // row 0: {1,"a"} NOT IN ({1,"a"}, {9,"z"}) → false
+ // row 1: {2,"b"} NOT IN ({9,"z"}, {2,"b"}) → false
+ // row 2: NULL NOT IN ({3,"c"}, {9,"z"}) → NULL
+ // row 3: {4,"d"} NOT IN ({4,"d"}, {9,"z"}) → false
+ assert_eq!(
+ result,
+ &BooleanArray::from(vec![Some(false), Some(false), None, Some(false)])
+ );
+ Ok(())
+ }
}