Skip to content

Apache DataFusion SQL Query Engine

test: count evaluations of volatile calls in sqllogictests (#25458)

Adrian Garcia Badaracco636590bf4f4b
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 88bf015d1e..a9a3d76291 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -60,6 +60,7 @@ use log::info;
use tempfile::TempDir;
mod custom_sort_pushdown;
+mod evaluation_counter;
mod range_partitioning;
/// Context for running tests
@@ -162,6 +163,7 @@ impl TestContext {
let state = state_builder.build();
let mut test_ctx = TestContext::new(SessionContext::new_with_state(state));
+ evaluation_counter::register_evaluation_counter(test_ctx.session_ctx());
match file_name {
"sort_pushdown.slt" => {
diff --git a/datafusion/sqllogictest/src/test_context/evaluation_counter.rs b/datafusion/sqllogictest/src/test_context/evaluation_counter.rs
new file mode 100644
index 0000000000..add3d802b2
--- /dev/null
+++ b/datafusion/sqllogictest/src/test_context/evaluation_counter.rs
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+//! Functions that count how many rows a call is evaluated for, so a test can
+//! check that the optimizer does not evaluate a call more (or fewer) times
+//! than the query says:
+//!
+//! - `counted_volatile(x)` returns `x` and is volatile, like `random()`.
+//! - `counted_immutable(x)` returns `x` and is immutable.
+//! - `evaluations()` returns the number of rows that the two functions above
+//! were evaluated for since the last call to `evaluations()`, and resets it.
+//!
+//! The count is per row, not per call: a function is invoked once per batch,
+//! and `CASE` evaluates its branches on a subset of a batch.
+//!
+//! ```text
+//! statement ok
+//! SELECT k FROM (SELECT counted_volatile(a) AS k FROM t) WHERE k > 1;
+//!
+//! query I
+//! SELECT evaluations();
+//! ----
+//! 3
+//! ```
+
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use arrow::datatypes::DataType;
+use datafusion::common::{Result, ScalarValue};
+use datafusion::logical_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion::prelude::SessionContext;
+
+/// Registers `counted_volatile`, `counted_immutable` and `evaluations` with `ctx`. Each
+/// context gets its own counter.
+pub fn register_evaluation_counter(ctx: &SessionContext) {
+ let counter = Arc::new(AtomicUsize::new(0));
+ for (name, volatility) in [
+ ("counted_volatile", Volatility::Volatile),
+ ("counted_immutable", Volatility::Immutable),
+ ] {
+ ctx.register_udf(ScalarUDF::new_from_impl(Counted {
+ name,
+ signature: Signature::any(1, volatility),
+ counter: Arc::clone(&counter),
+ }));
+ }
+ ctx.register_udf(ScalarUDF::new_from_impl(Evaluations {
+ signature: Signature::nullary(Volatility::Volatile),
+ counter,
+ }));
+}
+
+/// Returns its argument and adds the number of rows to `counter`.
+#[derive(Debug)]
+struct Counted {
+ name: &'static str,
+ signature: Signature,
+ counter: Arc<AtomicUsize>,
+}
+
+impl ScalarUDFImpl for Counted {
+ fn name(&self) -> &str {
+ self.name
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
+ }
+
+ fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ self.counter.fetch_add(args.number_rows, Ordering::SeqCst);
+ Ok(args.args.swap_remove(0))
+ }
+}
+
+/// Returns the count and resets it.
+#[derive(Debug)]
+struct Evaluations {
+ signature: Signature,
+ counter: Arc<AtomicUsize>,
+}
+
+impl ScalarUDFImpl for Evaluations {
+ fn name(&self) -> &str {
+ "evaluations"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Int64)
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let count = self.counter.swap(0, Ordering::SeqCst);
+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(
+ count as i64,
+ ))))
+ }
+}
+
+// Two instances are equal only if they share a counter.
+
+impl PartialEq for Counted {
+ fn eq(&self, other: &Self) -> bool {
+ self.name == other.name && Arc::ptr_eq(&self.counter, &other.counter)
+ }
+}
+
+impl Eq for Counted {}
+
+impl Hash for Counted {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.name.hash(state);
+ Arc::as_ptr(&self.counter).hash(state);
+ }
+}
+
+impl PartialEq for Evaluations {
+ fn eq(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.counter, &other.counter)
+ }
+}
+
+impl Eq for Evaluations {}
+
+impl Hash for Evaluations {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ Arc::as_ptr(&self.counter).hash(state);
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/aggregates_simplify.slt b/datafusion/sqllogictest/test_files/aggregates_simplify.slt
index b9c4ba147b..653c2125a9 100644
--- a/datafusion/sqllogictest/test_files/aggregates_simplify.slt
+++ b/datafusion/sqllogictest/test_files/aggregates_simplify.slt
@@ -259,11 +259,16 @@ physical_plan
01)AggregateExec: mode=Single, gby=[], aggr=[sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(1)), sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(0))]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
-# volatile aggregate arguments
-query B
-SELECT SUM(random() + 1) < SUM(random() + 2) FROM sum_simplify_t;
+# volatile aggregate arguments: each call is evaluated one time per row
+query II
+SELECT SUM(counted_volatile(column1) + 1), SUM(counted_volatile(column1) + 2) FROM sum_simplify_t;
+----
+7 10
+
+query I
+SELECT evaluations();
----
-true
+8
query TT
EXPLAIN SELECT SUM(random() + 1) < SUM(random() + 2) FROM sum_simplify_t;
diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.slt
index 8fd1c5518d..3b54d24f07 100644
--- a/datafusion/sqllogictest/test_files/coalesce.slt
+++ b/datafusion/sqllogictest/test_files/coalesce.slt
@@ -463,26 +463,34 @@ physical_plan
01)ProjectionExec: expr=[coalesce(nullif(floor(random() * 2), 0), -1) as c]
02)--PlaceholderRowExec
-# With two draws, about 25% of rows were NULL (and Arrow rejected them because
-# the `-1` fallback makes the result non-nullable). With one draw, every row is
-# either 1 or -1.
-statement ok
-create table volatile_t as select * from generate_series(1, 100000) as t(i);
+# A volatile first argument is evaluated one time per row, not once for the
+# null test and again for the value.
+query II
+select
+ count(coalesce(nullif(counted_volatile(value) % 2, 0), -1)),
+ sum(coalesce(nullif(counted_volatile(value) % 2, 0), -1))
+from generate_series(1, 10);
+----
+10 0
+
+query I
+select evaluations();
+----
+20
-query III
+query II
select
- count(coalesce(nullif(floor(random() * 2), 0), -1)),
- count(nvl(nullif(floor(random() * 2), 0), -1)),
- count(*) filter (where coalesce(nullif(floor(random() * 2), 0), -1) not in (1, -1))
-from volatile_t;
+ count(nvl(nullif(counted_volatile(value) % 2, 0), -1)),
+ sum(nvl(nullif(counted_volatile(value) % 2, 0), -1))
+from generate_series(1, 10);
----
-100000 100000 0
+10 0
-statement ok
-drop table volatile_t;
+query I
+select evaluations();
+----
+20
-# `case when random() >= 0 then x end` is volatile but always equals `x`, which
-# makes the results of the volatile code path deterministic.
statement ok
create table volatile_args(a int, b int, c int, s varchar, x int, y int) as values
(1, 9, 8, null, 0, 1),
@@ -492,11 +500,11 @@ create table volatile_args(a int, b int, c int, s varchar, x int, y int) as valu
query IIIIT
select
- coalesce(case when random() >= 0 then a end, b, c),
- coalesce(case when random() >= 0 then a end, -1),
- coalesce(case when random() >= 0 then a end, null::int),
- coalesce(a, case when random() >= 0 then b end, c),
- coalesce(case when random() >= 0 then s end, 'none')
+ coalesce(counted_volatile(a), b, c),
+ coalesce(counted_volatile(a), -1),
+ coalesce(counted_volatile(a), null::int),
+ coalesce(a, counted_volatile(b), c),
+ coalesce(counted_volatile(s), 'none')
from volatile_args;
----
1 1 1 1 none
@@ -504,13 +512,25 @@ from volatile_args;
3 -1 NULL 3 none
NULL -1 NULL NULL none
+# Four rows for each of the first three and the last call, and three for
+# `counted_volatile(b)`, which is reached only where `a` is null.
+query I
+select evaluations();
+----
+19
+
query TI
select
- coalesce(case when random() >= 0 then null::varchar end, 'datafusion', 'unused'),
- coalesce(case when random() >= 0 then null::int end, null::int);
+ coalesce(counted_volatile(null::varchar), 'datafusion', 'unused'),
+ coalesce(counted_volatile(null::int), null::int);
----
datafusion NULL
+query I
+select evaluations();
+----
+2
+
# Arguments before the first volatile one keep the CASE rewrite; only the rest
# is left to `coalesce`.
query TT
diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt
index a4a0f98ac3..e86ebcf1f6 100644
--- a/datafusion/sqllogictest/test_files/functions.slt
+++ b/datafusion/sqllogictest/test_files/functions.slt
@@ -1304,9 +1304,14 @@ ac
# Verify that multiple calls to volatile functions like `random()` are not combined / optimized away
query B
-SELECT r FROM (SELECT r1 == r2 r, r1, r2 FROM (SELECT random()+1 r1, random()+1 r2) WHERE r1 > 0 AND r2 > 0)
+SELECT r FROM (SELECT r1 == r2 r, r1, r2 FROM (SELECT counted_volatile(1)+1 r1, counted_volatile(1)+1 r2) WHERE r1 > 0 AND r2 > 0)
----
-false
+true
+
+query I
+SELECT evaluations();
+----
+2
# Verify that rand() is accepted as an alias for random()
query TB
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index b178a33da7..b5b001026d 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -2109,7 +2109,7 @@ SET datafusion.execution.target_partitions = 4;
# reused. Projection pushdown must not merge the outer projection into the file
# scan when doing so would inline and duplicate the volatile expression.
# Reproduces only against a file scan (not an in-memory table); if the volatile
-# expression is duplicated, the two references diverge and `x = y` is false.
+# expression is duplicated, it is evaluated two times per row.
#####################
statement ok
@@ -2121,15 +2121,20 @@ statement ok
CREATE EXTERNAL TABLE volatile_scan STORED AS PARQUET
LOCATION 'test_files/scratch/projection_pushdown/volatile.parquet';
-# The two references to the aliased `random()` value must be equal on every
-# row: the expression is evaluated once and reused, not inlined twice.
-query B rowsort
-SELECT s.x = s.y
-FROM (SELECT r AS x, r AS y FROM (SELECT random() AS r FROM volatile_scan) AS t) AS s;
+# The aliased volatile value is evaluated one time per row and reused, not
+# inlined twice.
+query II rowsort
+SELECT s.x, s.y
+FROM (SELECT r AS x, r AS y FROM (SELECT counted_volatile(id) AS r FROM volatile_scan) AS t) AS s;
----
-true
-true
-true
+1 1
+2 2
+3 3
+
+query I
+SELECT evaluations();
+----
+3
#####################
# Section: leaf expression extraction does not duplicate volatile expressions
@@ -2176,37 +2181,52 @@ logical_plan
statement ok
set datafusion.explain.logical_plan_only = false;
-# Every row's `field` is the `a` field of that row's own struct.
-query B
-SELECT bool_and(field = s['a'])
+# The struct is evaluated one time per row; `field` reads it.
+query II
+SELECT count(s), sum(field)
FROM (
SELECT s, s['a'] AS field
- FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))
+ FROM (SELECT named_struct('a', counted_volatile(value)) AS s FROM generate_series(1, 10))
);
----
-true
+10 55
+
+query I
+SELECT evaluations();
+----
+10
-# Same invariant when the extraction is routed into one side of a join.
-query B
-SELECT bool_and(field = s['a'])
+# Same when the extraction is routed into one side of a join.
+query II
+SELECT count(s), sum(field)
FROM (
SELECT l.s, l.s['a'] AS field
- FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000)) AS l
+ FROM (SELECT named_struct('a', counted_volatile(value)) AS s FROM generate_series(1, 10)) AS l
INNER JOIN (SELECT 1 AS k) AS r ON true
);
----
-true
+10 55
+
+query I
+SELECT evaluations();
+----
+10
-# Every surviving row satisfies the predicate it was filtered on.
-query B
-SELECT bool_and(s['a'] > 0.5)
+# The filter tests the struct that is returned, not a second evaluation.
+query I
+SELECT count(s)
FROM (
SELECT s
- FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))
- WHERE s['a'] > 0.5
+ FROM (SELECT named_struct('a', counted_volatile(value)) AS s FROM generate_series(1, 10))
+ WHERE s['a'] > 5
);
----
-true
+5
+
+query I
+SELECT evaluations();
+----
+10
#####################
# Section: expensive expressions are not re-inlined by projection pushdown
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
index c30d3c4389..8d18fb6430 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
@@ -696,20 +696,26 @@ physical_plan
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=10000, batch_size=8192]
-query B
-SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k;
+# The key is evaluated one time per input row, by the aggregate.
+query BI
+SELECT k, c FROM (SELECT counted_volatile(value) <= 3 AS k, count(*) AS c FROM generate_series(1, 10) GROUP BY counted_volatile(value) <= 3) WHERE k;
----
-true
+true 3
-query B
-SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY k) WHERE NOT k;
+query I
+SELECT evaluations();
+----
+10
+
+query BI
+SELECT k, c FROM (SELECT counted_volatile(value) <= 3 AS k, count(*) AS c FROM generate_series(1, 10) GROUP BY k) WHERE NOT k;
----
-false
+false 7
query I
-SELECT sum(c) FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k OR NOT k;
+SELECT evaluations();
----
-10000
+10
# Window filter pushdown over expression PARTITION BY keys.
#
diff --git a/datafusion/sqllogictest/test_files/volatile_evaluations.slt b/datafusion/sqllogictest/test_files/volatile_evaluations.slt
new file mode 100644
index 0000000000..ee8304910c
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/volatile_evaluations.slt
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+# The optimizer must not evaluate a volatile call more times than the query
+# says, because each evaluation gives a different answer.
+#
+# `counted_volatile(x)` returns `x`, is volatile like `random()`, and counts
+# the rows it is evaluated for. `evaluations()` returns that count and resets
+# it. `counted_immutable(x)` is the same, but immutable. See
+# `datafusion/sqllogictest/src/test_context/evaluation_counter.rs`.
+
+statement ok
+CREATE TABLE t (a INT, b DOUBLE, n INT) AS VALUES
+ (1, 1.5, 1), (2, 2.5, NULL), (3, 3.5, 3), (4, 4.5, NULL), (5, 5.5, 5),
+ (6, 6.5, 6), (7, 7.5, NULL), (8, 8.5, 8), (9, 9.5, 9), (10, 10.5, NULL);
+
+statement ok
+SELECT evaluations();
+
+# One evaluation per row.
+
+query I
+SELECT count(counted_volatile(a)) FROM t;
+----
+10
+
+query I
+SELECT evaluations();
+----
+10
+
+# A volatile column used two times is evaluated one time.
+
+query I
+SELECT count(*) FROM (SELECT r, r + 1 AS r1 FROM (SELECT counted_volatile(a) AS r FROM t)) WHERE r1 = r + 1;
+----
+10
+
+query I
+SELECT evaluations();
+----
+10
+
+# Two calls in the query are two evaluations: they must not be merged.
+
+query I
+SELECT count(x + y) FROM (SELECT counted_volatile(a) AS x, counted_volatile(a) AS y FROM t);
+----
+10
+
+query I
+SELECT evaluations();
+----
+20
+
+# A filter on a volatile column is not pushed below the projection that
+# computes it.
+
+query I
+SELECT count(*) FROM (SELECT counted_volatile(a) AS r FROM t) WHERE r > 5;
+----
+5
+
+query I
+SELECT evaluations();
+----
+10
+
+# The same, above a join.
+
+query I
+SELECT count(*) FROM (SELECT a, counted_volatile(a) AS r FROM t) x
+JOIN t y ON x.a = y.a WHERE x.r > 5;
+----
+5
+
+query I
+SELECT evaluations();
+----
+10
+
+# The same, above a UNION ALL.
+
+query I
+SELECT count(*) FROM (SELECT counted_volatile(a) AS r FROM t UNION ALL SELECT a FROM t)
+WHERE r > 5;
+----
+10
+
+query I
+SELECT evaluations();
+----
+10
+
+# HAVING on a volatile group key.
+
+query I
+SELECT count(*) FROM (SELECT r FROM (SELECT counted_volatile(a) AS r FROM t) GROUP BY r HAVING r > 5);
+----
+5
+
+query I
+SELECT evaluations();
+----
+10
+
+# https://github.com/apache/datafusion/issues/26910
+# Known bug: the preimage rewrite of `floor(x) = 3` copies `x`, so a volatile
+# `x` is evaluated two times. The count must be 10.
+
+query I
+SELECT count(*) FROM t WHERE floor(counted_volatile(b)) = 3;
+----
+1
+
+query I
+SELECT evaluations();
+----
+20
+
+# `x BETWEEN low AND high` evaluates `x` two times. This is accepted behaviour,
+# Postgres does the same: https://github.com/apache/datafusion/issues/25457
+
+query I
+SELECT count(*) FROM t WHERE counted_volatile(a) BETWEEN 3 AND 5;
+----
+3
+
+query I
+SELECT evaluations();
+----
+20
+
+# A repeated call that is not volatile is evaluated one time:
+# `CommonSubexprEliminate` computes it once and field accesses read the result.
+# A second evaluation would not change the answer, only the cost.
+
+# https://github.com/apache/datafusion/issues/23655
+query II
+SELECT sum(counted_immutable(s)['x']), sum(counted_immutable(s)['y'])
+FROM (SELECT named_struct('x', a, 'y', a) AS s FROM t);
+----
+55 55
+
+query I
+SELECT evaluations();
+----
+10
+
+# https://github.com/apache/datafusion/issues/25329
+query II
+SELECT sum(counted_immutable(s)['x']), count(counted_immutable(s))
+FROM (SELECT named_struct('x', a, 'y', a) AS s FROM t);
+----
+55 10
+
+query I
+SELECT evaluations();
+----
+10
+
+# https://github.com/apache/datafusion/issues/25329
+# Known bug: in a Filter, leaf extraction copies the call into the extracted
+# `get_field`, so it is evaluated two times. The count must be 10.
+query I
+SELECT count(*) FROM (SELECT named_struct('x', a, 'y', a) AS s FROM t)
+WHERE counted_immutable(s) IS NOT NULL AND counted_immutable(s)['x'] > 5;
+----
+5
+
+query I
+SELECT evaluations();
+----
+20
+
+statement ok
+DROP TABLE t;