Apache DataFusion SQL Query Engine
bench: add correlated-proxy case to the predicate_eval suite (#22919)
Adrian Garcia Badaracco073e9edcad79
diff --git a/benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q73.benchmark b/benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q73.benchmarknew file mode 100644index 0000000000..cc3f7bcf54--- /dev/null+++ b/benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q73.benchmark@@ -0,0 +1,7 @@+subgroup correlation++template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template+SUBGROUP=correlation+QPAD=73+DATASET=corrproxy+NAME=correlation_q73_redundant_proxydiff --git a/benchmarks/sql_benchmarks/predicate_eval/load/corrproxy.sql b/benchmarks/sql_benchmarks/predicate_eval/load/corrproxy.sqlnew file mode 100644index 0000000000..f06e68d38c--- /dev/null+++ b/benchmarks/sql_benchmarks/predicate_eval/load/corrproxy.sql@@ -0,0 +1,44 @@+-- Correlated-proxy dataset: a cheap integer predicate that is a perfect proxy+-- for three string predicates, plus one independent string predicate.+--+-- c0 = 1 for ~30% of rows (cheap proxy)+-- s1, s2, s3 each contain a marker exactly where c0 = 1 (correlated)+-- s4 contains a marker for an independent ~30% (independent)+--+-- The four string columns are deliberately *identical in shape*: same width,+-- the same single marker at the same offset, each matched by an equally cheap+-- regex with the same ~30% marginal selectivity. Marginally the four regex+-- predicates are therefore indistinguishable -- same cost, same selectivity, in+-- every position -- so neither a marginal cost/selectivity estimator nor+-- runtime timing can prefer one over another. Only their *conditional*+-- behaviour behind the proxy differs: after `c0 = 1`, the s1/s2/s3 regexes keep+-- every survivor (each re-tests the proxy's condition) while the s4 regex still+-- discards ~70%. Only joint statistics can see that; an independence assumption+-- prices all four regexes identically in every position.+--+-- PRED_FILL sets the filler width on each side of the marker (a non-matching+-- `regexp_like` must scan the whole value), and PRED_ROWS sizes the table.+CREATE TABLE t AS+WITH base AS (+ SELECT+ -- The cheap proxy and the independent control share one definition each, so+ -- the perfect-proxy / independence invariants can't drift apart silently.+ (value * 7) % 100 < 30 AS proxy, -- ~30%, drives c0 and s1/s2/s3+ (value * 13) % 100 < 30 AS indep -- ~30%, independent of proxy, drives s4+ FROM generate_series(1, ${PRED_ROWS:-1000000})+)+SELECT+ CASE WHEN proxy THEN 1 ELSE 0 END AS c0,+ repeat('q', ${PRED_FILL:-30})+ || CASE WHEN proxy THEN 'aaa' ELSE 'zzz' END+ || repeat('q', ${PRED_FILL:-30}) AS s1,+ repeat('q', ${PRED_FILL:-30})+ || CASE WHEN proxy THEN 'ccc' ELSE 'zzz' END+ || repeat('q', ${PRED_FILL:-30}) AS s2,+ repeat('q', ${PRED_FILL:-30})+ || CASE WHEN proxy THEN 'ddd' ELSE 'zzz' END+ || repeat('q', ${PRED_FILL:-30}) AS s3,+ repeat('q', ${PRED_FILL:-30})+ || CASE WHEN indep THEN 'bbb' ELSE 'zzz' END+ || repeat('q', ${PRED_FILL:-30}) AS s4+FROM base;diff --git a/benchmarks/sql_benchmarks/predicate_eval/queries/correlation/q73.sql b/benchmarks/sql_benchmarks/predicate_eval/queries/correlation/q73.sqlnew file mode 100644index 0000000000..5e1e822e92--- /dev/null+++ b/benchmarks/sql_benchmarks/predicate_eval/queries/correlation/q73.sql@@ -0,0 +1,14 @@+-- Hidden: `c0 = 1` is a perfect proxy for the s1/s2/s3 regexes -- after the+-- cheap proxy, each of those keeps every survivor while the equally selective+-- (~30%) s4 regex still discards ~70%. The optimal order is [c0, s4, s1/s2/s3]+-- (one informative regex on 30% of rows, the three redundant ones on 9%), but+-- the four regexes are marginally identical -- same width, same marker offset,+-- same cost, same selectivity -- so ranking them takes their *joint*+-- distribution with the proxy. Written with the redundant regexes first,+-- grouped with their proxy, as an author naturally would.+SELECT count(*) FROM t+WHERE c0 = 1+ AND regexp_like(s1, 'a.a')+ AND regexp_like(s2, 'c.c')+ AND regexp_like(s3, 'd.d')+ AND regexp_like(s4, 'b.b');