Apache DataFusion SQL Query Engine
fix: Estimate string equality and LIKE filter selectivity (#25835)
limadog9791660cd54f0
diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rsindex 2899acb3a1..235524d562 100644--- a/datafusion/core/tests/custom_sources_cases/statistics.rs+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs@@ -285,6 +285,128 @@ async fn sql_filter() -> Result<()> {Ok(())}+fn string_filter_ctx(+ data_type: DataType,+ distinct_count: Precision<usize>,+) -> Result<SessionContext> {+ init_ctx(+ Statistics {+ num_rows: Precision::Exact(1000),+ total_byte_size: Precision::Absent,+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Exact(200),+ distinct_count,+ ..ColumnStatistics::new_unknown()+ }],+ },+ Schema::new(vec![Field::new("c1", data_type, true)]),+ )+}++async fn string_filter_rows(+ ctx: &SessionContext,+ predicate: &str,+) -> Result<Precision<usize>> {+ let plan = ctx+ .sql(&format!("SELECT * FROM stats_table WHERE {predicate}"))+ .await?+ .create_physical_plan()+ .await?;+ Ok(StatisticsContext::new()+ .compute(plan.as_ref(), &StatisticsArgs::new())?+ .num_rows)+}++#[tokio::test]+async fn sql_string_filter_selectivity() -> Result<()> {+ // There are 800 non-null rows and 20 distinct strings. Equality estimates+ // 40 matches, and negated predicates exclude nulls as well as matches.+ // LIKE estimates distinguish unanchored patterns from patterns anchored+ // at one or both ends; escaped wildcards are literal characters.+ let cases = [+ ("c1 = 'foo'", 40),+ ("'foo' = c1", 40),+ ("c1 != 'foo'", 760),+ ("'foo' != c1", 760),+ ("c1 LIKE 'foo'", 40),+ ("c1 NOT LIKE 'foo'", 760),+ ("c1 LIKE '%foo%'", 160),+ ("c1 NOT LIKE '%foo%'", 640),+ ("c1 LIKE 'foo%'", 80),+ ("c1 NOT LIKE 'foo%'", 720),+ ("c1 LIKE '%foo'", 80),+ ("c1 NOT LIKE '%foo'", 720),+ ("c1 LIKE 'f_o'", 40),+ ("c1 NOT LIKE 'f_o'", 760),+ (r"c1 LIKE 'foo\%'", 40),+ (r"c1 NOT LIKE 'foo\%'", 760),+ (r"c1 LIKE 'foo\%%'", 80),+ (r"c1 NOT LIKE 'foo\%%'", 720),+ (r"c1 LIKE 'foo\'", 40),+ (r"c1 NOT LIKE 'foo\'", 760),+ (r"c1 LIKE '%foo\'", 80),+ (r"c1 NOT LIKE '%foo\'", 720),+ // Case-sensitive NDV cannot estimate a case-insensitive equality.+ ("c1 ILIKE 'foo'", 160),+ ("c1 NOT ILIKE 'foo'", 640),+ ];+ for data_type in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {+ let ctx = string_filter_ctx(data_type.clone(), Precision::Exact(20))?;+ for (predicate, expected_rows) in cases {+ assert_eq!(+ string_filter_rows(&ctx, predicate).await?,+ Precision::Inexact(expected_rows),+ "{data_type:?}: {predicate}"+ );+ }+ }+ Ok(())+}++#[tokio::test]+async fn sql_string_filter_without_distinct_count() -> Result<()> {+ let ctx = string_filter_ctx(DataType::Utf8View, Precision::Absent)?;+ // Without NDV, use the configured default (20%) over non-null rows.+ for (predicate, expected_rows) in [+ ("c1 = 'foo'", 160),+ ("c1 != 'foo'", 640),+ ("c1 LIKE 'foo'", 160),+ ("c1 NOT LIKE 'foo'", 640),+ ] {+ assert_eq!(+ string_filter_rows(&ctx, predicate).await?,+ Precision::Inexact(expected_rows),+ "{predicate}"+ );+ }+ Ok(())+}++#[tokio::test]+async fn sql_string_filter_custom_selectivity() -> Result<()> {+ let ctx = string_filter_ctx(DataType::Utf8View, Precision::Absent)?;+ ctx.sql("SET datafusion.optimizer.default_filter_selectivity = 40")+ .await?+ .collect()+ .await?;++ for (predicate, expected_rows) in [+ ("c1 = 'foo'", 320),+ ("c1 != 'foo'", 480),+ ("c1 LIKE '%foo%'", 320),+ ("c1 NOT LIKE '%foo%'", 480),+ ("c1 LIKE 'foo%'", 160),+ ("c1 NOT LIKE 'foo%'", 640),+ ] {+ assert_eq!(+ string_filter_rows(&ctx, predicate).await?,+ Precision::Inexact(expected_rows),+ "{predicate}"+ );+ }+ Ok(())+}+#[tokio::test]async fn sql_limit() -> Result<()> {let (stats, schema) = fully_defined();diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex bc1c1ce5d1..cc5e0846d3 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -64,7 +64,7 @@ use datafusion_execution::TaskContext;use datafusion_expr::Operator;use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::expressions::{- BinaryExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, Literal, lit,+ BinaryExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, LikeExpr, Literal, lit,};use datafusion_physical_expr::intervals::utils::check_support;use datafusion_physical_expr::utils::collect_columns;@@ -346,8 +346,8 @@ impl FilterExec {}/// Calculates `Statistics` for `FilterExec` by applying the filter's- /// selectivity (default, or estimated from interval analysis) to the input- /// statistics.+ /// selectivity (estimated from interval analysis or string statistics, with+ /// a configurable fallback) to the input statistics.////// The estimated output row count is used to keep the per-column statistics/// consistent with it:@@ -405,10 +405,17 @@ impl FilterExec {);(selectivity, filtered_num_rows, cs)} else {- // Without interval boundaries, use the default selectivity and- // apply the row-count constraints that still follow from the- // filter predicate.- let selectivity = default_selectivity as f64 / 100.0;+ // String predicates cannot use interval analysis. Estimate them+ // from distinct counts and pattern form when possible, then+ // apply the row-count constraints implied by the predicate.+ let default_selectivity = default_selectivity as f64 / 100.0;+ let selectivity = string_selectivity(+ predicate,+ schema,+ &input_stats,+ default_selectivity,+ )+ .unwrap_or(default_selectivity);let filtered_num_rows =input_num_rows.with_estimated_selectivity(selectivity);let mut cs = input_stats.to_inexact().column_statistics;@@ -1254,6 +1261,17 @@ fn collect_null_rejecting_columns(predicate: &Arc<dyn PhysicalExpr>) -> HashSet<let mut columns = HashSet::new();for expr in split_conjunction(predicate) {+ // LIKE (including ILIKE and their negations) returns NULL if either+ // operand is NULL.+ if let Some(like) = expr.downcast_ref::<LikeExpr>() {+ for operand in [like.expr(), like.pattern()] {+ if let Some(col) = operand.downcast_ref::<Column>() {+ columns.insert(col.index());+ }+ }+ continue;+ }+// `col IS NOT NULL` keeps only rows where `col` is non-null.if let Some(is_not_null) = expr.downcast_ref::<IsNotNullExpr>() {if let Some(col) = is_not_null.arg().downcast_ref::<Column>() {@@ -1280,6 +1298,135 @@ fn collect_null_rejecting_columns(predicate: &Arc<dyn PhysicalExpr>) -> HashSet<columns}+/// Estimates a single string comparison against a literal. Compound predicates+/// retain the fallback: multiplying estimates for predicates on the same column+/// would count null rejection (and possibly the same restriction) repeatedly.+fn string_selectivity(+ predicate: &Arc<dyn PhysicalExpr>,+ schema: &SchemaRef,+ input_stats: &Statistics,+ default_selectivity: f64,+) -> Option<f64> {+ let (column, literal, negated, like) =+ if let Some(binary) = predicate.downcast_ref::<BinaryExpr>() {+ if !matches!(binary.op(), Operator::Eq | Operator::NotEq) {+ return None;+ }+ let operands = [+ (binary.left(), binary.right()),+ (binary.right(), binary.left()),+ ];+ let (column, literal) = operands.into_iter().find_map(|(left, right)| {+ Some((+ left.downcast_ref::<Column>()?,+ right.downcast_ref::<Literal>()?,+ ))+ })?;+ (column, literal, *binary.op() == Operator::NotEq, None)+ } else {+ let like = predicate.downcast_ref::<LikeExpr>()?;+ (+ like.expr().downcast_ref::<Column>()?,+ like.pattern().downcast_ref::<Literal>()?,+ like.negated(),+ Some(like),+ )+ };++ if !matches!(+ schema.field(column.index()).data_type(),+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View+ ) {+ return None;+ }+ let Some(value) = literal.value().try_as_str()? else {+ // Both a comparison and its negation evaluate to NULL here.+ return Some(0.0);+ };+ let column_stats = &input_stats.column_statistics[column.index()];+ let equality_selectivity = column_stats+ .distinct_count+ .get_value()+ .filter(|&&ndv| ndv > 0)+ .map_or(default_selectivity, |&ndv| 1.0 / ndv as f64);+ let positive_selectivity = match like {+ Some(like) => like_selectivity(+ value,+ like.case_insensitive(),+ equality_selectivity,+ default_selectivity,+ ),+ None => equality_selectivity,+ };+ let selectivity = if negated {+ 1.0 - positive_selectivity+ } else {+ positive_selectivity+ };++ // Apply the complement within the non-null rows, not across all input+ // rows. Unknown null counts use the usual assumption of no nulls.+ match (+ input_stats.num_rows.get_value(),+ column_stats.null_count.get_value(),+ ) {+ (Some(&rows), Some(&nulls)) if rows > 0 => {+ Some(rows.saturating_sub(nulls) as f64 * selectivity / rows as f64)+ }+ (Some(0), _) => Some(0.0),+ _ => Some(selectivity),+ }+}++/// Classifies a LIKE pattern without treating escaped wildcards as operators.+/// In the absence of string histograms, wildcard patterns use the configured+/// fallback, halved for each anchored end. Thus prefix/suffix patterns are+/// estimated to match fewer rows than an unanchored contains pattern. These are+/// heuristics, not bounds on the number of matches.+fn like_selectivity(+ pattern: &str,+ case_insensitive: bool,+ equality_selectivity: f64,+ default_selectivity: f64,+) -> f64 {+ let mut chars = pattern.chars();+ let mut has_wildcard = false;+ let mut all_percent = !pattern.is_empty();+ let mut starts_with_percent = false;+ let mut ends_with_percent = false;+ let mut first = true;+ while let Some(ch) = chars.next() {+ let percent = ch == '%';+ if ch == '\\' {+ // Arrow treats a trailing backslash as a literal backslash too.+ chars.next();+ } else if matches!(ch, '%' | '_') {+ has_wildcard = true;+ }+ all_percent &= percent;+ if first {+ starts_with_percent = percent;+ first = false;+ }+ ends_with_percent = percent;+ }+ if all_percent {+ 1.0+ } else if !has_wildcard {+ // Case folding may merge several distinct input values, so the raw+ // NDV does not give the selectivity of a literal ILIKE pattern.+ if case_insensitive {+ default_selectivity+ } else {+ equality_selectivity+ }+ } else {+ let start_factor = if starts_with_percent { 1.0 } else { 0.5 };+ let end_factor = if ends_with_percent { 1.0 } else { 0.5 };+ default_selectivity * start_factor * end_factor+ }+}+/// Returns the checked column index, and whether the check is `IS NULL`, when/// `predicate` is a bare `IS NULL` or `IS NOT NULL` check on a column.pub(crate) fn null_check_column(@@ -4272,6 +4419,173 @@ mod tests {Ok(())}+ #[test]+ fn test_string_selectivity_non_string_literal() {+ let schema =+ Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, true)]));+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("name", 0)),+ Operator::Eq,+ lit(42_i32),+ ));+ let statistics = Statistics::new_unknown(&schema);++ // SQL coerces comparison operands, so exercise this defensive fallback directly.+ assert_eq!(+ string_selectivity(&predicate, &schema, &statistics, 0.2),+ None+ );+ }++ fn like_filter_statistics(+ num_rows: Precision<usize>,+ column: ColumnStatistics,+ pattern: Option<&str>,+ negated: bool,+ ) -> Arc<Statistics> {+ let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows,+ total_byte_size: Precision::Absent,+ column_statistics: vec![column],+ },+ schema.clone(),+ ));+ let predicate = Arc::new(LikeExpr::new(+ negated,+ false,+ col("name", &schema).expect("name column exists"),+ lit(ScalarValue::Utf8(pattern.map(str::to_owned))),+ ));+ let filter = FilterExec::try_new(predicate, input).expect("valid LIKE filter");+ StatisticsContext::new()+ .compute(&filter, &StatisticsArgs::new())+ .expect("LIKE filter statistics")+ }++ #[test]+ fn test_filter_statistics_like_literal_patterns() {+ // SQL simplification rewrites these patterns before FilterExec sees+ // them. Construct LIKE directly to exercise physical-plan callers too.+ for (pattern, positive_rows, negative_rows) in [+ (Some("%"), 800, 0),+ (Some("%%%"), 800, 0),+ (Some(""), 40, 760),+ (Some(r"\%"), 40, 760),+ (Some(r"\_"), 40, 760),+ (Some(r"\%%"), 80, 720),+ (Some(r"%\%"), 80, 720),+ (None, 0, 0),+ ] {+ for (negated, expected_rows) in+ [(false, positive_rows), (true, negative_rows)]+ {+ let statistics = like_filter_statistics(+ Precision::Exact(1000),+ ColumnStatistics {+ null_count: Precision::Exact(200),+ distinct_count: Precision::Exact(20),+ ..Default::default()+ },+ pattern,+ negated,+ );+ assert_eq!(+ statistics.num_rows,+ Precision::Inexact(expected_rows),+ "pattern={pattern:?}, negated={negated}"+ );+ assert_eq!(+ statistics.column_statistics[0].null_count,+ Precision::Exact(0)+ );+ }+ }+ }++ #[test]+ fn test_filter_statistics_like_missing_or_zero_counts() {+ use Precision::{Absent, Exact, Inexact};++ for (rows, nulls, distinct, positive_rows, negative_rows) in [+ (Exact(1000), Absent, Exact(20), Inexact(50), Inexact(950)),+ (+ Exact(1000),+ Exact(200),+ Exact(0),+ Inexact(160),+ Inexact(640),+ ),+ (Exact(1000), Exact(1000), Exact(0), Inexact(0), Inexact(0)),+ (Exact(0), Exact(0), Exact(0), Exact(0), Exact(0)),+ (Inexact(0), Absent, Absent, Inexact(0), Inexact(0)),+ (Absent, Absent, Exact(20), Absent, Absent),+ ] {+ for (negated, expected_rows) in+ [(false, positive_rows), (true, negative_rows)]+ {+ let statistics = like_filter_statistics(+ rows,+ ColumnStatistics {+ null_count: nulls,+ distinct_count: distinct,+ ..Default::default()+ },+ Some(""),+ negated,+ );+ assert_eq!(+ statistics.num_rows, expected_rows,+ "rows={rows:?}, nulls={nulls:?}, distinct={distinct:?}, negated={negated}"+ );+ assert_eq!(statistics.column_statistics[0].null_count, Exact(0));+ }+ }+ }++ #[test]+ fn test_filter_statistics_like_column_pattern_rejects_nulls() {+ let schema = Schema::new(vec![+ Field::new("name", DataType::Utf8, true),+ Field::new("pattern", DataType::Utf8, true),+ ]);+ for negated in [false, true] {+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Exact(1000),+ total_byte_size: Precision::Absent,+ column_statistics: [200, 300]+ .into_iter()+ .map(|nulls| ColumnStatistics {+ null_count: Precision::Exact(nulls),+ distinct_count: Precision::Exact(20),+ ..Default::default()+ })+ .collect(),+ },+ schema.clone(),+ ));+ let predicate = Arc::new(LikeExpr::new(+ negated,+ false,+ col("name", &schema).expect("name column exists"),+ col("pattern", &schema).expect("pattern column exists"),+ ));+ let filter =+ FilterExec::try_new(predicate, input).expect("valid LIKE filter");+ let statistics = StatisticsContext::new()+ .compute(&filter, &StatisticsArgs::new())+ .expect("LIKE filter statistics");+ // A varying pattern retains the fallback estimate, but neither+ // operand can be NULL in any surviving row.+ assert_eq!(statistics.num_rows, Precision::Inexact(200));+ for column in &statistics.column_statistics {+ assert_eq!(column.null_count, Precision::Exact(0));+ }+ }+ }+#[test]fn test_filter_statistics_fetch_preserves_singleton() -> Result<()> {use Precision::{Absent, Exact, Inexact};