Apache DataFusion SQL Query Engine
Fix pruning predicate for `LIKE` expressions with escape sequences (#22375)
Mason7b4d2e67a138
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rsindex 013a06812a..28d4fe9028 100644--- a/datafusion/pruning/src/pruning_predicate.rs+++ b/datafusion/pruning/src/pruning_predicate.rs@@ -1822,8 +1822,10 @@ fn extract_string_literal(expr: &Arc<dyn PhysicalExpr>) -> Option<&str> {fn build_like_match(expr_builder: &mut PruningExpressionBuilder,) -> Option<Arc<dyn PhysicalExpr>> {- // column LIKE literal => (min, max) LIKE literal split at % => min <= split literal && split literal <= max+ // column LIKE literal => (min, max) LIKE literal split at unescaped % => min <= split literal && split literal <= max// column LIKE 'foo%' => min <= 'foo' && 'foo' <= max+ // column LIKE 'foo\_%' => min <= 'foo_' && 'foo_' <= max (the _ is escaped)+ // column LIKE 'foo\%%' => min <= 'foo%' && 'foo%' <= max (the % is escaped)// column LIKE '%foo' => min <= '' && '' <= max => true// column LIKE '%foo%' => min <= '' && '' <= max => true// column LIKE 'foo' => min <= 'foo' && 'foo' <= max@@ -1836,24 +1838,25 @@ fn build_like_match(// check that the scalar is a string literallet s = extract_string_literal(scalar_expr)?;// ANSI SQL specifies two wildcards: % and _. % matches zero or more characters, _ matches exactly one character.- let first_wildcard_index = s.find(['%', '_']);- if first_wildcard_index == Some(0) {- // there's no filtering we could possibly do, return an error and have this be handled by the unhandled hook+ let (decoded_prefix, rest) = split_constant_prefix(s);+ let has_wildcard = !rest.is_empty();+ if has_wildcard && decoded_prefix.is_empty() {+ // there's no filtering we could possibly do, return None and have this be handled by the unhandled hookreturn None;}- let (lower_bound, upper_bound) = if let Some(wildcard_index) = first_wildcard_index {- let prefix = &s[..wildcard_index];+ let (lower_bound, upper_bound) = if has_wildcard {+ let incremented_prefix = increment_utf8(&decoded_prefix)?;let lower_bound_lit = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(- prefix.to_string(),+ decoded_prefix,))));let upper_bound_lit = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(- increment_utf8(prefix)?,+ incremented_prefix,))));(lower_bound_lit, upper_bound_lit)} else {// the like expression is a literal and can be converted into a comparisonlet bound = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(- s.to_string(),+ decoded_prefix,))));(Arc::clone(&bound), bound)};@@ -1934,19 +1937,20 @@ fn build_not_like_match(}/// Returns unescaped constant prefix of a LIKE pattern (possibly empty) and the remaining pattern (possibly empty)-fn split_constant_prefix(pattern: &str) -> (&str, &str) {- let char_indices = pattern.char_indices().collect::<Vec<_>>();- for i in 0..char_indices.len() {- let (idx, char) = char_indices[i];- if char == '%' || char == '_' {- if i != 0 && char_indices[i - 1].1 == '\\' {- // ecsaped by `\`- continue;- }- return (&pattern[..idx], &pattern[idx..]);+fn split_constant_prefix(pattern: &str) -> (String, &str) {+ let mut prefix = String::with_capacity(pattern.len());+ let mut iter = pattern.char_indices();+ while let Some((idx, c)) = iter.next() {+ match c {+ '%' | '_' => return (prefix, &pattern[idx..]),+ '\\' => match iter.next() {+ Some((_, escaped)) => prefix.push(escaped),+ None => prefix.push('\\'),+ },+ _ => prefix.push(c),}}- (pattern, "")+ (prefix, "")}/// Increment a UTF8 string by one, returning `None` if it can't be incremented.@@ -4816,6 +4820,174 @@ mod tests {prune_with_expr(expr, &schema, &statistics, expected_ret);}+ // `build_like_match()` must honor `\` escapes when scanning the pattern for+ // wildcards.+ #[test]+ fn prune_utf8_like_escaped_chars() {+ let schema = Arc::new(Schema::new(vec![Field::new("s1", DataType::Utf8, true)]));+ let statistics = TestStatistics::new().with(+ "s1",+ ContainerStats::new_utf8(+ vec![+ Some("foo_aaa"),+ Some(r#"foo\aaa"#),+ Some("foo"),+ Some("bar"),+ Some("foo%aaa"),+ Some("%foo_aaa"),+ ], // min+ vec![+ Some("foo_zzz"),+ Some(r#"foo\zzz"#),+ Some("foozzz"),+ Some("baz"),+ Some("foo%zzz"),+ Some("%foo_zzz"),+ ], // max+ ),+ );++ let expr = col("s1").like(lit(r#"foo\_%"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => every value starts with literal+ // "foo_" and matches the pattern; must keep.+ true,+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo", "foozzz"] => stats don't prove "foo_" is or isn't in+ // range; must conservatively keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ let expr = col("s1").like(lit(r#"foo\\%"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => every value starts with literal+ // "foo\" and matches the pattern; must keep.+ true,+ // s1 ["foo", "foozzz"] => stats don't prove "foo\" is or isn't in+ // range; must conservatively keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ let expr = col("s1").like(lit(r#"foo\%%"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo", "foozzz"] => range straddles "foo%"; must keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => every value starts with literal+ // "foo%" and matches the pattern; must keep.+ true,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ // No wildcard after escapes: pattern reduces to an equality check on+ // the literal "foo_".+ let expr = col("s1").like(lit(r#"foo\_"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo", "foozzz"] => "foo_" is within the range; must keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ // Leading escaped `%`: prefix is "%foo" (non-empty), so the guard+ // for "all wildcards" must NOT bail out here.+ let expr = col("s1").like(lit(r#"\%foo%"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo", "foozzz"] => no rows can pass (not keep)+ false,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => every value starts with literal+ // "%foo" and matches the pattern; must keep.+ true,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ // Two escaped wildcards, no real wildcard: equality on "foo%_".+ let expr = col("s1").like(lit(r#"foo\%\_"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo", "foozzz"] => "foo%_" is within the range; must keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);++ // Escaped backslash followed by more literal chars before the+ // wildcard: prefix is "foo\bar".+ let expr = col("s1").like(lit(r#"foo\\bar%"#));+ #[rustfmt::skip]+ let expected_ret = &[+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)+ false,+ // s1 ["foo\aaa", "foo\zzz"] => range straddles "foo\bar"; must+ // keep.+ true,+ // s1 ["foo", "foozzz"] => range straddles "foo\bar"; must keep.+ true,+ // s1 ["bar", "baz"] => no rows can pass (not keep)+ false,+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)+ false,+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)+ false,+ ];+ prune_with_expr(expr, &schema, &statistics, expected_ret);+ }+#[test]fn prune_utf8_not_like_one() {let (schema, statistics) = utf8_setup();