Skip to content

Apache DataFusion SQL Query Engine

Fix pruning predicate for `LIKE` expressions with escape sequences (#22375)

Mason7b4d2e67a138
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index 013a06812a..28d4fe9028 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -1822,8 +1822,10 @@ fn extract_string_literal(expr: &Arc<dyn PhysicalExpr>) -> Option<&str> {
fn build_like_match(
expr_builder: &mut PruningExpressionBuilder,
) -> Option<Arc<dyn PhysicalExpr>> {
- // column LIKE literal => (min, max) LIKE literal split at % => min <= split literal && split literal <= max
+ // column LIKE literal => (min, max) LIKE literal split at unescaped % => min <= split literal && split literal <= max
// column LIKE 'foo%' => min <= 'foo' && 'foo' <= max
+ // column LIKE 'foo\_%' => min <= 'foo_' && 'foo_' <= max (the _ is escaped)
+ // column LIKE 'foo\%%' => min <= 'foo%' && 'foo%' <= max (the % is escaped)
// column LIKE '%foo' => min <= '' && '' <= max => true
// column LIKE '%foo%' => min <= '' && '' <= max => true
// column LIKE 'foo' => min <= 'foo' && 'foo' <= max
@@ -1836,24 +1838,25 @@ fn build_like_match(
// check that the scalar is a string literal
let s = extract_string_literal(scalar_expr)?;
// ANSI SQL specifies two wildcards: % and _. % matches zero or more characters, _ matches exactly one character.
- let first_wildcard_index = s.find(['%', '_']);
- if first_wildcard_index == Some(0) {
- // there's no filtering we could possibly do, return an error and have this be handled by the unhandled hook
+ let (decoded_prefix, rest) = split_constant_prefix(s);
+ let has_wildcard = !rest.is_empty();
+ if has_wildcard && decoded_prefix.is_empty() {
+ // there's no filtering we could possibly do, return None and have this be handled by the unhandled hook
return None;
}
- let (lower_bound, upper_bound) = if let Some(wildcard_index) = first_wildcard_index {
- let prefix = &s[..wildcard_index];
+ let (lower_bound, upper_bound) = if has_wildcard {
+ let incremented_prefix = increment_utf8(&decoded_prefix)?;
let lower_bound_lit = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(
- prefix.to_string(),
+ decoded_prefix,
))));
let upper_bound_lit = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(
- increment_utf8(prefix)?,
+ incremented_prefix,
))));
(lower_bound_lit, upper_bound_lit)
} else {
// the like expression is a literal and can be converted into a comparison
let bound = Arc::new(phys_expr::Literal::new(ScalarValue::Utf8(Some(
- s.to_string(),
+ decoded_prefix,
))));
(Arc::clone(&bound), bound)
};
@@ -1934,19 +1937,20 @@ fn build_not_like_match(
}
/// Returns unescaped constant prefix of a LIKE pattern (possibly empty) and the remaining pattern (possibly empty)
-fn split_constant_prefix(pattern: &str) -> (&str, &str) {
- let char_indices = pattern.char_indices().collect::<Vec<_>>();
- for i in 0..char_indices.len() {
- let (idx, char) = char_indices[i];
- if char == '%' || char == '_' {
- if i != 0 && char_indices[i - 1].1 == '\\' {
- // ecsaped by `\`
- continue;
- }
- return (&pattern[..idx], &pattern[idx..]);
+fn split_constant_prefix(pattern: &str) -> (String, &str) {
+ let mut prefix = String::with_capacity(pattern.len());
+ let mut iter = pattern.char_indices();
+ while let Some((idx, c)) = iter.next() {
+ match c {
+ '%' | '_' => return (prefix, &pattern[idx..]),
+ '\\' => match iter.next() {
+ Some((_, escaped)) => prefix.push(escaped),
+ None => prefix.push('\\'),
+ },
+ _ => prefix.push(c),
}
}
- (pattern, "")
+ (prefix, "")
}
/// Increment a UTF8 string by one, returning `None` if it can't be incremented.
@@ -4816,6 +4820,174 @@ mod tests {
prune_with_expr(expr, &schema, &statistics, expected_ret);
}
+ // `build_like_match()` must honor `\` escapes when scanning the pattern for
+ // wildcards.
+ #[test]
+ fn prune_utf8_like_escaped_chars() {
+ let schema = Arc::new(Schema::new(vec![Field::new("s1", DataType::Utf8, true)]));
+ let statistics = TestStatistics::new().with(
+ "s1",
+ ContainerStats::new_utf8(
+ vec![
+ Some("foo_aaa"),
+ Some(r#"foo\aaa"#),
+ Some("foo"),
+ Some("bar"),
+ Some("foo%aaa"),
+ Some("%foo_aaa"),
+ ], // min
+ vec![
+ Some("foo_zzz"),
+ Some(r#"foo\zzz"#),
+ Some("foozzz"),
+ Some("baz"),
+ Some("foo%zzz"),
+ Some("%foo_zzz"),
+ ], // max
+ ),
+ );
+
+ let expr = col("s1").like(lit(r#"foo\_%"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => every value starts with literal
+ // "foo_" and matches the pattern; must keep.
+ true,
+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo", "foozzz"] => stats don't prove "foo_" is or isn't in
+ // range; must conservatively keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ let expr = col("s1").like(lit(r#"foo\\%"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => every value starts with literal
+ // "foo\" and matches the pattern; must keep.
+ true,
+ // s1 ["foo", "foozzz"] => stats don't prove "foo\" is or isn't in
+ // range; must conservatively keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ let expr = col("s1").like(lit(r#"foo\%%"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo", "foozzz"] => range straddles "foo%"; must keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => every value starts with literal
+ // "foo%" and matches the pattern; must keep.
+ true,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ // No wildcard after escapes: pattern reduces to an equality check on
+ // the literal "foo_".
+ let expr = col("s1").like(lit(r#"foo\_"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo", "foozzz"] => "foo_" is within the range; must keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ // Leading escaped `%`: prefix is "%foo" (non-empty), so the guard
+ // for "all wildcards" must NOT bail out here.
+ let expr = col("s1").like(lit(r#"\%foo%"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo", "foozzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => every value starts with literal
+ // "%foo" and matches the pattern; must keep.
+ true,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ // Two escaped wildcards, no real wildcard: equality on "foo%_".
+ let expr = col("s1").like(lit(r#"foo\%\_"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo", "foozzz"] => "foo%_" is within the range; must keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ // Escaped backslash followed by more literal chars before the
+ // wildcard: prefix is "foo\bar".
+ let expr = col("s1").like(lit(r#"foo\\bar%"#));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["foo_aaa", "foo_zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo\aaa", "foo\zzz"] => range straddles "foo\bar"; must
+ // keep.
+ true,
+ // s1 ["foo", "foozzz"] => range straddles "foo\bar"; must keep.
+ true,
+ // s1 ["bar", "baz"] => no rows can pass (not keep)
+ false,
+ // s1 ["foo%aaa", "foo%zzz"] => no rows can pass (not keep)
+ false,
+ // s1 ["%foo_aaa", "%foo_zzz"] => no rows can pass (not keep)
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+ }
+
#[test]
fn prune_utf8_not_like_one() {
let (schema, statistics) = utf8_setup();