Skip to content

Apache Lucene open-source search software

Collapse repeated wildcard asterisks (#16382)

Vismay0c64c02c63d7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b01c7b3003c..82b9b688f73 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -151,6 +151,10 @@ Improvements
Optimizations
---------------------
+* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*'
+ wildcards into a single any-string automaton, avoiding a quadratic blow-up in
+ automaton size for patterns with many repeated asterisks. (Vismay Tiwari)
+
* GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs via
TwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-values
not-equals filter), instead of advancing one doc at a time. (Jim Ferenczi)
diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
index 9c1e8c223ed..b92bdb9d082 100644
--- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
@@ -64,7 +64,6 @@ public class WildcardQuery extends AutomatonQuery {
*
* @lucene.internal
*/
- @SuppressWarnings("fallthrough")
public static Automaton toAutomaton(Term wildcardquery) {
List<Automaton> automata = new ArrayList<>();
@@ -74,22 +73,26 @@ public class WildcardQuery extends AutomatonQuery {
final int c = wildcardText.codePointAt(i);
int length = Character.charCount(c);
switch (c) {
- case WILDCARD_STRING:
- automata.add(Automata.makeAnyString());
- break;
- case WILDCARD_CHAR:
- automata.add(Automata.makeAnyChar());
- break;
- case WILDCARD_ESCAPE:
- // add the next codepoint instead, if it exists
+ case WILDCARD_STRING -> {
+ // Collapse consecutive '*' ("**" is equivalent to "*"): only append an
+ // any-string automaton when the previous segment isn't already one.
+ if (automata.isEmpty() || Operations.isTotal(automata.getLast()) == false) {
+ automata.add(Automata.makeAnyString());
+ }
+ }
+ case WILDCARD_CHAR -> automata.add(Automata.makeAnyChar());
+ case WILDCARD_ESCAPE -> {
+ // add the escaped codepoint as a literal if it exists; a trailing
+ // '\' is parsed leniently as a literal '\'
if (i + length < wildcardText.length()) {
final int nextChar = wildcardText.codePointAt(i + length);
length += Character.charCount(nextChar);
automata.add(Automata.makeChar(nextChar));
- break;
- } // else fallthru, lenient parsing with a trailing \
- default:
- automata.add(Automata.makeChar(c));
+ } else {
+ automata.add(Automata.makeChar(c));
+ }
+ }
+ default -> automata.add(Automata.makeChar(c));
}
i += length;
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
index c076367ce5b..d7f81ed71cd 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
@@ -38,8 +38,11 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
/** TestWildcardQuery tests the '*' and '?' wildcard characters. */
public class TestWildcardQuery extends LuceneTestCase {
@@ -235,6 +238,50 @@ public class TestWildcardQuery extends LuceneTestCase {
indexStore.close();
}
+ private static Automaton det(Automaton a) {
+ return Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ }
+
+ public void testRepeatedAsterisksCollapse() {
+ // Consecutive '*' must collapse to a single any-string automaton: same
+ // language as a single '*', and the same (tiny) shape, not a huge one.
+ Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*")));
+ Automaton repeated = det(WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000))));
+
+ assertTrue(AutomatonTestUtil.sameLanguage(single, repeated));
+ assertEquals(single.getNumStates(), repeated.getNumStates());
+ assertEquals(single.getNumTransitions(), repeated.getNumTransitions());
+ }
+
+ public void testRepeatedAsterisksCollapseWithEscapesAndLiterals() {
+ // Collapsing applies only to unescaped, consecutive '*'. An escaped '\*'
+ // stays a literal and the surrounding literals / '?' are untouched, so
+ // "a\*b***c?" matches the same language as its single-'*' form "a\*b*c?".
+ Automaton collapsed = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b***c?")));
+ Automaton reference = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b*c?")));
+
+ assertTrue(AutomatonTestUtil.sameLanguage(reference, collapsed));
+ assertEquals(reference.getNumStates(), collapsed.getNumStates());
+ assertEquals(reference.getNumTransitions(), collapsed.getNumTransitions());
+
+ // Escaped '\*' is a literal, so "\*\*\*" matches the string "***", not the
+ // any-string '*' — it must NOT be collapsed.
+ Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*")));
+ Automaton literalStars = det(WildcardQuery.toAutomaton(new Term("field", "\\*\\*\\*")));
+ assertFalse(AutomatonTestUtil.sameLanguage(single, literalStars));
+ }
+
+ public void testInterleavedStarQuestionDoesNotBlowUp() {
+ // '*?' repeated does not collapse (the '*' are not consecutive), but it
+ // must still build a linearly-sized automaton rather than exploding the way
+ // repeated '*' used to before this fix (GITHUB#16134).
+ int n = 5000;
+ Automaton a = WildcardQuery.toAutomaton(new Term("field", "*?".repeat(n)));
+ assertTrue(
+ "expected roughly linear size, got " + a.getNumTransitions() + " transitions for " + n,
+ a.getNumTransitions() < 100L * n);
+ }
+
private Directory getIndexStore(String field, String[] contents) throws IOException {
Directory indexStore = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);