Apache Lucene open-source search software
Collapse repeated wildcard asterisks (#16382)
Vismay0c64c02c63d7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex b01c7b3003c..82b9b688f73 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -151,6 +151,10 @@ ImprovementsOptimizations---------------------+* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*'+ wildcards into a single any-string automaton, avoiding a quadratic blow-up in+ automaton size for patterns with many repeated asterisks. (Vismay Tiwari)+* GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs viaTwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-valuesnot-equals filter), instead of advancing one doc at a time. (Jim Ferenczi)diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.javaindex 9c1e8c223ed..b92bdb9d082 100644--- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java@@ -64,7 +64,6 @@ public class WildcardQuery extends AutomatonQuery {** @lucene.internal*/- @SuppressWarnings("fallthrough")public static Automaton toAutomaton(Term wildcardquery) {List<Automaton> automata = new ArrayList<>();@@ -74,22 +73,26 @@ public class WildcardQuery extends AutomatonQuery {final int c = wildcardText.codePointAt(i);int length = Character.charCount(c);switch (c) {- case WILDCARD_STRING:- automata.add(Automata.makeAnyString());- break;- case WILDCARD_CHAR:- automata.add(Automata.makeAnyChar());- break;- case WILDCARD_ESCAPE:- // add the next codepoint instead, if it exists+ case WILDCARD_STRING -> {+ // Collapse consecutive '*' ("**" is equivalent to "*"): only append an+ // any-string automaton when the previous segment isn't already one.+ if (automata.isEmpty() || Operations.isTotal(automata.getLast()) == false) {+ automata.add(Automata.makeAnyString());+ }+ }+ case WILDCARD_CHAR -> automata.add(Automata.makeAnyChar());+ case WILDCARD_ESCAPE -> {+ // add the escaped codepoint as a literal if it exists; a trailing+ // '\' is parsed leniently as a literal '\'if (i + length < wildcardText.length()) {final int nextChar = wildcardText.codePointAt(i + length);length += Character.charCount(nextChar);automata.add(Automata.makeChar(nextChar));- break;- } // else fallthru, lenient parsing with a trailing \- default:- automata.add(Automata.makeChar(c));+ } else {+ automata.add(Automata.makeChar(c));+ }+ }+ default -> automata.add(Automata.makeChar(c));}i += length;}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.javaindex c076367ce5b..d7f81ed71cd 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java@@ -38,8 +38,11 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.automaton.Automaton;import org.apache.lucene.util.automaton.CompiledAutomaton;+import org.apache.lucene.util.automaton.Operations;/** TestWildcardQuery tests the '*' and '?' wildcard characters. */public class TestWildcardQuery extends LuceneTestCase {@@ -235,6 +238,50 @@ public class TestWildcardQuery extends LuceneTestCase {indexStore.close();}+ private static Automaton det(Automaton a) {+ return Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);+ }++ public void testRepeatedAsterisksCollapse() {+ // Consecutive '*' must collapse to a single any-string automaton: same+ // language as a single '*', and the same (tiny) shape, not a huge one.+ Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*")));+ Automaton repeated = det(WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000))));++ assertTrue(AutomatonTestUtil.sameLanguage(single, repeated));+ assertEquals(single.getNumStates(), repeated.getNumStates());+ assertEquals(single.getNumTransitions(), repeated.getNumTransitions());+ }++ public void testRepeatedAsterisksCollapseWithEscapesAndLiterals() {+ // Collapsing applies only to unescaped, consecutive '*'. An escaped '\*'+ // stays a literal and the surrounding literals / '?' are untouched, so+ // "a\*b***c?" matches the same language as its single-'*' form "a\*b*c?".+ Automaton collapsed = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b***c?")));+ Automaton reference = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b*c?")));++ assertTrue(AutomatonTestUtil.sameLanguage(reference, collapsed));+ assertEquals(reference.getNumStates(), collapsed.getNumStates());+ assertEquals(reference.getNumTransitions(), collapsed.getNumTransitions());++ // Escaped '\*' is a literal, so "\*\*\*" matches the string "***", not the+ // any-string '*' — it must NOT be collapsed.+ Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*")));+ Automaton literalStars = det(WildcardQuery.toAutomaton(new Term("field", "\\*\\*\\*")));+ assertFalse(AutomatonTestUtil.sameLanguage(single, literalStars));+ }++ public void testInterleavedStarQuestionDoesNotBlowUp() {+ // '*?' repeated does not collapse (the '*' are not consecutive), but it+ // must still build a linearly-sized automaton rather than exploding the way+ // repeated '*' used to before this fix (GITHUB#16134).+ int n = 5000;+ Automaton a = WildcardQuery.toAutomaton(new Term("field", "*?".repeat(n)));+ assertTrue(+ "expected roughly linear size, got " + a.getNumTransitions() + " transitions for " + n,+ a.getNumTransitions() < 100L * n);+ }+private Directory getIndexStore(String field, String[] contents) throws IOException {Directory indexStore = newDirectory();RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);