Skip to content

Apache Lucene open-source search software

Remove implicit determinization from WildcardQuery (#15961)

Dimitris Rempapis86593e51b28d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 94b05c92f88..4ba9ce4b205 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -82,6 +82,12 @@ API Changes
To force a classic DFA execution, use Operations.determinize() and AutomatonQuery.
(Dimitris Rempapis)
+* GITHUB#15961: Removed WildcardQuery constructor overloads that exposed determinization controls
+ (determinizeWorkLimit). Determinization is now done only as-needed.
+ To force a classic DFA execution, use Operations.determinize() and AutomatonQuery.
+ Also removed QueryParserBase.setDeterminizeWorkLimit/getDeterminizeWorkLimit, which are
+ no longer needed. (Dimitris Rempapis)
+
New Features
---------------------
* GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
index f1189c6d8d0..e7afaaedabd 100644
--- a/lucene/MIGRATE.md
+++ b/lucene/MIGRATE.md
@@ -159,9 +159,9 @@ iwc.getConfig().getCodec().compoundFormat().getShouldUseCompoundFile();
iwc.getConfig().getCodec().compoundFormat().getMaxCFSSegmentSizeMB();
```
-### Implicit determinization removed from RegexpQuery
+### Implicit determinization removed from RegexpQuery and WildcardQuery
-Previously, RegexpQuery would use DFA execution by default, even if it might be inefficient.
+Previously, RegexpQuery and WildcardQuery would use DFA execution by default, even if it might be inefficient.
RegexpQuery will now only [determinize as-needed](https://swtch.com/~rsc/regexp/regexp1.html). This might be
faster or slower depending upon your queries.
@@ -174,6 +174,18 @@ Automaton dfa = Operations.determinize(new RegExp(re).toAutomaton(), 10000);
Query query = new AutomatonQuery(new Term("myfield", re), dfa);
```
+Similarly for WildcardQuery, the `determinizeWorkLimit` parameter has been removed from `WildcardQuery` constructors and from
+`WildcardQuery.toAutomaton`. `QueryParserBase.setDeterminizeWorkLimit` and `getDeterminizeWorkLimit`
+have also been removed.
+
+To force the previous behavior, use:
+
+```java
+String pattern = "foo*bar";
+Automaton dfa = Operations.determinize(WildcardQuery.toAutomaton(new Term("myfield", pattern)), 10000);
+Query query = new AutomatonQuery(new Term("myfield", pattern), dfa);
+```
+
### CollectionStatistics and TermStatistics have been renamed to FieldStats and TermStats (GITHUB#15929)
Corresponding methods and parameters have been renamed accordingly.
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java
index 537f8fe8064..d184088f561 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java
@@ -31,7 +31,6 @@ import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.util.automaton.Operations;
/**
* A QueryMaker that uses common and uncommon actual Wikipedia queries for searching the English
@@ -132,10 +131,7 @@ public class EnwikiQueryMaker extends AbstractQueryMaker {
private static Query[] getPrebuiltQueries(String field) {
WildcardQuery wcq =
- new WildcardQuery(
- new Term(field, "fo*"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);
+ new WildcardQuery(new Term(field, "fo*"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);
// be wary of unanalyzed text
return new Query[] {
new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 5),
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index 845bae5238e..2d7bdc02ef7 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -42,7 +42,7 @@ import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
/**
* Implements search over a single IndexReader.
@@ -911,7 +911,7 @@ public class IndexSearcher {
@Override
public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ Query query, String field, Supplier<ByteRunnable> automaton) {
if (numClauses > maxClauseCount) {
throw new TooManyNestedClauses();
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java b/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java
index 644e21a371a..ac4074ba798 100644
--- a/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java
+++ b/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java
@@ -21,7 +21,7 @@ import java.util.Arrays;
import java.util.Set;
import java.util.function.Supplier;
import org.apache.lucene.index.Term;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
/**
* Allows recursion through a query tree
@@ -43,11 +43,10 @@ public abstract class QueryVisitor {
*
* @param query the leaf query
* @param field the field queried against
- * @param automaton a supplier for an automaton defining which terms match
+ * @param automaton a supplier for a runnable automaton defining which terms match
* @lucene.experimental
*/
- public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ public void consumeTermsMatching(Query query, String field, Supplier<ByteRunnable> automaton) {
visitLeaf(query); // default impl for backward compatibility
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
index 2aa438f84b7..9c1e8c223ed 100644
--- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
@@ -47,32 +47,16 @@ public class WildcardQuery extends AutomatonQuery {
/** Constructs a query for terms matching <code>term</code>. */
public WildcardQuery(Term term) {
- this(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ this(term, CONSTANT_SCORE_BLENDED_REWRITE);
}
/**
* Constructs a query for terms matching <code>term</code>.
*
- * @param determinizeWorkLimit maximum effort to spend while compiling the automaton from this
- * wildcard. Set higher to allow more complex queries and lower to prevent memory exhaustion.
- * Use {@link Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't
- * otherwise know what to specify.
- */
- public WildcardQuery(Term term, int determinizeWorkLimit) {
- this(term, determinizeWorkLimit, CONSTANT_SCORE_BLENDED_REWRITE);
- }
-
- /**
- * Constructs a query for terms matching <code>term</code>.
- *
- * @param determinizeWorkLimit maximum effort to spend while compiling the automaton from this
- * wildcard. Set higher to allow more complex queries and lower to prevent memory exhaustion.
- * Use {@link Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't
- * otherwise know what to specify.
* @param rewriteMethod the rewrite method to use when building the final query
*/
- public WildcardQuery(Term term, int determinizeWorkLimit, RewriteMethod rewriteMethod) {
- super(term, toAutomaton(term, determinizeWorkLimit), false, rewriteMethod);
+ public WildcardQuery(Term term, RewriteMethod rewriteMethod) {
+ super(term, toAutomaton(term), false, rewriteMethod);
}
/**
@@ -81,7 +65,7 @@ public class WildcardQuery extends AutomatonQuery {
* @lucene.internal
*/
@SuppressWarnings("fallthrough")
- public static Automaton toAutomaton(Term wildcardquery, int determinizeWorkLimit) {
+ public static Automaton toAutomaton(Term wildcardquery) {
List<Automaton> automata = new ArrayList<>();
String wildcardText = wildcardquery.text();
@@ -110,7 +94,7 @@ public class WildcardQuery extends AutomatonQuery {
i += length;
}
- return Operations.determinize(Operations.concatenate(automata), determinizeWorkLimit);
+ return Operations.concatenate(automata);
}
/** Returns the pattern term. */
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java
index cd50c359a37..2095c1e2f2c 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java
@@ -728,6 +728,36 @@ public class Automaton implements Accountable, TransitionAccessor {
return destState;
}
+ @Override
+ public int hashCode() {
+ int result = 1;
+ result = 31 * result + nextState;
+ result = 31 * result + nextTransition;
+ result = 31 * result + (deterministic ? 1231 : 1237);
+ result = 31 * result + isAccept.hashCode();
+ for (int i = 0; i < nextState; i++) {
+ result = 31 * result + states[i];
+ }
+ for (int i = 0; i < nextTransition; i++) {
+ result = 31 * result + transitions[i];
+ }
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ Automaton other = (Automaton) obj;
+ return nextState == other.nextState
+ && nextTransition == other.nextTransition
+ && deterministic == other.deterministic
+ && isAccept.equals(other.isAccept)
+ && Arrays.equals(states, 0, nextState, other.states, 0, other.nextState)
+ && Arrays.equals(
+ transitions, 0, nextTransition, other.transitions, 0, other.nextTransition);
+ }
+
/**
* Records new states and transitions and then {@link #finish} creates the {@link Automaton}. Use
* this when you cannot create the Automaton directly because it's too restrictive to have to add
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
index 22ea583ee08..f414b90b3e4 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
@@ -254,7 +254,7 @@ public class CompiledAutomaton implements Accountable {
this.automaton = null;
this.runAutomaton = null;
this.sinkState = -1;
- this.nfaRunAutomaton = new NFARunAutomaton(binary, 0xff);
+ this.nfaRunAutomaton = new NFARunAutomaton(binary, 0x100);
} else {
// We already had a DFA (or threw exception), according to mike UTF32toUTF8 won't "blow up"
binary = Operations.determinize(binary, Integer.MAX_VALUE);
@@ -366,7 +366,7 @@ public class CompiledAutomaton implements Accountable {
if (visitor.acceptField(field)) {
switch (type) {
case NORMAL:
- visitor.consumeTermsMatching(parent, field, () -> runAutomaton);
+ visitor.consumeTermsMatching(parent, field, this::getByteRunnable);
break;
case NONE:
break;
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
index b407cdd01a0..febfceb6aa8 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
@@ -89,6 +89,11 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accoun
}
}
+ /** Returns the wrapped NFA automaton. */
+ Automaton getAutomaton() {
+ return automaton;
+ }
+
/**
* For a given state and an incoming character (codepoint), return the next state
*
@@ -262,6 +267,19 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accoun
+ RamUsageEstimator.sizeOfObject(classmap);
}
+ @Override
+ public int hashCode() {
+ return automaton.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ NFARunAutomaton other = (NFARunAutomaton) obj;
+ return automaton.equals(other.automaton);
+ }
+
private class DState implements Accountable {
private final int[] nfaStates;
// this field is lazily init'd when first time caller wants to add a new transition
@@ -396,16 +414,15 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accoun
if (statesSet.size() > 0) {
assert lastPoint != -1;
int ord = findDState(new DState(statesSet.getArray()));
- while (points[charClass] < lastPoint) {
+ while (charClass < points.length && points[charClass] < lastPoint) {
assignTransition(charClass++, MISSING);
}
- assert points[charClass] == lastPoint;
+ assert charClass == points.length || points[charClass] == lastPoint;
while (charClass < points.length && points[charClass] < point) {
assert transitions[charClass] == NOT_COMPUTED || transitions[charClass] == ord;
assignTransition(charClass++, ord);
}
- assert (charClass == points.length && point == alphabetSize)
- || points[charClass] == point;
+ assert charClass == points.length || points[charClass] == point;
}
// process transitions that end on this point
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
index f09e808b3a2..b0ed0b89a5f 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
@@ -56,7 +56,6 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.NamedThreadFactory;
-import org.apache.lucene.util.automaton.Operations;
public class TestBooleanQuery extends LuceneTestCase {
@@ -265,10 +264,7 @@ public class TestBooleanQuery extends LuceneTestCase {
BooleanQuery.Builder query = new BooleanQuery.Builder(); // Query: +foo -ba*
query.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST);
WildcardQuery wildcardQuery =
- new WildcardQuery(
- new Term("field", "ba*"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.SCORING_BOOLEAN_REWRITE);
+ new WildcardQuery(new Term("field", "ba*"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
query.add(wildcardQuery, BooleanClause.Occur.MUST_NOT);
MultiReader multireader = new MultiReader(reader1, reader2);
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
index 22d4cd2deaf..682760b77e5 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
@@ -44,7 +44,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
/** Tests {@link FuzzyQuery}. */
@@ -801,9 +801,9 @@ public class TestFuzzyQuery extends LuceneTestCase {
new QueryVisitor() {
@Override
public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ Query query, String field, Supplier<ByteRunnable> automaton) {
visited.set(true);
- ByteRunAutomaton a = automaton.get();
+ ByteRunnable a = automaton.get();
assertMatches(a, "blob");
assertMatches(a, "bolb");
assertMatches(a, "blobby");
@@ -813,12 +813,12 @@ public class TestFuzzyQuery extends LuceneTestCase {
assertTrue(visited.get());
}
- private static void assertMatches(ByteRunAutomaton automaton, String text) {
+ private static void assertMatches(ByteRunnable automaton, String text) {
BytesRef b = newBytesRef(text);
assertTrue(automaton.run(b.bytes, b.offset, b.length));
}
- private static void assertNoMatches(ByteRunAutomaton automaton, String text) {
+ private static void assertNoMatches(ByteRunnable automaton, String text) {
BytesRef b = newBytesRef(text);
assertFalse(automaton.run(b.bytes, b.offset, b.length));
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
index 73842fa5d86..be8175f6da8 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
@@ -29,7 +29,6 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.QueryUtils;
-import org.apache.lucene.util.automaton.Operations;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -114,7 +113,7 @@ public class TestMultiTermConstantScore extends TestBaseRangeFilter {
/** macro for readability */
public static Query cswcq(Term wild, MultiTermQuery.RewriteMethod method) {
- return new WildcardQuery(wild, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, method);
+ return new WildcardQuery(wild, method);
}
@Test
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java
index 8d2d2a04c55..fcb165842e0 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java
@@ -54,7 +54,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
public class TestTermInSetQuery extends LuceneTestCase {
@@ -500,7 +500,7 @@ public class TestTermInSetQuery extends LuceneTestCase {
@Override
public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ Query query, String field, Supplier<ByteRunnable> automaton) {
fail("Singleton TermInSetQuery should not try to build ByteRunAutomaton");
}
});
@@ -520,8 +520,8 @@ public class TestTermInSetQuery extends LuceneTestCase {
@Override
public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
- ByteRunAutomaton a = automaton.get();
+ Query query, String field, Supplier<ByteRunnable> automaton) {
+ ByteRunnable a = automaton.get();
BytesRef test = newBytesRef("nonmatching");
assertFalse(a.run(test.bytes, test.offset, test.length));
for (BytesRef term : terms) {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
index 10a8a65f144..743787255c9 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java
@@ -33,7 +33,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.util.automaton.Operations;
/** TestWildcardQuery tests the '*' and '?' wildcard characters. */
public class TestWildcardQuery extends LuceneTestCase {
@@ -75,33 +74,25 @@ public class TestWildcardQuery extends LuceneTestCase {
Query q =
searcher.rewrite(
new WildcardQuery(
- new Term("field", "nowildcard"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.SCORING_BOOLEAN_REWRITE));
+ new Term("field", "nowildcard"), MultiTermQuery.SCORING_BOOLEAN_REWRITE));
assertThat(q, instanceOf(TermQuery.class));
q =
searcher.rewrite(
new WildcardQuery(
- new Term("field", "nowildcard"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.CONSTANT_SCORE_REWRITE));
+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_REWRITE));
assertThat(q, instanceOf(MultiTermQueryConstantScoreWrapper.class));
q =
searcher.rewrite(
new WildcardQuery(
- new Term("field", "nowildcard"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE));
+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE));
assertThat(q, instanceOf(MultiTermQueryConstantScoreBlendedWrapper.class));
q =
searcher.rewrite(
new WildcardQuery(
- new Term("field", "nowildcard"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE));
+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE));
assertThat(q, instanceOf(ConstantScoreQuery.class));
reader.close();
indexStore.close();
@@ -114,10 +105,7 @@ public class TestWildcardQuery extends LuceneTestCase {
IndexSearcher searcher = newSearcher(reader);
MultiTermQuery wq =
- new WildcardQuery(
- new Term("field", ""),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.SCORING_BOOLEAN_REWRITE);
+ new WildcardQuery(new Term("field", ""), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
assertMatches(searcher, wq, 0);
Query q = searcher.rewrite(wq);
assertThat(q, instanceOf(MatchNoDocsQuery.class));
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
index 447091bba36..2bf94194638 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
@@ -22,6 +22,11 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
@@ -159,4 +164,112 @@ public class TestCompiledAutomaton extends LuceneTestCase {
CompiledAutomaton ca = new CompiledAutomaton(a, true, true, false);
assertEquals(CompiledAutomaton.AUTOMATON_TYPE.SINGLE, ca.type);
}
+
+ public void testNfaCompiledAutomatonEqualsAndHashCode() {
+ Automaton nfa1 = makeSimpleNfa('a');
+ Automaton nfa2 = makeSimpleNfa('a');
+ assertNotSame(nfa1, nfa2);
+
+ CompiledAutomaton ca1 = new CompiledAutomaton(nfa1, false, true, true);
+ CompiledAutomaton ca2 = new CompiledAutomaton(nfa2, false, true, true);
+
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.NORMAL, ca1.type);
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.NORMAL, ca2.type);
+ assertNotNull(ca1.nfaRunAutomaton);
+ assertNotNull(ca2.nfaRunAutomaton);
+
+ assertEquals(ca1, ca2);
+ assertEquals(ca1.hashCode(), ca2.hashCode());
+ }
+
+ public void testNfaCompiledAutomatonHashCodeIsStableForStructuralEquals() {
+ CompiledAutomaton ca1 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);
+ CompiledAutomaton ca2 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);
+ CompiledAutomaton ca3 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);
+
+ assertEquals(ca1, ca2);
+ assertEquals(ca2, ca3);
+ assertEquals(ca1.hashCode(), ca2.hashCode());
+ assertEquals(ca2.hashCode(), ca3.hashCode());
+ }
+
+ public void testNfaCompiledAutomatonNotEquals() {
+ CompiledAutomaton ca1 = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);
+ CompiledAutomaton ca2 = new CompiledAutomaton(makeSimpleNfa('b'), false, true, true);
+
+ assertNotEquals(ca1, ca2);
+ }
+
+ public void testDfaCompiledAutomatonEqualsAndHashCode() {
+ CompiledAutomaton ca1 =
+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);
+ CompiledAutomaton ca2 =
+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);
+
+ assertNotNull(ca1.runAutomaton);
+ assertNotNull(ca2.runAutomaton);
+ assertNull(ca1.nfaRunAutomaton);
+ assertNull(ca2.nfaRunAutomaton);
+
+ assertEquals(ca1, ca2);
+ assertEquals(ca1.hashCode(), ca2.hashCode());
+ }
+
+ public void testDfaCompiledAutomatonNotEquals() {
+ CompiledAutomaton ca1 =
+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);
+ CompiledAutomaton ca2 =
+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'c'), false, false, true);
+
+ assertNotEquals(ca1, ca2);
+ }
+
+ public void testDfaAndNfaCompiledAutomataAreNotEqual() {
+ CompiledAutomaton dfa =
+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);
+ CompiledAutomaton nfa = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);
+
+ assertNotNull(dfa.runAutomaton);
+ assertNull(dfa.nfaRunAutomaton);
+ assertNull(nfa.runAutomaton);
+ assertNotNull(nfa.nfaRunAutomaton);
+ assertNotEquals(dfa, nfa);
+ assertNotEquals(nfa, dfa);
+ }
+
+ public void testVisitUsesMatchingCallbackForNfaAutomaton() {
+ CompiledAutomaton ca = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);
+ Query parent = new TermQuery(new Term("f", "x"));
+
+ final boolean[] matchingCalled = new boolean[1];
+ QueryVisitor visitor =
+ new QueryVisitor() {
+ @Override
+ public void consumeTermsMatching(
+ Query query, String field, Supplier<ByteRunnable> automaton) {
+ matchingCalled[0] = true;
+ assertNotNull(automaton.get());
+ }
+ };
+
+ ca.visit(visitor, parent, "f");
+ assertTrue(matchingCalled[0]);
+ }
+
+ private static Automaton makeSimpleNfa(char ch) {
+ return makeDeterministicNormalAutomaton(ch, ch);
+ }
+
+ private static Automaton makeDeterministicNormalAutomaton(char first, char second) {
+ Automaton automaton = new Automaton();
+ int start = automaton.createState();
+ int accept1 = automaton.createState();
+ int accept2 = automaton.createState();
+ automaton.setAccept(accept1, true);
+ automaton.setAccept(accept2, true);
+ automaton.addTransition(start, accept1, first, first);
+ automaton.addTransition(start, accept2, second, second);
+ automaton.finishState();
+ return automaton;
+ }
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java
index f8ef7cc5a46..6e98b96fce6 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java
@@ -18,7 +18,7 @@
package org.apache.lucene.search.uhighlight;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
/**
* Associates a label with a CharArrayMatcher to distinguish different sources for terms in
@@ -49,7 +49,7 @@ public interface LabelledCharArrayMatcher extends CharArrayMatcher {
}
/** Returns a representation of the automaton that matches char[] instead of byte[] */
- static LabelledCharArrayMatcher wrap(String label, ByteRunAutomaton runAutomaton) {
+ static LabelledCharArrayMatcher wrap(String label, ByteRunnable runAutomaton) {
return wrap(
label,
(chars, offset, length) -> {
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java
index 0e642f1431e..610783779f5 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java
@@ -26,7 +26,7 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.ByteRunnable;
/**
* Support for highlighting multi-term queries.
@@ -80,8 +80,7 @@ final class MultiTermHighlighting {
}
@Override
- public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ public void consumeTermsMatching(Query query, String field, Supplier<ByteRunnable> automaton) {
runAutomata.add(LabelledCharArrayMatcher.wrap(query.toString(), automaton.get()));
}
}
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java
index e5112c963b2..246124cc05b 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java
@@ -97,7 +97,6 @@ import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -1041,9 +1040,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte
numHighlights = 0;
WildcardQuery wildcardQuery =
new WildcardQuery(
- new Term(FIELD_NAME, "k?nnedy"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.SCORING_BOOLEAN_REWRITE);
+ new Term(FIELD_NAME, "k?nnedy"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
doSearching(wildcardQuery);
doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);
assertTrue(
@@ -1064,9 +1061,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte
numHighlights = 0;
WildcardQuery wildcardQuery =
new WildcardQuery(
- new Term(FIELD_NAME, "k*dy"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.SCORING_BOOLEAN_REWRITE);
+ new Term(FIELD_NAME, "k*dy"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
doSearching(wildcardQuery);
doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);
assertTrue(
@@ -1116,9 +1111,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte
query =
new WildcardQuery(
- new Term(FIELD_NAME, "ken*"),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);
+ new Term(FIELD_NAME, "ken*"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);
searcher = newSearcher(reader);
// can't rewrite ConstantScore if you want to highlight it -
// it rewrites to ConstantScoreQuery which cannot be highlighted
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
index e3808f406cf..79e525a8675 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
@@ -203,10 +203,7 @@ public final class Intervals {
* @see WildcardQuery for glob format
*/
public static IntervalsSource wildcard(BytesRef wildcard, int maxExpansions) {
- CompiledAutomaton ca =
- new CompiledAutomaton(
- WildcardQuery.toAutomaton(
- new Term("", wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
+ CompiledAutomaton ca = new CompiledAutomaton(WildcardQuery.toAutomaton(new Term("", wildcard)));
return new MultiTermIntervalsSource(ca, maxExpansions, wildcard.utf8ToString());
}
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
index e3c643b16f6..82511c48fd9 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
@@ -16,8 +16,6 @@
*/
package org.apache.lucene.queryparser.classic;
-import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
-
import java.io.StringReader;
import java.text.DateFormat;
import java.util.Calendar;
@@ -100,7 +98,6 @@ public abstract class QueryParserBase extends QueryBuilder
Map<String, DateTools.Resolution> fieldToDateResolution = null;
boolean autoGeneratePhraseQueries;
- int determinizeWorkLimit = DEFAULT_DETERMINIZE_WORK_LIMIT;
// So the generated QueryParser(CharStream) won't error out
protected QueryParserBase() {
@@ -347,22 +344,6 @@ public abstract class QueryParserBase extends QueryBuilder
return resolution;
}
- /**
- * @param determinizeWorkLimit the maximum effort that determinizing a regexp query can spend. If
- * the query requires more effort, a TooComplexToDeterminizeException is thrown.
- */
- public void setDeterminizeWorkLimit(int determinizeWorkLimit) {
- this.determinizeWorkLimit = determinizeWorkLimit;
- }
-
- /**
- * @return the maximum effort that determinizing a regexp query can spend. If the query requires
- * more effort, a TooComplexToDeterminizeException is thrown.
- */
- public int getDeterminizeWorkLimit() {
- return determinizeWorkLimit;
- }
-
protected void addClause(List<BooleanClause> clauses, int conj, int mods, Query q) {
boolean required, prohibited;
@@ -636,7 +617,7 @@ public abstract class QueryParserBase extends QueryBuilder
* @return new WildcardQuery instance
*/
protected Query newWildcardQuery(Term t) {
- return new WildcardQuery(t, determinizeWorkLimit, multiTermRewriteMethod);
+ return new WildcardQuery(t, multiTermRewriteMethod);
}
/**
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java
index 538962c2738..822a118c486 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java
@@ -23,7 +23,6 @@ import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode;
import org.apache.lucene.queryparser.flexible.standard.processors.MultiTermRewriteMethodProcessor;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.util.automaton.Operations;
/** Builds a {@link WildcardQuery} object from a {@link WildcardQueryNode} object. */
public class WildcardQueryNodeBuilder implements StandardQueryBuilder {
@@ -43,8 +42,6 @@ public class WildcardQueryNodeBuilder implements StandardQueryBuilder {
}
return new WildcardQuery(
- new Term(wildcardNode.getFieldAsString(), wildcardNode.getTextAsString()),
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
- method);
+ new Term(wildcardNode.getFieldAsString(), wildcardNode.getTextAsString()), method);
}
}
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java
index 27bcaf7c0fb..f7393edde50 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.Operations;
/** Query that matches wildcards */
public class SrndTruncQuery extends SimpleTerm {
@@ -34,11 +33,7 @@ public class SrndTruncQuery extends SimpleTerm {
this.truncated = truncated;
compiled =
new CompiledAutomaton(
- WildcardQuery.toAutomaton(
- new Term(null, truncated), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT),
- false,
- true,
- true);
+ WildcardQuery.toAutomaton(new Term(null, truncated)), false, true, true);
}
private final String truncated;
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
index 9e370863c4d..e00d3a5213a 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
@@ -51,7 +51,6 @@ import org.apache.lucene.tests.analysis.MockLowerCaseFilter;
import org.apache.lucene.tests.analysis.MockSynonymAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
/** Tests QueryParser. */
public class TestQueryParser extends QueryParserTestBase {
@@ -530,17 +529,6 @@ public class TestQueryParser extends QueryParserTestBase {
assertEquals(expected, qp.parse("\"中国\"~3^2"));
}
- /** LUCENE-6677: make sure wildcard query respects determinizeWorkLimit. */
- public void testWildcardDeterminizeWorkLimit() throws Exception {
- QueryParser qp = new QueryParser(FIELD, new MockAnalyzer(random()));
- qp.setDeterminizeWorkLimit(1);
- expectThrows(
- TooComplexToDeterminizeException.class,
- () -> {
- qp.parse("a*aaaaaaa");
- });
- }
-
// TODO: Remove this specialization once the flexible standard parser gets multi-word synonym
// support
@Override