Apache Lucene open-source search software
Remove implicit determinization from WildcardQuery (#15961)
Dimitris Rempapis86593e51b28d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 94b05c92f88..4ba9ce4b205 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -82,6 +82,12 @@ API ChangesTo force a classic DFA execution, use Operations.determinize() and AutomatonQuery.(Dimitris Rempapis)+* GITHUB#15961: Removed WildcardQuery constructor overloads that exposed determinization controls+ (determinizeWorkLimit). Determinization is now done only as-needed.+ To force a classic DFA execution, use Operations.determinize() and AutomatonQuery.+ Also removed QueryParserBase.setDeterminizeWorkLimit/getDeterminizeWorkLimit, which are+ no longer needed. (Dimitris Rempapis)+New Features---------------------* GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (anddiff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.mdindex f1189c6d8d0..e7afaaedabd 100644--- a/lucene/MIGRATE.md+++ b/lucene/MIGRATE.md@@ -159,9 +159,9 @@ iwc.getConfig().getCodec().compoundFormat().getShouldUseCompoundFile();iwc.getConfig().getCodec().compoundFormat().getMaxCFSSegmentSizeMB();```-### Implicit determinization removed from RegexpQuery+### Implicit determinization removed from RegexpQuery and WildcardQuery-Previously, RegexpQuery would use DFA execution by default, even if it might be inefficient.+Previously, RegexpQuery and WildcardQuery would use DFA execution by default, even if it might be inefficient.RegexpQuery will now only [determinize as-needed](https://swtch.com/~rsc/regexp/regexp1.html). This might befaster or slower depending upon your queries.@@ -174,6 +174,18 @@ Automaton dfa = Operations.determinize(new RegExp(re).toAutomaton(), 10000);Query query = new AutomatonQuery(new Term("myfield", re), dfa);```+Similarly for WildcardQuery, the `determinizeWorkLimit` parameter has been removed from `WildcardQuery` constructors and from+`WildcardQuery.toAutomaton`. `QueryParserBase.setDeterminizeWorkLimit` and `getDeterminizeWorkLimit`+have also been removed.++To force the previous behavior, use:++```java+String pattern = "foo*bar";+Automaton dfa = Operations.determinize(WildcardQuery.toAutomaton(new Term("myfield", pattern)), 10000);+Query query = new AutomatonQuery(new Term("myfield", pattern), dfa);+```+### CollectionStatistics and TermStatistics have been renamed to FieldStats and TermStats (GITHUB#15929)Corresponding methods and parameters have been renamed accordingly.diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.javaindex 537f8fe8064..d184088f561 100644--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java@@ -31,7 +31,6 @@ import org.apache.lucene.queryparser.classic.QueryParser;import org.apache.lucene.search.MultiTermQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.WildcardQuery;-import org.apache.lucene.util.automaton.Operations;/*** A QueryMaker that uses common and uncommon actual Wikipedia queries for searching the English@@ -132,10 +131,7 @@ public class EnwikiQueryMaker extends AbstractQueryMaker {private static Query[] getPrebuiltQueries(String field) {WildcardQuery wcq =- new WildcardQuery(- new Term(field, "fo*"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);+ new WildcardQuery(new Term(field, "fo*"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);// be wary of unanalyzed textreturn new Query[] {new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 5),diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.javaindex 845bae5238e..2d7bdc02ef7 100644--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java@@ -42,7 +42,7 @@ import org.apache.lucene.search.similarities.BM25Similarity;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.NIOFSDirectory;import org.apache.lucene.util.Bits;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;/*** Implements search over a single IndexReader.@@ -911,7 +911,7 @@ public class IndexSearcher {@Overridepublic void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ Query query, String field, Supplier<ByteRunnable> automaton) {if (numClauses > maxClauseCount) {throw new TooManyNestedClauses();}diff --git a/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java b/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.javaindex 644e21a371a..ac4074ba798 100644--- a/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java+++ b/lucene/core/src/java/org/apache/lucene/search/QueryVisitor.java@@ -21,7 +21,7 @@ import java.util.Arrays;import java.util.Set;import java.util.function.Supplier;import org.apache.lucene.index.Term;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;/*** Allows recursion through a query tree@@ -43,11 +43,10 @@ public abstract class QueryVisitor {** @param query the leaf query* @param field the field queried against- * @param automaton a supplier for an automaton defining which terms match+ * @param automaton a supplier for a runnable automaton defining which terms match* @lucene.experimental*/- public void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ public void consumeTermsMatching(Query query, String field, Supplier<ByteRunnable> automaton) {visitLeaf(query); // default impl for backward compatibility}diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.javaindex 2aa438f84b7..9c1e8c223ed 100644--- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java@@ -47,32 +47,16 @@ public class WildcardQuery extends AutomatonQuery {/** Constructs a query for terms matching <code>term</code>. */public WildcardQuery(Term term) {- this(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);+ this(term, CONSTANT_SCORE_BLENDED_REWRITE);}/*** Constructs a query for terms matching <code>term</code>.*- * @param determinizeWorkLimit maximum effort to spend while compiling the automaton from this- * wildcard. Set higher to allow more complex queries and lower to prevent memory exhaustion.- * Use {@link Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't- * otherwise know what to specify.- */- public WildcardQuery(Term term, int determinizeWorkLimit) {- this(term, determinizeWorkLimit, CONSTANT_SCORE_BLENDED_REWRITE);- }-- /**- * Constructs a query for terms matching <code>term</code>.- *- * @param determinizeWorkLimit maximum effort to spend while compiling the automaton from this- * wildcard. Set higher to allow more complex queries and lower to prevent memory exhaustion.- * Use {@link Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't- * otherwise know what to specify.* @param rewriteMethod the rewrite method to use when building the final query*/- public WildcardQuery(Term term, int determinizeWorkLimit, RewriteMethod rewriteMethod) {- super(term, toAutomaton(term, determinizeWorkLimit), false, rewriteMethod);+ public WildcardQuery(Term term, RewriteMethod rewriteMethod) {+ super(term, toAutomaton(term), false, rewriteMethod);}/**@@ -81,7 +65,7 @@ public class WildcardQuery extends AutomatonQuery {* @lucene.internal*/@SuppressWarnings("fallthrough")- public static Automaton toAutomaton(Term wildcardquery, int determinizeWorkLimit) {+ public static Automaton toAutomaton(Term wildcardquery) {List<Automaton> automata = new ArrayList<>();String wildcardText = wildcardquery.text();@@ -110,7 +94,7 @@ public class WildcardQuery extends AutomatonQuery {i += length;}- return Operations.determinize(Operations.concatenate(automata), determinizeWorkLimit);+ return Operations.concatenate(automata);}/** Returns the pattern term. */diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.javaindex cd50c359a37..2095c1e2f2c 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java@@ -728,6 +728,36 @@ public class Automaton implements Accountable, TransitionAccessor {return destState;}+ @Override+ public int hashCode() {+ int result = 1;+ result = 31 * result + nextState;+ result = 31 * result + nextTransition;+ result = 31 * result + (deterministic ? 1231 : 1237);+ result = 31 * result + isAccept.hashCode();+ for (int i = 0; i < nextState; i++) {+ result = 31 * result + states[i];+ }+ for (int i = 0; i < nextTransition; i++) {+ result = 31 * result + transitions[i];+ }+ return result;+ }++ @Override+ public boolean equals(Object obj) {+ if (this == obj) return true;+ if (obj == null || getClass() != obj.getClass()) return false;+ Automaton other = (Automaton) obj;+ return nextState == other.nextState+ && nextTransition == other.nextTransition+ && deterministic == other.deterministic+ && isAccept.equals(other.isAccept)+ && Arrays.equals(states, 0, nextState, other.states, 0, other.nextState)+ && Arrays.equals(+ transitions, 0, nextTransition, other.transitions, 0, other.nextTransition);+ }+/*** Records new states and transitions and then {@link #finish} creates the {@link Automaton}. Use* this when you cannot create the Automaton directly because it's too restrictive to have to adddiff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.javaindex 22ea583ee08..f414b90b3e4 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java@@ -254,7 +254,7 @@ public class CompiledAutomaton implements Accountable {this.automaton = null;this.runAutomaton = null;this.sinkState = -1;- this.nfaRunAutomaton = new NFARunAutomaton(binary, 0xff);+ this.nfaRunAutomaton = new NFARunAutomaton(binary, 0x100);} else {// We already had a DFA (or threw exception), according to mike UTF32toUTF8 won't "blow up"binary = Operations.determinize(binary, Integer.MAX_VALUE);@@ -366,7 +366,7 @@ public class CompiledAutomaton implements Accountable {if (visitor.acceptField(field)) {switch (type) {case NORMAL:- visitor.consumeTermsMatching(parent, field, () -> runAutomaton);+ visitor.consumeTermsMatching(parent, field, this::getByteRunnable);break;case NONE:break;diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.javaindex b407cdd01a0..febfceb6aa8 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java@@ -89,6 +89,11 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accoun}}+ /** Returns the wrapped NFA automaton. */+ Automaton getAutomaton() {+ return automaton;+ }+/*** For a given state and an incoming character (codepoint), return the next state*@@ -262,6 +267,19 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accoun+ RamUsageEstimator.sizeOfObject(classmap);}+ @Override+ public int hashCode() {+ return automaton.hashCode();+ }++ @Override+ public boolean equals(Object obj) {+ if (this == obj) return true;+ if (obj == null || getClass() != obj.getClass()) return false;+ NFARunAutomaton other = (NFARunAutomaton) obj;+ return automaton.equals(other.automaton);+ }+private class DState implements Accountable {private final int[] nfaStates;// this field is lazily init'd when first time caller wants to add a new transition@@ -396,16 +414,15 @@ public class NFARunAutomaton implements ByteRunnable, TransitionAccessor, Accounif (statesSet.size() > 0) {assert lastPoint != -1;int ord = findDState(new DState(statesSet.getArray()));- while (points[charClass] < lastPoint) {+ while (charClass < points.length && points[charClass] < lastPoint) {assignTransition(charClass++, MISSING);}- assert points[charClass] == lastPoint;+ assert charClass == points.length || points[charClass] == lastPoint;while (charClass < points.length && points[charClass] < point) {assert transitions[charClass] == NOT_COMPUTED || transitions[charClass] == ord;assignTransition(charClass++, ord);}- assert (charClass == points.length && point == alphabetSize)- || points[charClass] == point;+ assert charClass == points.length || points[charClass] == point;}// process transitions that end on this pointdiff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.javaindex f09e808b3a2..b0ed0b89a5f 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java@@ -56,7 +56,6 @@ import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.NamedThreadFactory;-import org.apache.lucene.util.automaton.Operations;public class TestBooleanQuery extends LuceneTestCase {@@ -265,10 +264,7 @@ public class TestBooleanQuery extends LuceneTestCase {BooleanQuery.Builder query = new BooleanQuery.Builder(); // Query: +foo -ba*query.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST);WildcardQuery wildcardQuery =- new WildcardQuery(- new Term("field", "ba*"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ new WildcardQuery(new Term("field", "ba*"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);query.add(wildcardQuery, BooleanClause.Occur.MUST_NOT);MultiReader multireader = new MultiReader(reader1, reader2);diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.javaindex 22d4cd2deaf..682760b77e5 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java@@ -44,7 +44,7 @@ import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.IntsRef;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;import org.apache.lucene.util.automaton.LevenshteinAutomata;/** Tests {@link FuzzyQuery}. */@@ -801,9 +801,9 @@ public class TestFuzzyQuery extends LuceneTestCase {new QueryVisitor() {@Overridepublic void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ Query query, String field, Supplier<ByteRunnable> automaton) {visited.set(true);- ByteRunAutomaton a = automaton.get();+ ByteRunnable a = automaton.get();assertMatches(a, "blob");assertMatches(a, "bolb");assertMatches(a, "blobby");@@ -813,12 +813,12 @@ public class TestFuzzyQuery extends LuceneTestCase {assertTrue(visited.get());}- private static void assertMatches(ByteRunAutomaton automaton, String text) {+ private static void assertMatches(ByteRunnable automaton, String text) {BytesRef b = newBytesRef(text);assertTrue(automaton.run(b.bytes, b.offset, b.length));}- private static void assertNoMatches(ByteRunAutomaton automaton, String text) {+ private static void assertNoMatches(ByteRunnable automaton, String text) {BytesRef b = newBytesRef(text);assertFalse(automaton.run(b.bytes, b.offset, b.length));}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.javaindex 73842fa5d86..be8175f6da8 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java@@ -29,7 +29,6 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.analysis.MockTokenizer;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.search.QueryUtils;-import org.apache.lucene.util.automaton.Operations;import org.junit.AfterClass;import org.junit.Assert;import org.junit.BeforeClass;@@ -114,7 +113,7 @@ public class TestMultiTermConstantScore extends TestBaseRangeFilter {/** macro for readability */public static Query cswcq(Term wild, MultiTermQuery.RewriteMethod method) {- return new WildcardQuery(wild, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, method);+ return new WildcardQuery(wild, method);}@Testdiff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.javaindex 8d2d2a04c55..fcb165842e0 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java@@ -54,7 +54,7 @@ import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefIterator;import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;public class TestTermInSetQuery extends LuceneTestCase {@@ -500,7 +500,7 @@ public class TestTermInSetQuery extends LuceneTestCase {@Overridepublic void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ Query query, String field, Supplier<ByteRunnable> automaton) {fail("Singleton TermInSetQuery should not try to build ByteRunAutomaton");}});@@ -520,8 +520,8 @@ public class TestTermInSetQuery extends LuceneTestCase {@Overridepublic void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {- ByteRunAutomaton a = automaton.get();+ Query query, String field, Supplier<ByteRunnable> automaton) {+ ByteRunnable a = automaton.get();BytesRef test = newBytesRef("nonmatching");assertFalse(a.run(test.bytes, test.offset, test.length));for (BytesRef term : terms) {diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.javaindex 10a8a65f144..743787255c9 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java@@ -33,7 +33,6 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.util.automaton.Operations;/** TestWildcardQuery tests the '*' and '?' wildcard characters. */public class TestWildcardQuery extends LuceneTestCase {@@ -75,33 +74,25 @@ public class TestWildcardQuery extends LuceneTestCase {Query q =searcher.rewrite(new WildcardQuery(- new Term("field", "nowildcard"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.SCORING_BOOLEAN_REWRITE));+ new Term("field", "nowildcard"), MultiTermQuery.SCORING_BOOLEAN_REWRITE));assertThat(q, instanceOf(TermQuery.class));q =searcher.rewrite(new WildcardQuery(- new Term("field", "nowildcard"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.CONSTANT_SCORE_REWRITE));+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_REWRITE));assertThat(q, instanceOf(MultiTermQueryConstantScoreWrapper.class));q =searcher.rewrite(new WildcardQuery(- new Term("field", "nowildcard"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE));+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE));assertThat(q, instanceOf(MultiTermQueryConstantScoreBlendedWrapper.class));q =searcher.rewrite(new WildcardQuery(- new Term("field", "nowildcard"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE));+ new Term("field", "nowildcard"), MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE));assertThat(q, instanceOf(ConstantScoreQuery.class));reader.close();indexStore.close();@@ -114,10 +105,7 @@ public class TestWildcardQuery extends LuceneTestCase {IndexSearcher searcher = newSearcher(reader);MultiTermQuery wq =- new WildcardQuery(- new Term("field", ""),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ new WildcardQuery(new Term("field", ""), MultiTermQuery.SCORING_BOOLEAN_REWRITE);assertMatches(searcher, wq, 0);Query q = searcher.rewrite(wq);assertThat(q, instanceOf(MatchNoDocsQuery.class));diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.javaindex 447091bba36..2bf94194638 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java@@ -22,6 +22,11 @@ import java.util.Collections;import java.util.HashSet;import java.util.List;import java.util.Set;+import java.util.function.Supplier;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.QueryVisitor;+import org.apache.lucene.search.TermQuery;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;@@ -159,4 +164,112 @@ public class TestCompiledAutomaton extends LuceneTestCase {CompiledAutomaton ca = new CompiledAutomaton(a, true, true, false);assertEquals(CompiledAutomaton.AUTOMATON_TYPE.SINGLE, ca.type);}++ public void testNfaCompiledAutomatonEqualsAndHashCode() {+ Automaton nfa1 = makeSimpleNfa('a');+ Automaton nfa2 = makeSimpleNfa('a');+ assertNotSame(nfa1, nfa2);++ CompiledAutomaton ca1 = new CompiledAutomaton(nfa1, false, true, true);+ CompiledAutomaton ca2 = new CompiledAutomaton(nfa2, false, true, true);++ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.NORMAL, ca1.type);+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.NORMAL, ca2.type);+ assertNotNull(ca1.nfaRunAutomaton);+ assertNotNull(ca2.nfaRunAutomaton);++ assertEquals(ca1, ca2);+ assertEquals(ca1.hashCode(), ca2.hashCode());+ }++ public void testNfaCompiledAutomatonHashCodeIsStableForStructuralEquals() {+ CompiledAutomaton ca1 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);+ CompiledAutomaton ca2 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);+ CompiledAutomaton ca3 = new CompiledAutomaton(makeSimpleNfa('z'), false, true, true);++ assertEquals(ca1, ca2);+ assertEquals(ca2, ca3);+ assertEquals(ca1.hashCode(), ca2.hashCode());+ assertEquals(ca2.hashCode(), ca3.hashCode());+ }++ public void testNfaCompiledAutomatonNotEquals() {+ CompiledAutomaton ca1 = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);+ CompiledAutomaton ca2 = new CompiledAutomaton(makeSimpleNfa('b'), false, true, true);++ assertNotEquals(ca1, ca2);+ }++ public void testDfaCompiledAutomatonEqualsAndHashCode() {+ CompiledAutomaton ca1 =+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);+ CompiledAutomaton ca2 =+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);++ assertNotNull(ca1.runAutomaton);+ assertNotNull(ca2.runAutomaton);+ assertNull(ca1.nfaRunAutomaton);+ assertNull(ca2.nfaRunAutomaton);++ assertEquals(ca1, ca2);+ assertEquals(ca1.hashCode(), ca2.hashCode());+ }++ public void testDfaCompiledAutomatonNotEquals() {+ CompiledAutomaton ca1 =+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);+ CompiledAutomaton ca2 =+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'c'), false, false, true);++ assertNotEquals(ca1, ca2);+ }++ public void testDfaAndNfaCompiledAutomataAreNotEqual() {+ CompiledAutomaton dfa =+ new CompiledAutomaton(makeDeterministicNormalAutomaton('a', 'b'), false, false, true);+ CompiledAutomaton nfa = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);++ assertNotNull(dfa.runAutomaton);+ assertNull(dfa.nfaRunAutomaton);+ assertNull(nfa.runAutomaton);+ assertNotNull(nfa.nfaRunAutomaton);+ assertNotEquals(dfa, nfa);+ assertNotEquals(nfa, dfa);+ }++ public void testVisitUsesMatchingCallbackForNfaAutomaton() {+ CompiledAutomaton ca = new CompiledAutomaton(makeSimpleNfa('a'), false, true, true);+ Query parent = new TermQuery(new Term("f", "x"));++ final boolean[] matchingCalled = new boolean[1];+ QueryVisitor visitor =+ new QueryVisitor() {+ @Override+ public void consumeTermsMatching(+ Query query, String field, Supplier<ByteRunnable> automaton) {+ matchingCalled[0] = true;+ assertNotNull(automaton.get());+ }+ };++ ca.visit(visitor, parent, "f");+ assertTrue(matchingCalled[0]);+ }++ private static Automaton makeSimpleNfa(char ch) {+ return makeDeterministicNormalAutomaton(ch, ch);+ }++ private static Automaton makeDeterministicNormalAutomaton(char first, char second) {+ Automaton automaton = new Automaton();+ int start = automaton.createState();+ int accept1 = automaton.createState();+ int accept2 = automaton.createState();+ automaton.setAccept(accept1, true);+ automaton.setAccept(accept2, true);+ automaton.addTransition(start, accept1, first, first);+ automaton.addTransition(start, accept2, second, second);+ automaton.finishState();+ return automaton;+ }}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.javaindex f8ef7cc5a46..6e98b96fce6 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java@@ -18,7 +18,7 @@package org.apache.lucene.search.uhighlight;import org.apache.lucene.util.UnicodeUtil;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;/*** Associates a label with a CharArrayMatcher to distinguish different sources for terms in@@ -49,7 +49,7 @@ public interface LabelledCharArrayMatcher extends CharArrayMatcher {}/** Returns a representation of the automaton that matches char[] instead of byte[] */- static LabelledCharArrayMatcher wrap(String label, ByteRunAutomaton runAutomaton) {+ static LabelledCharArrayMatcher wrap(String label, ByteRunnable runAutomaton) {return wrap(label,(chars, offset, length) -> {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.javaindex 0e642f1431e..610783779f5 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java@@ -26,7 +26,7 @@ import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.FuzzyQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.QueryVisitor;-import org.apache.lucene.util.automaton.ByteRunAutomaton;+import org.apache.lucene.util.automaton.ByteRunnable;/*** Support for highlighting multi-term queries.@@ -80,8 +80,7 @@ final class MultiTermHighlighting {}@Override- public void consumeTermsMatching(- Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ public void consumeTermsMatching(Query query, String field, Supplier<ByteRunnable> automaton) {runAutomata.add(LabelledCharArrayMatcher.wrap(query.toString(), automaton.get()));}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.javaindex e5112c963b2..246124cc05b 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java@@ -97,7 +97,6 @@ import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.automaton.Automata;import org.apache.lucene.util.automaton.CharacterRunAutomaton;-import org.apache.lucene.util.automaton.Operations;import org.apache.lucene.util.automaton.RegExp;import org.w3c.dom.Element;import org.w3c.dom.NodeList;@@ -1041,9 +1040,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormattenumHighlights = 0;WildcardQuery wildcardQuery =new WildcardQuery(- new Term(FIELD_NAME, "k?nnedy"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ new Term(FIELD_NAME, "k?nnedy"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);doSearching(wildcardQuery);doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);assertTrue(@@ -1064,9 +1061,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormattenumHighlights = 0;WildcardQuery wildcardQuery =new WildcardQuery(- new Term(FIELD_NAME, "k*dy"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ new Term(FIELD_NAME, "k*dy"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);doSearching(wildcardQuery);doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);assertTrue(@@ -1116,9 +1111,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattequery =new WildcardQuery(- new Term(FIELD_NAME, "ken*"),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);+ new Term(FIELD_NAME, "ken*"), MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE);searcher = newSearcher(reader);// can't rewrite ConstantScore if you want to highlight it -// it rewrites to ConstantScoreQuery which cannot be highlighteddiff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.javaindex e3808f406cf..79e525a8675 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java@@ -203,10 +203,7 @@ public final class Intervals {* @see WildcardQuery for glob format*/public static IntervalsSource wildcard(BytesRef wildcard, int maxExpansions) {- CompiledAutomaton ca =- new CompiledAutomaton(- WildcardQuery.toAutomaton(- new Term("", wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));+ CompiledAutomaton ca = new CompiledAutomaton(WildcardQuery.toAutomaton(new Term("", wildcard)));return new MultiTermIntervalsSource(ca, maxExpansions, wildcard.utf8ToString());}diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.javaindex e3c643b16f6..82511c48fd9 100644--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java@@ -16,8 +16,6 @@*/package org.apache.lucene.queryparser.classic;-import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;-import java.io.StringReader;import java.text.DateFormat;import java.util.Calendar;@@ -100,7 +98,6 @@ public abstract class QueryParserBase extends QueryBuilderMap<String, DateTools.Resolution> fieldToDateResolution = null;boolean autoGeneratePhraseQueries;- int determinizeWorkLimit = DEFAULT_DETERMINIZE_WORK_LIMIT;// So the generated QueryParser(CharStream) won't error outprotected QueryParserBase() {@@ -347,22 +344,6 @@ public abstract class QueryParserBase extends QueryBuilderreturn resolution;}- /**- * @param determinizeWorkLimit the maximum effort that determinizing a regexp query can spend. If- * the query requires more effort, a TooComplexToDeterminizeException is thrown.- */- public void setDeterminizeWorkLimit(int determinizeWorkLimit) {- this.determinizeWorkLimit = determinizeWorkLimit;- }-- /**- * @return the maximum effort that determinizing a regexp query can spend. If the query requires- * more effort, a TooComplexToDeterminizeException is thrown.- */- public int getDeterminizeWorkLimit() {- return determinizeWorkLimit;- }-protected void addClause(List<BooleanClause> clauses, int conj, int mods, Query q) {boolean required, prohibited;@@ -636,7 +617,7 @@ public abstract class QueryParserBase extends QueryBuilder* @return new WildcardQuery instance*/protected Query newWildcardQuery(Term t) {- return new WildcardQuery(t, determinizeWorkLimit, multiTermRewriteMethod);+ return new WildcardQuery(t, multiTermRewriteMethod);}/**diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.javaindex 538962c2738..822a118c486 100644--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java@@ -23,7 +23,6 @@ import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode;import org.apache.lucene.queryparser.flexible.standard.processors.MultiTermRewriteMethodProcessor;import org.apache.lucene.search.MultiTermQuery;import org.apache.lucene.search.WildcardQuery;-import org.apache.lucene.util.automaton.Operations;/** Builds a {@link WildcardQuery} object from a {@link WildcardQueryNode} object. */public class WildcardQueryNodeBuilder implements StandardQueryBuilder {@@ -43,8 +42,6 @@ public class WildcardQueryNodeBuilder implements StandardQueryBuilder {}return new WildcardQuery(- new Term(wildcardNode.getFieldAsString(), wildcardNode.getTextAsString()),- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,- method);+ new Term(wildcardNode.getFieldAsString(), wildcardNode.getTextAsString()), method);}}diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.javaindex 27bcaf7c0fb..f7393edde50 100644--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java@@ -25,7 +25,6 @@ import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.WildcardQuery;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.automaton.CompiledAutomaton;-import org.apache.lucene.util.automaton.Operations;/** Query that matches wildcards */public class SrndTruncQuery extends SimpleTerm {@@ -34,11 +33,7 @@ public class SrndTruncQuery extends SimpleTerm {this.truncated = truncated;compiled =new CompiledAutomaton(- WildcardQuery.toAutomaton(- new Term(null, truncated), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT),- false,- true,- true);+ WildcardQuery.toAutomaton(new Term(null, truncated)), false, true, true);}private final String truncated;diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.javaindex 9e370863c4d..e00d3a5213a 100644--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java@@ -51,7 +51,6 @@ import org.apache.lucene.tests.analysis.MockLowerCaseFilter;import org.apache.lucene.tests.analysis.MockSynonymAnalyzer;import org.apache.lucene.tests.analysis.MockTokenizer;import org.apache.lucene.tests.index.RandomIndexWriter;-import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;/** Tests QueryParser. */public class TestQueryParser extends QueryParserTestBase {@@ -530,17 +529,6 @@ public class TestQueryParser extends QueryParserTestBase {assertEquals(expected, qp.parse("\"中国\"~3^2"));}- /** LUCENE-6677: make sure wildcard query respects determinizeWorkLimit. */- public void testWildcardDeterminizeWorkLimit() throws Exception {- QueryParser qp = new QueryParser(FIELD, new MockAnalyzer(random()));- qp.setDeterminizeWorkLimit(1);- expectThrows(- TooComplexToDeterminizeException.class,- () -> {- qp.parse("a*aaaaaaa");- });- }-// TODO: Remove this specialization once the flexible standard parser gets multi-word synonym// support@Override