Skip to content

Apache Lucene open-source search software

Enable executing using NFA in RegexpQuery (#12767)

Patrick Zhai904a994f6670
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 470666c822b..613c5d85823 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -72,6 +72,8 @@ New Features
* LUCENE-10010 Introduce NFARunAutomaton to run NFA directly. (Patrick Zhai)
+* GITHUB-12767: Add a flag to enable executing using NFA in RegexpQuery. (Patrick Zhai)
+
* LUCENE-10626 Hunspell: add tools to aid dictionary editing:
analysis introspection, stem expansion and stem/flag suggestion (Peter Gromov)
diff --git a/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java b/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java
index 218d65f36a1..f64f17d94b5 100644
--- a/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.AutomatonProvider;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
@@ -139,15 +140,60 @@ public class RegexpQuery extends AutomatonQuery {
AutomatonProvider provider,
int determinizeWorkLimit,
RewriteMethod rewriteMethod) {
+ this(term, syntax_flags, match_flags, provider, determinizeWorkLimit, rewriteMethod, true);
+ }
+
+ /**
+ * Constructs a query for terms matching <code>term</code>.
+ *
+ * @param term regular expression.
+ * @param syntax_flags optional RegExp features from {@link RegExp}
+ * @param match_flags boolean 'or' of match behavior options such as case insensitivity
+ * @param provider custom AutomatonProvider for named automata
+ * @param determinizeWorkLimit maximum effort to spend while compiling the automaton from this
+ * regexp. Set higher to allow more complex queries and lower to prevent memory exhaustion.
+ * Use {@link Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't
+ * otherwise know what to specify.
+ * @param rewriteMethod the rewrite method to use to build the final query
+ * @param doDeterminization whether do determinization to force the query to use DFA as
+ * runAutomaton, if false, the query will not try to determinize the generated automaton from
+ * regexp such that it might or might not be a DFA. In case it is an NFA, the query will
+ * eventually use {@link org.apache.lucene.util.automaton.NFARunAutomaton} to execute. Notice
+ * that {@link org.apache.lucene.util.automaton.NFARunAutomaton} is not thread-safe, so better
+ * to avoid rewritten method like {@link #CONSTANT_SCORE_BLENDED_REWRITE} when searcher is
+ * configured with an executor service
+ */
+ public RegexpQuery(
+ Term term,
+ int syntax_flags,
+ int match_flags,
+ AutomatonProvider provider,
+ int determinizeWorkLimit,
+ RewriteMethod rewriteMethod,
+ boolean doDeterminization) {
super(
term,
- Operations.determinize(
- new RegExp(term.text(), syntax_flags, match_flags).toAutomaton(provider),
- determinizeWorkLimit),
+ toAutomaton(
+ new RegExp(term.text(), syntax_flags, match_flags),
+ determinizeWorkLimit,
+ provider,
+ doDeterminization),
false,
rewriteMethod);
}
+ private static Automaton toAutomaton(
+ RegExp regexp,
+ int determinizeWorkLimit,
+ AutomatonProvider provider,
+ boolean doDeterminization) {
+ if (doDeterminization) {
+ return Operations.determinize(regexp.toAutomaton(provider), determinizeWorkLimit);
+ } else {
+ return regexp.toAutomaton(provider);
+ }
+ }
+
/** Returns the regexp of this query wrapped in a Term. */
public Term getRegexp() {
return term;
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
index a358c2b416f..6ff52baebbc 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/NFARunAutomaton.java
@@ -25,9 +25,11 @@ import org.apache.lucene.util.hppc.BitMixer;
/**
* A RunAutomaton that does not require DFA. It will lazily determinize on-demand, memorizing the
- * generated DFA states that has been explored
+ * generated DFA states that has been explored. Note: the current implementation is NOT thread-safe
*
* <p>implemented based on: https://swtch.com/~rsc/regexp/regexp1.html
+ *
+ * @lucene.internal
*/
public class NFARunAutomaton implements ByteRunnable, TransitionAccessor {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
index 00803103655..89d70da9d9b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
@@ -80,7 +80,10 @@ public class TestRegexpQuery extends LuceneTestCase {
newTerm(regex),
RegExp.ALL,
RegExp.ASCII_CASE_INSENSITIVE,
- Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ RegexpQuery.DEFAULT_PROVIDER,
+ Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
+ MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
+ random().nextBoolean());
return searcher.count(query);
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
index 6c2909d0449..bf977cbe746 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
@@ -52,6 +52,7 @@ import org.apache.lucene.util.automaton.RegExp;
public class TestRegexpRandom2 extends LuceneTestCase {
protected IndexSearcher searcher1;
protected IndexSearcher searcher2;
+ protected IndexSearcher searcher3;
private IndexReader reader;
private Directory dir;
protected String fieldName;
@@ -95,6 +96,7 @@ public class TestRegexpRandom2 extends LuceneTestCase {
reader = writer.getReader();
searcher1 = newSearcher(reader);
searcher2 = newSearcher(reader);
+ searcher3 = newSearcher(reader);
writer.close();
}
@@ -172,11 +174,22 @@ public class TestRegexpRandom2 extends LuceneTestCase {
/** check that the # of hits is the same as from a very simple regexpquery implementation. */
protected void assertSame(String regexp) throws IOException {
RegexpQuery smart = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
+ RegexpQuery nfaQuery =
+ new RegexpQuery(
+ new Term(fieldName, regexp),
+ RegExp.NONE,
+ 0,
+ RegexpQuery.DEFAULT_PROVIDER,
+ 0,
+ MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
+ false);
DumbRegexpQuery dumb = new DumbRegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
TopDocs smartDocs = searcher1.search(smart, 25);
TopDocs dumbDocs = searcher2.search(dumb, 25);
+ TopDocs nfaDocs = searcher3.search(nfaQuery, 25);
CheckHits.checkEqual(smart, smartDocs.scoreDocs, dumbDocs.scoreDocs);
+ CheckHits.checkEqual(nfaQuery, nfaDocs.scoreDocs, dumbDocs.scoreDocs);
}
}