Apache Lucene open-source search software
Add IntervalsSource for range and regexp queries (#13562)
Mayya Sharipova5e52b8094aca
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d67318da678..14a6d0eb773 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -261,6 +261,9 @@ Improvements* GITHUB#13548: Refactor and javadoc update for KNN vector writer classes. (Patrick Zhai)+* GITHUB#13562: Add Intervals.regexp and Intervals.range methods to produce IntervalsSource+ for regexp and range queries. (Mayya Sharipova)+Optimizations---------------------diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.javaindex a16e721b5c8..7b9c933c167 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java@@ -27,11 +27,15 @@ import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.index.Term;import org.apache.lucene.search.FuzzyQuery;import org.apache.lucene.search.PrefixQuery;+import org.apache.lucene.search.RegexpQuery;+import org.apache.lucene.search.TermRangeQuery;import org.apache.lucene.search.WildcardQuery;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.automaton.Automaton;import org.apache.lucene.util.automaton.CompiledAutomaton;import org.apache.lucene.util.automaton.LevenshteinAutomata;import org.apache.lucene.util.automaton.Operations;+import org.apache.lucene.util.automaton.RegExp;/*** Factory functions for creating {@link IntervalsSource interval sources}.@@ -206,6 +210,88 @@ public final class Intervals {return new MultiTermIntervalsSource(ca, maxExpansions, wildcard.utf8ToString());}+ /**+ * Return an {@link IntervalsSource} over the disjunction of all terms that match a regular+ * expression+ *+ * @param regexp regular expression+ * @throws IllegalStateException if the regex expands to more than {@link #DEFAULT_MAX_EXPANSIONS}+ * terms+ * @see RegexpQuery for regexp format+ */+ public static IntervalsSource regexp(BytesRef regexp) {+ return regexp(regexp, DEFAULT_MAX_EXPANSIONS);+ }++ /**+ * Expert: Return an {@link IntervalsSource} over the disjunction of all terms that match a+ * regular expression+ *+ * <p>WARNING: Setting {@code maxExpansions} to higher than the default value of {@link+ * #DEFAULT_MAX_EXPANSIONS} can be both slow and memory-intensive+ *+ * @param regexp regular expression+ * @param maxExpansions the maximum number of terms to expand to+ * @throws IllegalStateException if the regex expands to more than {@link #DEFAULT_MAX_EXPANSIONS}+ * terms+ * @see RegexpQuery for regexp format+ */+ public static IntervalsSource regexp(BytesRef regexp, int maxExpansions) {+ Automaton automaton = new RegExp(new Term("", regexp).text()).toAutomaton();+ CompiledAutomaton ca = new CompiledAutomaton(automaton, false, true, false);+ return new MultiTermIntervalsSource(ca, maxExpansions, regexp.utf8ToString());+ }++ /**+ * Return an {@link IntervalsSource} over the disjunction of all terms that fall within the given+ * range+ *+ * @param lowerTerm The term text at the lower end of the range+ * @param upperTerm The term text at the upper end of the range+ * @param includeLower If true, the <code>lowerTerm</code> is included in the range+ * @param includeUpper If true, the <code>upperTerm</code> is included in the range+ * @throws IllegalStateException if the range expands to more than {@link #DEFAULT_MAX_EXPANSIONS}+ * terms+ */+ public static IntervalsSource range(+ BytesRef lowerTerm, BytesRef upperTerm, boolean includeLower, boolean includeUpper) {+ return range(lowerTerm, upperTerm, includeLower, includeUpper, DEFAULT_MAX_EXPANSIONS);+ }++ /**+ * Expert: Return an {@link IntervalsSource} over the disjunction of all terms that fall within+ * the given range+ *+ * <p>WARNING: Setting {@code maxExpansions} to higher than the default value of {@link+ * #DEFAULT_MAX_EXPANSIONS} can be both slow and memory-intensive+ *+ * @param lowerTerm The term text at the lower end of the range+ * @param upperTerm The term text at the upper end of the range+ * @param includeLower If true, the <code>lowerTerm</code> is included in the range+ * @param includeUpper If true, the <code>upperTerm</code> is included in the range+ * @param maxExpansions the maximum number of terms to expand to+ * @throws IllegalStateException if the wildcard glob expands to more than {@code maxExpansions}+ * terms+ */+ public static IntervalsSource range(+ BytesRef lowerTerm,+ BytesRef upperTerm,+ boolean includeLower,+ boolean includeUpper,+ int maxExpansions) {+ Automaton automaton =+ TermRangeQuery.toAutomaton(lowerTerm, upperTerm, includeLower, includeUpper);+ CompiledAutomaton ca = new CompiledAutomaton(automaton, false, true, true);++ StringBuilder buffer = new StringBuilder();+ buffer.append("{");+ buffer.append(lowerTerm.utf8ToString());+ buffer.append(",");+ buffer.append(upperTerm.utf8ToString());+ buffer.append("}");+ return new MultiTermIntervalsSource(ca, maxExpansions, buffer.toString());+ }+/*** A fuzzy term {@link IntervalsSource} matches the disjunction of intervals of terms that are* within the specified {@code maxEdits} from the provided term.diff --git a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.javaindex 6d1413efabd..944530937ce 100644--- a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java+++ b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java@@ -1028,6 +1028,40 @@ public class TestIntervals extends LuceneTestCase {checkVisits(Intervals.wildcard(new BytesRef("p??")), 1);}+ public void testRegexp() throws IOException {+ IntervalsSource source = Intervals.regexp(new BytesRef(".ot"));+ checkIntervals(+ source,+ "field1",+ 4,+ new int[][] {+ {},+ {2, 2, 10, 10, 17, 17, 27, 27},+ {5, 5, 10, 10, 21, 21},+ {3, 3},+ {2, 2, 10, 10, 17, 17},+ {}+ });+ MatchesIterator mi = getMatches(source, 4, "field1");+ assertNotNull(mi);+ assertMatch(mi, 2, 2, 15, 18);+ assertMatch(mi, 10, 10, 63, 66);+ assertMatch(mi, 17, 17, 97, 100);++ IllegalStateException e =+ expectThrows(+ IllegalStateException.class,+ () -> {+ IntervalsSource s = Intervals.regexp(new BytesRef(".ot"), 1);+ for (LeafReaderContext ctx : searcher.getIndexReader().leaves()) {+ s.intervals("field1", ctx);+ }+ });+ assertEquals("Automaton [.ot] expanded to too many terms (limit 1)", e.getMessage());++ checkVisits(Intervals.regexp(new BytesRef("p.*")), 1);+ }+public void testFuzzyTerm() throws IOException {IntervalsSource source = Intervals.fuzzyTerm("kot", 1); // matches 'pot'checkIntervals(@@ -1069,6 +1103,41 @@ public class TestIntervals extends LuceneTestCase {checkVisits(Intervals.fuzzyTerm("kot", FuzzyQuery.defaultMaxEdits), 1);}+ public void testRange() throws IOException {+ IntervalsSource source = Intervals.range(new BytesRef("cold"), new BytesRef("hot"), true, true);+ checkIntervals(+ source,+ "field1",+ 6,+ new int[][] {+ {5, 5},+ {2, 2, 5, 5, 12, 12, 17, 17, 21, 21, 29, 29},+ {2, 2, 5, 5, 12, 12, 17, 17, 21, 21, 27, 27},+ {1, 1, 3, 3, 4, 4},+ {2, 2, 5, 5, 17, 17},+ {2, 2}+ });+ MatchesIterator mi = getMatches(source, 3, "field1");+ assertNotNull(mi);+ assertMatch(mi, 1, 1, 4, 8);+ assertMatch(mi, 3, 3, 15, 18);+ assertMatch(mi, 4, 4, 19, 24);++ IllegalStateException e =+ expectThrows(+ IllegalStateException.class,+ () -> {+ IntervalsSource s =+ Intervals.range(new BytesRef("cold"), new BytesRef("hot"), true, true, 1);+ for (LeafReaderContext ctx : searcher.getIndexReader().leaves()) {+ s.intervals("field1", ctx);+ }+ });+ assertEquals("Automaton [{cold,hot}] expanded to too many terms (limit 1)", e.getMessage());++ checkVisits(source, 1);+ }+public void testWrappedFilters() throws IOException {IntervalsSource source =Intervals.or(