Skip to content

Apache Lucene open-source search software

Add Automata.makeCharSet/makeCharClass to optimize regexp (#14193)

Robert Muirfe42efc59188
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
index 9ecf748418f..9b6198fb04c 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
@@ -32,6 +32,7 @@ package org.apache.lucene.util.automaton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Objects;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.StringHelper;
@@ -140,6 +141,32 @@ public final class Automata {
return a;
}
+ /** Returns a new minimal automaton that accepts any of the provided codepoints */
+ public static Automaton makeCharSet(int[] codepoints) {
+ return makeCharClass(codepoints, codepoints);
+ }
+
+ /** Returns a new minimal automaton that accepts any of the codepoint ranges */
+ public static Automaton makeCharClass(int[] starts, int[] ends) {
+ Objects.requireNonNull(starts);
+ Objects.requireNonNull(ends);
+ if (starts.length != ends.length) {
+ throw new IllegalArgumentException("starts must match ends");
+ }
+ if (starts.length == 0) {
+ return makeEmpty();
+ }
+ Automaton a = new Automaton();
+ int s1 = a.createState();
+ int s2 = a.createState();
+ a.setAccept(s2, true);
+ for (int i = 0; i < starts.length; i++) {
+ a.addTransition(s1, s2, starts[i], ends[i]);
+ }
+ a.finishState();
+ return a;
+ }
+
/**
* Constructs sub-automaton corresponding to decimal numbers of length x.substring(n).length().
*/
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
index 7c2b164aa10..fb255f8a02e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
@@ -148,8 +148,7 @@ public final class Operations {
}
result.finishState();
-
- return result;
+ return Operations.removeDeadStates(result);
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
index 92bfe41b462..aaf50c8c180 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.BooleanSupplier;
@@ -374,6 +375,8 @@ public class RegExp {
REGEXP_CHAR,
/** A Character range */
REGEXP_CHAR_RANGE,
+ /** A Character class (list of ranges) */
+ REGEXP_CHAR_CLASS,
/** Any Character allowed */
REGEXP_ANYCHAR,
/** An empty expression */
@@ -386,8 +389,6 @@ public class RegExp {
REGEXP_AUTOMATON,
/** An Interval expression */
REGEXP_INTERVAL,
- /** An expression for a pre-defined class e.g. \w */
- REGEXP_PRE_CLASS,
/**
* The complement of an expression.
*
@@ -453,7 +454,7 @@ public class RegExp {
public final int min, max, digits;
/** Extents for range type expressions */
- public final int from, to;
+ public final int from[], to[];
// Parser variables
private final String originalString;
@@ -528,8 +529,8 @@ public class RegExp {
int min,
int max,
int digits,
- int from,
- int to) {
+ int from[],
+ int to[]) {
this.originalString = null;
this.kind = kind;
this.flags = flags;
@@ -546,17 +547,17 @@ public class RegExp {
// Simplified construction of container nodes
static RegExp newContainerNode(int flags, Kind kind, RegExp exp1, RegExp exp2) {
- return new RegExp(flags, kind, exp1, exp2, null, 0, 0, 0, 0, 0, 0);
+ return new RegExp(flags, kind, exp1, exp2, null, 0, 0, 0, 0, null, null);
}
// Simplified construction of repeating nodes
static RegExp newRepeatingNode(int flags, Kind kind, RegExp exp, int min, int max) {
- return new RegExp(flags, kind, exp, null, null, 0, min, max, 0, 0, 0);
+ return new RegExp(flags, kind, exp, null, null, 0, min, max, 0, null, null);
}
// Simplified construction of leaf nodes
static RegExp newLeafNode(
- int flags, Kind kind, String s, int c, int min, int max, int digits, int from, int to) {
+ int flags, Kind kind, String s, int c, int min, int max, int digits, int from[], int to[]) {
return new RegExp(flags, kind, null, null, s, c, min, max, digits, from, to);
}
@@ -598,10 +599,6 @@ public class RegExp {
List<Automaton> list;
Automaton a = null;
switch (kind) {
- case REGEXP_PRE_CLASS:
- RegExp expanded = expandPredefined();
- a = expanded.toAutomaton(automata, automaton_provider);
- break;
case REGEXP_UNION:
list = new ArrayList<>();
findLeaves(exp1, Kind.REGEXP_UNION, list, automata, automaton_provider);
@@ -648,13 +645,16 @@ public class RegExp {
break;
case REGEXP_CHAR:
if (check(ASCII_CASE_INSENSITIVE)) {
- a = toCaseInsensitiveChar(c);
+ a = Automata.makeCharSet(toCaseInsensitiveChar(c));
} else {
a = Automata.makeChar(c);
}
break;
case REGEXP_CHAR_RANGE:
- a = Automata.makeCharRange(from, to);
+ a = Automata.makeCharRange(from[0], to[0]);
+ break;
+ case REGEXP_CHAR_CLASS:
+ a = Automata.makeCharClass(from, to);
break;
case REGEXP_ANYCHAR:
a = Automata.makeAnyChar();
@@ -696,23 +696,20 @@ public class RegExp {
return a;
}
- private Automaton toCaseInsensitiveChar(int codepoint) {
- Automaton case1 = Automata.makeChar(codepoint);
+ private int[] toCaseInsensitiveChar(int codepoint) {
// For now we only work with ASCII characters
if (codepoint > 128) {
- return case1;
+ return new int[] {codepoint};
}
int altCase =
Character.isLowerCase(codepoint)
? Character.toUpperCase(codepoint)
: Character.toLowerCase(codepoint);
- Automaton result;
if (altCase != codepoint) {
- result = Operations.union(case1, Automata.makeChar(altCase));
+ return new int[] {codepoint, altCase};
} else {
- result = case1;
+ return new int[] {codepoint};
}
- return result;
}
private Automaton toCaseInsensitiveString() {
@@ -720,7 +717,8 @@ public class RegExp {
Iterator<Integer> iter = s.codePoints().iterator();
while (iter.hasNext()) {
- list.add(toCaseInsensitiveChar(iter.next()));
+ int points[] = toCaseInsensitiveChar(iter.next());
+ list.add(Automata.makeCharSet(points));
}
return Operations.concatenate(list);
}
@@ -802,7 +800,19 @@ public class RegExp {
b.append("\\").appendCodePoint(c);
break;
case REGEXP_CHAR_RANGE:
- b.append("[\\").appendCodePoint(from).append("-\\").appendCodePoint(to).append("]");
+ b.append("[\\").appendCodePoint(from[0]).append("-\\").appendCodePoint(to[0]).append("]");
+ break;
+ case REGEXP_CHAR_CLASS:
+ b.append("[");
+ for (int i = 0; i < from.length; i++) {
+ if (from[i] == to[i]) {
+ b.append("\\").appendCodePoint(from[i]);
+ } else {
+ b.append("\\").appendCodePoint(from[i]);
+ b.append("-\\").appendCodePoint(to[i]);
+ }
+ }
+ b.append("]");
break;
case REGEXP_ANYCHAR:
b.append(".");
@@ -828,13 +838,10 @@ public class RegExp {
if (digits > 0) for (int i = s2.length(); i < digits; i++) b.append('0');
b.append(s2).append(">");
break;
- case REGEXP_PRE_CLASS:
- b.append("\\").appendCodePoint(from);
- break;
}
}
- /** Like to string, but more verbose (shows the higherchy more clearly). */
+ /** Like to string, but more verbose (shows the hierarchy more clearly). */
public String toStringTree() {
StringBuilder b = new StringBuilder();
toStringTree(b, "");
@@ -888,20 +895,22 @@ public class RegExp {
b.appendCodePoint(c);
b.append('\n');
break;
- case REGEXP_PRE_CLASS:
+ case REGEXP_CHAR_RANGE:
b.append(indent);
b.append(kind);
- b.append(" class=\\");
- b.appendCodePoint(from);
+ b.append(" from=");
+ b.appendCodePoint(from[0]);
+ b.append(" to=");
+ b.appendCodePoint(to[0]);
b.append('\n');
break;
- case REGEXP_CHAR_RANGE:
+ case REGEXP_CHAR_CLASS:
b.append(indent);
b.append(kind);
- b.append(" from=");
- b.appendCodePoint(from);
- b.append(" to=");
- b.appendCodePoint(to);
+ b.append(" starts=");
+ b.append(toHexString(from));
+ b.append(" ends=");
+ b.append(toHexString(to));
b.append('\n');
break;
case REGEXP_ANYCHAR:
@@ -942,6 +951,20 @@ public class RegExp {
}
}
+ /** prints like <code>[U+002A U+FD72 U+1FFFF]</code> */
+ private StringBuilder toHexString(int[] range) {
+ StringBuilder sb = new StringBuilder();
+ sb.append('[');
+ for (int codepoint : range) {
+ if (sb.length() > 1) {
+ sb.append(' ');
+ }
+ sb.append(String.format(Locale.ROOT, "U+%04X", codepoint));
+ }
+ sb.append(']');
+ return sb;
+ }
+
/** Returns set of automaton identifiers that occur in this regular expression. */
public Set<String> getIdentifiers() {
HashSet<String> set = new HashSet<>();
@@ -972,9 +995,9 @@ public class RegExp {
case REGEXP_ANYSTRING:
case REGEXP_CHAR:
case REGEXP_CHAR_RANGE:
+ case REGEXP_CHAR_CLASS:
case REGEXP_EMPTY:
case REGEXP_INTERVAL:
- case REGEXP_PRE_CLASS:
case REGEXP_STRING:
default:
}
@@ -1050,14 +1073,33 @@ public class RegExp {
}
static RegExp makeChar(int flags, int c) {
- return newLeafNode(flags, Kind.REGEXP_CHAR, null, c, 0, 0, 0, 0, 0);
+ return newLeafNode(flags, Kind.REGEXP_CHAR, null, c, 0, 0, 0, null, null);
}
static RegExp makeCharRange(int flags, int from, int to) {
if (from > to)
throw new IllegalArgumentException(
"invalid range: from (" + from + ") cannot be > to (" + to + ")");
- return newLeafNode(flags, Kind.REGEXP_CHAR_RANGE, null, 0, 0, 0, 0, from, to);
+ return newLeafNode(
+ flags, Kind.REGEXP_CHAR_RANGE, null, 0, 0, 0, 0, new int[] {from}, new int[] {to});
+ }
+
+ static RegExp makeCharClass(int flags, int from[], int to[]) {
+ if (from.length != to.length) {
+ throw new IllegalStateException(
+ String.format(
+ Locale.ROOT,
+ "invalid class: from.length (%d) != to.length (%d)",
+ from.length,
+ to.length));
+ }
+ for (int i = 0; i < from.length; i++) {
+ if (from[i] > to[i]) {
+ throw new IllegalArgumentException(
+ "invalid range: from (" + from[i] + ") cannot be > to (" + to[i] + ")");
+ }
+ }
+ return newLeafNode(flags, Kind.REGEXP_CHAR_CLASS, null, 0, 0, 0, 0, from, to);
}
static RegExp makeAnyChar(int flags) {
@@ -1069,7 +1111,7 @@ public class RegExp {
}
static RegExp makeString(int flags, String s) {
- return newLeafNode(flags, Kind.REGEXP_STRING, s, 0, 0, 0, 0, 0, 0);
+ return newLeafNode(flags, Kind.REGEXP_STRING, s, 0, 0, 0, 0, null, null);
}
static RegExp makeAnyString(int flags) {
@@ -1077,11 +1119,11 @@ public class RegExp {
}
static RegExp makeAutomaton(int flags, String s) {
- return newLeafNode(flags, Kind.REGEXP_AUTOMATON, s, 0, 0, 0, 0, 0, 0);
+ return newLeafNode(flags, Kind.REGEXP_AUTOMATON, s, 0, 0, 0, 0, null, null);
}
static RegExp makeInterval(int flags, int min, int max, int digits) {
- return newLeafNode(flags, Kind.REGEXP_INTERVAL, null, 0, min, max, digits, 0, 0);
+ return newLeafNode(flags, Kind.REGEXP_INTERVAL, null, 0, min, max, digits, null, null);
}
private boolean peek(String s) {
@@ -1195,60 +1237,132 @@ public class RegExp {
}
final RegExp parseCharClasses() throws IllegalArgumentException {
- RegExp e = parseCharClass();
- while (more() && !peek("]")) e = makeUnion(flags, e, parseCharClass());
- return e;
- }
+ ArrayList<Integer> starts = new ArrayList<>();
+ ArrayList<Integer> ends = new ArrayList<>();
- final RegExp parseCharClass() throws IllegalArgumentException {
- RegExp predefinedExp = matchPredefinedCharacterClass();
- if (predefinedExp != null) {
- return predefinedExp;
- }
-
- int c = parseCharExp();
- if (match('-')) return makeCharRange(flags, c, parseCharExp());
- else return makeChar(flags, c);
- }
-
- RegExp expandPredefined() {
- // See https://docs.oracle.com/javase/tutorial/essential/regex/pre_char_classes.html
- switch (from) {
- case 'd':
- return new RegExp("[0-9]"); // digit
- case 'D':
- return new RegExp("[^0-9]"); // non-digit
- case 's':
- return new RegExp("[ \t\n\r]"); // whitespace
- case 'S':
- return new RegExp("[^\\s]"); // non-whitespace
- case 'w':
- return new RegExp("[a-zA-Z_0-9]"); // word
- case 'W':
- return new RegExp("[^\\w]"); // non-word
- default:
- throw new IllegalArgumentException("invalid character class " + from);
- }
- }
-
- final RegExp matchPredefinedCharacterClass() {
- // See https://docs.oracle.com/javase/tutorial/essential/regex/pre_char_classes.html
- if (match('\\')) {
- if (peek("dDwWsS")) {
- return newLeafNode(flags, Kind.REGEXP_PRE_CLASS, null, 0, 0, 0, 0, next(), 0);
+ do {
+ // look for escape
+ if (match('\\')) {
+ expandPreDefined(starts, ends);
+ } else {
+ // parse a character
+ int c = parseCharExp();
+
+ if (match('-')) {
+ // range from c-d
+ starts.add(c);
+ ends.add(parseCharExp());
+ } else if (check(ASCII_CASE_INSENSITIVE)) {
+ // single case-insensitive character
+ for (int form : toCaseInsensitiveChar(c)) {
+ starts.add(form);
+ ends.add(form);
+ }
+ } else {
+ // single character
+ starts.add(c);
+ ends.add(c);
+ }
}
+ } while (more() && !peek("]"));
- if (peek("\\")) {
- return makeChar(flags, next());
+ // not sure why we bother optimizing nodes, same automaton...
+ // definitely saves time vs fixing toString()-based tests.
+ if (starts.size() == 1) {
+ if (starts.get(0).intValue() == ends.get(0).intValue()) {
+ return makeChar(flags, starts.get(0));
+ } else {
+ return makeCharRange(flags, starts.get(0), ends.get(0));
}
+ } else {
+ return makeCharClass(
+ flags,
+ starts.stream().mapToInt(Integer::intValue).toArray(),
+ ends.stream().mapToInt(Integer::intValue).toArray());
+ }
+ }
+ void expandPreDefined(List<Integer> starts, List<Integer> ends) {
+ if (peek("\\")) {
+ // escape
+ starts.add((int) '\\');
+ ends.add((int) '\\');
+ next();
+ } else if (peek("d")) {
+ // digit: [0-9]
+ starts.add((int) '0');
+ ends.add((int) '9');
+ next();
+ } else if (peek("D")) {
+ // non-digit: [^0-9]
+ starts.add(Character.MIN_CODE_POINT);
+ ends.add('0' - 1);
+ starts.add('9' + 1);
+ ends.add(Character.MAX_CODE_POINT);
+ next();
+ } else if (peek("s")) {
+ // whitespace: [\t-\n\r ]
+ starts.add((int) '\t');
+ ends.add((int) '\n');
+ starts.add((int) '\r');
+ ends.add((int) '\r');
+ starts.add((int) ' ');
+ ends.add((int) ' ');
+ next();
+ } else if (peek("S")) {
+ // non-whitespace: [^\t-\n\r ]
+ starts.add(Character.MIN_CODE_POINT);
+ ends.add('\t' - 1);
+ starts.add('\n' + 1);
+ ends.add('\r' - 1);
+ starts.add('\r' + 1);
+ ends.add(' ' - 1);
+ starts.add(' ' + 1);
+ ends.add(Character.MAX_CODE_POINT);
+ next();
+ } else if (peek("w")) {
+ // word: [0-9A-Z_a-z]
+ starts.add((int) '0');
+ ends.add((int) '9');
+ starts.add((int) 'A');
+ ends.add((int) 'Z');
+ starts.add((int) '_');
+ ends.add((int) '_');
+ starts.add((int) 'a');
+ ends.add((int) 'z');
+ next();
+ } else if (peek("W")) {
+ // non-word: [^0-9A-Z_a-z]
+ starts.add(Character.MIN_CODE_POINT);
+ ends.add('0' - 1);
+ starts.add('9' + 1);
+ ends.add('A' - 1);
+ starts.add('Z' + 1);
+ ends.add('_' - 1);
+ starts.add('_' + 1);
+ ends.add('a' - 1);
+ starts.add('z' + 1);
+ ends.add(Character.MAX_CODE_POINT);
+ next();
+ } else if (peek("abcefghijklmnopqrtuvxyz") || peek("ABCEFGHIJKLMNOPQRTUVXYZ")) {
// From https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#bs
// "It is an error to use a backslash prior to any alphabetic character that does not denote
// an escaped
// construct;"
- if (peek("abcefghijklmnopqrtuvxyz") || peek("ABCEFGHIJKLMNOPQRTUVXYZ")) {
- throw new IllegalArgumentException("invalid character class \\" + next());
- }
+ throw new IllegalArgumentException("invalid character class \\" + next());
+ }
+ }
+
+ final RegExp matchPredefinedCharacterClass() {
+ // See https://docs.oracle.com/javase/tutorial/essential/regex/pre_char_classes.html
+ if (match('\\') && peek("\\ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")) {
+ var starts = new ArrayList<Integer>();
+ var ends = new ArrayList<Integer>();
+ expandPreDefined(starts, ends);
+ return makeCharClass(
+ flags,
+ starts.stream().mapToInt(Integer::intValue).toArray(),
+ ends.stream().mapToInt(Integer::intValue).toArray());
}
return null;
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
index 3c7d6eea198..e9e35416db4 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
@@ -667,11 +667,14 @@ public class TestAutomaton extends LuceneTestCase {
}
public void testRemoveDeadStates() throws Exception {
- Automaton a =
- Operations.concatenate(Arrays.asList(Automata.makeString("x"), Automata.makeString("y")));
- assertEquals(4, a.getNumStates());
+ Automaton a = new Automaton();
+ int s1 = a.createState();
+ a.createState(); // create dead state
+ a.setAccept(s1, true);
+ a.finishState();
+ assertEquals(2, a.getNumStates());
a = Operations.removeDeadStates(a);
- assertEquals(3, a.getNumStates());
+ assertEquals(1, a.getNumStates());
}
public void testRemoveDeadStatesEmpty1() throws Exception {
@@ -1682,4 +1685,40 @@ public class TestAutomaton extends LuceneTestCase {
Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
});
}
+
+ public void testMakeCharSetEmpty() {
+ Automaton expected = Automata.makeEmpty();
+ Automaton actual = Automata.makeCharSet(new int[] {});
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ assertTrue(actual.isDeterministic());
+ assertEquals(0, actual.getNumStates());
+ assertEquals(0, actual.getNumTransitions());
+ }
+
+ public void testMakeCharSetOne() {
+ Automaton expected = Automata.makeChar('a');
+ Automaton actual = Automata.makeCharSet(new int[] {'a'});
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ assertTrue(actual.isDeterministic());
+ assertEquals(2, actual.getNumStates());
+ assertEquals(1, actual.getNumTransitions());
+ }
+
+ public void testMakeCharSetTwo() {
+ Automaton expected = Operations.union(Automata.makeChar('a'), Automata.makeChar('A'));
+ Automaton actual = Automata.makeCharSet(new int[] {'a', 'A'});
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ assertTrue(actual.isDeterministic());
+ assertEquals(2, actual.getNumStates());
+ assertEquals(2, actual.getNumTransitions());
+ }
+
+ public void testMakeCharSetDups() {
+ Automaton expected = Automata.makeChar('a');
+ Automaton actual = Automata.makeCharSet(new int[] {'a', 'a', 'a'});
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ assertTrue(actual.isDeterministic());
+ assertEquals(2, actual.getNumStates());
+ assertEquals(1, actual.getNumTransitions());
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
index 74fb08cb718..e0aec1e5351 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
@@ -78,6 +78,20 @@ public class TestRegExpParsing extends LuceneTestCase {
assertSameLanguage(expected, actual);
}
+ // individual characters (only) inside a class are treated as case insensitive.
+ public void testCaseInsensitiveClassChar() {
+ RegExp re = new RegExp("[c]", RegExp.NONE, RegExp.ASCII_CASE_INSENSITIVE);
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0063 U+0043] ends=[U+0063 U+0043]\n", re.toStringTree());
+ }
+
+ // ranges aren't treated as case-insensitive, but maybe ok with charclass
+ // instead of adding range, expand it: iterate each codepoint, adding its alternatives
+ public void testCaseInsensitiveClassRange() {
+ RegExp re = new RegExp("[c-d]", RegExp.NONE, RegExp.ASCII_CASE_INSENSITIVE);
+ assertEquals("REGEXP_CHAR_RANGE from=c to=d\n", re.toStringTree());
+ }
+
public void testCaseInsensitiveCharUpper() {
RegExp re = new RegExp("C", RegExp.NONE, RegExp.ASCII_CASE_INSENSITIVE);
assertEquals("\\C", re.toString());
@@ -136,6 +150,22 @@ public class TestRegExpParsing extends LuceneTestCase {
assertSameLanguage(expected, actual);
}
+ public void testNegatedClass() {
+ RegExp re = new RegExp("[^c-da]");
+ assertEquals(
+ String.join(
+ "\n",
+ "REGEXP_INTERSECTION",
+ " REGEXP_ANYCHAR",
+ " REGEXP_COMPLEMENT",
+ " REGEXP_CHAR_CLASS starts=[U+0063 U+0061] ends=[U+0064 U+0061]\n"),
+ re.toStringTree());
+
+ Automaton actual = re.toAutomaton();
+ assertTrue(actual.isDeterministic());
+ assertEquals(2, actual.getNumStates());
+ }
+
public void testCharRange() {
RegExp re = new RegExp("[b-d]");
assertEquals("[\\b-\\d]", re.toString());
@@ -180,8 +210,8 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassDigit() {
RegExp re = new RegExp("[\\d]");
- assertEquals("\\d", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\d\n", re.toStringTree());
+ assertEquals("[\\0-\\9]", re.toString());
+ assertEquals("REGEXP_CHAR_RANGE from=0 to=9\n", re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -192,8 +222,8 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassNonDigit() {
RegExp re = new RegExp("[\\D]");
- assertEquals("\\D", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\D\n", re.toStringTree());
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0000 U+003A] ends=[U+002F U+10FFFF]\n", re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -208,8 +238,9 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassWhitespace() {
RegExp re = new RegExp("[\\s]");
- assertEquals("\\s", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\s\n", re.toStringTree());
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0009 U+000D U+0020] ends=[U+000A U+000D U+0020]\n",
+ re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -223,8 +254,9 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassNonWhitespace() {
RegExp re = new RegExp("[\\S]");
- assertEquals("\\S", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\S\n", re.toStringTree());
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0000 U+000B U+000E U+0021] ends=[U+0008 U+000C U+001F U+10FFFF]\n",
+ re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -247,8 +279,10 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassWord() {
RegExp re = new RegExp("[\\w]");
- assertEquals("\\w", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\w\n", re.toStringTree());
+ assertEquals("[\\0-\\9\\A-\\Z\\_\\a-\\z]", re.toString());
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0030 U+0041 U+005F U+0061] ends=[U+0039 U+005A U+005F U+007A]\n",
+ re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -262,8 +296,9 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCharClassNonWord() {
RegExp re = new RegExp("[\\W]");
- assertEquals("\\W", re.toString());
- assertEquals("REGEXP_PRE_CLASS class=\\W\n", re.toStringTree());
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0000 U+003A U+005B U+0060 U+007B] ends=[U+002F U+0040 U+005E U+0060 U+10FFFF]\n",
+ re.toStringTree());
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
@@ -284,6 +319,17 @@ public class TestRegExpParsing extends LuceneTestCase {
assertSameLanguage(expected, actual);
}
+ // char class with a couple of ranges, predefined,and individual chars
+ public void testJumboCharClass() {
+ RegExp re = new RegExp("[0-5a\\sbc-d]");
+ assertEquals(
+ "REGEXP_CHAR_CLASS starts=[U+0030 U+0061 U+0009 U+000D U+0020 U+0062 U+0063] ends=[U+0035 U+0061 U+000A U+000D U+0020 U+0062 U+0064]\n",
+ re.toStringTree());
+ Automaton actual = re.toAutomaton();
+ assertTrue(actual.isDeterministic());
+ assertEquals(2, actual.getNumStates());
+ }
+
public void testTruncatedCharClass() {
expectThrows(
IllegalArgumentException.class,