Skip to content

Apache Lucene open-source search software

Clean up more dead states creation by Automata/Operations (#14218)

Robert Muirde1ed71261d5
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
index 9b6198fb04c..6b9a7fcc556 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
@@ -491,6 +491,7 @@ public final class Automata {
*/
public static Automaton makeDecimalInterval(int min, int max, int digits)
throws IllegalArgumentException {
+ // TODO: can this be improved to always return a DFA?
String x = Integer.toString(min);
String y = Integer.toString(max);
if (min > max || (digits > 0 && y.length() > digits)) {
@@ -533,7 +534,7 @@ public final class Automata {
a1.finishState();
}
- return a1;
+ return Operations.removeDeadStates(a1);
}
/** Returns a new (deterministic) automaton that accepts the single given string. */
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
index fdd6242abcc..6974cd5137b 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
@@ -89,8 +89,8 @@ public final class Operations {
// First pass: create all states
for (Automaton a : list) {
if (a.getNumStates() == 0) {
- result.finishState();
- return result;
+ // concatenation with empty is empty
+ return Automata.makeEmpty();
}
int numStates = a.getNumStates();
for (int s = 0; s < numStates; s++) {
@@ -331,7 +331,7 @@ public final class Operations {
prevAcceptStates = toSet(a, numStates);
}
- return builder.finish();
+ return Operations.removeDeadStates(builder.finish());
}
private static IntHashSet toSet(Automaton a, int offset) {
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
index 6f9c1aa04b6..6ca567d0aeb 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
@@ -85,15 +85,14 @@ public class TestAutomaton extends LuceneTestCase {
public void testSameLanguage() throws Exception {
Automaton a1 = Automata.makeString("foobar");
Automaton a2 =
- Operations.removeDeadStates(
- Operations.concatenate(
- List.of(Automata.makeString("foo"), Automata.makeString("bar"))));
+ Operations.concatenate(List.of(Automata.makeString("foo"), Automata.makeString("bar")));
assertTrue(AutomatonTestUtil.sameLanguage(a1, a2));
}
public void testCommonPrefixString() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("foobar"), Automata.makeAnyString()));
+ AutomatonTestUtil.assertCleanDFA(a); // not minimal
assertEquals("foobar", Operations.getCommonPrefix(a));
}
@@ -122,24 +121,28 @@ public class TestAutomaton extends LuceneTestCase {
public void testCommonPrefixLeadingWildcard() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeAnyChar(), Automata.makeString("boo")));
+ AutomatonTestUtil.assertMinimalDFA(a);
assertEquals("", Operations.getCommonPrefix(a));
}
public void testCommonPrefixTrailingWildcard() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyChar()));
+ AutomatonTestUtil.assertMinimalDFA(a);
assertEquals("boo", Operations.getCommonPrefix(a));
}
public void testCommonPrefixLeadingKleenStar() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));
+ AutomatonTestUtil.assertCleanNFA(a);
assertEquals("", Operations.getCommonPrefix(a));
}
public void testCommonPrefixTrailingKleenStar() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyString()));
+ AutomatonTestUtil.assertCleanDFA(a); // not minimal
assertEquals("boo", Operations.getCommonPrefix(a));
}
@@ -191,6 +194,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testConcatenate1() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("m"), Automata.makeAnyString()));
+ AutomatonTestUtil.assertCleanDFA(a); // not minimal
assertTrue(Operations.run(a, "m"));
assertTrue(Operations.run(a, "me"));
assertTrue(Operations.run(a, "me too"));
@@ -204,6 +208,7 @@ public class TestAutomaton extends LuceneTestCase {
Automata.makeAnyString(),
Automata.makeString("n"),
Automata.makeAnyString()));
+ AutomatonTestUtil.assertCleanNFA(a);
a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
assertTrue(Operations.run(a, "mn"));
assertTrue(Operations.run(a, "mone"));
@@ -215,7 +220,7 @@ public class TestAutomaton extends LuceneTestCase {
Automaton a =
Operations.union(
Arrays.asList(Automata.makeString("foobar"), Automata.makeString("barbaz")));
- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, "foobar"));
assertTrue(Operations.run(a, "barbaz"));
@@ -229,7 +234,7 @@ public class TestAutomaton extends LuceneTestCase {
Automata.makeString("foobar"),
Automata.makeString(""),
Automata.makeString("barbaz")));
- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, "foobar"));
assertTrue(Operations.run(a, "barbaz"));
assertTrue(Operations.run(a, ""));
@@ -258,7 +263,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testReverse() throws Exception {
Automaton a = Automata.makeString("foobar");
Automaton ra = Operations.reverse(a);
- Automaton a2 = Operations.determinize(Operations.reverse(ra), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertMinimalDFA(a);
+ Automaton a2 = Operations.reverse(ra);
+ AutomatonTestUtil.assertMinimalDFA(a2);
assertTrue(AutomatonTestUtil.sameLanguage(a, a2));
}
@@ -266,7 +273,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testOptional() throws Exception {
Automaton a = Automata.makeString("foobar");
Automaton a2 = Operations.optional(a);
- a2 = Operations.determinize(a2, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertMinimalDFA(a2);
assertTrue(Operations.run(a, "foobar"));
assertFalse(Operations.run(a, ""));
@@ -276,7 +283,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testRepeatAny() throws Exception {
Automaton a = Automata.makeString("zee");
- Automaton a2 = Operations.determinize(Operations.repeat(a), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a2 = Operations.repeat(a);
+ AutomatonTestUtil.assertMinimalDFA(a2);
+
assertTrue(Operations.run(a2, ""));
assertTrue(Operations.run(a2, "zee"));
assertTrue(Operations.run(a2, "zeezee"));
@@ -285,7 +294,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testRepeatMin() throws Exception {
Automaton a = Automata.makeString("zee");
- Automaton a2 = Operations.determinize(Operations.repeat(a, 2), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a2 = Operations.repeat(a, 2);
+ AutomatonTestUtil.assertCleanDFA(a2); // not minimal
+
assertFalse(Operations.run(a2, ""));
assertFalse(Operations.run(a2, "zee"));
assertTrue(Operations.run(a2, "zeezee"));
@@ -294,8 +305,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testRepeatMinMax1() throws Exception {
Automaton a = Automata.makeString("zee");
- Automaton a2 =
- Operations.determinize(Operations.repeat(a, 0, 2), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a2 = Operations.repeat(a, 0, 2);
+ AutomatonTestUtil.assertMinimalDFA(a2);
+
assertTrue(Operations.run(a2, ""));
assertTrue(Operations.run(a2, "zee"));
assertTrue(Operations.run(a2, "zeezee"));
@@ -304,8 +316,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testRepeatMinMax2() throws Exception {
Automaton a = Automata.makeString("zee");
- Automaton a2 =
- Operations.determinize(Operations.repeat(a, 2, 4), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a2 = Operations.repeat(a, 2, 4);
+ AutomatonTestUtil.assertMinimalDFA(a2);
+
assertFalse(Operations.run(a2, ""));
assertFalse(Operations.run(a2, "zee"));
assertTrue(Operations.run(a2, "zeezee"));
@@ -316,10 +329,9 @@ public class TestAutomaton extends LuceneTestCase {
public void testComplement() throws Exception {
Automaton a = Automata.makeString("zee");
- Automaton a2 =
- Operations.determinize(
- Operations.complement(a, DEFAULT_DETERMINIZE_WORK_LIMIT),
- DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a2 = Operations.complement(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertMinimalDFA(a2);
+
assertTrue(Operations.run(a2, ""));
assertFalse(Operations.run(a2, "zee"));
assertTrue(Operations.run(a2, "zeezee"));
@@ -327,9 +339,9 @@ public class TestAutomaton extends LuceneTestCase {
}
public void testInterval() throws Exception {
- Automaton a =
- Operations.determinize(
- Automata.makeDecimalInterval(17, 100, 3), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a = Automata.makeDecimalInterval(17, 100, 3);
+ AutomatonTestUtil.assertCleanDFA(a); // not minimal
+
assertFalse(Operations.run(a, ""));
assertTrue(Operations.run(a, "017"));
assertTrue(Operations.run(a, "100"));
@@ -359,24 +371,28 @@ public class TestAutomaton extends LuceneTestCase {
public void testCommonSuffixTrailingWildcard() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyChar()));
+ AutomatonTestUtil.assertMinimalDFA(a);
assertEquals(newBytesRef(), Operations.getCommonSuffixBytesRef(a));
}
public void testCommonSuffixLeadingKleenStar() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));
+ AutomatonTestUtil.assertCleanNFA(a);
assertEquals(newBytesRef("boo"), Operations.getCommonSuffixBytesRef(a));
}
public void testCommonSuffixTrailingKleenStar() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyString()));
+ AutomatonTestUtil.assertCleanDFA(a); // not minimal
assertEquals(newBytesRef(), Operations.getCommonSuffixBytesRef(a));
}
public void testCommonSuffixUnicode() throws Exception {
Automaton a =
Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("booπŸ˜‚πŸ˜‚πŸ˜‚")));
+ AutomatonTestUtil.assertCleanNFA(a);
Automaton binary = new UTF32ToUTF8().convert(a);
assertEquals(newBytesRef("booπŸ˜‚πŸ˜‚πŸ˜‚"), Operations.getCommonSuffixBytesRef(binary));
}
@@ -431,7 +447,8 @@ public class TestAutomaton extends LuceneTestCase {
}
public void testAnyStringEmptyString() throws Exception {
- Automaton a = Operations.determinize(Automata.makeAnyString(), DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a = Automata.makeAnyString();
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, ""));
}
@@ -528,36 +545,28 @@ public class TestAutomaton extends LuceneTestCase {
Automaton a2 = Automata.makeString("boobar");
Automaton a3 = Automata.makeString("beebar");
Automaton a = Operations.union(Arrays.asList(a1, a2, a3));
- if (random().nextBoolean()) {
- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
- } else if (random().nextBoolean()) {
- a = MinimizationOperations.minimize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
- }
+ AutomatonTestUtil.assertCleanNFA(a); // this is why we have makeStringUnion()
assertMatches(a, "foobar", "beebar", "boobar");
- Automaton a4 =
- Operations.determinize(
- Operations.minus(a, a2, DEFAULT_DETERMINIZE_WORK_LIMIT),
- DEFAULT_DETERMINIZE_WORK_LIMIT);
+ Automaton a4 = Operations.minus(a, a2, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertCleanDFA(a4);
assertTrue(Operations.run(a4, "foobar"));
assertFalse(Operations.run(a4, "boobar"));
assertTrue(Operations.run(a4, "beebar"));
assertMatches(a4, "foobar", "beebar");
- a4 =
- Operations.determinize(
- Operations.minus(a4, a1, DEFAULT_DETERMINIZE_WORK_LIMIT),
- DEFAULT_DETERMINIZE_WORK_LIMIT);
+ a4 = Operations.minus(a4, a1, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertCleanDFA(a4);
+
assertFalse(Operations.run(a4, "foobar"));
assertFalse(Operations.run(a4, "boobar"));
assertTrue(Operations.run(a4, "beebar"));
assertMatches(a4, "beebar");
- a4 =
- Operations.determinize(
- Operations.minus(a4, a3, DEFAULT_DETERMINIZE_WORK_LIMIT),
- DEFAULT_DETERMINIZE_WORK_LIMIT);
+ a4 = Operations.minus(a4, a3, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertCleanDFA(a4);
+
assertFalse(Operations.run(a4, "foobar"));
assertFalse(Operations.run(a4, "boobar"));
assertFalse(Operations.run(a4, "beebar"));
@@ -566,6 +575,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testOneInterval() throws Exception {
Automaton a = Automata.makeDecimalInterval(999, 1032, 0);
+ AutomatonTestUtil.assertCleanNFA(a);
a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
assertTrue(Operations.run(a, "0999"));
assertTrue(Operations.run(a, "00999"));
@@ -574,7 +584,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testAnotherInterval() throws Exception {
Automaton a = Automata.makeDecimalInterval(1, 2, 0);
- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);
+ AutomatonTestUtil.assertCleanDFA(a);
assertTrue(Operations.run(a, "01"));
}
@@ -640,17 +650,16 @@ public class TestAutomaton extends LuceneTestCase {
expected.add(Util.toUTF32(s, ints));
}
- assertEquals(
- expected,
- TestOperations.getFiniteStrings(Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT)));
+ assertEquals(expected, TestOperations.getFiniteStrings(a));
}
public void testConcatenatePreservesDet() throws Exception {
Automaton a1 = Automata.makeString("foobar");
- assertTrue(a1.isDeterministic());
+ AutomatonTestUtil.assertMinimalDFA(a1);
Automaton a2 = Automata.makeString("baz");
- assertTrue(a2.isDeterministic());
- assertTrue((Operations.concatenate(Arrays.asList(a1, a2)).isDeterministic()));
+ AutomatonTestUtil.assertMinimalDFA(a2);
+ Automaton a3 = Operations.concatenate(Arrays.asList(a1, a2));
+ AutomatonTestUtil.assertMinimalDFA(a3);
}
public void testRemoveDeadStates() throws Exception {
@@ -690,9 +699,11 @@ public class TestAutomaton extends LuceneTestCase {
public void testConcatEmpty() throws Exception {
// If you concat empty automaton to anything the result should still be empty:
Automaton a = Operations.concatenate(List.of(Automata.makeEmpty(), Automata.makeString("foo")));
+ AutomatonTestUtil.assertMinimalDFA(a);
assertEquals(new HashSet<IntsRef>(), TestOperations.getFiniteStrings(a));
a = Operations.concatenate(List.of(Automata.makeString("foo"), Automata.makeEmpty()));
+ AutomatonTestUtil.assertMinimalDFA(a);
assertEquals(new HashSet<IntsRef>(), TestOperations.getFiniteStrings(a));
}
@@ -724,9 +735,7 @@ public class TestAutomaton extends LuceneTestCase {
int state = a2.createState();
a2.addTransition(0, state, 'a');
a2.finishState();
- assertTrue(
- AutomatonTestUtil.sameLanguage(
- Operations.removeDeadStates(a), Operations.removeDeadStates(a2)));
+ assertTrue(AutomatonTestUtil.sameLanguage(a, Operations.removeDeadStates(a2)));
}
private Automaton randomNoOp(Automaton a) {
@@ -1356,6 +1365,7 @@ public class TestAutomaton extends LuceneTestCase {
byte[] zeros = new byte[3];
Automaton a =
makeBinaryInterval(newBytesRef(zeros, 0, 1), true, newBytesRef(zeros, 0, 2), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.isFinite(a));
assertFalse(accepts(a, newBytesRef()));
assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));
@@ -1364,6 +1374,7 @@ public class TestAutomaton extends LuceneTestCase {
// '' (incl) - 00 (incl)
a = makeBinaryInterval(newBytesRef(), true, newBytesRef(zeros, 0, 2), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.isFinite(a));
assertTrue(accepts(a, newBytesRef()));
assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));
@@ -1372,6 +1383,7 @@ public class TestAutomaton extends LuceneTestCase {
// '' (excl) - 00 (incl)
a = makeBinaryInterval(newBytesRef(), false, newBytesRef(zeros, 0, 2), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.isFinite(a));
assertFalse(accepts(a, newBytesRef()));
assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));
@@ -1380,6 +1392,7 @@ public class TestAutomaton extends LuceneTestCase {
// 0 (excl) - 00 (incl)
a = makeBinaryInterval(newBytesRef(zeros, 0, 1), false, newBytesRef(zeros, 0, 2), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.isFinite(a));
assertFalse(accepts(a, newBytesRef()));
assertFalse(accepts(a, newBytesRef(zeros, 0, 1)));
@@ -1388,6 +1401,7 @@ public class TestAutomaton extends LuceneTestCase {
// 0 (excl) - 00 (excl)
a = makeBinaryInterval(newBytesRef(zeros, 0, 1), false, newBytesRef(zeros, 0, 2), false);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.isFinite(a));
assertFalse(accepts(a, newBytesRef()));
assertFalse(accepts(a, newBytesRef(zeros, 0, 1)));
@@ -1505,6 +1519,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalBasic() throws Exception {
Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("foo"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, intsRef("bar")));
assertTrue(Operations.run(a, intsRef("foo")));
assertTrue(Operations.run(a, intsRef("beep")));
@@ -1514,6 +1529,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalLowerBoundEmptyString() throws Exception {
Automaton a = Automata.makeBinaryInterval(newBytesRef(""), true, newBytesRef("bar"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, intsRef("")));
assertTrue(Operations.run(a, intsRef("a")));
assertTrue(Operations.run(a, intsRef("bar")));
@@ -1521,6 +1537,7 @@ public class TestAutomaton extends LuceneTestCase {
assertFalse(Operations.run(a, intsRef("baz")));
a = Automata.makeBinaryInterval(newBytesRef(""), false, newBytesRef("bar"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("")));
assertTrue(Operations.run(a, intsRef("a")));
assertTrue(Operations.run(a, intsRef("bar")));
@@ -1530,6 +1547,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalEqual() throws Exception {
Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("bar"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, intsRef("bar")));
assertTrue(AutomatonTestUtil.isFinite(a));
assertEquals(1, TestOperations.getFiniteStrings(a).size());
@@ -1538,6 +1556,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalCommonPrefix() throws Exception {
Automaton a =
Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("barfoo"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("bam")));
assertTrue(Operations.run(a, intsRef("bar")));
assertTrue(Operations.run(a, intsRef("bara")));
@@ -1551,12 +1570,14 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryExceptEmpty() throws Exception {
Automaton a = Automata.makeNonEmptyBinary();
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("")));
assertTrue(Operations.run(a, intsRef(TestUtil.randomRealisticUnicodeString(random(), 1, 10))));
}
public void testMakeBinaryIntervalOpenMax() throws Exception {
Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, null, true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("bam")));
assertTrue(Operations.run(a, intsRef("bar")));
assertTrue(Operations.run(a, intsRef("bara")));
@@ -1572,11 +1593,13 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalOpenMaxZeroLengthMin() throws Exception {
// when including min, automaton should accept "a"
Automaton a = Automata.makeBinaryInterval(newBytesRef(""), true, null, true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, intsRef("")));
assertTrue(Operations.run(a, intsRef("a")));
assertTrue(Operations.run(a, intsRef("aaaaaa")));
// excluding min should still accept "a"
a = Automata.makeBinaryInterval(newBytesRef(""), false, null, true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("")));
assertTrue(Operations.run(a, intsRef("a")));
assertTrue(Operations.run(a, intsRef("aaaaaa")));
@@ -1584,6 +1607,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalOpenMin() throws Exception {
Automaton a = Automata.makeBinaryInterval(null, true, newBytesRef("foo"), true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertFalse(Operations.run(a, intsRef("foz")));
assertFalse(Operations.run(a, intsRef("zzz")));
assertTrue(Operations.run(a, intsRef("foo")));
@@ -1595,6 +1619,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeBinaryIntervalOpenBoth() throws Exception {
Automaton a = Automata.makeBinaryInterval(null, true, null, true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(Operations.run(a, intsRef("foz")));
assertTrue(Operations.run(a, intsRef("zzz")));
assertTrue(Operations.run(a, intsRef("foo")));
@@ -1606,6 +1631,7 @@ public class TestAutomaton extends LuceneTestCase {
public void testAcceptAllEmptyStringMin() throws Exception {
Automaton a = Automata.makeBinaryInterval(newBytesRef(), true, null, true);
+ AutomatonTestUtil.assertMinimalDFA(a);
assertTrue(AutomatonTestUtil.sameLanguage(Automata.makeAnyBinary(), a));
}
@@ -1678,36 +1704,28 @@ public class TestAutomaton extends LuceneTestCase {
public void testMakeCharSetEmpty() {
Automaton expected = Automata.makeEmpty();
Automaton actual = Automata.makeCharSet(new int[] {});
+ AutomatonTestUtil.assertMinimalDFA(actual);
assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
- assertTrue(actual.isDeterministic());
- assertEquals(0, actual.getNumStates());
- assertEquals(0, actual.getNumTransitions());
}
public void testMakeCharSetOne() {
Automaton expected = Automata.makeChar('a');
Automaton actual = Automata.makeCharSet(new int[] {'a'});
+ AutomatonTestUtil.assertMinimalDFA(actual);
assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
- assertTrue(actual.isDeterministic());
- assertEquals(2, actual.getNumStates());
- assertEquals(1, actual.getNumTransitions());
}
public void testMakeCharSetTwo() {
Automaton expected = Operations.union(List.of(Automata.makeChar('a'), Automata.makeChar('A')));
Automaton actual = Automata.makeCharSet(new int[] {'a', 'A'});
+ AutomatonTestUtil.assertMinimalDFA(actual);
assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
- assertTrue(actual.isDeterministic());
- assertEquals(2, actual.getNumStates());
- assertEquals(2, actual.getNumTransitions());
}
public void testMakeCharSetDups() {
Automaton expected = Automata.makeChar('a');
Automaton actual = Automata.makeCharSet(new int[] {'a', 'a', 'a'});
+ AutomatonTestUtil.assertMinimalDFA(actual);
assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
- assertTrue(actual.isDeterministic());
- assertEquals(2, actual.getNumStates());
- assertEquals(1, actual.getNumTransitions());
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
index aedac499ecc..dc6fbed5e87 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
@@ -67,6 +67,7 @@ public class TestOperations extends LuceneTestCase {
public void testEmptyLanguageConcatenate() {
Automaton concat =
Operations.concatenate(List.of(Automata.makeString("a"), Automata.makeEmpty()));
+ AutomatonTestUtil.assertMinimalDFA(concat);
assertTrue(Operations.isEmpty(concat));
}
@@ -113,8 +114,11 @@ public class TestOperations extends LuceneTestCase {
// an NFA (two transitions for 't' from initial state)
Automaton nfa =
Operations.union(List.of(Automata.makeString("this"), Automata.makeString("three")));
+ AutomatonTestUtil.assertCleanNFA(nfa);
Automaton concat1 = Operations.concatenate(List.of(expandedSingleton, nfa));
+ AutomatonTestUtil.assertCleanNFA(concat1);
Automaton concat2 = Operations.concatenate(List.of(singleton, nfa));
+ AutomatonTestUtil.assertCleanNFA(concat2);
assertFalse(concat2.isDeterministic());
assertTrue(
AutomatonTestUtil.sameLanguage(
@@ -297,58 +301,114 @@ public class TestOperations extends LuceneTestCase {
return a;
}
- public void testRepeat() {
- Automaton emptyLanguage = Automata.makeEmpty();
- assertSame(emptyLanguage, Operations.repeat(emptyLanguage));
+ public void testRepeatEmptyLanguage() {
+ Automaton expected = Automata.makeEmpty();
+ Automaton actual = Operations.repeat(expected);
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertSame(expected, actual);
+ }
+
+ public void testRepeatEmptyString() {
+ Automaton expected = Automata.makeEmptyString();
+ Automaton actual = Operations.repeat(expected);
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertSame(expected, actual);
+ }
- Automaton emptyString = Automata.makeEmptyString();
- assertSame(emptyString, Operations.repeat(emptyString));
+ public void testRepeatChar() {
+ Automaton actual = Operations.repeat(Automata.makeChar('a'));
+ AutomatonTestUtil.assertMinimalDFA(actual);
- Automaton a = Automata.makeChar('a');
- Automaton as = new Automaton();
- as.createState();
- as.setAccept(0, true);
- as.addTransition(0, 0, 'a');
- as.finishState();
- assertTrue(AutomatonTestUtil.sameLanguage(as, Operations.repeat(a)));
- assertSame(as, Operations.repeat(as));
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.addTransition(0, 0, 'a');
+ expected.finishState();
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+ public void testRepeatOptionalChar() {
Automaton aOrEmpty = new Automaton();
aOrEmpty.createState();
aOrEmpty.setAccept(0, true);
aOrEmpty.createState();
aOrEmpty.setAccept(1, true);
aOrEmpty.addTransition(0, 1, 'a');
- assertTrue(AutomatonTestUtil.sameLanguage(as, Operations.repeat(aOrEmpty)));
-
- Automaton ab = Automata.makeString("ab");
- Automaton abs = new Automaton();
- abs.createState();
- abs.createState();
- abs.setAccept(0, true);
- abs.addTransition(0, 1, 'a');
- abs.finishState();
- abs.addTransition(1, 0, 'b');
- abs.finishState();
- assertTrue(AutomatonTestUtil.sameLanguage(abs, Operations.repeat(ab)));
- assertSame(abs, Operations.repeat(abs));
+ Automaton actual = Operations.repeat(aOrEmpty);
+ AutomatonTestUtil.assertMinimalDFA(actual);
+
+ Automaton expected = Operations.repeat(Automata.makeChar('a'));
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+
+ public void testRepeatTwoChar() {
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.addTransition(0, 1, 'a');
+ expected.finishState();
+ expected.addTransition(1, 0, 'b');
+ expected.finishState();
+ Automaton actual = Operations.repeat(Automata.makeString("ab"));
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+
+ public void testRepeatOptionalTwoChar() {
+ Automaton expected = Operations.repeat(Automata.makeString("ab"));
+ Automaton actual = Operations.repeat(expected);
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+
+ public void testRepeatConcatenation() {
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.createState();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.addTransition(0, 1, 'a');
+ expected.addTransition(0, 0, 'c');
+ expected.finishState();
+ expected.addTransition(1, 2, 'b');
+ expected.finishState();
+ expected.addTransition(2, 1, 'a');
+ expected.addTransition(2, 0, 'c');
+ expected.finishState();
+
+ Automaton abs = Operations.repeat(Automata.makeString("ab"));
+ Automaton absThenC = Operations.concatenate(List.of(abs, Automata.makeChar('c')));
+ Automaton actual = Operations.repeat(absThenC);
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+ public void testRepeatOptionalConcatenation() {
+ Automaton abs = Operations.repeat(Automata.makeString("ab"));
Automaton absThenC = Operations.concatenate(List.of(abs, Automata.makeChar('c')));
- Automaton absThenCs = new Automaton();
- absThenCs.createState();
- absThenCs.createState();
- absThenCs.createState();
- absThenCs.setAccept(0, true);
- absThenCs.addTransition(0, 1, 'a');
- absThenCs.addTransition(0, 0, 'c');
- absThenCs.finishState();
- absThenCs.addTransition(1, 2, 'b');
- absThenCs.finishState();
- absThenCs.addTransition(2, 1, 'a');
- absThenCs.addTransition(2, 0, 'c');
- absThenCs.finishState();
- assertTrue(AutomatonTestUtil.sameLanguage(absThenCs, Operations.repeat(absThenC)));
- assertSame(absThenCs, Operations.repeat(absThenCs));
+
+ Automaton expected = Operations.repeat(absThenC);
+ Automaton actual = Operations.repeat(expected);
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertSame(expected, Operations.repeat(actual));
+ }
+
+ public void testRepeatConcatenateOptional() {
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.addTransition(0, 0, 'a');
+ expected.addTransition(0, 1, 'a');
+ expected.finishState();
+ expected.addTransition(1, 0, 'b');
+ expected.finishState();
+ expected = Operations.determinize(expected, Integer.MAX_VALUE);
Automaton aOrAb = new Automaton();
aOrAb.createState();
@@ -360,19 +420,10 @@ public class TestOperations extends LuceneTestCase {
aOrAb.finishState();
aOrAb.addTransition(1, 2, 'b');
aOrAb.finishState();
- Automaton aOrAbs = new Automaton();
- aOrAbs.createState();
- aOrAbs.createState();
- aOrAbs.setAccept(0, true);
- aOrAbs.addTransition(0, 0, 'a');
- aOrAbs.addTransition(0, 1, 'a');
- aOrAbs.finishState();
- aOrAbs.addTransition(1, 0, 'b');
- aOrAbs.finishState();
- assertTrue(
- AutomatonTestUtil.sameLanguage(
- Operations.determinize(aOrAbs, Integer.MAX_VALUE),
- Operations.determinize(Operations.repeat(aOrAb), Integer.MAX_VALUE)));
+ Automaton actual = Operations.repeat(aOrAb);
+ AutomatonTestUtil.assertMinimalDFA(actual);
+
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
}
public void testMergeAcceptStatesWithNoTransition() {
@@ -468,22 +519,45 @@ public class TestOperations extends LuceneTestCase {
}
public void testOptional() {
- Automaton a = Automata.makeChar('a');
-
- Automaton optionalA = new Automaton();
- optionalA.createState();
- optionalA.setAccept(0, true);
- optionalA.finishState();
- optionalA.createState();
- optionalA.setAccept(1, true);
- optionalA.addTransition(0, 1, 'a');
- optionalA.finishState();
-
- assertTrue(AutomatonTestUtil.sameLanguage(Operations.optional(a), optionalA));
- assertSame(optionalA, Operations.optional(optionalA));
-
- // Now test an automaton that has a transition to state 0. a(ba)*
- a = new Automaton();
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.finishState();
+ expected.createState();
+ expected.setAccept(1, true);
+ expected.addTransition(0, 1, 'a');
+ expected.finishState();
+
+ Automaton actual = Operations.optional(Automata.makeChar('a'));
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+
+ public void testOptionalOptional() {
+ Automaton expected = Operations.optional(Automata.makeChar('a'));
+ Automaton actual = Operations.optional(expected);
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
+
+ // test an automaton that has a transition to state 0. a(ba)*
+ public void testOptionalAcceptsState0() {
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.setAccept(0, true);
+ expected.createState();
+ expected.createState();
+ expected.setAccept(2, true);
+ expected.addTransition(0, 2, 'a');
+ expected.finishState();
+ expected.addTransition(1, 2, 'a');
+ expected.finishState();
+ expected.addTransition(2, 1, 'b');
+ expected.finishState();
+
+ Automaton a = new Automaton();
a.createState();
a.createState();
a.setAccept(1, true);
@@ -491,22 +565,26 @@ public class TestOperations extends LuceneTestCase {
a.finishState();
a.addTransition(1, 0, 'b');
a.finishState();
+ Automaton actual = Operations.optional(a);
+
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
+ }
- optionalA = new Automaton();
- optionalA.createState();
- optionalA.setAccept(0, true);
- optionalA.createState();
- optionalA.createState();
- optionalA.setAccept(2, true);
- optionalA.addTransition(0, 2, 'a');
- optionalA.finishState();
- optionalA.addTransition(1, 2, 'a');
- optionalA.finishState();
- optionalA.addTransition(2, 1, 'b');
- optionalA.finishState();
-
- assertTrue(AutomatonTestUtil.sameLanguage(Operations.optional(a), optionalA));
- assertSame(optionalA, Operations.optional(optionalA));
+ public void TestOptionalOptionalAcceptsState0() {
+ Automaton expected = new Automaton();
+ expected.createState();
+ expected.createState();
+ expected.setAccept(1, true);
+ expected.addTransition(0, 1, 'a');
+ expected.finishState();
+ expected.addTransition(1, 0, 'b');
+ expected.finishState();
+ expected = Operations.optional(expected);
+
+ Automaton actual = Operations.optional(expected);
+ AutomatonTestUtil.assertMinimalDFA(actual);
+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));
}
public void testDuelOptional() {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java
index f0f78064a13..8109b847d19 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java
@@ -37,6 +37,7 @@ import org.apache.lucene.util.automaton.RegExp;
import org.apache.lucene.util.automaton.StatePair;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
import org.apache.lucene.util.automaton.Transition;
+import org.junit.Assert;
/**
* Utilities for testing automata.
@@ -44,7 +45,7 @@ import org.apache.lucene.util.automaton.Transition;
* <p>Capable of generating random regular expressions, and automata, and also provides a number of
* very basic unoptimized implementations (*slow) for testing.
*/
-public class AutomatonTestUtil {
+public class AutomatonTestUtil extends Assert {
/** Default maximum number of states that {@link Operations#determinize} should create. */
public static final int DEFAULT_MAX_DETERMINIZED_STATES = 1000000;
@@ -390,6 +391,27 @@ public class AutomatonTestUtil {
return a;
}
+ /** Asserts that an automaton is a minimal DFA. */
+ public static void assertMinimalDFA(Automaton automaton) {
+ assertCleanDFA(automaton);
+ Automaton minimized = minimizeSimple(automaton);
+ assertEquals(minimized.getNumStates(), automaton.getNumStates());
+ }
+
+ /** Asserts that an automaton is a DFA with no dead states */
+ public static void assertCleanDFA(Automaton automaton) {
+ assertCleanNFA(automaton);
+ assertTrue("must be deterministic", automaton.isDeterministic());
+ }
+
+ /** Asserts that an automaton has no dead states */
+ public static void assertCleanNFA(Automaton automaton) {
+ assertFalse(
+ "has dead states reachable from initial", Operations.hasDeadStatesFromInitial(automaton));
+ assertFalse("has dead states leading to accept", Operations.hasDeadStatesToAccept(automaton));
+ assertFalse("has unreachable dead states (ghost states)", Operations.hasDeadStates(automaton));
+ }
+
/** Simple, original brics implementation of determinize() */
public static Automaton determinizeSimple(Automaton a) {
Set<Integer> initialset = new HashSet<>();
@@ -607,7 +629,7 @@ public class AutomatonTestUtil {
assert Operations.hasDeadStatesFromInitial(a1) == false;
assert Operations.hasDeadStatesFromInitial(a2) == false;
if (a1.getNumStates() == 0) {
- // Empty language is alwyas a subset of any other language
+ // Empty language is always a subset of any other language
return true;
} else if (a2.getNumStates() == 0) {
return Operations.isEmpty(a1);