Apache Lucene open-source search software
Clean up more dead states creation by Automata/Operations (#14218)
Robert Muirde1ed71261d5
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.javaindex 9b6198fb04c..6b9a7fcc556 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java@@ -491,6 +491,7 @@ public final class Automata {*/public static Automaton makeDecimalInterval(int min, int max, int digits)throws IllegalArgumentException {+ // TODO: can this be improved to always return a DFA?String x = Integer.toString(min);String y = Integer.toString(max);if (min > max || (digits > 0 && y.length() > digits)) {@@ -533,7 +534,7 @@ public final class Automata {a1.finishState();}- return a1;+ return Operations.removeDeadStates(a1);}/** Returns a new (deterministic) automaton that accepts the single given string. */diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.javaindex fdd6242abcc..6974cd5137b 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java@@ -89,8 +89,8 @@ public final class Operations {// First pass: create all statesfor (Automaton a : list) {if (a.getNumStates() == 0) {- result.finishState();- return result;+ // concatenation with empty is empty+ return Automata.makeEmpty();}int numStates = a.getNumStates();for (int s = 0; s < numStates; s++) {@@ -331,7 +331,7 @@ public final class Operations {prevAcceptStates = toSet(a, numStates);}- return builder.finish();+ return Operations.removeDeadStates(builder.finish());}private static IntHashSet toSet(Automaton a, int offset) {diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.javaindex 6f9c1aa04b6..6ca567d0aeb 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java@@ -85,15 +85,14 @@ public class TestAutomaton extends LuceneTestCase {public void testSameLanguage() throws Exception {Automaton a1 = Automata.makeString("foobar");Automaton a2 =- Operations.removeDeadStates(- Operations.concatenate(- List.of(Automata.makeString("foo"), Automata.makeString("bar"))));+ Operations.concatenate(List.of(Automata.makeString("foo"), Automata.makeString("bar")));assertTrue(AutomatonTestUtil.sameLanguage(a1, a2));}public void testCommonPrefixString() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("foobar"), Automata.makeAnyString()));+ AutomatonTestUtil.assertCleanDFA(a); // not minimalassertEquals("foobar", Operations.getCommonPrefix(a));}@@ -122,24 +121,28 @@ public class TestAutomaton extends LuceneTestCase {public void testCommonPrefixLeadingWildcard() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeAnyChar(), Automata.makeString("boo")));+ AutomatonTestUtil.assertMinimalDFA(a);assertEquals("", Operations.getCommonPrefix(a));}public void testCommonPrefixTrailingWildcard() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyChar()));+ AutomatonTestUtil.assertMinimalDFA(a);assertEquals("boo", Operations.getCommonPrefix(a));}public void testCommonPrefixLeadingKleenStar() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));+ AutomatonTestUtil.assertCleanNFA(a);assertEquals("", Operations.getCommonPrefix(a));}public void testCommonPrefixTrailingKleenStar() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyString()));+ AutomatonTestUtil.assertCleanDFA(a); // not minimalassertEquals("boo", Operations.getCommonPrefix(a));}@@ -191,6 +194,7 @@ public class TestAutomaton extends LuceneTestCase {public void testConcatenate1() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("m"), Automata.makeAnyString()));+ AutomatonTestUtil.assertCleanDFA(a); // not minimalassertTrue(Operations.run(a, "m"));assertTrue(Operations.run(a, "me"));assertTrue(Operations.run(a, "me too"));@@ -204,6 +208,7 @@ public class TestAutomaton extends LuceneTestCase {Automata.makeAnyString(),Automata.makeString("n"),Automata.makeAnyString()));+ AutomatonTestUtil.assertCleanNFA(a);a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);assertTrue(Operations.run(a, "mn"));assertTrue(Operations.run(a, "mone"));@@ -215,7 +220,7 @@ public class TestAutomaton extends LuceneTestCase {Automaton a =Operations.union(Arrays.asList(Automata.makeString("foobar"), Automata.makeString("barbaz")));- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, "foobar"));assertTrue(Operations.run(a, "barbaz"));@@ -229,7 +234,7 @@ public class TestAutomaton extends LuceneTestCase {Automata.makeString("foobar"),Automata.makeString(""),Automata.makeString("barbaz")));- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, "foobar"));assertTrue(Operations.run(a, "barbaz"));assertTrue(Operations.run(a, ""));@@ -258,7 +263,9 @@ public class TestAutomaton extends LuceneTestCase {public void testReverse() throws Exception {Automaton a = Automata.makeString("foobar");Automaton ra = Operations.reverse(a);- Automaton a2 = Operations.determinize(Operations.reverse(ra), DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertMinimalDFA(a);+ Automaton a2 = Operations.reverse(ra);+ AutomatonTestUtil.assertMinimalDFA(a2);assertTrue(AutomatonTestUtil.sameLanguage(a, a2));}@@ -266,7 +273,7 @@ public class TestAutomaton extends LuceneTestCase {public void testOptional() throws Exception {Automaton a = Automata.makeString("foobar");Automaton a2 = Operations.optional(a);- a2 = Operations.determinize(a2, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertMinimalDFA(a2);assertTrue(Operations.run(a, "foobar"));assertFalse(Operations.run(a, ""));@@ -276,7 +283,9 @@ public class TestAutomaton extends LuceneTestCase {public void testRepeatAny() throws Exception {Automaton a = Automata.makeString("zee");- Automaton a2 = Operations.determinize(Operations.repeat(a), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a2 = Operations.repeat(a);+ AutomatonTestUtil.assertMinimalDFA(a2);+assertTrue(Operations.run(a2, ""));assertTrue(Operations.run(a2, "zee"));assertTrue(Operations.run(a2, "zeezee"));@@ -285,7 +294,9 @@ public class TestAutomaton extends LuceneTestCase {public void testRepeatMin() throws Exception {Automaton a = Automata.makeString("zee");- Automaton a2 = Operations.determinize(Operations.repeat(a, 2), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a2 = Operations.repeat(a, 2);+ AutomatonTestUtil.assertCleanDFA(a2); // not minimal+assertFalse(Operations.run(a2, ""));assertFalse(Operations.run(a2, "zee"));assertTrue(Operations.run(a2, "zeezee"));@@ -294,8 +305,9 @@ public class TestAutomaton extends LuceneTestCase {public void testRepeatMinMax1() throws Exception {Automaton a = Automata.makeString("zee");- Automaton a2 =- Operations.determinize(Operations.repeat(a, 0, 2), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a2 = Operations.repeat(a, 0, 2);+ AutomatonTestUtil.assertMinimalDFA(a2);+assertTrue(Operations.run(a2, ""));assertTrue(Operations.run(a2, "zee"));assertTrue(Operations.run(a2, "zeezee"));@@ -304,8 +316,9 @@ public class TestAutomaton extends LuceneTestCase {public void testRepeatMinMax2() throws Exception {Automaton a = Automata.makeString("zee");- Automaton a2 =- Operations.determinize(Operations.repeat(a, 2, 4), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a2 = Operations.repeat(a, 2, 4);+ AutomatonTestUtil.assertMinimalDFA(a2);+assertFalse(Operations.run(a2, ""));assertFalse(Operations.run(a2, "zee"));assertTrue(Operations.run(a2, "zeezee"));@@ -316,10 +329,9 @@ public class TestAutomaton extends LuceneTestCase {public void testComplement() throws Exception {Automaton a = Automata.makeString("zee");- Automaton a2 =- Operations.determinize(- Operations.complement(a, DEFAULT_DETERMINIZE_WORK_LIMIT),- DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a2 = Operations.complement(a, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertMinimalDFA(a2);+assertTrue(Operations.run(a2, ""));assertFalse(Operations.run(a2, "zee"));assertTrue(Operations.run(a2, "zeezee"));@@ -327,9 +339,9 @@ public class TestAutomaton extends LuceneTestCase {}public void testInterval() throws Exception {- Automaton a =- Operations.determinize(- Automata.makeDecimalInterval(17, 100, 3), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a = Automata.makeDecimalInterval(17, 100, 3);+ AutomatonTestUtil.assertCleanDFA(a); // not minimal+assertFalse(Operations.run(a, ""));assertTrue(Operations.run(a, "017"));assertTrue(Operations.run(a, "100"));@@ -359,24 +371,28 @@ public class TestAutomaton extends LuceneTestCase {public void testCommonSuffixTrailingWildcard() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyChar()));+ AutomatonTestUtil.assertMinimalDFA(a);assertEquals(newBytesRef(), Operations.getCommonSuffixBytesRef(a));}public void testCommonSuffixLeadingKleenStar() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));+ AutomatonTestUtil.assertCleanNFA(a);assertEquals(newBytesRef("boo"), Operations.getCommonSuffixBytesRef(a));}public void testCommonSuffixTrailingKleenStar() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeString("boo"), Automata.makeAnyString()));+ AutomatonTestUtil.assertCleanDFA(a); // not minimalassertEquals(newBytesRef(), Operations.getCommonSuffixBytesRef(a));}public void testCommonSuffixUnicode() throws Exception {Automaton a =Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("booπππ")));+ AutomatonTestUtil.assertCleanNFA(a);Automaton binary = new UTF32ToUTF8().convert(a);assertEquals(newBytesRef("booπππ"), Operations.getCommonSuffixBytesRef(binary));}@@ -431,7 +447,8 @@ public class TestAutomaton extends LuceneTestCase {}public void testAnyStringEmptyString() throws Exception {- Automaton a = Operations.determinize(Automata.makeAnyString(), DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a = Automata.makeAnyString();+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, ""));}@@ -528,36 +545,28 @@ public class TestAutomaton extends LuceneTestCase {Automaton a2 = Automata.makeString("boobar");Automaton a3 = Automata.makeString("beebar");Automaton a = Operations.union(Arrays.asList(a1, a2, a3));- if (random().nextBoolean()) {- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);- } else if (random().nextBoolean()) {- a = MinimizationOperations.minimize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);- }+ AutomatonTestUtil.assertCleanNFA(a); // this is why we have makeStringUnion()assertMatches(a, "foobar", "beebar", "boobar");- Automaton a4 =- Operations.determinize(- Operations.minus(a, a2, DEFAULT_DETERMINIZE_WORK_LIMIT),- DEFAULT_DETERMINIZE_WORK_LIMIT);+ Automaton a4 = Operations.minus(a, a2, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertCleanDFA(a4);assertTrue(Operations.run(a4, "foobar"));assertFalse(Operations.run(a4, "boobar"));assertTrue(Operations.run(a4, "beebar"));assertMatches(a4, "foobar", "beebar");- a4 =- Operations.determinize(- Operations.minus(a4, a1, DEFAULT_DETERMINIZE_WORK_LIMIT),- DEFAULT_DETERMINIZE_WORK_LIMIT);+ a4 = Operations.minus(a4, a1, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertCleanDFA(a4);+assertFalse(Operations.run(a4, "foobar"));assertFalse(Operations.run(a4, "boobar"));assertTrue(Operations.run(a4, "beebar"));assertMatches(a4, "beebar");- a4 =- Operations.determinize(- Operations.minus(a4, a3, DEFAULT_DETERMINIZE_WORK_LIMIT),- DEFAULT_DETERMINIZE_WORK_LIMIT);+ a4 = Operations.minus(a4, a3, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertCleanDFA(a4);+assertFalse(Operations.run(a4, "foobar"));assertFalse(Operations.run(a4, "boobar"));assertFalse(Operations.run(a4, "beebar"));@@ -566,6 +575,7 @@ public class TestAutomaton extends LuceneTestCase {public void testOneInterval() throws Exception {Automaton a = Automata.makeDecimalInterval(999, 1032, 0);+ AutomatonTestUtil.assertCleanNFA(a);a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);assertTrue(Operations.run(a, "0999"));assertTrue(Operations.run(a, "00999"));@@ -574,7 +584,7 @@ public class TestAutomaton extends LuceneTestCase {public void testAnotherInterval() throws Exception {Automaton a = Automata.makeDecimalInterval(1, 2, 0);- a = Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT);+ AutomatonTestUtil.assertCleanDFA(a);assertTrue(Operations.run(a, "01"));}@@ -640,17 +650,16 @@ public class TestAutomaton extends LuceneTestCase {expected.add(Util.toUTF32(s, ints));}- assertEquals(- expected,- TestOperations.getFiniteStrings(Operations.determinize(a, DEFAULT_DETERMINIZE_WORK_LIMIT)));+ assertEquals(expected, TestOperations.getFiniteStrings(a));}public void testConcatenatePreservesDet() throws Exception {Automaton a1 = Automata.makeString("foobar");- assertTrue(a1.isDeterministic());+ AutomatonTestUtil.assertMinimalDFA(a1);Automaton a2 = Automata.makeString("baz");- assertTrue(a2.isDeterministic());- assertTrue((Operations.concatenate(Arrays.asList(a1, a2)).isDeterministic()));+ AutomatonTestUtil.assertMinimalDFA(a2);+ Automaton a3 = Operations.concatenate(Arrays.asList(a1, a2));+ AutomatonTestUtil.assertMinimalDFA(a3);}public void testRemoveDeadStates() throws Exception {@@ -690,9 +699,11 @@ public class TestAutomaton extends LuceneTestCase {public void testConcatEmpty() throws Exception {// If you concat empty automaton to anything the result should still be empty:Automaton a = Operations.concatenate(List.of(Automata.makeEmpty(), Automata.makeString("foo")));+ AutomatonTestUtil.assertMinimalDFA(a);assertEquals(new HashSet<IntsRef>(), TestOperations.getFiniteStrings(a));a = Operations.concatenate(List.of(Automata.makeString("foo"), Automata.makeEmpty()));+ AutomatonTestUtil.assertMinimalDFA(a);assertEquals(new HashSet<IntsRef>(), TestOperations.getFiniteStrings(a));}@@ -724,9 +735,7 @@ public class TestAutomaton extends LuceneTestCase {int state = a2.createState();a2.addTransition(0, state, 'a');a2.finishState();- assertTrue(- AutomatonTestUtil.sameLanguage(- Operations.removeDeadStates(a), Operations.removeDeadStates(a2)));+ assertTrue(AutomatonTestUtil.sameLanguage(a, Operations.removeDeadStates(a2)));}private Automaton randomNoOp(Automaton a) {@@ -1356,6 +1365,7 @@ public class TestAutomaton extends LuceneTestCase {byte[] zeros = new byte[3];Automaton a =makeBinaryInterval(newBytesRef(zeros, 0, 1), true, newBytesRef(zeros, 0, 2), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.isFinite(a));assertFalse(accepts(a, newBytesRef()));assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));@@ -1364,6 +1374,7 @@ public class TestAutomaton extends LuceneTestCase {// '' (incl) - 00 (incl)a = makeBinaryInterval(newBytesRef(), true, newBytesRef(zeros, 0, 2), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.isFinite(a));assertTrue(accepts(a, newBytesRef()));assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));@@ -1372,6 +1383,7 @@ public class TestAutomaton extends LuceneTestCase {// '' (excl) - 00 (incl)a = makeBinaryInterval(newBytesRef(), false, newBytesRef(zeros, 0, 2), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.isFinite(a));assertFalse(accepts(a, newBytesRef()));assertTrue(accepts(a, newBytesRef(zeros, 0, 1)));@@ -1380,6 +1392,7 @@ public class TestAutomaton extends LuceneTestCase {// 0 (excl) - 00 (incl)a = makeBinaryInterval(newBytesRef(zeros, 0, 1), false, newBytesRef(zeros, 0, 2), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.isFinite(a));assertFalse(accepts(a, newBytesRef()));assertFalse(accepts(a, newBytesRef(zeros, 0, 1)));@@ -1388,6 +1401,7 @@ public class TestAutomaton extends LuceneTestCase {// 0 (excl) - 00 (excl)a = makeBinaryInterval(newBytesRef(zeros, 0, 1), false, newBytesRef(zeros, 0, 2), false);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.isFinite(a));assertFalse(accepts(a, newBytesRef()));assertFalse(accepts(a, newBytesRef(zeros, 0, 1)));@@ -1505,6 +1519,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalBasic() throws Exception {Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("foo"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, intsRef("bar")));assertTrue(Operations.run(a, intsRef("foo")));assertTrue(Operations.run(a, intsRef("beep")));@@ -1514,6 +1529,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalLowerBoundEmptyString() throws Exception {Automaton a = Automata.makeBinaryInterval(newBytesRef(""), true, newBytesRef("bar"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, intsRef("")));assertTrue(Operations.run(a, intsRef("a")));assertTrue(Operations.run(a, intsRef("bar")));@@ -1521,6 +1537,7 @@ public class TestAutomaton extends LuceneTestCase {assertFalse(Operations.run(a, intsRef("baz")));a = Automata.makeBinaryInterval(newBytesRef(""), false, newBytesRef("bar"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("")));assertTrue(Operations.run(a, intsRef("a")));assertTrue(Operations.run(a, intsRef("bar")));@@ -1530,6 +1547,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalEqual() throws Exception {Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("bar"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, intsRef("bar")));assertTrue(AutomatonTestUtil.isFinite(a));assertEquals(1, TestOperations.getFiniteStrings(a).size());@@ -1538,6 +1556,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalCommonPrefix() throws Exception {Automaton a =Automata.makeBinaryInterval(newBytesRef("bar"), true, newBytesRef("barfoo"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("bam")));assertTrue(Operations.run(a, intsRef("bar")));assertTrue(Operations.run(a, intsRef("bara")));@@ -1551,12 +1570,14 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryExceptEmpty() throws Exception {Automaton a = Automata.makeNonEmptyBinary();+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("")));assertTrue(Operations.run(a, intsRef(TestUtil.randomRealisticUnicodeString(random(), 1, 10))));}public void testMakeBinaryIntervalOpenMax() throws Exception {Automaton a = Automata.makeBinaryInterval(newBytesRef("bar"), true, null, true);+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("bam")));assertTrue(Operations.run(a, intsRef("bar")));assertTrue(Operations.run(a, intsRef("bara")));@@ -1572,11 +1593,13 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalOpenMaxZeroLengthMin() throws Exception {// when including min, automaton should accept "a"Automaton a = Automata.makeBinaryInterval(newBytesRef(""), true, null, true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, intsRef("")));assertTrue(Operations.run(a, intsRef("a")));assertTrue(Operations.run(a, intsRef("aaaaaa")));// excluding min should still accept "a"a = Automata.makeBinaryInterval(newBytesRef(""), false, null, true);+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("")));assertTrue(Operations.run(a, intsRef("a")));assertTrue(Operations.run(a, intsRef("aaaaaa")));@@ -1584,6 +1607,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalOpenMin() throws Exception {Automaton a = Automata.makeBinaryInterval(null, true, newBytesRef("foo"), true);+ AutomatonTestUtil.assertMinimalDFA(a);assertFalse(Operations.run(a, intsRef("foz")));assertFalse(Operations.run(a, intsRef("zzz")));assertTrue(Operations.run(a, intsRef("foo")));@@ -1595,6 +1619,7 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeBinaryIntervalOpenBoth() throws Exception {Automaton a = Automata.makeBinaryInterval(null, true, null, true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(Operations.run(a, intsRef("foz")));assertTrue(Operations.run(a, intsRef("zzz")));assertTrue(Operations.run(a, intsRef("foo")));@@ -1606,6 +1631,7 @@ public class TestAutomaton extends LuceneTestCase {public void testAcceptAllEmptyStringMin() throws Exception {Automaton a = Automata.makeBinaryInterval(newBytesRef(), true, null, true);+ AutomatonTestUtil.assertMinimalDFA(a);assertTrue(AutomatonTestUtil.sameLanguage(Automata.makeAnyBinary(), a));}@@ -1678,36 +1704,28 @@ public class TestAutomaton extends LuceneTestCase {public void testMakeCharSetEmpty() {Automaton expected = Automata.makeEmpty();Automaton actual = Automata.makeCharSet(new int[] {});+ AutomatonTestUtil.assertMinimalDFA(actual);assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));- assertTrue(actual.isDeterministic());- assertEquals(0, actual.getNumStates());- assertEquals(0, actual.getNumTransitions());}public void testMakeCharSetOne() {Automaton expected = Automata.makeChar('a');Automaton actual = Automata.makeCharSet(new int[] {'a'});+ AutomatonTestUtil.assertMinimalDFA(actual);assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));- assertTrue(actual.isDeterministic());- assertEquals(2, actual.getNumStates());- assertEquals(1, actual.getNumTransitions());}public void testMakeCharSetTwo() {Automaton expected = Operations.union(List.of(Automata.makeChar('a'), Automata.makeChar('A')));Automaton actual = Automata.makeCharSet(new int[] {'a', 'A'});+ AutomatonTestUtil.assertMinimalDFA(actual);assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));- assertTrue(actual.isDeterministic());- assertEquals(2, actual.getNumStates());- assertEquals(2, actual.getNumTransitions());}public void testMakeCharSetDups() {Automaton expected = Automata.makeChar('a');Automaton actual = Automata.makeCharSet(new int[] {'a', 'a', 'a'});+ AutomatonTestUtil.assertMinimalDFA(actual);assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));- assertTrue(actual.isDeterministic());- assertEquals(2, actual.getNumStates());- assertEquals(1, actual.getNumTransitions());}}diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.javaindex aedac499ecc..dc6fbed5e87 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java@@ -67,6 +67,7 @@ public class TestOperations extends LuceneTestCase {public void testEmptyLanguageConcatenate() {Automaton concat =Operations.concatenate(List.of(Automata.makeString("a"), Automata.makeEmpty()));+ AutomatonTestUtil.assertMinimalDFA(concat);assertTrue(Operations.isEmpty(concat));}@@ -113,8 +114,11 @@ public class TestOperations extends LuceneTestCase {// an NFA (two transitions for 't' from initial state)Automaton nfa =Operations.union(List.of(Automata.makeString("this"), Automata.makeString("three")));+ AutomatonTestUtil.assertCleanNFA(nfa);Automaton concat1 = Operations.concatenate(List.of(expandedSingleton, nfa));+ AutomatonTestUtil.assertCleanNFA(concat1);Automaton concat2 = Operations.concatenate(List.of(singleton, nfa));+ AutomatonTestUtil.assertCleanNFA(concat2);assertFalse(concat2.isDeterministic());assertTrue(AutomatonTestUtil.sameLanguage(@@ -297,58 +301,114 @@ public class TestOperations extends LuceneTestCase {return a;}- public void testRepeat() {- Automaton emptyLanguage = Automata.makeEmpty();- assertSame(emptyLanguage, Operations.repeat(emptyLanguage));+ public void testRepeatEmptyLanguage() {+ Automaton expected = Automata.makeEmpty();+ Automaton actual = Operations.repeat(expected);+ AutomatonTestUtil.assertMinimalDFA(actual);+ assertSame(expected, actual);+ }++ public void testRepeatEmptyString() {+ Automaton expected = Automata.makeEmptyString();+ Automaton actual = Operations.repeat(expected);+ AutomatonTestUtil.assertMinimalDFA(actual);+ assertSame(expected, actual);+ }- Automaton emptyString = Automata.makeEmptyString();- assertSame(emptyString, Operations.repeat(emptyString));+ public void testRepeatChar() {+ Automaton actual = Operations.repeat(Automata.makeChar('a'));+ AutomatonTestUtil.assertMinimalDFA(actual);- Automaton a = Automata.makeChar('a');- Automaton as = new Automaton();- as.createState();- as.setAccept(0, true);- as.addTransition(0, 0, 'a');- as.finishState();- assertTrue(AutomatonTestUtil.sameLanguage(as, Operations.repeat(a)));- assertSame(as, Operations.repeat(as));+ Automaton expected = new Automaton();+ expected.createState();+ expected.setAccept(0, true);+ expected.addTransition(0, 0, 'a');+ expected.finishState();+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }+ public void testRepeatOptionalChar() {Automaton aOrEmpty = new Automaton();aOrEmpty.createState();aOrEmpty.setAccept(0, true);aOrEmpty.createState();aOrEmpty.setAccept(1, true);aOrEmpty.addTransition(0, 1, 'a');- assertTrue(AutomatonTestUtil.sameLanguage(as, Operations.repeat(aOrEmpty)));-- Automaton ab = Automata.makeString("ab");- Automaton abs = new Automaton();- abs.createState();- abs.createState();- abs.setAccept(0, true);- abs.addTransition(0, 1, 'a');- abs.finishState();- abs.addTransition(1, 0, 'b');- abs.finishState();- assertTrue(AutomatonTestUtil.sameLanguage(abs, Operations.repeat(ab)));- assertSame(abs, Operations.repeat(abs));+ Automaton actual = Operations.repeat(aOrEmpty);+ AutomatonTestUtil.assertMinimalDFA(actual);++ Automaton expected = Operations.repeat(Automata.makeChar('a'));+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }++ public void testRepeatTwoChar() {+ Automaton expected = new Automaton();+ expected.createState();+ expected.createState();+ expected.setAccept(0, true);+ expected.addTransition(0, 1, 'a');+ expected.finishState();+ expected.addTransition(1, 0, 'b');+ expected.finishState();+ Automaton actual = Operations.repeat(Automata.makeString("ab"));++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }++ public void testRepeatOptionalTwoChar() {+ Automaton expected = Operations.repeat(Automata.makeString("ab"));+ Automaton actual = Operations.repeat(expected);++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }++ public void testRepeatConcatenation() {+ Automaton expected = new Automaton();+ expected.createState();+ expected.createState();+ expected.createState();+ expected.setAccept(0, true);+ expected.addTransition(0, 1, 'a');+ expected.addTransition(0, 0, 'c');+ expected.finishState();+ expected.addTransition(1, 2, 'b');+ expected.finishState();+ expected.addTransition(2, 1, 'a');+ expected.addTransition(2, 0, 'c');+ expected.finishState();++ Automaton abs = Operations.repeat(Automata.makeString("ab"));+ Automaton absThenC = Operations.concatenate(List.of(abs, Automata.makeChar('c')));+ Automaton actual = Operations.repeat(absThenC);++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }+ public void testRepeatOptionalConcatenation() {+ Automaton abs = Operations.repeat(Automata.makeString("ab"));Automaton absThenC = Operations.concatenate(List.of(abs, Automata.makeChar('c')));- Automaton absThenCs = new Automaton();- absThenCs.createState();- absThenCs.createState();- absThenCs.createState();- absThenCs.setAccept(0, true);- absThenCs.addTransition(0, 1, 'a');- absThenCs.addTransition(0, 0, 'c');- absThenCs.finishState();- absThenCs.addTransition(1, 2, 'b');- absThenCs.finishState();- absThenCs.addTransition(2, 1, 'a');- absThenCs.addTransition(2, 0, 'c');- absThenCs.finishState();- assertTrue(AutomatonTestUtil.sameLanguage(absThenCs, Operations.repeat(absThenC)));- assertSame(absThenCs, Operations.repeat(absThenCs));++ Automaton expected = Operations.repeat(absThenC);+ Automaton actual = Operations.repeat(expected);++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertSame(expected, Operations.repeat(actual));+ }++ public void testRepeatConcatenateOptional() {+ Automaton expected = new Automaton();+ expected.createState();+ expected.createState();+ expected.setAccept(0, true);+ expected.addTransition(0, 0, 'a');+ expected.addTransition(0, 1, 'a');+ expected.finishState();+ expected.addTransition(1, 0, 'b');+ expected.finishState();+ expected = Operations.determinize(expected, Integer.MAX_VALUE);Automaton aOrAb = new Automaton();aOrAb.createState();@@ -360,19 +420,10 @@ public class TestOperations extends LuceneTestCase {aOrAb.finishState();aOrAb.addTransition(1, 2, 'b');aOrAb.finishState();- Automaton aOrAbs = new Automaton();- aOrAbs.createState();- aOrAbs.createState();- aOrAbs.setAccept(0, true);- aOrAbs.addTransition(0, 0, 'a');- aOrAbs.addTransition(0, 1, 'a');- aOrAbs.finishState();- aOrAbs.addTransition(1, 0, 'b');- aOrAbs.finishState();- assertTrue(- AutomatonTestUtil.sameLanguage(- Operations.determinize(aOrAbs, Integer.MAX_VALUE),- Operations.determinize(Operations.repeat(aOrAb), Integer.MAX_VALUE)));+ Automaton actual = Operations.repeat(aOrAb);+ AutomatonTestUtil.assertMinimalDFA(actual);++ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));}public void testMergeAcceptStatesWithNoTransition() {@@ -468,22 +519,45 @@ public class TestOperations extends LuceneTestCase {}public void testOptional() {- Automaton a = Automata.makeChar('a');-- Automaton optionalA = new Automaton();- optionalA.createState();- optionalA.setAccept(0, true);- optionalA.finishState();- optionalA.createState();- optionalA.setAccept(1, true);- optionalA.addTransition(0, 1, 'a');- optionalA.finishState();-- assertTrue(AutomatonTestUtil.sameLanguage(Operations.optional(a), optionalA));- assertSame(optionalA, Operations.optional(optionalA));-- // Now test an automaton that has a transition to state 0. a(ba)*- a = new Automaton();+ Automaton expected = new Automaton();+ expected.createState();+ expected.setAccept(0, true);+ expected.finishState();+ expected.createState();+ expected.setAccept(1, true);+ expected.addTransition(0, 1, 'a');+ expected.finishState();++ Automaton actual = Operations.optional(Automata.makeChar('a'));++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }++ public void testOptionalOptional() {+ Automaton expected = Operations.optional(Automata.makeChar('a'));+ Automaton actual = Operations.optional(expected);++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }++ // test an automaton that has a transition to state 0. a(ba)*+ public void testOptionalAcceptsState0() {+ Automaton expected = new Automaton();+ expected.createState();+ expected.setAccept(0, true);+ expected.createState();+ expected.createState();+ expected.setAccept(2, true);+ expected.addTransition(0, 2, 'a');+ expected.finishState();+ expected.addTransition(1, 2, 'a');+ expected.finishState();+ expected.addTransition(2, 1, 'b');+ expected.finishState();++ Automaton a = new Automaton();a.createState();a.createState();a.setAccept(1, true);@@ -491,22 +565,26 @@ public class TestOperations extends LuceneTestCase {a.finishState();a.addTransition(1, 0, 'b');a.finishState();+ Automaton actual = Operations.optional(a);++ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));+ }- optionalA = new Automaton();- optionalA.createState();- optionalA.setAccept(0, true);- optionalA.createState();- optionalA.createState();- optionalA.setAccept(2, true);- optionalA.addTransition(0, 2, 'a');- optionalA.finishState();- optionalA.addTransition(1, 2, 'a');- optionalA.finishState();- optionalA.addTransition(2, 1, 'b');- optionalA.finishState();-- assertTrue(AutomatonTestUtil.sameLanguage(Operations.optional(a), optionalA));- assertSame(optionalA, Operations.optional(optionalA));+ public void TestOptionalOptionalAcceptsState0() {+ Automaton expected = new Automaton();+ expected.createState();+ expected.createState();+ expected.setAccept(1, true);+ expected.addTransition(0, 1, 'a');+ expected.finishState();+ expected.addTransition(1, 0, 'b');+ expected.finishState();+ expected = Operations.optional(expected);++ Automaton actual = Operations.optional(expected);+ AutomatonTestUtil.assertMinimalDFA(actual);+ assertTrue(AutomatonTestUtil.sameLanguage(expected, actual));}public void testDuelOptional() {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.javaindex f0f78064a13..8109b847d19 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java@@ -37,6 +37,7 @@ import org.apache.lucene.util.automaton.RegExp;import org.apache.lucene.util.automaton.StatePair;import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;import org.apache.lucene.util.automaton.Transition;+import org.junit.Assert;/*** Utilities for testing automata.@@ -44,7 +45,7 @@ import org.apache.lucene.util.automaton.Transition;* <p>Capable of generating random regular expressions, and automata, and also provides a number of* very basic unoptimized implementations (*slow) for testing.*/-public class AutomatonTestUtil {+public class AutomatonTestUtil extends Assert {/** Default maximum number of states that {@link Operations#determinize} should create. */public static final int DEFAULT_MAX_DETERMINIZED_STATES = 1000000;@@ -390,6 +391,27 @@ public class AutomatonTestUtil {return a;}+ /** Asserts that an automaton is a minimal DFA. */+ public static void assertMinimalDFA(Automaton automaton) {+ assertCleanDFA(automaton);+ Automaton minimized = minimizeSimple(automaton);+ assertEquals(minimized.getNumStates(), automaton.getNumStates());+ }++ /** Asserts that an automaton is a DFA with no dead states */+ public static void assertCleanDFA(Automaton automaton) {+ assertCleanNFA(automaton);+ assertTrue("must be deterministic", automaton.isDeterministic());+ }++ /** Asserts that an automaton has no dead states */+ public static void assertCleanNFA(Automaton automaton) {+ assertFalse(+ "has dead states reachable from initial", Operations.hasDeadStatesFromInitial(automaton));+ assertFalse("has dead states leading to accept", Operations.hasDeadStatesToAccept(automaton));+ assertFalse("has unreachable dead states (ghost states)", Operations.hasDeadStates(automaton));+ }+/** Simple, original brics implementation of determinize() */public static Automaton determinizeSimple(Automaton a) {Set<Integer> initialset = new HashSet<>();@@ -607,7 +629,7 @@ public class AutomatonTestUtil {assert Operations.hasDeadStatesFromInitial(a1) == false;assert Operations.hasDeadStatesFromInitial(a2) == false;if (a1.getNumStates() == 0) {- // Empty language is alwyas a subset of any other language+ // Empty language is always a subset of any other languagereturn true;} else if (a2.getNumStates() == 0) {return Operations.isEmpty(a1);