Apache Lucene open-source search software
Fix Operations.reverse() to not add non-deterministic dead states (#14212)
Robert Muirad7ff1febb01
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.javaindex 34a18446191..cd50c359a37 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java@@ -129,8 +129,14 @@ public class Automaton implements Accountable, TransitionAccessor {return transitions;}- /** Returns accept states. If the bit is set then that state is an accept state. */- BitSet getAcceptStates() {+ /**+ * Returns accept states. If the bit is set then that state is an accept state.+ *+ * <p>expert: Use {@link #isAccept(int)} instead, unless you really need to scan bits.+ *+ * @lucene.internal This method signature may change in the future+ */+ public BitSet getAcceptStates() {return isAccept;}diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.javaindex 2a1a0086fdc..3a87fe40733 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java@@ -37,7 +37,6 @@ import java.util.Collection;import java.util.HashMap;import java.util.List;import java.util.Map;-import java.util.Set;import org.apache.lucene.internal.hppc.BitMixer;import org.apache.lucene.internal.hppc.IntCursor;import org.apache.lucene.internal.hppc.IntHashSet;@@ -979,7 +978,7 @@ public final class Operations {private static BitSet getLiveStatesToAccept(Automaton a) {Automaton.Builder builder = new Automaton.Builder();- // NOTE: not quite the same thing as what SpecialOperations.reverse does:+ // NOTE: not quite the same thing as what reverse() does:Transition t = new Transition();int numStates = a.getNumStates();for (int s = 0; s < numStates; s++) {@@ -1187,7 +1186,7 @@ public final class Operations {*/public static BytesRef getCommonSuffixBytesRef(Automaton a) {// reverse the language of the automaton, then reverse its common prefix.- Automaton r = removeDeadStates(reverse(a));+ Automaton r = reverse(a);BytesRef ref = getCommonPrefixBytesRef(r);reverseBytes(ref);return ref;@@ -1205,12 +1204,6 @@ public final class Operations {/** Returns an automaton accepting the reverse language. */public static Automaton reverse(Automaton a) {- return reverse(a, null);- }-- /** Reverses the automaton, returning the new initial states. */- public static Automaton reverse(Automaton a, Set<Integer> initialStates) {-if (Operations.isEmpty(a)) {return new Automaton();}@@ -1246,15 +1239,12 @@ public final class Operations {BitSet acceptStates = a.getAcceptStates();while (s < numStates && (s = acceptStates.nextSetBit(s)) != -1) {result.addEpsilon(0, s + 1);- if (initialStates != null) {- initialStates.add(s + 1);- }s++;}result.finishState();- return result;+ return removeDeadStates(result);}/**diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.javaindex b3adaad65d1..6f9c1aa04b6 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java@@ -143,32 +143,6 @@ public class TestAutomaton extends LuceneTestCase {assertEquals("boo", Operations.getCommonPrefix(a));}- public void testCommonPrefixDeadStates() throws Exception {- Automaton a =- Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));- // reverse it twice, to create some dead states- // TODO: is it possible to fix reverse() to not create dead states?!- Automaton withDeadStates = Operations.reverse(Operations.reverse(a));- IllegalArgumentException expected =- expectThrows(- IllegalArgumentException.class,- () -> {- Operations.getCommonPrefix(withDeadStates);- });- assertEquals("input automaton has dead states", expected.getMessage());- }-- public void testCommonPrefixRemoveDeadStates() throws Exception {- Automaton a =- Operations.concatenate(List.of(Automata.makeAnyString(), Automata.makeString("boo")));- // reverse it twice, to create some dead states- // TODO: is it possible to fix reverse() to not create dead states?!- Automaton withDeadStates = Operations.reverse(Operations.reverse(a));- // now remove the deadstates- Automaton withoutDeadStates = Operations.removeDeadStates(withDeadStates);- assertEquals("", Operations.getCommonPrefix(withoutDeadStates));- }-public void testCommonPrefixOptional() throws Exception {Automaton a = new Automaton();int init = a.createState();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.javaindex 686c12323a4..f0f78064a13 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/automaton/AutomatonTestUtil.java@@ -329,12 +329,64 @@ public class AutomatonTestUtil {* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.*/+ /**+ * Original brics implementation of reverse(). It tries to satisfy multiple use-cases by+ * populating a set of initial states too.+ */+ public static Automaton reverseOriginal(Automaton a, Set<Integer> initialStates) {++ if (Operations.isEmpty(a)) {+ return new Automaton();+ }++ int numStates = a.getNumStates();++ // Build a new automaton with all edges reversed+ Automaton.Builder builder = new Automaton.Builder();++ // Initial node; we'll add epsilon transitions in the end:+ builder.createState();++ for (int s = 0; s < numStates; s++) {+ builder.createState();+ }++ // Old initial state becomes new accept state:+ builder.setAccept(1, true);++ Transition t = new Transition();+ for (int s = 0; s < numStates; s++) {+ int numTransitions = a.getNumTransitions(s);+ a.initTransition(s, t);+ for (int i = 0; i < numTransitions; i++) {+ a.getNextTransition(t);+ builder.addTransition(t.dest + 1, s + 1, t.min, t.max);+ }+ }++ Automaton result = builder.finish();++ int s = 0;+ BitSet acceptStates = a.getAcceptStates();+ while (s < numStates && (s = acceptStates.nextSetBit(s)) != -1) {+ result.addEpsilon(0, s + 1);+ if (initialStates != null) {+ initialStates.add(s + 1);+ }+ s++;+ }++ result.finishState();++ return result;+ }+/** Simple, original brics implementation of Brzozowski minimize() */public static Automaton minimizeSimple(Automaton a) {Set<Integer> initialSet = new HashSet<Integer>();- a = determinizeSimple(Operations.reverse(a, initialSet), initialSet);+ a = determinizeSimple(reverseOriginal(a, initialSet), initialSet);initialSet.clear();- a = determinizeSimple(Operations.reverse(a, initialSet), initialSet);+ a = determinizeSimple(reverseOriginal(a, initialSet), initialSet);return a;}