Apache Lucene open-source search software
hunspell: check for aff file wellformedness more strictly (#12468)
Peter Gromov32ec38271e6b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 7f31158c3f5..49e10862d10 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -79,6 +79,8 @@ Improvements* GITHUB#12464: Hunspell: allow customizing the hash table load factor (Peter Gromov)+* GITHUB#12468: Hunspell: check for aff file wellformedness more strictly (Peter Gromov)+Optimizations---------------------diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex ae2c7b05439..1811c3b9bc8 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -676,15 +676,27 @@ public class Dictionary {} catch (@SuppressWarnings("unused")NumberFormatException e) {- return;+ if (tolerateAffixRuleCountMismatches()) {+ return;+ }+ throw new ParseException("Affix rule header expected; got " + header, reader.getLineNumber());}affixData = ArrayUtil.grow(affixData, currentAffix * 4 + numLines * 4);for (int i = 0; i < numLines; i++) {String line = reader.readLine();+ if (line == null) {+ throw new ParseException("Premature end of rules for " + header, reader.getLineNumber());+ }+// from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);+ if (!ruleArgs[1].equals(args[1])) {+ throw new ParseException(+ "Affix rule mismatch. Header: " + header + "; rule: " + line, reader.getLineNumber());+ }+char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];String affixArg = ruleArgs[3];@@ -1215,6 +1227,14 @@ public class Dictionary {return 1.0;}+ /**+ * Whether incorrect PFX/SFX rule counts should be silently ignored. False by default: a {@link+ * ParseException} will happen.+ */+ protected boolean tolerateAffixRuleCountMismatches() {+ return false;+ }+char[] allNonSuggestibleFlags() {return Dictionary.toSortedCharArray(Stream.of(HIDDEN_FLAG, noSuggest, forbiddenword, onlyincompound, subStandard)diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.javaindex 5c9db6fbdfb..708a52b6ed6 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java@@ -74,7 +74,12 @@ public class TestAllDictionaries extends LuceneTestCase {try (InputStream dictionary = Files.newInputStream(dic);InputStream affix = Files.newInputStream(aff);BaseDirectoryWrapper tempDir = newDirectory()) {- return new Dictionary(tempDir, "dictionary", affix, dictionary);+ return new Dictionary(tempDir, "dictionary", affix, dictionary) {+ @Override+ protected boolean tolerateAffixRuleCountMismatches() {+ return true;+ }+ };}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.javaindex a7335a0c3f5..f08fbfbd2d6 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java@@ -149,6 +149,19 @@ public class TestDictionary extends LuceneTestCase {expectThrows(ParseException.class, () -> loadDictionary("broken.aff", "simple.dic"));assertTrue(expected.getMessage().startsWith("Invalid syntax"));assertEquals(24, expected.getErrorOffset());++ List<String> names =+ List.of(+ "broken_missingAffRule.aff",+ "broken_extraAffRule.aff",+ "broken_extraAffRule_last.aff",+ "broken_extraAffRule_beforeAnother.aff",+ "broken_mismatchedAffix.aff");++ for (String name : names) {+ String msg = "Expected ParseException on " + name;+ expectThrows(ParseException.class, msg, () -> loadDictionary(name, "simple.dic"));+ }}public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseException {@@ -166,12 +179,13 @@ public class TestDictionary extends LuceneTestCase {}public void testForgivableErrors() throws Exception {- Dictionary dictionary = loadDictionary("forgivable-errors.aff", "forgivable-errors.dic");+ Dictionary dictionary =+ loadForgivingDictionary("forgivable-errors.aff", "forgivable-errors.dic");assertEquals(1, dictionary.repTable.size());assertEquals(2, dictionary.compoundMax);- loadDictionary("forgivable-errors-long.aff", "single-word.dic");- loadDictionary("forgivable-errors-num.aff", "single-word.dic");+ loadForgivingDictionary("forgivable-errors-long.aff", "single-word.dic");+ loadForgivingDictionary("forgivable-errors-num.aff", "single-word.dic");}private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {@@ -182,6 +196,20 @@ public class TestDictionary extends LuceneTestCase {}}+ private Dictionary loadForgivingDictionary(String aff, String dic)+ throws IOException, ParseException {+ try (InputStream affixStream = getClass().getResourceAsStream(aff);+ InputStream dicStream = getClass().getResourceAsStream(dic);+ Directory tempDir = getDirectory()) {+ return new Dictionary(tempDir, "dictionary", affixStream, dicStream) {+ @Override+ protected boolean tolerateAffixRuleCountMismatches() {+ return true;+ }+ };+ }+ }+private static class CloseCheckInputStream extends FilterInputStream {private boolean closed = false;diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.affnew file mode 100644index 00000000000..ca3efc0b502--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.aff@@ -0,0 +1,8 @@+SFX A Y 2+SFX A 0 e n+SFX A 0 e t+SFX A 0 e h++SFX C Y 2+SFX C 0 d/C c+SFX C 0 c bdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.affnew file mode 100644index 00000000000..8254f253e04--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.aff@@ -0,0 +1,7 @@++SFX A Y 4+SFX A 0 e n+SFX A 0 e t+SFX A 0 e h++ONLYINCOMPOUND o\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.affnew file mode 100644index 00000000000..a925d5a76ad--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.aff@@ -0,0 +1,5 @@++SFX A Y 4+SFX A 0 e n+SFX A 0 e t+SFX A 0 e hdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.affnew file mode 100644index 00000000000..e33653fbae1--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.aff@@ -0,0 +1,5 @@++SFX A Y 3+SFX A 0 e n+SFX B 0 e t+SFX A 0 e h\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.affnew file mode 100644index 00000000000..db5d6391ecc--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.aff@@ -0,0 +1,8 @@+SFX A Y 4+SFX A 0 e n+SFX A 0 e t+SFX A 0 e h++SFX C Y 2+SFX C 0 d/C c+SFX C 0 c b