Skip to content

Apache Lucene open-source search software

hunspell: check for aff file wellformedness more strictly (#12468)

Peter Gromov32ec38271e6b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7f31158c3f5..49e10862d10 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -79,6 +79,8 @@ Improvements
* GITHUB#12464: Hunspell: allow customizing the hash table load factor (Peter Gromov)
+* GITHUB#12468: Hunspell: check for aff file wellformedness more strictly (Peter Gromov)
+
Optimizations
---------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index ae2c7b05439..1811c3b9bc8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -676,15 +676,27 @@ public class Dictionary {
} catch (
@SuppressWarnings("unused")
NumberFormatException e) {
- return;
+ if (tolerateAffixRuleCountMismatches()) {
+ return;
+ }
+ throw new ParseException("Affix rule header expected; got " + header, reader.getLineNumber());
}
affixData = ArrayUtil.grow(affixData, currentAffix * 4 + numLines * 4);
for (int i = 0; i < numLines; i++) {
String line = reader.readLine();
+ if (line == null) {
+ throw new ParseException("Premature end of rules for " + header, reader.getLineNumber());
+ }
+
// from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]
String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);
+ if (!ruleArgs[1].equals(args[1])) {
+ throw new ParseException(
+ "Affix rule mismatch. Header: " + header + "; rule: " + line, reader.getLineNumber());
+ }
+
char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);
String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];
String affixArg = ruleArgs[3];
@@ -1215,6 +1227,14 @@ public class Dictionary {
return 1.0;
}
+ /**
+ * Whether incorrect PFX/SFX rule counts should be silently ignored. False by default: a {@link
+ * ParseException} will happen.
+ */
+ protected boolean tolerateAffixRuleCountMismatches() {
+ return false;
+ }
+
char[] allNonSuggestibleFlags() {
return Dictionary.toSortedCharArray(
Stream.of(HIDDEN_FLAG, noSuggest, forbiddenword, onlyincompound, subStandard)
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
index 5c9db6fbdfb..708a52b6ed6 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
@@ -74,7 +74,12 @@ public class TestAllDictionaries extends LuceneTestCase {
try (InputStream dictionary = Files.newInputStream(dic);
InputStream affix = Files.newInputStream(aff);
BaseDirectoryWrapper tempDir = newDirectory()) {
- return new Dictionary(tempDir, "dictionary", affix, dictionary);
+ return new Dictionary(tempDir, "dictionary", affix, dictionary) {
+ @Override
+ protected boolean tolerateAffixRuleCountMismatches() {
+ return true;
+ }
+ };
}
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index a7335a0c3f5..f08fbfbd2d6 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -149,6 +149,19 @@ public class TestDictionary extends LuceneTestCase {
expectThrows(ParseException.class, () -> loadDictionary("broken.aff", "simple.dic"));
assertTrue(expected.getMessage().startsWith("Invalid syntax"));
assertEquals(24, expected.getErrorOffset());
+
+ List<String> names =
+ List.of(
+ "broken_missingAffRule.aff",
+ "broken_extraAffRule.aff",
+ "broken_extraAffRule_last.aff",
+ "broken_extraAffRule_beforeAnother.aff",
+ "broken_mismatchedAffix.aff");
+
+ for (String name : names) {
+ String msg = "Expected ParseException on " + name;
+ expectThrows(ParseException.class, msg, () -> loadDictionary(name, "simple.dic"));
+ }
}
public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseException {
@@ -166,12 +179,13 @@ public class TestDictionary extends LuceneTestCase {
}
public void testForgivableErrors() throws Exception {
- Dictionary dictionary = loadDictionary("forgivable-errors.aff", "forgivable-errors.dic");
+ Dictionary dictionary =
+ loadForgivingDictionary("forgivable-errors.aff", "forgivable-errors.dic");
assertEquals(1, dictionary.repTable.size());
assertEquals(2, dictionary.compoundMax);
- loadDictionary("forgivable-errors-long.aff", "single-word.dic");
- loadDictionary("forgivable-errors-num.aff", "single-word.dic");
+ loadForgivingDictionary("forgivable-errors-long.aff", "single-word.dic");
+ loadForgivingDictionary("forgivable-errors-num.aff", "single-word.dic");
}
private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {
@@ -182,6 +196,20 @@ public class TestDictionary extends LuceneTestCase {
}
}
+ private Dictionary loadForgivingDictionary(String aff, String dic)
+ throws IOException, ParseException {
+ try (InputStream affixStream = getClass().getResourceAsStream(aff);
+ InputStream dicStream = getClass().getResourceAsStream(dic);
+ Directory tempDir = getDirectory()) {
+ return new Dictionary(tempDir, "dictionary", affixStream, dicStream) {
+ @Override
+ protected boolean tolerateAffixRuleCountMismatches() {
+ return true;
+ }
+ };
+ }
+ }
+
private static class CloseCheckInputStream extends FilterInputStream {
private boolean closed = false;
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.aff
new file mode 100644
index 00000000000..ca3efc0b502
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule.aff
@@ -0,0 +1,8 @@
+SFX A Y 2
+SFX A 0 e n
+SFX A 0 e t
+SFX A 0 e h
+
+SFX C Y 2
+SFX C 0 d/C c
+SFX C 0 c b
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.aff
new file mode 100644
index 00000000000..8254f253e04
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_beforeAnother.aff
@@ -0,0 +1,7 @@
+
+SFX A Y 4
+SFX A 0 e n
+SFX A 0 e t
+SFX A 0 e h
+
+ONLYINCOMPOUND o
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.aff
new file mode 100644
index 00000000000..a925d5a76ad
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_extraAffRule_last.aff
@@ -0,0 +1,5 @@
+
+SFX A Y 4
+SFX A 0 e n
+SFX A 0 e t
+SFX A 0 e h
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.aff
new file mode 100644
index 00000000000..e33653fbae1
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_mismatchedAffix.aff
@@ -0,0 +1,5 @@
+
+SFX A Y 3
+SFX A 0 e n
+SFX B 0 e t
+SFX A 0 e h
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.aff
new file mode 100644
index 00000000000..db5d6391ecc
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken_missingAffRule.aff
@@ -0,0 +1,8 @@
+SFX A Y 4
+SFX A 0 e n
+SFX A 0 e t
+SFX A 0 e h
+
+SFX C Y 2
+SFX C 0 d/C c
+SFX C 0 c b