Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9724: Hunspell: tolerate existing aff/dic file typos (#2307)

Peter Gromov1cc26b6bb448
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index d71e714876a..ae1a3a1cef4 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -376,7 +376,7 @@ public class Dictionary {
Arrays.sort(ignore);
needsInputCleaning = true;
} else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {
- int num = Integer.parseInt(singleArgument(reader, line));
+ int num = parseNum(reader, line);
FST<CharsRef> res = parseConversions(reader, num);
if (line.startsWith("I")) {
iconv = res;
@@ -397,9 +397,9 @@ public class Dictionary {
} else if ("TRY".equals(firstWord)) {
tryChars = singleArgument(reader, line);
} else if ("REP".equals(firstWord)) {
- int count = Integer.parseInt(singleArgument(reader, line));
+ int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
- String[] parts = splitBySpace(reader, reader.readLine(), 3);
+ String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);
repTable.add(new RepEntry(parts[1], parts[2]));
}
} else if ("KEY".equals(firstWord)) {
@@ -409,11 +409,11 @@ public class Dictionary {
} else if ("FORBIDDENWORD".equals(firstWord)) {
forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDMIN".equals(firstWord)) {
- compoundMin = Math.max(1, Integer.parseInt(singleArgument(reader, line)));
+ compoundMin = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDWORDMAX".equals(firstWord)) {
- compoundMax = Math.max(1, Integer.parseInt(singleArgument(reader, line)));
+ compoundMax = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDRULE".equals(firstWord)) {
- compoundRules = parseCompoundRules(reader, Integer.parseInt(singleArgument(reader, line)));
+ compoundRules = parseCompoundRules(reader, parseNum(reader, line));
} else if ("COMPOUNDFLAG".equals(firstWord)) {
compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDBEGIN".equals(firstWord)) {
@@ -437,7 +437,7 @@ public class Dictionary {
} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {
simplifiedTriple = true;
} else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {
- int count = Integer.parseInt(singleArgument(reader, line));
+ int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
checkCompoundPatterns.add(
new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));
@@ -481,16 +481,24 @@ public class Dictionary {
return underscore < 0 ? isoCode : isoCode.substring(0, underscore);
}
+ private int parseNum(LineNumberReader reader, String line) throws ParseException {
+ return Integer.parseInt(splitBySpace(reader, line, 2, Integer.MAX_VALUE)[1]);
+ }
+
private String singleArgument(LineNumberReader reader, String line) throws ParseException {
return splitBySpace(reader, line, 2)[1];
}
private String[] splitBySpace(LineNumberReader reader, String line, int expectedParts)
throws ParseException {
+ return splitBySpace(reader, line, expectedParts, expectedParts);
+ }
+
+ private String[] splitBySpace(LineNumberReader reader, String line, int minParts, int maxParts)
+ throws ParseException {
String[] parts = line.split("\\s+");
- if (parts.length < expectedParts
- || parts.length > expectedParts && !parts[expectedParts].startsWith("#")) {
- throw new ParseException("Invalid syntax", reader.getLineNumber());
+ if (parts.length < minParts || parts.length > maxParts && !parts[maxParts].startsWith("#")) {
+ throw new ParseException("Invalid syntax: " + line, reader.getLineNumber());
}
return parts;
}
@@ -509,7 +517,7 @@ public class Dictionary {
Set<String> starting = new LinkedHashSet<>();
Set<String> ending = new LinkedHashSet<>();
Set<String> middle = new LinkedHashSet<>();
- int num = Integer.parseInt(singleArgument(reader, line));
+ int num = parseNum(reader, line);
for (int i = 0; i < num; i++) {
String breakStr = singleArgument(reader, reader.readLine());
if (breakStr.startsWith("^")) {
@@ -590,15 +598,8 @@ public class Dictionary {
for (int i = 0; i < numLines; i++) {
String line = reader.readLine();
- String[] ruleArgs = line.split("\\s+");
-
// from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]
- // condition is optional
- if (ruleArgs.length < 4) {
- throw new ParseException(
- "The affix file contains a rule with less than four elements: " + line,
- reader.getLineNumber());
- }
+ String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);
char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);
String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];
@@ -654,9 +655,11 @@ public class Dictionary {
"Too many patterns, please report this to [email protected]");
}
seenPatterns.put(regex, patternIndex);
- CharacterRunAutomaton pattern =
- new CharacterRunAutomaton(new RegExp(regex, RegExp.NONE).toAutomaton());
- patterns.add(pattern);
+ try {
+ patterns.add(new CharacterRunAutomaton(conditionRegexp(regex).toAutomaton()));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("On line " + reader.getLineNumber() + ": " + line, e);
+ }
}
Integer stripOrd = seenStrips.get(strip);
@@ -706,6 +709,17 @@ public class Dictionary {
}
}
+ private static RegExp conditionRegexp(String regex) {
+ try {
+ return new RegExp(regex, RegExp.NONE);
+ } catch (IllegalArgumentException e) {
+ if (e.getMessage().contains("expected ']'")) {
+ return conditionRegexp(regex + "]");
+ }
+ throw e;
+ }
+ }
+
char affixData(int affixIndex, int offset) {
return affixData[affixIndex * 4 + offset];
}
@@ -752,6 +766,8 @@ public class Dictionary {
LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, streamCharset));
String line;
while ((line = reader.readLine()) != null) {
+ if (line.isBlank()) continue;
+
String firstWord = line.split("\\s")[0];
if ("SET".equals(firstWord)) {
decoder = getDecoder(singleArgument(reader, line));
@@ -767,11 +783,12 @@ public class Dictionary {
*
* @return {@code true} if the sequence matched and has been consumed.
*/
+ @SuppressWarnings("SameParameterValue")
private static boolean maybeConsume(BufferedInputStream stream, byte[] bytes) throws IOException {
stream.mark(bytes.length);
- for (int i = 0; i < bytes.length; i++) {
+ for (byte b : bytes) {
int nextByte = stream.read();
- if (nextByte != (bytes[i] & 0xff)) { // covers EOF (-1) as well.
+ if (nextByte != (b & 0xff)) { // covers EOF (-1) as well.
stream.reset();
return false;
}
@@ -1344,6 +1361,9 @@ public class Dictionary {
/** Abstraction of the process of parsing flags taken from the affix and dic files */
abstract static class FlagParsingStrategy {
+ // we don't check the flag count, as Hunspell accepts longer sequences
+ // https://github.com/hunspell/hunspell/issues/707
+ static final boolean checkFlags = false;
/**
* Parses the given String into a single flag
@@ -1353,7 +1373,7 @@ public class Dictionary {
*/
char parseFlag(String rawFlag) {
char[] flags = parseFlags(rawFlag);
- if (flags.length != 1) {
+ if (checkFlags && flags.length != 1) {
throw new IllegalArgumentException("expected only one flag, got: " + rawFlag);
}
return flags[0];
@@ -1406,7 +1426,8 @@ public class Dictionary {
continue;
}
int flag = Integer.parseInt(replacement);
- if (flag == FLAG_UNSET || flag >= Character.MAX_VALUE) { // read default flags as well
+ if (flag >= Character.MAX_VALUE) { // read default flags as well
+ // accept 0 due to https://github.com/hunspell/hunspell/issues/708
throw new IllegalArgumentException(
"Num flags should be between 0 and " + DEFAULT_FLAGS + ", found " + flag);
}
@@ -1428,28 +1449,21 @@ public class Dictionary {
@Override
public char[] parseFlags(String rawFlags) {
- if (rawFlags.length() == 0) {
- return new char[0];
- }
-
- StringBuilder builder = new StringBuilder();
- if (rawFlags.length() % 2 == 1) {
+ if (checkFlags && rawFlags.length() % 2 == 1) {
throw new IllegalArgumentException(
"Invalid flags (should be even number of characters): " + rawFlags);
}
- for (int i = 0; i < rawFlags.length(); i += 2) {
- char f1 = rawFlags.charAt(i);
- char f2 = rawFlags.charAt(i + 1);
+
+ char[] flags = new char[rawFlags.length() / 2];
+ for (int i = 0; i < flags.length; i++) {
+ char f1 = rawFlags.charAt(i * 2);
+ char f2 = rawFlags.charAt(i * 2 + 1);
if (f1 >= 256 || f2 >= 256) {
throw new IllegalArgumentException(
"Invalid flags (LONG flags must be double ASCII): " + rawFlags);
}
- char combined = (char) (f1 << 8 | f2);
- builder.append(combined);
+ flags[i] = (char) (f1 << 8 | f2);
}
-
- char[] flags = new char[builder.length()];
- builder.getChars(0, builder.length(), flags, 0);
return flags;
}
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index 289f4ed81ab..8c4bc305026 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -39,11 +39,7 @@ import org.junit.Test;
public class TestDictionary extends LuceneTestCase {
public void testSimpleDictionary() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("simple.aff");
- InputStream dictStream = getClass().getResourceAsStream("simple.dic");
- Directory tempDir = getDirectory();
-
- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
+ Dictionary dictionary = loadDictionary("simple.aff", "simple.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
@@ -60,85 +56,44 @@ public class TestDictionary extends LuceneTestCase {
assertEquals(1, ordList.length);
flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);
-
- affixStream.close();
- dictStream.close();
- tempDir.close();
}
public void testCompressedDictionary() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("compressed.aff");
- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
-
- Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
+ Dictionary dictionary = loadDictionary("compressed.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);
-
- affixStream.close();
- dictStream.close();
- tempDir.close();
}
public void testCompressedBeforeSetDictionary() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
- Directory tempDir = getDirectory();
-
- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
+ Dictionary dictionary = loadDictionary("compressed-before-set.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);
-
- affixStream.close();
- dictStream.close();
- tempDir.close();
}
public void testCompressedEmptyAliasDictionary() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
- Directory tempDir = getDirectory();
-
- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
+ Dictionary dictionary = loadDictionary("compressed-empty-alias.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);
-
- affixStream.close();
- dictStream.close();
- tempDir.close();
}
// malformed rule causes ParseException
- public void testInvalidData() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("broken.aff");
- InputStream dictStream = getClass().getResourceAsStream("simple.dic");
- Directory tempDir = getDirectory();
-
+ public void testInvalidData() {
ParseException expected =
- expectThrows(
- ParseException.class,
- () -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));
- assertTrue(
- expected
- .getMessage()
- .startsWith("The affix file contains a rule with less than four elements"));
+ expectThrows(ParseException.class, () -> loadDictionary("broken.aff", "simple.dic"));
+ assertTrue(expected.getMessage().startsWith("Invalid syntax"));
assertEquals(24, expected.getErrorOffset());
-
- affixStream.close();
- dictStream.close();
- tempDir.close();
}
public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseException {
@@ -155,20 +110,21 @@ public class TestDictionary extends LuceneTestCase {
assertEquals(42, dictionary.keepcase);
}
- // malformed flags causes ParseException
- public void testInvalidFlags() throws Exception {
- InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");
- InputStream dictStream = getClass().getResourceAsStream("simple.dic");
- Directory tempDir = getDirectory();
+ public void testForgivableErrors() throws Exception {
+ Dictionary dictionary = loadDictionary("forgivable-errors.aff", "simple.dic");
+ assertEquals(1, dictionary.repTable.size());
+ assertEquals(2, dictionary.compoundMax);
- Exception expected =
- expectThrows(
- Exception.class, () -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));
- assertTrue(expected.getMessage().startsWith("expected only one flag"));
+ loadDictionary("forgivable-errors-long.aff", "single-word.dic");
+ loadDictionary("forgivable-errors-num.aff", "single-word.dic");
+ }
- affixStream.close();
- dictStream.close();
- tempDir.close();
+ private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {
+ try (InputStream affixStream = getClass().getResourceAsStream(aff);
+ InputStream dicStream = getClass().getResourceAsStream(dic);
+ Directory tempDir = getDirectory()) {
+ return new Dictionary(tempDir, "dictionary", affixStream, dicStream);
+ }
}
private static class CloseCheckInputStream extends FilterInputStream {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff
deleted file mode 100644
index 0c189c845ce..00000000000
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff
+++ /dev/null
@@ -1,21 +0,0 @@
-SET UTF-8
-TRY abcdefghijklmopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ
-
-SFX A Y 3
-SFX A 0 e n
-SFX A 0 e t
-SFX A 0 e h
-
-SFX C Y 2
-SFX C 0 d/C c
-SFX C 0 c b
-
-SFX D Y 1
-SFX D 0 s o
-
-SFX E Y 1
-SFX E 0 d o
-
-# broken, the flag has too much in it
-PFX B0 Y 1
-PFX B0 0 s o
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff
new file mode 100644
index 00000000000..e3e57332af5
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff
@@ -0,0 +1,4 @@
+FLAG long
+
+SFX A10 Y 1
+SFX A10 nout l .
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff
new file mode 100644
index 00000000000..172c7a7127f
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff
@@ -0,0 +1,4 @@
+FLAG num
+
+SFX 0 Y 1
+SFX 0 nout l .
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.aff
new file mode 100644
index 00000000000..8d17b4e8e82
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.aff
@@ -0,0 +1,9 @@
+REP 1
+REP foo bar goo doo zoo
+
+COMPOUNDWORDMAX 2 y
+
+KEEPCASE Aa
+
+SFX A Y 1
+SFX A nout l [aeiouyáéíóúýůěr][^aeiouyáéíóúýůěrl][^aeiouy
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dic
new file mode 100644
index 00000000000..856497b4efa
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dic
@@ -0,0 +1,2 @@
+1
+foo
\ No newline at end of file