Apache Solr open-source search software
LUCENE-9724: Hunspell: tolerate existing aff/dic file typos (#2307)
Peter Gromov1cc26b6bb448
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex d71e714876a..ae1a3a1cef4 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -376,7 +376,7 @@ public class Dictionary {Arrays.sort(ignore);needsInputCleaning = true;} else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {- int num = Integer.parseInt(singleArgument(reader, line));+ int num = parseNum(reader, line);FST<CharsRef> res = parseConversions(reader, num);if (line.startsWith("I")) {iconv = res;@@ -397,9 +397,9 @@ public class Dictionary {} else if ("TRY".equals(firstWord)) {tryChars = singleArgument(reader, line);} else if ("REP".equals(firstWord)) {- int count = Integer.parseInt(singleArgument(reader, line));+ int count = parseNum(reader, line);for (int i = 0; i < count; i++) {- String[] parts = splitBySpace(reader, reader.readLine(), 3);+ String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);repTable.add(new RepEntry(parts[1], parts[2]));}} else if ("KEY".equals(firstWord)) {@@ -409,11 +409,11 @@ public class Dictionary {} else if ("FORBIDDENWORD".equals(firstWord)) {forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));} else if ("COMPOUNDMIN".equals(firstWord)) {- compoundMin = Math.max(1, Integer.parseInt(singleArgument(reader, line)));+ compoundMin = Math.max(1, parseNum(reader, line));} else if ("COMPOUNDWORDMAX".equals(firstWord)) {- compoundMax = Math.max(1, Integer.parseInt(singleArgument(reader, line)));+ compoundMax = Math.max(1, parseNum(reader, line));} else if ("COMPOUNDRULE".equals(firstWord)) {- compoundRules = parseCompoundRules(reader, Integer.parseInt(singleArgument(reader, line)));+ compoundRules = parseCompoundRules(reader, parseNum(reader, line));} else if ("COMPOUNDFLAG".equals(firstWord)) {compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));} else if ("COMPOUNDBEGIN".equals(firstWord)) {@@ -437,7 +437,7 @@ public class Dictionary {} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {simplifiedTriple = true;} else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {- int count = Integer.parseInt(singleArgument(reader, line));+ int count = parseNum(reader, line);for (int i = 0; i < count; i++) {checkCompoundPatterns.add(new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));@@ -481,16 +481,24 @@ public class Dictionary {return underscore < 0 ? isoCode : isoCode.substring(0, underscore);}+ private int parseNum(LineNumberReader reader, String line) throws ParseException {+ return Integer.parseInt(splitBySpace(reader, line, 2, Integer.MAX_VALUE)[1]);+ }+private String singleArgument(LineNumberReader reader, String line) throws ParseException {return splitBySpace(reader, line, 2)[1];}private String[] splitBySpace(LineNumberReader reader, String line, int expectedParts)throws ParseException {+ return splitBySpace(reader, line, expectedParts, expectedParts);+ }++ private String[] splitBySpace(LineNumberReader reader, String line, int minParts, int maxParts)+ throws ParseException {String[] parts = line.split("\\s+");- if (parts.length < expectedParts- || parts.length > expectedParts && !parts[expectedParts].startsWith("#")) {- throw new ParseException("Invalid syntax", reader.getLineNumber());+ if (parts.length < minParts || parts.length > maxParts && !parts[maxParts].startsWith("#")) {+ throw new ParseException("Invalid syntax: " + line, reader.getLineNumber());}return parts;}@@ -509,7 +517,7 @@ public class Dictionary {Set<String> starting = new LinkedHashSet<>();Set<String> ending = new LinkedHashSet<>();Set<String> middle = new LinkedHashSet<>();- int num = Integer.parseInt(singleArgument(reader, line));+ int num = parseNum(reader, line);for (int i = 0; i < num; i++) {String breakStr = singleArgument(reader, reader.readLine());if (breakStr.startsWith("^")) {@@ -590,15 +598,8 @@ public class Dictionary {for (int i = 0; i < numLines; i++) {String line = reader.readLine();- String[] ruleArgs = line.split("\\s+");-// from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]- // condition is optional- if (ruleArgs.length < 4) {- throw new ParseException(- "The affix file contains a rule with less than four elements: " + line,- reader.getLineNumber());- }+ String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];@@ -654,9 +655,11 @@ public class Dictionary {"Too many patterns, please report this to [email protected]");}seenPatterns.put(regex, patternIndex);- CharacterRunAutomaton pattern =- new CharacterRunAutomaton(new RegExp(regex, RegExp.NONE).toAutomaton());- patterns.add(pattern);+ try {+ patterns.add(new CharacterRunAutomaton(conditionRegexp(regex).toAutomaton()));+ } catch (IllegalArgumentException e) {+ throw new IllegalArgumentException("On line " + reader.getLineNumber() + ": " + line, e);+ }}Integer stripOrd = seenStrips.get(strip);@@ -706,6 +709,17 @@ public class Dictionary {}}+ private static RegExp conditionRegexp(String regex) {+ try {+ return new RegExp(regex, RegExp.NONE);+ } catch (IllegalArgumentException e) {+ if (e.getMessage().contains("expected ']'")) {+ return conditionRegexp(regex + "]");+ }+ throw e;+ }+ }+char affixData(int affixIndex, int offset) {return affixData[affixIndex * 4 + offset];}@@ -752,6 +766,8 @@ public class Dictionary {LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, streamCharset));String line;while ((line = reader.readLine()) != null) {+ if (line.isBlank()) continue;+String firstWord = line.split("\\s")[0];if ("SET".equals(firstWord)) {decoder = getDecoder(singleArgument(reader, line));@@ -767,11 +783,12 @@ public class Dictionary {** @return {@code true} if the sequence matched and has been consumed.*/+ @SuppressWarnings("SameParameterValue")private static boolean maybeConsume(BufferedInputStream stream, byte[] bytes) throws IOException {stream.mark(bytes.length);- for (int i = 0; i < bytes.length; i++) {+ for (byte b : bytes) {int nextByte = stream.read();- if (nextByte != (bytes[i] & 0xff)) { // covers EOF (-1) as well.+ if (nextByte != (b & 0xff)) { // covers EOF (-1) as well.stream.reset();return false;}@@ -1344,6 +1361,9 @@ public class Dictionary {/** Abstraction of the process of parsing flags taken from the affix and dic files */abstract static class FlagParsingStrategy {+ // we don't check the flag count, as Hunspell accepts longer sequences+ // https://github.com/hunspell/hunspell/issues/707+ static final boolean checkFlags = false;/*** Parses the given String into a single flag@@ -1353,7 +1373,7 @@ public class Dictionary {*/char parseFlag(String rawFlag) {char[] flags = parseFlags(rawFlag);- if (flags.length != 1) {+ if (checkFlags && flags.length != 1) {throw new IllegalArgumentException("expected only one flag, got: " + rawFlag);}return flags[0];@@ -1406,7 +1426,8 @@ public class Dictionary {continue;}int flag = Integer.parseInt(replacement);- if (flag == FLAG_UNSET || flag >= Character.MAX_VALUE) { // read default flags as well+ if (flag >= Character.MAX_VALUE) { // read default flags as well+ // accept 0 due to https://github.com/hunspell/hunspell/issues/708throw new IllegalArgumentException("Num flags should be between 0 and " + DEFAULT_FLAGS + ", found " + flag);}@@ -1428,28 +1449,21 @@ public class Dictionary {@Overridepublic char[] parseFlags(String rawFlags) {- if (rawFlags.length() == 0) {- return new char[0];- }-- StringBuilder builder = new StringBuilder();- if (rawFlags.length() % 2 == 1) {+ if (checkFlags && rawFlags.length() % 2 == 1) {throw new IllegalArgumentException("Invalid flags (should be even number of characters): " + rawFlags);}- for (int i = 0; i < rawFlags.length(); i += 2) {- char f1 = rawFlags.charAt(i);- char f2 = rawFlags.charAt(i + 1);++ char[] flags = new char[rawFlags.length() / 2];+ for (int i = 0; i < flags.length; i++) {+ char f1 = rawFlags.charAt(i * 2);+ char f2 = rawFlags.charAt(i * 2 + 1);if (f1 >= 256 || f2 >= 256) {throw new IllegalArgumentException("Invalid flags (LONG flags must be double ASCII): " + rawFlags);}- char combined = (char) (f1 << 8 | f2);- builder.append(combined);+ flags[i] = (char) (f1 << 8 | f2);}-- char[] flags = new char[builder.length()];- builder.getChars(0, builder.length(), flags, 0);return flags;}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.javaindex 289f4ed81ab..8c4bc305026 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java@@ -39,11 +39,7 @@ import org.junit.Test;public class TestDictionary extends LuceneTestCase {public void testSimpleDictionary() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("simple.aff");- InputStream dictStream = getClass().getResourceAsStream("simple.dic");- Directory tempDir = getDirectory();-- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);+ Dictionary dictionary = loadDictionary("simple.aff", "simple.dic");assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);@@ -60,85 +56,44 @@ public class TestDictionary extends LuceneTestCase {assertEquals(1, ordList.length);flags = dictionary.decodeFlags(ordList.ints[0], ref);assertEquals(1, flags.length);-- affixStream.close();- dictStream.close();- tempDir.close();}public void testCompressedDictionary() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("compressed.aff");- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");-- Directory tempDir = getDirectory();- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);+ Dictionary dictionary = loadDictionary("compressed.aff", "compressed.dic");assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);BytesRef ref = new BytesRef();char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);assertEquals(1, flags.length);-- affixStream.close();- dictStream.close();- tempDir.close();}public void testCompressedBeforeSetDictionary() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");- Directory tempDir = getDirectory();-- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);+ Dictionary dictionary = loadDictionary("compressed-before-set.aff", "compressed.dic");assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);BytesRef ref = new BytesRef();char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);assertEquals(1, flags.length);-- affixStream.close();- dictStream.close();- tempDir.close();}public void testCompressedEmptyAliasDictionary() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");- InputStream dictStream = getClass().getResourceAsStream("compressed.dic");- Directory tempDir = getDirectory();-- Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);+ Dictionary dictionary = loadDictionary("compressed-empty-alias.aff", "compressed.dic");assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);BytesRef ref = new BytesRef();char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);assertEquals(1, flags.length);-- affixStream.close();- dictStream.close();- tempDir.close();}// malformed rule causes ParseException- public void testInvalidData() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("broken.aff");- InputStream dictStream = getClass().getResourceAsStream("simple.dic");- Directory tempDir = getDirectory();-+ public void testInvalidData() {ParseException expected =- expectThrows(- ParseException.class,- () -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));- assertTrue(- expected- .getMessage()- .startsWith("The affix file contains a rule with less than four elements"));+ expectThrows(ParseException.class, () -> loadDictionary("broken.aff", "simple.dic"));+ assertTrue(expected.getMessage().startsWith("Invalid syntax"));assertEquals(24, expected.getErrorOffset());-- affixStream.close();- dictStream.close();- tempDir.close();}public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseException {@@ -155,20 +110,21 @@ public class TestDictionary extends LuceneTestCase {assertEquals(42, dictionary.keepcase);}- // malformed flags causes ParseException- public void testInvalidFlags() throws Exception {- InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");- InputStream dictStream = getClass().getResourceAsStream("simple.dic");- Directory tempDir = getDirectory();+ public void testForgivableErrors() throws Exception {+ Dictionary dictionary = loadDictionary("forgivable-errors.aff", "simple.dic");+ assertEquals(1, dictionary.repTable.size());+ assertEquals(2, dictionary.compoundMax);- Exception expected =- expectThrows(- Exception.class, () -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));- assertTrue(expected.getMessage().startsWith("expected only one flag"));+ loadDictionary("forgivable-errors-long.aff", "single-word.dic");+ loadDictionary("forgivable-errors-num.aff", "single-word.dic");+ }- affixStream.close();- dictStream.close();- tempDir.close();+ private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {+ try (InputStream affixStream = getClass().getResourceAsStream(aff);+ InputStream dicStream = getClass().getResourceAsStream(dic);+ Directory tempDir = getDirectory()) {+ return new Dictionary(tempDir, "dictionary", affixStream, dicStream);+ }}private static class CloseCheckInputStream extends FilterInputStream {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.affdeleted file mode 100644index 0c189c845ce..00000000000--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff+++ /dev/null@@ -1,21 +0,0 @@-SET UTF-8-TRY abcdefghijklmopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ--SFX A Y 3-SFX A 0 e n-SFX A 0 e t-SFX A 0 e h--SFX C Y 2-SFX C 0 d/C c-SFX C 0 c b--SFX D Y 1-SFX D 0 s o--SFX E Y 1-SFX E 0 d o--# broken, the flag has too much in it-PFX B0 Y 1-PFX B0 0 s odiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.affnew file mode 100644index 00000000000..e3e57332af5--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff@@ -0,0 +1,4 @@+FLAG long++SFX A10 Y 1+SFX A10 nout l .\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.affnew file mode 100644index 00000000000..172c7a7127f--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff@@ -0,0 +1,4 @@+FLAG num++SFX 0 Y 1+SFX 0 nout l .\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.affnew file mode 100644index 00000000000..8d17b4e8e82--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors.aff@@ -0,0 +1,9 @@+REP 1+REP foo bar goo doo zoo++COMPOUNDWORDMAX 2 y++KEEPCASE Aa++SFX A Y 1+SFX A nout l [aeiouyáéíóúýůěr][^aeiouyáéíóúýůěrl][^aeiouy\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dicnew file mode 100644index 00000000000..856497b4efa--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/single-word.dic@@ -0,0 +1,2 @@+1+foo\ No newline at end of file