Apache Solr open-source search software
LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)
Peter Gromov7a7949aed2a1
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex 9de67eb2f18..58b6f220f32 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -83,7 +83,7 @@ public class Dictionary {// TODO: really for suffixes we should reverse the automaton and run them backwardsprivate static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";- static final String DEFAULT_ENCODING = StandardCharsets.ISO_8859_1.name();+ static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;FST<IntsRef> prefixes;FST<IntsRef> suffixes;@@ -347,7 +347,7 @@ public class Dictionary {} else if ("FLAG".equals(firstWord)) {// Assume that the FLAG line comes before any prefix or suffixes// Store the strategy so it can be used when parsing the dic file- flagParsingStrategy = getFlagParsingStrategy(line);+ flagParsingStrategy = getFlagParsingStrategy(line, decoder.charset());} else if (line.equals("COMPLEXPREFIXES")) {complexPrefixes =true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix@@ -725,7 +725,7 @@ public class Dictionary {// this test only at the end as ineffective but would allow lines only containing spaces:encoding.toString().trim().length() == 0) {if (ch < 0) {- return DEFAULT_ENCODING;+ return DEFAULT_CHARSET.name();}continue;}@@ -734,7 +734,7 @@ public class Dictionary {int last = matcher.end();return encoding.substring(last).trim();}- return DEFAULT_ENCODING;+ return DEFAULT_CHARSET.name();}}@@ -768,7 +768,7 @@ public class Dictionary {* @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG* definition*/- static FlagParsingStrategy getFlagParsingStrategy(String flagLine) {+ static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset charset) {String[] parts = flagLine.split("\\s+");if (parts.length != 2) {throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine);@@ -778,6 +778,9 @@ public class Dictionary {if ("num".equals(flagType)) {return new NumFlagParsingStrategy();} else if ("UTF-8".equals(flagType)) {+ if (DEFAULT_CHARSET.equals(charset)) {+ return new DefaultAsUtf8FlagParsingStrategy();+ }return new SimpleFlagParsingStrategy();} else if ("long".equals(flagType)) {return new DoubleASCIIFlagParsingStrategy();@@ -1271,6 +1274,19 @@ public class Dictionary {}}+ /** Used to read flags as UTF-8 even if the rest of the file is in the default (8-bit) encoding */+ private static class DefaultAsUtf8FlagParsingStrategy extends FlagParsingStrategy {+ @Override+ public char[] parseFlags(String rawFlags) {+ return new String(rawFlags.getBytes(DEFAULT_CHARSET), StandardCharsets.UTF_8).toCharArray();+ }++ @Override+ void appendFlag(char flag, StringBuilder to) {+ to.append(new String(String.valueOf(flag).getBytes(StandardCharsets.UTF_8), DEFAULT_CHARSET));+ }+ }+/*** Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its* numerical form. In the case of multiple flags, each number is separated by a comma.diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.javaindex a0ece7809e3..faee6f06347 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java@@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream;import java.io.FilterInputStream;import java.io.IOException;import java.io.InputStream;+import java.nio.charset.Charset;import java.nio.charset.StandardCharsets;import java.text.ParseException;import java.util.Random;@@ -261,13 +262,24 @@ public class TestDictionary extends LuceneTestCase {Dictionary.getDictionaryEncoding(new ByteArrayInputStream("\uFEFFSET\tUTF-8\r\n".getBytes(StandardCharsets.UTF_8))));assertEquals(- Dictionary.DEFAULT_ENCODING,+ Dictionary.DEFAULT_CHARSET.name(),Dictionary.getDictionaryEncoding(new ByteArrayInputStream(new byte[0])));}public void testFlagWithCrazyWhitespace() {- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", StandardCharsets.UTF_8));+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8", StandardCharsets.UTF_8));+ }++ @Test+ public void testUtf8Flag() {+ Dictionary.FlagParsingStrategy strategy =+ Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", Dictionary.DEFAULT_CHARSET);++ String src = "привет";+ String asAscii = new String(src.getBytes(StandardCharsets.UTF_8), Dictionary.DEFAULT_CHARSET);+ assertNotEquals(src, asAscii);+ assertEquals(src, new String(strategy.parseFlags(asAscii)));}@Test@@ -275,19 +287,23 @@ public class TestDictionary extends LuceneTestCase {Random r = random();char[] flags = new char[r.nextInt(10)];for (int i = 0; i < flags.length; i++) {- flags[i] = (char) r.nextInt(Character.MAX_VALUE);+ flags[i] = (char) r.nextInt(Character.MIN_SURROGATE);}String[] flagLines = {"FLAG long", "FLAG UTF-8", "FLAG num"};+ Charset[] charsets = {StandardCharsets.UTF_8, Dictionary.DEFAULT_CHARSET};for (String flagLine : flagLines) {- Dictionary.FlagParsingStrategy strategy = Dictionary.getFlagParsingStrategy(flagLine);- StringBuilder serialized = new StringBuilder();- for (char flag : flags) {- strategy.appendFlag(flag, serialized);+ for (Charset charset : charsets) {+ Dictionary.FlagParsingStrategy strategy =+ Dictionary.getFlagParsingStrategy(flagLine, charset);+ StringBuilder serialized = new StringBuilder();+ for (char flag : flags) {+ strategy.appendFlag(flag, serialized);+ }++ char[] deserialized = strategy.parseFlags(serialized.toString());+ assertEquals(new String(flags), new String(deserialized));}-- char[] deserialized = strategy.parseFlags(serialized.toString());- assertEquals(new String(flags), new String(deserialized));}}