Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9708: Hunspell: support FLAG UTF-8 in absence of SET UTF-8 (#2270)

Peter Gromov7a7949aed2a1
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 9de67eb2f18..58b6f220f32 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -83,7 +83,7 @@ public class Dictionary {
// TODO: really for suffixes we should reverse the automaton and run them backwards
private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";
private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";
- static final String DEFAULT_ENCODING = StandardCharsets.ISO_8859_1.name();
+ static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
FST<IntsRef> prefixes;
FST<IntsRef> suffixes;
@@ -347,7 +347,7 @@ public class Dictionary {
} else if ("FLAG".equals(firstWord)) {
// Assume that the FLAG line comes before any prefix or suffixes
// Store the strategy so it can be used when parsing the dic file
- flagParsingStrategy = getFlagParsingStrategy(line);
+ flagParsingStrategy = getFlagParsingStrategy(line, decoder.charset());
} else if (line.equals("COMPLEXPREFIXES")) {
complexPrefixes =
true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
@@ -725,7 +725,7 @@ public class Dictionary {
// this test only at the end as ineffective but would allow lines only containing spaces:
encoding.toString().trim().length() == 0) {
if (ch < 0) {
- return DEFAULT_ENCODING;
+ return DEFAULT_CHARSET.name();
}
continue;
}
@@ -734,7 +734,7 @@ public class Dictionary {
int last = matcher.end();
return encoding.substring(last).trim();
}
- return DEFAULT_ENCODING;
+ return DEFAULT_CHARSET.name();
}
}
@@ -768,7 +768,7 @@ public class Dictionary {
* @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG
* definition
*/
- static FlagParsingStrategy getFlagParsingStrategy(String flagLine) {
+ static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset charset) {
String[] parts = flagLine.split("\\s+");
if (parts.length != 2) {
throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine);
@@ -778,6 +778,9 @@ public class Dictionary {
if ("num".equals(flagType)) {
return new NumFlagParsingStrategy();
} else if ("UTF-8".equals(flagType)) {
+ if (DEFAULT_CHARSET.equals(charset)) {
+ return new DefaultAsUtf8FlagParsingStrategy();
+ }
return new SimpleFlagParsingStrategy();
} else if ("long".equals(flagType)) {
return new DoubleASCIIFlagParsingStrategy();
@@ -1271,6 +1274,19 @@ public class Dictionary {
}
}
+ /** Used to read flags as UTF-8 even if the rest of the file is in the default (8-bit) encoding */
+ private static class DefaultAsUtf8FlagParsingStrategy extends FlagParsingStrategy {
+ @Override
+ public char[] parseFlags(String rawFlags) {
+ return new String(rawFlags.getBytes(DEFAULT_CHARSET), StandardCharsets.UTF_8).toCharArray();
+ }
+
+ @Override
+ void appendFlag(char flag, StringBuilder to) {
+ to.append(new String(String.valueOf(flag).getBytes(StandardCharsets.UTF_8), DEFAULT_CHARSET));
+ }
+ }
+
/**
* Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its
* numerical form. In the case of multiple flags, each number is separated by a comma.
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index a0ece7809e3..faee6f06347 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Random;
@@ -261,13 +262,24 @@ public class TestDictionary extends LuceneTestCase {
Dictionary.getDictionaryEncoding(
new ByteArrayInputStream("\uFEFFSET\tUTF-8\r\n".getBytes(StandardCharsets.UTF_8))));
assertEquals(
- Dictionary.DEFAULT_ENCODING,
+ Dictionary.DEFAULT_CHARSET.name(),
Dictionary.getDictionaryEncoding(new ByteArrayInputStream(new byte[0])));
}
public void testFlagWithCrazyWhitespace() {
- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
- assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", StandardCharsets.UTF_8));
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8", StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testUtf8Flag() {
+ Dictionary.FlagParsingStrategy strategy =
+ Dictionary.getFlagParsingStrategy("FLAG\tUTF-8", Dictionary.DEFAULT_CHARSET);
+
+ String src = "привет";
+ String asAscii = new String(src.getBytes(StandardCharsets.UTF_8), Dictionary.DEFAULT_CHARSET);
+ assertNotEquals(src, asAscii);
+ assertEquals(src, new String(strategy.parseFlags(asAscii)));
}
@Test
@@ -275,19 +287,23 @@ public class TestDictionary extends LuceneTestCase {
Random r = random();
char[] flags = new char[r.nextInt(10)];
for (int i = 0; i < flags.length; i++) {
- flags[i] = (char) r.nextInt(Character.MAX_VALUE);
+ flags[i] = (char) r.nextInt(Character.MIN_SURROGATE);
}
String[] flagLines = {"FLAG long", "FLAG UTF-8", "FLAG num"};
+ Charset[] charsets = {StandardCharsets.UTF_8, Dictionary.DEFAULT_CHARSET};
for (String flagLine : flagLines) {
- Dictionary.FlagParsingStrategy strategy = Dictionary.getFlagParsingStrategy(flagLine);
- StringBuilder serialized = new StringBuilder();
- for (char flag : flags) {
- strategy.appendFlag(flag, serialized);
+ for (Charset charset : charsets) {
+ Dictionary.FlagParsingStrategy strategy =
+ Dictionary.getFlagParsingStrategy(flagLine, charset);
+ StringBuilder serialized = new StringBuilder();
+ for (char flag : flags) {
+ strategy.appendFlag(flag, serialized);
+ }
+
+ char[] deserialized = strategy.parseFlags(serialized.toString());
+ assertEquals(new String(flags), new String(deserialized));
}
-
- char[] deserialized = strategy.parseFlags(serialized.toString());
- assertEquals(new String(flags), new String(deserialized));
}
}