Apache Lucene open-source search software
hunspell: allow in-memory entry sorting for faster dictionary loading (#12834)
Peter Gromovf460d612b5e4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 19d5c4949c8..8c85e5bce33 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -88,7 +88,8 @@ Optimizations* GITHUB#11857, GITHUB#11859, GITHUB#11893, GITHUB#11909: Hunspell: improved suggestion performance (Peter Gromov)-* GITHUB#12825: Hunspell: improved dictionary loading performance (Peter Gromov)+* GITHUB#12825, GITHUB#12834: Hunspell: improved dictionary loading performance, allowed in-memory entry sorting.+ (Peter Gromov)* GITHUB#12372: Reduce allocation during HNSW construction (Jonathan Ellis)diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex d453404bb54..4768cced9ab 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -50,19 +50,12 @@ import java.util.Set;import java.util.TreeMap;import java.util.stream.Collectors;import java.util.stream.Stream;-import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.analysis.hunspell.SortingStrategy.EntryAccumulator;+import org.apache.lucene.analysis.hunspell.SortingStrategy.EntrySupplier;import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefComparator;-import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.IntsRef;import org.apache.lucene.util.IntsRefBuilder;-import org.apache.lucene.util.OfflineSorter;-import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;-import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;import org.apache.lucene.util.fst.FST;import org.apache.lucene.util.fst.FSTCompiler;import org.apache.lucene.util.fst.IntSequenceOutputs;@@ -216,6 +209,25 @@ public class Dictionary {List<InputStream> dictionaries,boolean ignoreCase)throws IOException, ParseException {+ this(affix, dictionaries, ignoreCase, SortingStrategy.offline(tempDir, tempFileNamePrefix));+ }++ /**+ * Creates a new Dictionary containing the information read from the provided InputStreams to+ * hunspell affix and dictionary files. You have to close the provided InputStreams yourself.+ *+ * @param affix InputStream for reading the hunspell affix file (won't be closed).+ * @param dictionaries InputStream for reading the hunspell dictionary files (won't be closed).+ * @param sortingStrategy the entry strategy for the dictionary loading+ * @throws IOException Can be thrown while reading from the InputStreams+ * @throws ParseException Can be thrown if the content of the files does not meet expected formats+ */+ public Dictionary(+ InputStream affix,+ List<InputStream> dictionaries,+ boolean ignoreCase,+ SortingStrategy sortingStrategy)+ throws IOException, ParseException {this.ignoreCase = ignoreCase;try (BufferedInputStream affixStream =@@ -251,10 +263,11 @@ public class Dictionary {readAffixFile(affixStream, decoder, flagEnumerator);// read dictionary entries- IndexOutput unsorted = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT);- int wordCount = mergeDictionaries(dictionaries, decoder, unsorted);- String sortedFile = sortWordsOffline(tempDir, tempFileNamePrefix, unsorted);- words = readSortedDictionaries(tempDir, sortedFile, flagEnumerator, wordCount);+ EntryAccumulator acc = sortingStrategy.start();+ mergeDictionaries(dictionaries, decoder, acc);+ try (EntrySupplier sorted = acc.finishAndSort()) {+ words = readSortedDictionaries(flagEnumerator, sorted);+ }flagLookup = flagEnumerator.finish();aliases = null; // no longer neededmorphAliases = null; // no longer needed@@ -986,52 +999,43 @@ public class Dictionary {}}- private int mergeDictionaries(- List<InputStream> dictionaries, CharsetDecoder decoder, IndexOutput output)+ private void mergeDictionaries(+ List<InputStream> dictionaries, CharsetDecoder decoder, EntryAccumulator acc)throws IOException {StringBuilder sb = new StringBuilder();- int wordCount = 0;- try (ByteSequencesWriter writer = new ByteSequencesWriter(output)) {- for (InputStream dictionary : dictionaries) {- BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));- lines.readLine(); // first line is number of entries (approximately, sometimes)-- String line;- while ((line = lines.readLine()) != null) {- // wild and unpredictable code comment rules- if (line.isEmpty() || line.charAt(0) == '#' || line.charAt(0) == '\t') {- continue;- }- line = unescapeEntry(line);- // if we haven't seen any custom morphological data, try to parse one- if (!hasCustomMorphData) {- int morphStart = line.indexOf(MORPH_SEPARATOR);- if (morphStart >= 0) {- String data = line.substring(morphStart + 1);- hasCustomMorphData =- splitMorphData(data).stream().anyMatch(s -> !s.startsWith("ph:"));- }+ for (InputStream dictionary : dictionaries) {+ BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));+ lines.readLine(); // first line is number of entries (approximately, sometimes)++ String line;+ while ((line = lines.readLine()) != null) {+ // wild and unpredictable code comment rules+ if (line.isEmpty() || line.charAt(0) == '#' || line.charAt(0) == '\t') {+ continue;+ }+ line = unescapeEntry(line);+ // if we haven't seen any custom morphological data, try to parse one+ if (!hasCustomMorphData) {+ int morphStart = line.indexOf(MORPH_SEPARATOR);+ if (morphStart >= 0) {+ String data = line.substring(morphStart + 1);+ hasCustomMorphData = splitMorphData(data).stream().anyMatch(s -> !s.startsWith("ph:"));}-- wordCount += writeNormalizedWordEntry(sb, writer, line);}++ writeNormalizedWordEntry(sb, line, acc);}- CodecUtil.writeFooter(output);}- return wordCount;}- /**- * @return the number of word entries written- */- private int writeNormalizedWordEntry(StringBuilder reuse, ByteSequencesWriter writer, String line)+ private void writeNormalizedWordEntry(StringBuilder reuse, String line, EntryAccumulator acc)throws IOException {int flagSep = line.indexOf(FLAG_SEPARATOR);int morphSep = line.indexOf(MORPH_SEPARATOR);assert morphSep > 0;assert morphSep > flagSep;int sep = flagSep < 0 ? morphSep : flagSep;- if (sep == 0) return 0;+ if (sep == 0) return;CharSequence toWrite;String beforeSep = line.substring(0, sep);@@ -1045,19 +1049,16 @@ public class Dictionary {String written = toWrite.toString();sep = written.length() - (line.length() - sep);- writer.write(written.getBytes(StandardCharsets.UTF_8));+ acc.addEntry(written);WordCase wordCase = WordCase.caseOf(written, sep);if (wordCase == WordCase.MIXED || wordCase == WordCase.UPPER && flagSep > 0) {- addHiddenCapitalizedWord(reuse, writer, written.substring(0, sep), written.substring(sep));- return 2;+ addHiddenCapitalizedWord(reuse, acc, written.substring(0, sep), written.substring(sep));}- return 1;}private void addHiddenCapitalizedWord(- StringBuilder reuse, ByteSequencesWriter writer, String word, String afterSep)- throws IOException {+ StringBuilder reuse, EntryAccumulator acc, String word, String afterSep) throws IOException {reuse.setLength(0);reuse.append(Character.toUpperCase(word.charAt(0)));for (int i = 1; i < word.length(); i++) {@@ -1066,7 +1067,7 @@ public class Dictionary {reuse.append(FLAG_SEPARATOR);reuse.append(HIDDEN_FLAG);reuse.append(afterSep, afterSep.charAt(0) == FLAG_SEPARATOR ? 1 : 0, afterSep.length());- writer.write(reuse.toString().getBytes(StandardCharsets.UTF_8));+ acc.addEntry(reuse.toString());}String toLowerCase(String word) {@@ -1086,102 +1087,66 @@ public class Dictionary {return new String(chars);}- private String sortWordsOffline(- Directory tempDir, String tempFileNamePrefix, IndexOutput unsorted) throws IOException {- var sorter = new OfflineSorter(tempDir, tempFileNamePrefix, BytesRefComparator.NATURAL);-- String sorted;- boolean success = false;- try {- sorted = sorter.sort(unsorted.getName());- success = true;- } finally {- if (success) {- tempDir.deleteFile(unsorted.getName());- } else {- IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());- }- }- return sorted;- }-- private WordStorage readSortedDictionaries(- Directory tempDir, String sorted, FlagEnumerator flags, int wordCount) throws IOException {- boolean success = false;-+ private WordStorage readSortedDictionaries(FlagEnumerator flags, EntrySupplier sorted)+ throws IOException {Map<String, Integer> morphIndices = new HashMap<>();WordStorage.Builder builder =new WordStorage.Builder(- wordCount, hashFactor(), hasCustomMorphData, flags, allNonSuggestibleFlags());+ sorted.wordCount(), hashFactor(), hasCustomMorphData, flags, allNonSuggestibleFlags());- try (ByteSequencesReader reader =- new ByteSequencesReader(tempDir.openChecksumInput(sorted), sorted)) {+ // TODO: the flags themselves can be double-chars (long) or also numeric+ // either way the trick is to encode them as char... but they must be parsed differently- // TODO: the flags themselves can be double-chars (long) or also numeric- // either way the trick is to encode them as char... but they must be parsed differently+ while (true) {+ String line = sorted.next();+ if (line == null) break;- while (true) {- BytesRef scratch = reader.next();- if (scratch == null) {- break;- }-- String line = scratch.utf8ToString();- String entry;- char[] wordForm;- int end;-- int flagSep = line.indexOf(FLAG_SEPARATOR);- if (flagSep == -1) {- wordForm = NOFLAGS;- end = line.indexOf(MORPH_SEPARATOR);- entry = line.substring(0, end);- } else {- end = line.indexOf(MORPH_SEPARATOR);- boolean hidden = line.charAt(flagSep + 1) == HIDDEN_FLAG;- String flagPart = line.substring(flagSep + (hidden ? 2 : 1), end).strip();- if (aliasCount > 0 && !flagPart.isEmpty()) {- flagPart = getAliasValue(Integer.parseInt(flagPart));- }+ String entry;+ char[] wordForm;+ int end;- wordForm = flagParsingStrategy.parseFlags(flagPart);- if (hidden) {- wordForm = ArrayUtil.growExact(wordForm, wordForm.length + 1);- wordForm[wordForm.length - 1] = HIDDEN_FLAG;- }- entry = line.substring(0, flagSep);+ int flagSep = line.indexOf(FLAG_SEPARATOR);+ if (flagSep == -1) {+ wordForm = NOFLAGS;+ end = line.indexOf(MORPH_SEPARATOR);+ entry = line.substring(0, end);+ } else {+ end = line.indexOf(MORPH_SEPARATOR);+ boolean hidden = line.charAt(flagSep + 1) == HIDDEN_FLAG;+ String flagPart = line.substring(flagSep + (hidden ? 2 : 1), end).strip();+ if (aliasCount > 0 && !flagPart.isEmpty()) {+ flagPart = getAliasValue(Integer.parseInt(flagPart));}- if (entry.isEmpty()) continue;-- int morphDataID = 0;- if (end + 1 < line.length()) {- List<String> morphFields = readMorphFields(entry, line.substring(end + 1));- if (!morphFields.isEmpty()) {- morphFields.sort(Comparator.naturalOrder());- morphDataID = addMorphFields(morphIndices, String.join(" ", morphFields));- }+ wordForm = flagParsingStrategy.parseFlags(flagPart);+ if (hidden) {+ wordForm = ArrayUtil.growExact(wordForm, wordForm.length + 1);+ wordForm[wordForm.length - 1] = HIDDEN_FLAG;}-- builder.add(entry, wordForm, morphDataID);+ entry = line.substring(0, flagSep);}- // finalize last entry- success = true;- return new WordStorage(builder) {- @Override- char caseFold(char c) {- return Dictionary.this.caseFold(c);+ if (entry.isEmpty()) continue;++ int morphDataID = 0;+ if (end + 1 < line.length()) {+ List<String> morphFields = readMorphFields(entry, line.substring(end + 1));+ if (!morphFields.isEmpty()) {+ morphFields.sort(Comparator.naturalOrder());+ morphDataID = addMorphFields(morphIndices, String.join(" ", morphFields));}- };- } finally {- if (success) {- tempDir.deleteFile(sorted);- } else {- IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);}++ builder.add(entry, wordForm, morphDataID);}++ return new WordStorage(builder) {+ @Override+ char caseFold(char c) {+ return Dictionary.this.caseFold(c);+ }+ };}/**diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SortingStrategy.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SortingStrategy.javanew file mode 100644index 00000000000..e1c59020d44--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SortingStrategy.java@@ -0,0 +1,181 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.hunspell;++import java.io.Closeable;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.List;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.BytesRefComparator;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.OfflineSorter;+import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;+import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;++/**+ * The strategy defining how a Hunspell dictionary should be loaded, with different tradeoffs. The+ * entries should be sorted in a special way, and this can be done either in-memory (faster, but+ * temporarily allocating more memory) or using disk (slower, but not needing much memory).+ *+ * @see #offline(Directory, String)+ * @see #inMemory()+ */+public abstract class SortingStrategy {++ abstract EntryAccumulator start() throws IOException;++ interface EntryAccumulator {++ void addEntry(String entry) throws IOException;++ EntrySupplier finishAndSort() throws IOException;+ }++ interface EntrySupplier extends Closeable {+ int wordCount();++ /** The next line or {@code null} if the end is reached */+ String next() throws IOException;+ }++ /**+ * An "offline" strategy that creates temporary files in the given directory and uses them for+ * sorting with {@link OfflineSorter}. It's slower than {@link #inMemory()}, but doesn't need to+ * load the entire dictionary into memory.+ */+ public static SortingStrategy offline(Directory tempDir, String tempFileNamePrefix) {+ return new SortingStrategy() {+ @Override+ EntryAccumulator start() throws IOException {+ IndexOutput output = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT);+ ByteSequencesWriter writer = new ByteSequencesWriter(output);+ return new EntryAccumulator() {+ int wordCount = 0;++ @Override+ public void addEntry(String entry) throws IOException {+ wordCount++;+ writer.write(entry.getBytes(StandardCharsets.UTF_8));+ }++ @Override+ public EntrySupplier finishAndSort() throws IOException {+ CodecUtil.writeFooter(output);+ writer.close();+ String sortedFile = sortWordsOffline();+ ByteSequencesReader reader =+ new ByteSequencesReader(tempDir.openChecksumInput(sortedFile), sortedFile);+ return new EntrySupplier() {+ boolean success = false;++ @Override+ public int wordCount() {+ return wordCount;+ }++ @Override+ public String next() throws IOException {+ BytesRef scratch = reader.next();+ if (scratch == null) {+ success = true;+ return null;+ }+ return scratch.utf8ToString();+ }++ @Override+ public void close() throws IOException {+ reader.close();+ if (success) {+ tempDir.deleteFile(sortedFile);+ } else {+ IOUtils.deleteFilesIgnoringExceptions(tempDir, sortedFile);+ }+ }+ };+ }++ private String sortWordsOffline() throws IOException {+ var sorter = new OfflineSorter(tempDir, tempFileNamePrefix, BytesRefComparator.NATURAL);++ String sorted;+ boolean success = false;+ try {+ sorted = sorter.sort(output.getName());+ success = true;+ } finally {+ if (success) {+ tempDir.deleteFile(output.getName());+ } else {+ IOUtils.deleteFilesIgnoringExceptions(tempDir, output.getName());+ }+ }+ return sorted;+ }+ };+ }+ };+ }++ /**+ * The strategy that loads all entries as {@link String} objects and sorts them in memory. The+ * entries are then stored in a more compressed way, and the strings are gc-ed, but the loading+ * itself needs {@code O(dictionary_size)} memory.+ */+ public static SortingStrategy inMemory() {+ return new SortingStrategy() {+ @Override+ EntryAccumulator start() {+ List<String> entries = new ArrayList<>();+ return new EntryAccumulator() {+ @Override+ public void addEntry(String entry) {+ entries.add(entry);+ }++ @Override+ public EntrySupplier finishAndSort() {+ entries.sort(Comparator.naturalOrder());+ return new EntrySupplier() {+ int i = 0;++ @Override+ public int wordCount() {+ return entries.size();+ }++ @Override+ public String next() {+ return i < entries.size() ? entries.get(i++) : null;+ }++ @Override+ public void close() {}+ };+ }+ };+ }+ };+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.javaindex 708a52b6ed6..68f6922c758 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java@@ -41,7 +41,6 @@ import java.util.concurrent.atomic.AtomicLong;import java.util.function.Function;import java.util.stream.Collectors;import java.util.stream.Stream;-import org.apache.lucene.tests.store.BaseDirectoryWrapper;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;import org.apache.lucene.tests.util.RamUsageTester;@@ -72,9 +71,8 @@ public class TestAllDictionaries extends LuceneTestCase {Path dic = Path.of(affPath.substring(0, affPath.length() - 4) + ".dic");assert Files.exists(dic) : dic;try (InputStream dictionary = Files.newInputStream(dic);- InputStream affix = Files.newInputStream(aff);- BaseDirectoryWrapper tempDir = newDirectory()) {- return new Dictionary(tempDir, "dictionary", affix, dictionary) {+ InputStream affix = Files.newInputStream(aff)) {+ return new Dictionary(affix, List.of(dictionary), false, SortingStrategy.inMemory()) {@Overrideprotected boolean tolerateAffixRuleCountMismatches() {return true;diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.javaindex cbb6f21f0cf..4ca262799c6 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java@@ -256,15 +256,22 @@ public class TestSpellChecking extends LuceneTestCase {}static void checkSpellCheckerExpectations(Path basePath) throws IOException, ParseException {- InputStream affixStream = Files.newInputStream(Path.of(basePath.toString() + ".aff"));+ checkSpellCheckerExpectations(+ basePath, SortingStrategy.offline(new ByteBuffersDirectory(), "dictionary"));+ checkSpellCheckerExpectations(basePath, SortingStrategy.inMemory());+ }++ private static void checkSpellCheckerExpectations(Path basePath, SortingStrategy strategy)+ throws IOException, ParseException {+ Path affFile = Path.of(basePath + ".aff");Path dicFile = Path.of(basePath + ".dic");+ InputStream affixStream = Files.newInputStream(affFile);InputStream dictStream = Files.newInputStream(dicFile);Hunspell speller;Map<String, Suggester> suggesters = new LinkedHashMap<>();try {- Dictionary dictionary =- new Dictionary(new ByteBuffersDirectory(), "dictionary", affixStream, dictStream);+ Dictionary dictionary = new Dictionary(affixStream, List.of(dictStream), false, strategy);speller = new Hunspell(dictionary, TimeoutPolicy.NO_TIMEOUT, () -> {});Suggester suggester = new Suggester(dictionary);suggesters.put("default", suggester);