Apache Solr open-source search software
LUCENE-9728: Hunspell: add a performance test (#2296)
Peter Gromov82f8d7ba1d02
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policyindex 1103dd326c2..e17af8e78b6 100644--- a/gradle/testing/randomization/policies/tests.policy+++ b/gradle/testing/randomization/policies/tests.policy@@ -94,6 +94,7 @@ grant {// Some Hunspell tests may read from external files specified in system propertiespermission java.io.FilePermission "${hunspell.repo.path}${/}-", "read";permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";+ permission java.io.FilePermission "${hunspell.corpora}${/}-", "read";};// Permissions to support ant builddiff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex ebb76ee9eab..c4d902ef425 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -109,6 +109,8 @@ public class Dictionary {char[] stripData;int[] stripOffsets;+ String wordChars = "";+// 4 chars per affix, each char representing an unsigned 2-byte integerchar[] affixData = new char[32];private int currentAffix = 0;@@ -384,11 +386,12 @@ public class Dictionary {fullStrip = true;} else if ("LANG".equals(firstWord)) {language = singleArgument(reader, line);- int underscore = language.indexOf("_");- String langCode = underscore < 0 ? language : language.substring(0, underscore);+ String langCode = extractLanguageCode(language);alternateCasing = langCode.equals("tr") || langCode.equals("az");} else if ("BREAK".equals(firstWord)) {breaks = parseBreaks(reader, line);+ } else if ("WORDCHARS".equals(firstWord)) {+ wordChars = singleArgument(reader, line);} else if ("TRY".equals(firstWord)) {tryChars = singleArgument(reader, line);} else if ("REP".equals(firstWord)) {@@ -460,6 +463,11 @@ public class Dictionary {stripOffsets[currentIndex] = currentOffset;}+ static String extractLanguageCode(String isoCode) {+ int underscore = isoCode.indexOf("_");+ return underscore < 0 ? isoCode : isoCode.substring(0, underscore);+ }+private String singleArgument(LineNumberReader reader, String line) throws ParseException {return splitBySpace(reader, line, 2)[1];}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.javaindex ad621af8961..886272c396d 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java@@ -21,8 +21,8 @@ import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Path;import java.text.ParseException;-import java.util.List;import java.util.stream.Collectors;+import java.util.stream.Stream;import org.apache.lucene.store.BaseDirectoryWrapper;import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;@@ -41,15 +41,13 @@ import org.junit.Ignore;@SuppressSysoutChecks(bugUrl = "prints important memory utilization stats per dictionary")public class TestAllDictionaries extends LuceneTestCase {- private static List<Path> findAllAffixFiles() throws IOException {+ static Stream<Path> findAllAffixFiles() throws IOException {String dicDir = System.getProperty("hunspell.dictionaries");Assume.assumeFalse("Missing -Dhunspell.dictionaries=...", dicDir == null);- return Files.walk(Path.of(dicDir), 2)- .filter(f -> f.toString().endsWith(".aff"))- .collect(Collectors.toList());+ return Files.walk(Path.of(dicDir), 2).filter(f -> f.toString().endsWith(".aff"));}- private static Dictionary loadDictionary(Path aff) throws IOException, ParseException {+ static Dictionary loadDictionary(Path aff) throws IOException, ParseException {String affPath = aff.toString();Path dic = Path.of(affPath.substring(0, affPath.length() - 4) + ".dic");assert Files.exists(dic) : dic;@@ -62,7 +60,7 @@ public class TestAllDictionaries extends LuceneTestCase {public void testDictionariesLoadSuccessfully() throws Exception {int failures = 0;- for (Path aff : findAllAffixFiles()) {+ for (Path aff : findAllAffixFiles().collect(Collectors.toList())) {try {System.out.println(aff + "\t" + memoryUsage(loadDictionary(aff)));} catch (Throwable e) {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestPerformance.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestPerformance.javanew file mode 100644index 00000000000..33da1ca0299--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestPerformance.java@@ -0,0 +1,168 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.hunspell;++import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;+import java.io.BufferedReader;+import java.io.IOException;+import java.io.InputStream;+import java.io.InputStreamReader;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.function.Consumer;+import java.util.regex.Pattern;+import org.apache.lucene.util.LuceneTestCase;+import org.junit.Assume;+import org.junit.Ignore;+import org.junit.Test;++/**+ * A test that runs various Hunspell APIs on real dictionaries and relatively large corpora for+ * specific languages and prints the execution times. The dictionaries should be set up as in {@link+ * TestAllDictionaries}, the corpora should be in files named {@code langCode.txt} (e.g. {@code+ * en.txt}) in a directory specified in {@code -Dhunspell.corpora=...}+ */+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)+@Ignore("enable manually")+public class TestPerformance extends LuceneTestCase {++ @Test+ public void en() throws Exception {+ checkPerformance("en", 500_000);+ }++ @Test+ public void de() throws Exception {+ checkPerformance("de", 100_000);+ }++ @Test+ public void fr() throws Exception {+ checkPerformance("fr", 20_000);+ }++ private void checkPerformance(String code, int wordCount) throws Exception {+ Path aff = findAffFile(code);+ Dictionary dictionary = TestAllDictionaries.loadDictionary(aff);+ System.out.println("Loaded " + aff);++ List<String> words = loadWords(code, wordCount, dictionary);++ Stemmer stemmer = new Stemmer(dictionary);+ SpellChecker speller = new SpellChecker(dictionary);+ measure(+ "Stemming " + code,+ blackHole -> {+ for (String word : words) {+ blackHole.accept(stemmer.stem(word));+ }+ });+ measure(+ "Spellchecking " + code,+ blackHole -> {+ for (String word : words) {+ blackHole.accept(speller.spell(word));+ }+ });+ System.out.println();+ }++ private Path findAffFile(String code) throws IOException {+ return TestAllDictionaries.findAllAffixFiles()+ .filter(+ path -> {+ String parentName = path.getParent().getFileName().toString();+ return code.equals(Dictionary.extractLanguageCode(parentName));+ })+ .findFirst()+ .orElseThrow(() -> new IllegalArgumentException("Cannot find aff/dic for " + code));+ }++ private List<String> loadWords(String code, int wordCount, Dictionary dictionary)+ throws IOException {+ String corpusDir = System.getProperty("hunspell.corpora");+ Assume.assumeFalse("", corpusDir == null);++ Path dataPath = Path.of(corpusDir).resolve(code + ".txt");+ List<String> words = new ArrayList<>();+ try (InputStream stream = Files.newInputStream(dataPath)) {+ BufferedReader reader =+ new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));+ while (true) {+ String line = reader.readLine();+ if (line == null) break;++ for (String token : line.split("[^a-zA-Z" + Pattern.quote(dictionary.wordChars) + "]+")) {+ String word = stripPunctuation(token);+ if (word != null) {+ words.add(word);+ if (words.size() == wordCount) {+ return words;+ }+ }+ }+ }+ }+ return words;+ }++ private void measure(String what, Iteration iteration) {+ Consumer<Object> consumer =+ o -> {+ if (o == null) {+ throw new AssertionError();+ }+ };++ // warmup+ for (int i = 0; i < 2; i++) {+ iteration.run(consumer);+ }++ List<Long> times = new ArrayList<>();+ for (int i = 0; i < 7; i++) {+ long start = System.currentTimeMillis();+ iteration.run(consumer);+ times.add(System.currentTimeMillis() - start);+ }+ System.out.println(+ what+ + ": average "+ + times.stream().mapToLong(Long::longValue).average().orElseThrow()+ + ", all times = "+ + times);+ }++ private interface Iteration {+ void run(Consumer<Object> blackHole);+ }++ static String stripPunctuation(String token) {+ int start = 0;+ int end = token.length();+ while (start < end && isPunctuation(token.charAt(start))) start++;+ while (start < end - 1 && isPunctuation(token.charAt(end - 1))) end--;+ return start < end ? token.substring(start, end) : null;+ }++ private static boolean isPunctuation(char c) {+ return ".!?,\"'’‘".indexOf(c) >= 0;+ }+}