Apache Lucene open-source search software
Add CaseFoldingFilter with flat lookup table (#16371)
Costin Leauba69bfb1ea9a
diff --git a/.gitattributes b/.gitattributesindex e7881fb2524..ff8d5fa571a 100644--- a/.gitattributes+++ b/.gitattributes@@ -23,6 +23,8 @@ lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene*/ForUtilucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene*/ForDeltaUtil.java linguist-generatedlucene/core/src/generated/jdk/*.jar linguist-generatedlucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java linguist-generated+lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java linguist-generated+lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java linguist-generatedlucene/core/src/java/org/apache/lucene/util/automaton/Lev*ParametricDescription.java linguist-generatedlucene/core/src/java/org/apache/lucene/util/packed/BulkOperation*.java linguist-generatedlucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java linguist-generateddiff --git a/gradle/regenerate/icu/GenerateCaseFolding.groovy b/gradle/regenerate/icu/GenerateCaseExpansion.groovysimilarity index 95%rename from gradle/regenerate/icu/GenerateCaseFolding.groovyrename to gradle/regenerate/icu/GenerateCaseExpansion.groovyindex 06297d94363..f4549ffdc2b 100644--- a/gradle/regenerate/icu/GenerateCaseFolding.groovy+++ b/gradle/regenerate/icu/GenerateCaseExpansion.groovy@@ -35,7 +35,7 @@ def generateSwitch() {}def code = """-// DO NOT EDIT THIS FILE! Use "gradlew generateUnicodeProps tidy" to recreate.+// DO NOT EDIT THIS FILE! Use "gradlew regenerateCaseExpansion tidy" to recreate./** Licensed to the Apache Software Foundation (ASF) under one or more@@ -62,8 +62,8 @@ import java.util.function.IntConsumer;* This file contains unicode properties used by {@code RegExp}.* The data was generated using ICU4J v${icuVersion}, unicode version: ${unicodeVersion}.*/-final class CaseFolding {- private CaseFolding() {}+final class CaseExpansion {+ private CaseExpansion() {}/*** Calls {@code fn} consumer with {@code c} itself and its {@code scf} mappings.diff --git a/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovy b/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovynew file mode 100644index 00000000000..c9de96f7b10--- /dev/null+++ b/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovy@@ -0,0 +1,87 @@+import com.ibm.icu.lang.UCharacter+import com.ibm.icu.lang.UScript+import com.ibm.icu.util.VersionInfo+import java.nio.file.Paths+import java.util.Locale++def icuVersion = VersionInfo.ICU_VERSION.toString()+def unicodeVersion = UCharacter.getUnicodeVersion().toString()++def outputFile = Paths.get(args[0])++def generateSwitchCases() {+ StringBuilder sb = new StringBuilder()+ int lastScript = -1+ int count = 0++ for (int c = 0; c <= Character.MAX_CODE_POINT; c++) {+ int icuFolded = UCharacter.foldCase(c, true)+ int jdkLower = Character.toLowerCase(c)++ if (icuFolded != jdkLower) {+ int script = UScript.getScript(c)+ String scriptName = UScript.getName(script)++ if (script != lastScript) {+ if (count > 0) {+ sb.append("\n")+ }+ sb.append("\n // ${scriptName}")+ lastScript = script+ }+ String fromChar = new String(Character.toChars(c))+ String toChar = new String(Character.toChars(icuFolded))+ sb.append(String.format(Locale.ROOT, "\n case 0x%04X:\n return 0x%04X; // %s → %s", c, icuFolded, fromChar, toChar))+ count+++ }+ }+ return [sb.toString(), count]+}++def (switchCases, caseCount) = generateSwitchCases()++def code = """\+// DO NOT EDIT THIS FILE! Use "gradlew regenerateSimpleCaseFolding tidy" to recreate.++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util;++/**+ * Switch-based Unicode simple case folding for the ${caseCount} codepoints where folding differs from {@link+ * Character#toLowerCase(int)}. Generated using ICU4J v${icuVersion}, unicode version: ${unicodeVersion}.+ *+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding+ * mappings</a>+ */+final class SimpleCaseFolding {+ private SimpleCaseFolding() {}++ /**+ * Returns the simple case fold of {@code c}, falling back to {@link Character#toLowerCase(int)}+ * for codepoints where fold equals lowercase.+ */+ static int fold(int c) {+ switch (c) {${switchCases}+ default:+ return Character.toLowerCase(c);+ }+ }+}+"""+outputFile.setText(code, "UTF-8")diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 90d5a039325..0e07088673d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -331,6 +331,12 @@ New Featuresand document length instead of corpus statistics such as document frequency. It also supports k3query-term frequency saturation. (Tianxiao Wei)+* GITHUB#16383: Add fp16 vector encoding support. (Pulkit Gupta)++* GITHUB#16371, GITHUB#14389: Add CaseFoldingFilter for Unicode simple case folding without+ requiring the ICU analysis plugin. Uses a generated switch for fold exceptions with+ Character.toLowerCase() as the default fallback. (Robert Muir, Costin Leau)+Improvements---------------------diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.javaindex 857663f8f0c..025ea08c6df 100644--- a/lucene/analysis/common/src/java/module-info.java+++ b/lucene/analysis/common/src/java/module-info.java@@ -169,6 +169,7 @@ module org.apache.lucene.analysis.common {org.apache.lucene.analysis.commongrams.CommonGramsQueryFilterFactory,org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilterFactory,org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilterFactory,+ org.apache.lucene.analysis.core.CaseFoldingFilterFactory,org.apache.lucene.analysis.core.DecimalDigitFilterFactory,org.apache.lucene.analysis.core.LowerCaseFilterFactory,org.apache.lucene.analysis.core.StopFilterFactory,diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.javanew file mode 100644index 00000000000..de2ae3f45af--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.java@@ -0,0 +1,65 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.core;++import java.util.Map;+import org.apache.lucene.analysis.CaseFoldingFilter;+import org.apache.lucene.analysis.TokenFilterFactory;+import org.apache.lucene.analysis.TokenStream;++/**+ * Factory for {@link CaseFoldingFilter}.+ *+ * <pre><code class="language-xml">+ * <fieldType name="text_fold" class="solr.TextField" positionIncrementGap="100">+ * <analyzer>+ * <tokenizer class="solr.WhitespaceTokenizerFactory"/>+ * <filter class="solr.CaseFoldingFilterFactory"/>+ * </analyzer>+ * </fieldType></code></pre>+ *+ * @since 10.6.0+ * @lucene.spi {@value #NAME}+ */+public class CaseFoldingFilterFactory extends TokenFilterFactory {++ /** SPI name */+ public static final String NAME = "caseFolding";++ /** Creates a new CaseFoldingFilterFactory */+ public CaseFoldingFilterFactory(Map<String, String> args) {+ super(args);+ if (!args.isEmpty()) {+ throw new IllegalArgumentException("Unknown parameters: " + args);+ }+ }++ /** Default ctor for compatibility with SPI */+ public CaseFoldingFilterFactory() {+ throw defaultCtorException();+ }++ @Override+ public TokenStream create(TokenStream input) {+ return new CaseFoldingFilter(input);+ }++ @Override+ public TokenStream normalize(TokenStream input) {+ return create(input);+ }+}diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactoryindex 7d7fb83d7f7..1b33f2603b3 100644--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory@@ -30,6 +30,7 @@ org.apache.lucene.analysis.commongrams.CommonGramsFilterFactoryorg.apache.lucene.analysis.commongrams.CommonGramsQueryFilterFactoryorg.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilterFactoryorg.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilterFactory+org.apache.lucene.analysis.core.CaseFoldingFilterFactoryorg.apache.lucene.analysis.core.DecimalDigitFilterFactoryorg.apache.lucene.analysis.core.LowerCaseFilterFactoryorg.apache.lucene.analysis.core.StopFilterFactorydiff --git a/lucene/analysis/icu/build.gradle b/lucene/analysis/icu/build.gradleindex 9f1b8629055..edacc185779 100644--- a/lucene/analysis/icu/build.gradle+++ b/lucene/analysis/icu/build.gradle@@ -284,10 +284,10 @@ configure(project(":lucene:analysis:common")) {configure(project(":lucene:core")) {setupIcuDependencies(project)- tasks.register("generateUnicodeProps", {+ tasks.register("regenerateCaseExpansion", {def icuConfig = configurations.icu_currentdef groovyConfig = configurations.groovy- def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")+ def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseExpansion.java")description = "Regenerate ${outputFile} using ICU"@@ -304,7 +304,34 @@ configure(project(":lucene:core")) {args = ["--encoding","UTF-8",- file("${resources}/GenerateCaseFolding.groovy"),+ file("${resources}/GenerateCaseExpansion.groovy"),+ outputFile+ ]+ })+ }+ })++ tasks.register("regenerateSimpleCaseFolding", {+ def icuConfig = configurations.icu_current+ def groovyConfig = configurations.groovy+ def outputFile = file("src/java/org/apache/lucene/util/SimpleCaseFolding.java")++ description = "Regenerate ${outputFile} using ICU"++ dependsOn icuConfig++ inputs.property "icuConfig", icu4jVersion+ outputs.file outputFile++ doFirst {+ buildGlobals.execOps.javaexec({+ mainClass = "groovy.lang.GroovyShell"+ classpath icuConfig, groovyConfig++ args = [+ "--encoding",+ "UTF-8",+ file("${resources}/GenerateSimpleCaseFolding.groovy"),outputFile]})diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.javanew file mode 100644index 00000000000..8965476c003--- /dev/null+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.java@@ -0,0 +1,260 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.benchmark.jmh;++import java.util.Random;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.analysis.CharacterUtils;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Param;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.Warmup;+import org.openjdk.jmh.infra.Blackhole;++@BenchmarkMode(Mode.Throughput)+@OutputTimeUnit(TimeUnit.MICROSECONDS)+@State(Scope.Thread)+@Warmup(iterations = 5, time = 1)+@Measurement(iterations = 10, time = 1)+@Fork(3)+public class CaseFoldBenchmark {++ @Param({"256"})+ int tokenCount;++ @Param({+ "english",+ "german",+ "russian",+ "turkish",+ "greek",+ "armenian",+ "japanese",+ "cherokee",+ "mixed"+ })+ String distribution;++ private char[][] workBuffers;+ private int[] workLengths;+ private char[][] masterBuffers;+ private int[] masterLengths;++ private static final char[] RUSSIAN_UPPER = new char[32];+ private static final char[] RUSSIAN_LOWER = new char[32];+ private static final char[] TURKISH_SPECIAL = {+ 'İ', 'ı', 'Ş', 'ş', 'Ğ', 'ğ', 'Ç', 'ç', 'Ö', 'ö', 'Ü', 'ü'+ };+ private static final char[] ARMENIAN_UPPER = new char[38];+ private static final char[] ARMENIAN_LOWER = new char[38];+ private static final char[] JAPANESE_KATAKANA = new char[86];++ static {+ for (int i = 0; i < 32; i++) {+ RUSSIAN_UPPER[i] = (char) (0x0410 + i);+ RUSSIAN_LOWER[i] = (char) (0x0430 + i);+ }+ for (int i = 0; i < 38; i++) {+ ARMENIAN_UPPER[i] = (char) (0x0531 + i);+ ARMENIAN_LOWER[i] = (char) (0x0561 + i);+ }+ for (int i = 0; i < 86; i++) {+ JAPANESE_KATAKANA[i] = (char) (0x30A1 + i);+ }+ }++ @Setup(Level.Trial)+ public void setup() {+ Random rng = new Random(42);+ masterBuffers = new char[tokenCount][];+ masterLengths = new int[tokenCount];++ for (int t = 0; t < tokenCount; t++) {+ int len = 3 + rng.nextInt(8);+ masterBuffers[t] = generateToken(rng, len);+ masterLengths[t] = masterBuffers[t].length;+ }++ workBuffers = new char[tokenCount][];+ workLengths = new int[tokenCount];+ for (int t = 0; t < tokenCount; t++) {+ workBuffers[t] = new char[masterBuffers[t].length];+ workLengths[t] = masterLengths[t];+ }+ }++ private char[] generateToken(Random rng, int len) {+ return switch (distribution) {+ case "english" -> generateAsciiDominant(rng, len, 0.98);+ case "german" -> generateGerman(rng, len);+ case "russian" -> generateRussian(rng, len);+ case "turkish" -> generateTurkish(rng, len);+ case "greek" -> generateGreek(rng, len);+ case "armenian" -> generateArmenian(rng, len);+ case "japanese" -> generateJapanese(rng, len);+ case "cherokee" -> generateCherokee(rng, len);+ case "mixed" -> generateMixed(rng, len);+ default -> throw new IllegalArgumentException(distribution);+ };+ }++ @Setup(Level.Invocation)+ public void resetBuffers() {+ for (int t = 0; t < tokenCount; t++) {+ System.arraycopy(masterBuffers[t], 0, workBuffers[t], 0, masterLengths[t]);+ }+ }++ @Benchmark+ public void simpleCaseFold(Blackhole bh) {+ for (int t = 0; t < tokenCount; t++) {+ CharacterUtils.simpleCaseFold(workBuffers[t], 0, workLengths[t]);+ bh.consume(workBuffers[t]);+ }+ }++ private static char[] generateAsciiDominant(Random rng, int len, double asciiRatio) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ if (rng.nextDouble() < asciiRatio) {+ buf[i] =+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));+ } else {+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended+ }+ }+ return buf;+ }++ private static char[] generateGerman(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ double r = rng.nextDouble();+ if (r < 0.80) {+ buf[i] =+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));+ } else if (r < 0.95) {+ char[] umlauts = {'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß'};+ buf[i] = umlauts[rng.nextInt(umlauts.length)];+ } else {+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended+ }+ }+ return buf;+ }++ private static char[] generateRussian(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ if (rng.nextDouble() < 0.45) {+ buf[i] = RUSSIAN_UPPER[rng.nextInt(RUSSIAN_UPPER.length)];+ } else {+ buf[i] = RUSSIAN_LOWER[rng.nextInt(RUSSIAN_LOWER.length)];+ }+ }+ return buf;+ }++ private static char[] generateTurkish(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ double r = rng.nextDouble();+ if (r < 0.70) {+ buf[i] =+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));+ } else if (r < 0.90) {+ buf[i] = TURKISH_SPECIAL[rng.nextInt(TURKISH_SPECIAL.length)];+ } else {+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended+ }+ }+ return buf;+ }++ private static char[] generateGreek(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ double r = rng.nextDouble();+ if (r < 0.45) {+ buf[i] = (char) (0x0391 + rng.nextInt(25)); // Greek uppercase+ } else if (r < 0.90) {+ buf[i] = (char) (0x03B1 + rng.nextInt(25)); // Greek lowercase+ } else {+ char[] specials = {'ς', 'ϐ', 'ϑ', 'ϕ', 'ϖ', 'ϰ', 'ϱ', 'ϵ', 'µ'}; // fold exceptions+ buf[i] = specials[rng.nextInt(specials.length)];+ }+ }+ return buf;+ }++ private static char[] generateArmenian(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ buf[i] =+ rng.nextBoolean()+ ? ARMENIAN_UPPER[rng.nextInt(ARMENIAN_UPPER.length)]+ : ARMENIAN_LOWER[rng.nextInt(ARMENIAN_LOWER.length)];+ }+ return buf;+ }++ private static char[] generateJapanese(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ buf[i] = JAPANESE_KATAKANA[rng.nextInt(JAPANESE_KATAKANA.length)];+ }+ return buf;+ }++ private static char[] generateCherokee(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ if (rng.nextBoolean()) {+ buf[i] = (char) (0x13A0 + rng.nextInt(86)); // Cherokee uppercase+ } else {+ buf[i] = (char) (0xAB70 + rng.nextInt(80)); // Cherokee small+ }+ }+ return buf;+ }++ private static char[] generateMixed(Random rng, int len) {+ char[] buf = new char[len];+ for (int i = 0; i < len; i++) {+ double r = rng.nextDouble();+ if (r < 0.40) {+ buf[i] =+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));+ } else if (r < 0.60) {+ buf[i] = (char) (0x00C0 + rng.nextInt(60)); // Latin Extended+ } else if (r < 0.80) {+ buf[i] = (char) (0x0391 + rng.nextInt(50)); // Greek+ } else {+ buf[i] = (char) (0x0410 + rng.nextInt(64)); // Cyrillic+ }+ }+ return buf;+ }+}diff --git a/lucene/core/src/generated/checksums/regenerateCaseExpansion.json b/lucene/core/src/generated/checksums/regenerateCaseExpansion.jsonnew file mode 100644index 00000000000..5060fed86e4--- /dev/null+++ b/lucene/core/src/generated/checksums/regenerateCaseExpansion.json@@ -0,0 +1,4 @@+{+ "lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java": "6e491bb17d9482622a924f587ac9cd89a7fb8d69",+ "property:icuConfig": "78.3"+}diff --git a/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.json b/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.jsonnew file mode 100644index 00000000000..7ab79d7f318--- /dev/null+++ b/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.json@@ -0,0 +1,4 @@+{+ "lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java": "9c589fe1f829550b829aeab5ba36850fb1b684bd",+ "property:icuConfig": "78.3"+}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.java b/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.javanew file mode 100644index 00000000000..94943578497--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.java@@ -0,0 +1,50 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis;++import java.io.IOException;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;++/**+ * Normalizes token text by applying Unicode simple case folding.+ *+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding+ * mappings</a>+ * @since 10.6.0+ */+public class CaseFoldingFilter extends TokenFilter {+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);++ /**+ * Create a new CaseFoldingFilter.+ *+ * @param in TokenStream to filter+ */+ public CaseFoldingFilter(TokenStream in) {+ super(in);+ }++ @Override+ public final boolean incrementToken() throws IOException {+ if (input.incrementToken()) {+ CharacterUtils.simpleCaseFold(termAtt.buffer(), 0, termAtt.length());+ return true;+ } else {+ return false;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java b/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.javaindex 8ed7e5c9cc5..9fd4d79cec2 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java@@ -18,6 +18,7 @@ package org.apache.lucene.analysis;import java.io.IOException;import java.io.Reader;+import org.apache.lucene.util.UnicodeUtil;/*** Utility class to write tokenizers or token filters.@@ -68,6 +69,31 @@ public final class CharacterUtils {}}+ /**+ * Applies Unicode simple case folding to each codepoint starting at the given offset.+ *+ * @param buffer the char buffer to fold+ * @param offset the offset to start at+ * @param limit the max char in the buffer to fold+ */+ public static void simpleCaseFold(final char[] buffer, final int offset, final int limit) {+ assert buffer.length >= limit;+ assert 0 <= offset && offset <= buffer.length;+ for (int i = offset; i < limit; i++) {+ char c = buffer[i];+ if (c > 127) { // non-ASCII: switch to full Unicode path from here onward+ while (i < limit) {+ i +=+ Character.toChars(+ UnicodeUtil.foldCase(Character.codePointAt(buffer, i, limit)), buffer, i);+ }+ return;+ } else if (c >= 'A' && c <= 'Z') {+ buffer[i] = (char) (c | 32); // set bit 5 to lowercase ASCII+ }+ }+ }+/*** Converts each unicode codepoint to UpperCase via {@link Character#toUpperCase(int)} starting at* the given offset.diff --git a/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.javanew file mode 100644index 00000000000..004fe585510--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java@@ -0,0 +1,517 @@+// DO NOT EDIT THIS FILE! Use "gradlew regenerateSimpleCaseFolding tidy" to recreate.++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util;++/**+ * Switch-based Unicode simple case folding for the 226 codepoints where folding differs from {@link+ * Character#toLowerCase(int)}. Generated using ICU4J v78.3.0.0, unicode version: 17.0.0.0.+ *+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding+ * mappings</a>+ */+final class SimpleCaseFolding {+ private SimpleCaseFolding() {}++ /**+ * Returns the simple case fold of {@code c}, falling back to {@link Character#toLowerCase(int)}+ * for codepoints where fold equals lowercase.+ */+ static int fold(int c) {+ switch (c) {+ // Common+ case 0x00B5:+ return 0x03BC; // µ → μ++ // Latin+ case 0x0130:+ return 0x0130; // İ → İ+ case 0x017F:+ return 0x0073; // ſ → s++ // Inherited+ case 0x0345:+ return 0x03B9; // ͅ → ι++ // Greek+ case 0x03C2:+ return 0x03C3; // ς → σ+ case 0x03D0:+ return 0x03B2; // ϐ → β+ case 0x03D1:+ return 0x03B8; // ϑ → θ+ case 0x03D5:+ return 0x03C6; // ϕ → φ+ case 0x03D6:+ return 0x03C0; // ϖ → π+ case 0x03F0:+ return 0x03BA; // ϰ → κ+ case 0x03F1:+ return 0x03C1; // ϱ → ρ+ case 0x03F5:+ return 0x03B5; // ϵ → ε++ // Cherokee+ case 0x13A0:+ return 0x13A0; // Ꭰ → Ꭰ+ case 0x13A1:+ return 0x13A1; // Ꭱ → Ꭱ+ case 0x13A2:+ return 0x13A2; // Ꭲ → Ꭲ+ case 0x13A3:+ return 0x13A3; // Ꭳ → Ꭳ+ case 0x13A4:+ return 0x13A4; // Ꭴ → Ꭴ+ case 0x13A5:+ return 0x13A5; // Ꭵ → Ꭵ+ case 0x13A6:+ return 0x13A6; // Ꭶ → Ꭶ+ case 0x13A7:+ return 0x13A7; // Ꭷ → Ꭷ+ case 0x13A8:+ return 0x13A8; // Ꭸ → Ꭸ+ case 0x13A9:+ return 0x13A9; // Ꭹ → Ꭹ+ case 0x13AA:+ return 0x13AA; // Ꭺ → Ꭺ+ case 0x13AB:+ return 0x13AB; // Ꭻ → Ꭻ+ case 0x13AC:+ return 0x13AC; // Ꭼ → Ꭼ+ case 0x13AD:+ return 0x13AD; // Ꭽ → Ꭽ+ case 0x13AE:+ return 0x13AE; // Ꭾ → Ꭾ+ case 0x13AF:+ return 0x13AF; // Ꭿ → Ꭿ+ case 0x13B0:+ return 0x13B0; // Ꮀ → Ꮀ+ case 0x13B1:+ return 0x13B1; // Ꮁ → Ꮁ+ case 0x13B2:+ return 0x13B2; // Ꮂ → Ꮂ+ case 0x13B3:+ return 0x13B3; // Ꮃ → Ꮃ+ case 0x13B4:+ return 0x13B4; // Ꮄ → Ꮄ+ case 0x13B5:+ return 0x13B5; // Ꮅ → Ꮅ+ case 0x13B6:+ return 0x13B6; // Ꮆ → Ꮆ+ case 0x13B7:+ return 0x13B7; // Ꮇ → Ꮇ+ case 0x13B8:+ return 0x13B8; // Ꮈ → Ꮈ+ case 0x13B9:+ return 0x13B9; // Ꮉ → Ꮉ+ case 0x13BA:+ return 0x13BA; // Ꮊ → Ꮊ+ case 0x13BB:+ return 0x13BB; // Ꮋ → Ꮋ+ case 0x13BC:+ return 0x13BC; // Ꮌ → Ꮌ+ case 0x13BD:+ return 0x13BD; // Ꮍ → Ꮍ+ case 0x13BE:+ return 0x13BE; // Ꮎ → Ꮎ+ case 0x13BF:+ return 0x13BF; // Ꮏ → Ꮏ+ case 0x13C0:+ return 0x13C0; // Ꮐ → Ꮐ+ case 0x13C1:+ return 0x13C1; // Ꮑ → Ꮑ+ case 0x13C2:+ return 0x13C2; // Ꮒ → Ꮒ+ case 0x13C3:+ return 0x13C3; // Ꮓ → Ꮓ+ case 0x13C4:+ return 0x13C4; // Ꮔ → Ꮔ+ case 0x13C5:+ return 0x13C5; // Ꮕ → Ꮕ+ case 0x13C6:+ return 0x13C6; // Ꮖ → Ꮖ+ case 0x13C7:+ return 0x13C7; // Ꮗ → Ꮗ+ case 0x13C8:+ return 0x13C8; // Ꮘ → Ꮘ+ case 0x13C9:+ return 0x13C9; // Ꮙ → Ꮙ+ case 0x13CA:+ return 0x13CA; // Ꮚ → Ꮚ+ case 0x13CB:+ return 0x13CB; // Ꮛ → Ꮛ+ case 0x13CC:+ return 0x13CC; // Ꮜ → Ꮜ+ case 0x13CD:+ return 0x13CD; // Ꮝ → Ꮝ+ case 0x13CE:+ return 0x13CE; // Ꮞ → Ꮞ+ case 0x13CF:+ return 0x13CF; // Ꮟ → Ꮟ+ case 0x13D0:+ return 0x13D0; // Ꮠ → Ꮠ+ case 0x13D1:+ return 0x13D1; // Ꮡ → Ꮡ+ case 0x13D2:+ return 0x13D2; // Ꮢ → Ꮢ+ case 0x13D3:+ return 0x13D3; // Ꮣ → Ꮣ+ case 0x13D4:+ return 0x13D4; // Ꮤ → Ꮤ+ case 0x13D5:+ return 0x13D5; // Ꮥ → Ꮥ+ case 0x13D6:+ return 0x13D6; // Ꮦ → Ꮦ+ case 0x13D7:+ return 0x13D7; // Ꮧ → Ꮧ+ case 0x13D8:+ return 0x13D8; // Ꮨ → Ꮨ+ case 0x13D9:+ return 0x13D9; // Ꮩ → Ꮩ+ case 0x13DA:+ return 0x13DA; // Ꮪ → Ꮪ+ case 0x13DB:+ return 0x13DB; // Ꮫ → Ꮫ+ case 0x13DC:+ return 0x13DC; // Ꮬ → Ꮬ+ case 0x13DD:+ return 0x13DD; // Ꮭ → Ꮭ+ case 0x13DE:+ return 0x13DE; // Ꮮ → Ꮮ+ case 0x13DF:+ return 0x13DF; // Ꮯ → Ꮯ+ case 0x13E0:+ return 0x13E0; // Ꮰ → Ꮰ+ case 0x13E1:+ return 0x13E1; // Ꮱ → Ꮱ+ case 0x13E2:+ return 0x13E2; // Ꮲ → Ꮲ+ case 0x13E3:+ return 0x13E3; // Ꮳ → Ꮳ+ case 0x13E4:+ return 0x13E4; // Ꮴ → Ꮴ+ case 0x13E5:+ return 0x13E5; // Ꮵ → Ꮵ+ case 0x13E6:+ return 0x13E6; // Ꮶ → Ꮶ+ case 0x13E7:+ return 0x13E7; // Ꮷ → Ꮷ+ case 0x13E8:+ return 0x13E8; // Ꮸ → Ꮸ+ case 0x13E9:+ return 0x13E9; // Ꮹ → Ꮹ+ case 0x13EA:+ return 0x13EA; // Ꮺ → Ꮺ+ case 0x13EB:+ return 0x13EB; // Ꮻ → Ꮻ+ case 0x13EC:+ return 0x13EC; // Ꮼ → Ꮼ+ case 0x13ED:+ return 0x13ED; // Ꮽ → Ꮽ+ case 0x13EE:+ return 0x13EE; // Ꮾ → Ꮾ+ case 0x13EF:+ return 0x13EF; // Ꮿ → Ꮿ+ case 0x13F0:+ return 0x13F0; // Ᏸ → Ᏸ+ case 0x13F1:+ return 0x13F1; // Ᏹ → Ᏹ+ case 0x13F2:+ return 0x13F2; // Ᏺ → Ᏺ+ case 0x13F3:+ return 0x13F3; // Ᏻ → Ᏻ+ case 0x13F4:+ return 0x13F4; // Ᏼ → Ᏼ+ case 0x13F5:+ return 0x13F5; // Ᏽ → Ᏽ+ case 0x13F8:+ return 0x13F0; // ᏸ → Ᏸ+ case 0x13F9:+ return 0x13F1; // ᏹ → Ᏹ+ case 0x13FA:+ return 0x13F2; // ᏺ → Ᏺ+ case 0x13FB:+ return 0x13F3; // ᏻ → Ᏻ+ case 0x13FC:+ return 0x13F4; // ᏼ → Ᏼ+ case 0x13FD:+ return 0x13F5; // ᏽ → Ᏽ++ // Cyrillic+ case 0x1C80:+ return 0x0432; // ᲀ → в+ case 0x1C81:+ return 0x0434; // ᲁ → д+ case 0x1C82:+ return 0x043E; // ᲂ → о+ case 0x1C83:+ return 0x0441; // ᲃ → с+ case 0x1C84:+ return 0x0442; // ᲄ → т+ case 0x1C85:+ return 0x0442; // ᲅ → т+ case 0x1C86:+ return 0x044A; // ᲆ → ъ+ case 0x1C87:+ return 0x0463; // ᲇ → ѣ+ case 0x1C88:+ return 0xA64B; // ᲈ → ꙋ++ // Latin+ case 0x1E9B:+ return 0x1E61; // ẛ → ṡ++ // Greek+ case 0x1FBE:+ return 0x03B9; // ι → ι+ case 0x1FD3:+ return 0x0390; // ΐ → ΐ+ case 0x1FE3:+ return 0x03B0; // ΰ → ΰ++ // Latin+ case 0xA7CE:+ return 0xA7CF; // → + case 0xA7D2:+ return 0xA7D3; // → ꟓ+ case 0xA7D4:+ return 0xA7D5; // → ꟕ++ // Cherokee+ case 0xAB70:+ return 0x13A0; // ꭰ → Ꭰ+ case 0xAB71:+ return 0x13A1; // ꭱ → Ꭱ+ case 0xAB72:+ return 0x13A2; // ꭲ → Ꭲ+ case 0xAB73:+ return 0x13A3; // ꭳ → Ꭳ+ case 0xAB74:+ return 0x13A4; // ꭴ → Ꭴ+ case 0xAB75:+ return 0x13A5; // ꭵ → Ꭵ+ case 0xAB76:+ return 0x13A6; // ꭶ → Ꭶ+ case 0xAB77:+ return 0x13A7; // ꭷ → Ꭷ+ case 0xAB78:+ return 0x13A8; // ꭸ → Ꭸ+ case 0xAB79:+ return 0x13A9; // ꭹ → Ꭹ+ case 0xAB7A:+ return 0x13AA; // ꭺ → Ꭺ+ case 0xAB7B:+ return 0x13AB; // ꭻ → Ꭻ+ case 0xAB7C:+ return 0x13AC; // ꭼ → Ꭼ+ case 0xAB7D:+ return 0x13AD; // ꭽ → Ꭽ+ case 0xAB7E:+ return 0x13AE; // ꭾ → Ꭾ+ case 0xAB7F:+ return 0x13AF; // ꭿ → Ꭿ+ case 0xAB80:+ return 0x13B0; // ꮀ → Ꮀ+ case 0xAB81:+ return 0x13B1; // ꮁ → Ꮁ+ case 0xAB82:+ return 0x13B2; // ꮂ → Ꮂ+ case 0xAB83:+ return 0x13B3; // ꮃ → Ꮃ+ case 0xAB84:+ return 0x13B4; // ꮄ → Ꮄ+ case 0xAB85:+ return 0x13B5; // ꮅ → Ꮅ+ case 0xAB86:+ return 0x13B6; // ꮆ → Ꮆ+ case 0xAB87:+ return 0x13B7; // ꮇ → Ꮇ+ case 0xAB88:+ return 0x13B8; // ꮈ → Ꮈ+ case 0xAB89:+ return 0x13B9; // ꮉ → Ꮉ+ case 0xAB8A:+ return 0x13BA; // ꮊ → Ꮊ+ case 0xAB8B:+ return 0x13BB; // ꮋ → Ꮋ+ case 0xAB8C:+ return 0x13BC; // ꮌ → Ꮌ+ case 0xAB8D:+ return 0x13BD; // ꮍ → Ꮍ+ case 0xAB8E:+ return 0x13BE; // ꮎ → Ꮎ+ case 0xAB8F:+ return 0x13BF; // ꮏ → Ꮏ+ case 0xAB90:+ return 0x13C0; // ꮐ → Ꮐ+ case 0xAB91:+ return 0x13C1; // ꮑ → Ꮑ+ case 0xAB92:+ return 0x13C2; // ꮒ → Ꮒ+ case 0xAB93:+ return 0x13C3; // ꮓ → Ꮓ+ case 0xAB94:+ return 0x13C4; // ꮔ → Ꮔ+ case 0xAB95:+ return 0x13C5; // ꮕ → Ꮕ+ case 0xAB96:+ return 0x13C6; // ꮖ → Ꮖ+ case 0xAB97:+ return 0x13C7; // ꮗ → Ꮗ+ case 0xAB98:+ return 0x13C8; // ꮘ → Ꮘ+ case 0xAB99:+ return 0x13C9; // ꮙ → Ꮙ+ case 0xAB9A:+ return 0x13CA; // ꮚ → Ꮚ+ case 0xAB9B:+ return 0x13CB; // ꮛ → Ꮛ+ case 0xAB9C:+ return 0x13CC; // ꮜ → Ꮜ+ case 0xAB9D:+ return 0x13CD; // ꮝ → Ꮝ+ case 0xAB9E:+ return 0x13CE; // ꮞ → Ꮞ+ case 0xAB9F:+ return 0x13CF; // ꮟ → Ꮟ+ case 0xABA0:+ return 0x13D0; // ꮠ → Ꮠ+ case 0xABA1:+ return 0x13D1; // ꮡ → Ꮡ+ case 0xABA2:+ return 0x13D2; // ꮢ → Ꮢ+ case 0xABA3:+ return 0x13D3; // ꮣ → Ꮣ+ case 0xABA4:+ return 0x13D4; // ꮤ → Ꮤ+ case 0xABA5:+ return 0x13D5; // ꮥ → Ꮥ+ case 0xABA6:+ return 0x13D6; // ꮦ → Ꮦ+ case 0xABA7:+ return 0x13D7; // ꮧ → Ꮧ+ case 0xABA8:+ return 0x13D8; // ꮨ → Ꮨ+ case 0xABA9:+ return 0x13D9; // ꮩ → Ꮩ+ case 0xABAA:+ return 0x13DA; // ꮪ → Ꮪ+ case 0xABAB:+ return 0x13DB; // ꮫ → Ꮫ+ case 0xABAC:+ return 0x13DC; // ꮬ → Ꮬ+ case 0xABAD:+ return 0x13DD; // ꮭ → Ꮭ+ case 0xABAE:+ return 0x13DE; // ꮮ → Ꮮ+ case 0xABAF:+ return 0x13DF; // ꮯ → Ꮯ+ case 0xABB0:+ return 0x13E0; // ꮰ → Ꮰ+ case 0xABB1:+ return 0x13E1; // ꮱ → Ꮱ+ case 0xABB2:+ return 0x13E2; // ꮲ → Ꮲ+ case 0xABB3:+ return 0x13E3; // ꮳ → Ꮳ+ case 0xABB4:+ return 0x13E4; // ꮴ → Ꮴ+ case 0xABB5:+ return 0x13E5; // ꮵ → Ꮵ+ case 0xABB6:+ return 0x13E6; // ꮶ → Ꮶ+ case 0xABB7:+ return 0x13E7; // ꮷ → Ꮷ+ case 0xABB8:+ return 0x13E8; // ꮸ → Ꮸ+ case 0xABB9:+ return 0x13E9; // ꮹ → Ꮹ+ case 0xABBA:+ return 0x13EA; // ꮺ → Ꮺ+ case 0xABBB:+ return 0x13EB; // ꮻ → Ꮻ+ case 0xABBC:+ return 0x13EC; // ꮼ → Ꮼ+ case 0xABBD:+ return 0x13ED; // ꮽ → Ꮽ+ case 0xABBE:+ return 0x13EE; // ꮾ → Ꮾ+ case 0xABBF:+ return 0x13EF; // ꮿ → Ꮿ++ // Latin+ case 0xFB05:+ return 0xFB06; // ſt → st++ // Beria_Erfe+ case 0x16EA0:+ return 0x16EBB; // → + case 0x16EA1:+ return 0x16EBC; // → + case 0x16EA2:+ return 0x16EBD; // → + case 0x16EA3:+ return 0x16EBE; // → + case 0x16EA4:+ return 0x16EBF; // → + case 0x16EA5:+ return 0x16EC0; // → + case 0x16EA6:+ return 0x16EC1; // → + case 0x16EA7:+ return 0x16EC2; // → + case 0x16EA8:+ return 0x16EC3; // → + case 0x16EA9:+ return 0x16EC4; // → + case 0x16EAA:+ return 0x16EC5; // → + case 0x16EAB:+ return 0x16EC6; // → + case 0x16EAC:+ return 0x16EC7; // → + case 0x16EAD:+ return 0x16EC8; // → + case 0x16EAE:+ return 0x16EC9; // → + case 0x16EAF:+ return 0x16ECA; // → + case 0x16EB0:+ return 0x16ECB; // → + case 0x16EB1:+ return 0x16ECC; // → + case 0x16EB2:+ return 0x16ECD; // → + case 0x16EB3:+ return 0x16ECE; // → + case 0x16EB4:+ return 0x16ECF; // → + case 0x16EB5:+ return 0x16ED0; // → + case 0x16EB6:+ return 0x16ED1; // → + case 0x16EB7:+ return 0x16ED2; // → + case 0x16EB8:+ return 0x16ED3; // → + default:+ return Character.toLowerCase(c);+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.javaindex 6f493c0556d..758c9f0cb68 100644--- a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java@@ -686,4 +686,9 @@ public final class UnicodeUtil {public static int UTF8toUTF16(BytesRef bytesRef, char[] chars) {return UTF8toUTF16(bytesRef.bytes, bytesRef.offset, bytesRef.length, chars);}++ /** Returns the Unicode simple case folding of {@code codepoint}. */+ public static int foldCase(int codepoint) {+ return SimpleCaseFolding.fold(codepoint);+ }}diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.javaindex e359b6ddb63..3b36b9bcc39 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java@@ -699,7 +699,7 @@ public final class Automata {*/private static int[] toCaseInsensitiveChar(int codepoint) {IntArrayList list = new IntArrayList();- CaseFolding.expand(+ CaseExpansion.expand(codepoint,(int variant) -> {list.add(variant);diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.javarename to lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.javaindex f21e0ea3834..3477a610cf1 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java@@ -1,4 +1,4 @@-// DO NOT EDIT THIS FILE! Use "gradlew generateUnicodeProps tidy" to recreate.+// DO NOT EDIT THIS FILE! Use "gradlew regenerateCaseExpansion tidy" to recreate./** Licensed to the Apache Software Foundation (ASF) under one or more@@ -23,10 +23,10 @@ import java.util.function.IntConsumer;/*** This file contains unicode properties used by {@code RegExp}. The data was generated using ICU4J- * v77.1.0.0, unicode version: 16.0.0.0.+ * v78.3.0.0, unicode version: 17.0.0.0.*/-final class CaseFolding {- private CaseFolding() {}+final class CaseExpansion {+ private CaseExpansion() {}/** Calls {@code fn} consumer with {@code c} itself and its {@code scf} mappings. */static void expand(int c, IntConsumer fn) {diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.javaindex ce4d2d2d951..8367f81f0df 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java@@ -742,7 +742,7 @@ public class RegExp {*/private int[] toCaseInsensitiveChar(int codepoint) {IntArrayList list = new IntArrayList();- CaseFolding.expand(+ CaseExpansion.expand(codepoint,(int variant) -> {list.add(variant);@@ -770,7 +770,7 @@ public class RegExp {// iterate over range, adding codepoint and any alternatives as transitionsfor (int i = start; i <= end; i++) {- CaseFolding.expand(+ CaseExpansion.expand(i,(int ch) -> {scratch.addTransition(state, state, ch);diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java b/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.javaindex 43352537d2a..e0de632d223 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.CharacterUtils.CharacterBuffer;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.UnicodeUtil;/** TestCase for the {@link CharacterUtils} class. */public class TestCharacterUtils extends LuceneTestCase {@@ -109,4 +110,44 @@ public class TestCharacterUtils extends LuceneTestCase {assertFalse(CharacterUtils.fill(buffer, reader));assertEquals(0, buffer.getLength());}++ public void testAsciiFoldMatchesToLowerCase() {+ for (int c = 0; c < 128; c++) {+ assertEquals(Character.toLowerCase(c), UnicodeUtil.foldCase(c));+ }+ }++ public void testFoldExceptions() {+ assertEquals(0x03BC, UnicodeUtil.foldCase(0x00B5));+ assertEquals(0x0073, UnicodeUtil.foldCase(0x017F));+ assertEquals(0x03C3, UnicodeUtil.foldCase(0x03C2));+ assertEquals('0', UnicodeUtil.foldCase('0'));+ assertEquals('a', UnicodeUtil.foldCase('a'));+ assertEquals('a', UnicodeUtil.foldCase('A'));+ }++ public void testFoldIdempotent() {+ for (int c = 0; c <= 0xFFFF; c++) {+ int folded = UnicodeUtil.foldCase(c);+ assertEquals(+ "Not idempotent at U+" + Integer.toHexString(c), folded, UnicodeUtil.foldCase(folded));+ }+ }++ public void testSimpleCaseFold() {+ char[] buf = "ABcΣσς".toCharArray();+ CharacterUtils.simpleCaseFold(buf, 0, buf.length);+ assertEquals("abcσσσ", new String(buf));+ }++ public void testSimpleCaseFoldRandom() {+ for (int iter = 0; iter < 100; iter++) {+ String s = TestUtil.randomUnicodeString(random(), 100);+ char[] buf = s.toCharArray();+ CharacterUtils.simpleCaseFold(buf, 0, buf.length);+ char[] buf2 = new String(buf).toCharArray();+ CharacterUtils.simpleCaseFold(buf2, 0, buf2.length);+ assertArrayEquals(buf, buf2);+ }+ }}