Skip to content

Apache Lucene open-source search software

Add CaseFoldingFilter with flat lookup table (#16371)

Costin Leauba69bfb1ea9a
diff --git a/.gitattributes b/.gitattributes
index e7881fb2524..ff8d5fa571a 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -23,6 +23,8 @@ lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene*/ForUti
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene*/ForDeltaUtil.java linguist-generated
lucene/core/src/generated/jdk/*.jar linguist-generated
lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java linguist-generated
+lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java linguist-generated
+lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java linguist-generated
lucene/core/src/java/org/apache/lucene/util/automaton/Lev*ParametricDescription.java linguist-generated
lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation*.java linguist-generated
lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java linguist-generated
diff --git a/gradle/regenerate/icu/GenerateCaseFolding.groovy b/gradle/regenerate/icu/GenerateCaseExpansion.groovy
similarity index 95%
rename from gradle/regenerate/icu/GenerateCaseFolding.groovy
rename to gradle/regenerate/icu/GenerateCaseExpansion.groovy
index 06297d94363..f4549ffdc2b 100644
--- a/gradle/regenerate/icu/GenerateCaseFolding.groovy
+++ b/gradle/regenerate/icu/GenerateCaseExpansion.groovy
@@ -35,7 +35,7 @@ def generateSwitch() {
}
def code = """
-// DO NOT EDIT THIS FILE! Use "gradlew generateUnicodeProps tidy" to recreate.
+// DO NOT EDIT THIS FILE! Use "gradlew regenerateCaseExpansion tidy" to recreate.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -62,8 +62,8 @@ import java.util.function.IntConsumer;
* This file contains unicode properties used by {@code RegExp}.
* The data was generated using ICU4J v${icuVersion}, unicode version: ${unicodeVersion}.
*/
-final class CaseFolding {
- private CaseFolding() {}
+final class CaseExpansion {
+ private CaseExpansion() {}
/**
* Calls {@code fn} consumer with {@code c} itself and its {@code scf} mappings.
diff --git a/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovy b/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovy
new file mode 100644
index 00000000000..c9de96f7b10
--- /dev/null
+++ b/gradle/regenerate/icu/GenerateSimpleCaseFolding.groovy
@@ -0,0 +1,87 @@
+import com.ibm.icu.lang.UCharacter
+import com.ibm.icu.lang.UScript
+import com.ibm.icu.util.VersionInfo
+import java.nio.file.Paths
+import java.util.Locale
+
+def icuVersion = VersionInfo.ICU_VERSION.toString()
+def unicodeVersion = UCharacter.getUnicodeVersion().toString()
+
+def outputFile = Paths.get(args[0])
+
+def generateSwitchCases() {
+ StringBuilder sb = new StringBuilder()
+ int lastScript = -1
+ int count = 0
+
+ for (int c = 0; c <= Character.MAX_CODE_POINT; c++) {
+ int icuFolded = UCharacter.foldCase(c, true)
+ int jdkLower = Character.toLowerCase(c)
+
+ if (icuFolded != jdkLower) {
+ int script = UScript.getScript(c)
+ String scriptName = UScript.getName(script)
+
+ if (script != lastScript) {
+ if (count > 0) {
+ sb.append("\n")
+ }
+ sb.append("\n // ${scriptName}")
+ lastScript = script
+ }
+ String fromChar = new String(Character.toChars(c))
+ String toChar = new String(Character.toChars(icuFolded))
+ sb.append(String.format(Locale.ROOT, "\n case 0x%04X:\n return 0x%04X; // %s → %s", c, icuFolded, fromChar, toChar))
+ count++
+ }
+ }
+ return [sb.toString(), count]
+}
+
+def (switchCases, caseCount) = generateSwitchCases()
+
+def code = """\
+// DO NOT EDIT THIS FILE! Use "gradlew regenerateSimpleCaseFolding tidy" to recreate.
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util;
+
+/**
+ * Switch-based Unicode simple case folding for the ${caseCount} codepoints where folding differs from {@link
+ * Character#toLowerCase(int)}. Generated using ICU4J v${icuVersion}, unicode version: ${unicodeVersion}.
+ *
+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding
+ * mappings</a>
+ */
+final class SimpleCaseFolding {
+ private SimpleCaseFolding() {}
+
+ /**
+ * Returns the simple case fold of {@code c}, falling back to {@link Character#toLowerCase(int)}
+ * for codepoints where fold equals lowercase.
+ */
+ static int fold(int c) {
+ switch (c) {${switchCases}
+ default:
+ return Character.toLowerCase(c);
+ }
+ }
+}
+"""
+outputFile.setText(code, "UTF-8")
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 90d5a039325..0e07088673d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -331,6 +331,12 @@ New Features
and document length instead of corpus statistics such as document frequency. It also supports k3
query-term frequency saturation. (Tianxiao Wei)
+* GITHUB#16383: Add fp16 vector encoding support. (Pulkit Gupta)
+
+* GITHUB#16371, GITHUB#14389: Add CaseFoldingFilter for Unicode simple case folding without
+ requiring the ICU analysis plugin. Uses a generated switch for fold exceptions with
+ Character.toLowerCase() as the default fallback. (Robert Muir, Costin Leau)
+
Improvements
---------------------
diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.java
index 857663f8f0c..025ea08c6df 100644
--- a/lucene/analysis/common/src/java/module-info.java
+++ b/lucene/analysis/common/src/java/module-info.java
@@ -169,6 +169,7 @@ module org.apache.lucene.analysis.common {
org.apache.lucene.analysis.commongrams.CommonGramsQueryFilterFactory,
org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilterFactory,
org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilterFactory,
+ org.apache.lucene.analysis.core.CaseFoldingFilterFactory,
org.apache.lucene.analysis.core.DecimalDigitFilterFactory,
org.apache.lucene.analysis.core.LowerCaseFilterFactory,
org.apache.lucene.analysis.core.StopFilterFactory,
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.java
new file mode 100644
index 00000000000..de2ae3f45af
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/CaseFoldingFilterFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.core;
+
+import java.util.Map;
+import org.apache.lucene.analysis.CaseFoldingFilter;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Factory for {@link CaseFoldingFilter}.
+ *
+ * <pre><code class="language-xml">
+ * &lt;fieldType name="text_fold" class="solr.TextField" positionIncrementGap="100"&gt;
+ * &lt;analyzer&gt;
+ * &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
+ * &lt;filter class="solr.CaseFoldingFilterFactory"/&gt;
+ * &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</code></pre>
+ *
+ * @since 10.6.0
+ * @lucene.spi {@value #NAME}
+ */
+public class CaseFoldingFilterFactory extends TokenFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "caseFolding";
+
+ /** Creates a new CaseFoldingFilterFactory */
+ public CaseFoldingFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public CaseFoldingFilterFactory() {
+ throw defaultCtorException();
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new CaseFoldingFilter(input);
+ }
+
+ @Override
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
+ }
+}
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
index 7d7fb83d7f7..1b33f2603b3 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
@@ -30,6 +30,7 @@ org.apache.lucene.analysis.commongrams.CommonGramsFilterFactory
org.apache.lucene.analysis.commongrams.CommonGramsQueryFilterFactory
org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilterFactory
org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilterFactory
+org.apache.lucene.analysis.core.CaseFoldingFilterFactory
org.apache.lucene.analysis.core.DecimalDigitFilterFactory
org.apache.lucene.analysis.core.LowerCaseFilterFactory
org.apache.lucene.analysis.core.StopFilterFactory
diff --git a/lucene/analysis/icu/build.gradle b/lucene/analysis/icu/build.gradle
index 9f1b8629055..edacc185779 100644
--- a/lucene/analysis/icu/build.gradle
+++ b/lucene/analysis/icu/build.gradle
@@ -284,10 +284,10 @@ configure(project(":lucene:analysis:common")) {
configure(project(":lucene:core")) {
setupIcuDependencies(project)
- tasks.register("generateUnicodeProps", {
+ tasks.register("regenerateCaseExpansion", {
def icuConfig = configurations.icu_current
def groovyConfig = configurations.groovy
- def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")
+ def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseExpansion.java")
description = "Regenerate ${outputFile} using ICU"
@@ -304,7 +304,34 @@ configure(project(":lucene:core")) {
args = [
"--encoding",
"UTF-8",
- file("${resources}/GenerateCaseFolding.groovy"),
+ file("${resources}/GenerateCaseExpansion.groovy"),
+ outputFile
+ ]
+ })
+ }
+ })
+
+ tasks.register("regenerateSimpleCaseFolding", {
+ def icuConfig = configurations.icu_current
+ def groovyConfig = configurations.groovy
+ def outputFile = file("src/java/org/apache/lucene/util/SimpleCaseFolding.java")
+
+ description = "Regenerate ${outputFile} using ICU"
+
+ dependsOn icuConfig
+
+ inputs.property "icuConfig", icu4jVersion
+ outputs.file outputFile
+
+ doFirst {
+ buildGlobals.execOps.javaexec({
+ mainClass = "groovy.lang.GroovyShell"
+ classpath icuConfig, groovyConfig
+
+ args = [
+ "--encoding",
+ "UTF-8",
+ file("${resources}/GenerateSimpleCaseFolding.groovy"),
outputFile
]
})
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.java
new file mode 100644
index 00000000000..8965476c003
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CaseFoldBenchmark.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.analysis.CharacterUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(3)
+public class CaseFoldBenchmark {
+
+ @Param({"256"})
+ int tokenCount;
+
+ @Param({
+ "english",
+ "german",
+ "russian",
+ "turkish",
+ "greek",
+ "armenian",
+ "japanese",
+ "cherokee",
+ "mixed"
+ })
+ String distribution;
+
+ private char[][] workBuffers;
+ private int[] workLengths;
+ private char[][] masterBuffers;
+ private int[] masterLengths;
+
+ private static final char[] RUSSIAN_UPPER = new char[32];
+ private static final char[] RUSSIAN_LOWER = new char[32];
+ private static final char[] TURKISH_SPECIAL = {
+ 'İ', 'ı', 'Ş', 'ş', 'Ğ', 'ğ', 'Ç', 'ç', 'Ö', 'ö', 'Ü', 'ü'
+ };
+ private static final char[] ARMENIAN_UPPER = new char[38];
+ private static final char[] ARMENIAN_LOWER = new char[38];
+ private static final char[] JAPANESE_KATAKANA = new char[86];
+
+ static {
+ for (int i = 0; i < 32; i++) {
+ RUSSIAN_UPPER[i] = (char) (0x0410 + i);
+ RUSSIAN_LOWER[i] = (char) (0x0430 + i);
+ }
+ for (int i = 0; i < 38; i++) {
+ ARMENIAN_UPPER[i] = (char) (0x0531 + i);
+ ARMENIAN_LOWER[i] = (char) (0x0561 + i);
+ }
+ for (int i = 0; i < 86; i++) {
+ JAPANESE_KATAKANA[i] = (char) (0x30A1 + i);
+ }
+ }
+
+ @Setup(Level.Trial)
+ public void setup() {
+ Random rng = new Random(42);
+ masterBuffers = new char[tokenCount][];
+ masterLengths = new int[tokenCount];
+
+ for (int t = 0; t < tokenCount; t++) {
+ int len = 3 + rng.nextInt(8);
+ masterBuffers[t] = generateToken(rng, len);
+ masterLengths[t] = masterBuffers[t].length;
+ }
+
+ workBuffers = new char[tokenCount][];
+ workLengths = new int[tokenCount];
+ for (int t = 0; t < tokenCount; t++) {
+ workBuffers[t] = new char[masterBuffers[t].length];
+ workLengths[t] = masterLengths[t];
+ }
+ }
+
+ private char[] generateToken(Random rng, int len) {
+ return switch (distribution) {
+ case "english" -> generateAsciiDominant(rng, len, 0.98);
+ case "german" -> generateGerman(rng, len);
+ case "russian" -> generateRussian(rng, len);
+ case "turkish" -> generateTurkish(rng, len);
+ case "greek" -> generateGreek(rng, len);
+ case "armenian" -> generateArmenian(rng, len);
+ case "japanese" -> generateJapanese(rng, len);
+ case "cherokee" -> generateCherokee(rng, len);
+ case "mixed" -> generateMixed(rng, len);
+ default -> throw new IllegalArgumentException(distribution);
+ };
+ }
+
+ @Setup(Level.Invocation)
+ public void resetBuffers() {
+ for (int t = 0; t < tokenCount; t++) {
+ System.arraycopy(masterBuffers[t], 0, workBuffers[t], 0, masterLengths[t]);
+ }
+ }
+
+ @Benchmark
+ public void simpleCaseFold(Blackhole bh) {
+ for (int t = 0; t < tokenCount; t++) {
+ CharacterUtils.simpleCaseFold(workBuffers[t], 0, workLengths[t]);
+ bh.consume(workBuffers[t]);
+ }
+ }
+
+ private static char[] generateAsciiDominant(Random rng, int len, double asciiRatio) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ if (rng.nextDouble() < asciiRatio) {
+ buf[i] =
+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));
+ } else {
+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateGerman(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ double r = rng.nextDouble();
+ if (r < 0.80) {
+ buf[i] =
+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));
+ } else if (r < 0.95) {
+ char[] umlauts = {'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß'};
+ buf[i] = umlauts[rng.nextInt(umlauts.length)];
+ } else {
+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateRussian(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ if (rng.nextDouble() < 0.45) {
+ buf[i] = RUSSIAN_UPPER[rng.nextInt(RUSSIAN_UPPER.length)];
+ } else {
+ buf[i] = RUSSIAN_LOWER[rng.nextInt(RUSSIAN_LOWER.length)];
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateTurkish(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ double r = rng.nextDouble();
+ if (r < 0.70) {
+ buf[i] =
+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));
+ } else if (r < 0.90) {
+ buf[i] = TURKISH_SPECIAL[rng.nextInt(TURKISH_SPECIAL.length)];
+ } else {
+ buf[i] = (char) (0x00C0 + rng.nextInt(30)); // Latin Extended
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateGreek(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ double r = rng.nextDouble();
+ if (r < 0.45) {
+ buf[i] = (char) (0x0391 + rng.nextInt(25)); // Greek uppercase
+ } else if (r < 0.90) {
+ buf[i] = (char) (0x03B1 + rng.nextInt(25)); // Greek lowercase
+ } else {
+ char[] specials = {'ς', 'ϐ', 'ϑ', 'ϕ', 'ϖ', 'ϰ', 'ϱ', 'ϵ', 'µ'}; // fold exceptions
+ buf[i] = specials[rng.nextInt(specials.length)];
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateArmenian(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ buf[i] =
+ rng.nextBoolean()
+ ? ARMENIAN_UPPER[rng.nextInt(ARMENIAN_UPPER.length)]
+ : ARMENIAN_LOWER[rng.nextInt(ARMENIAN_LOWER.length)];
+ }
+ return buf;
+ }
+
+ private static char[] generateJapanese(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ buf[i] = JAPANESE_KATAKANA[rng.nextInt(JAPANESE_KATAKANA.length)];
+ }
+ return buf;
+ }
+
+ private static char[] generateCherokee(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ if (rng.nextBoolean()) {
+ buf[i] = (char) (0x13A0 + rng.nextInt(86)); // Cherokee uppercase
+ } else {
+ buf[i] = (char) (0xAB70 + rng.nextInt(80)); // Cherokee small
+ }
+ }
+ return buf;
+ }
+
+ private static char[] generateMixed(Random rng, int len) {
+ char[] buf = new char[len];
+ for (int i = 0; i < len; i++) {
+ double r = rng.nextDouble();
+ if (r < 0.40) {
+ buf[i] =
+ rng.nextBoolean() ? (char) ('a' + rng.nextInt(26)) : (char) ('A' + rng.nextInt(26));
+ } else if (r < 0.60) {
+ buf[i] = (char) (0x00C0 + rng.nextInt(60)); // Latin Extended
+ } else if (r < 0.80) {
+ buf[i] = (char) (0x0391 + rng.nextInt(50)); // Greek
+ } else {
+ buf[i] = (char) (0x0410 + rng.nextInt(64)); // Cyrillic
+ }
+ }
+ return buf;
+ }
+}
diff --git a/lucene/core/src/generated/checksums/regenerateCaseExpansion.json b/lucene/core/src/generated/checksums/regenerateCaseExpansion.json
new file mode 100644
index 00000000000..5060fed86e4
--- /dev/null
+++ b/lucene/core/src/generated/checksums/regenerateCaseExpansion.json
@@ -0,0 +1,4 @@
+{
+ "lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java": "6e491bb17d9482622a924f587ac9cd89a7fb8d69",
+ "property:icuConfig": "78.3"
+}
diff --git a/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.json b/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.json
new file mode 100644
index 00000000000..7ab79d7f318
--- /dev/null
+++ b/lucene/core/src/generated/checksums/regenerateSimpleCaseFolding.json
@@ -0,0 +1,4 @@
+{
+ "lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java": "9c589fe1f829550b829aeab5ba36850fb1b684bd",
+ "property:icuConfig": "78.3"
+}
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.java b/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.java
new file mode 100644
index 00000000000..94943578497
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/analysis/CaseFoldingFilter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * Normalizes token text by applying Unicode simple case folding.
+ *
+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding
+ * mappings</a>
+ * @since 10.6.0
+ */
+public class CaseFoldingFilter extends TokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ /**
+ * Create a new CaseFoldingFilter.
+ *
+ * @param in TokenStream to filter
+ */
+ public CaseFoldingFilter(TokenStream in) {
+ super(in);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ CharacterUtils.simpleCaseFold(termAtt.buffer(), 0, termAtt.length());
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java b/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java
index 8ed7e5c9cc5..9fd4d79cec2 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/CharacterUtils.java
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis;
import java.io.IOException;
import java.io.Reader;
+import org.apache.lucene.util.UnicodeUtil;
/**
* Utility class to write tokenizers or token filters.
@@ -68,6 +69,31 @@ public final class CharacterUtils {
}
}
+ /**
+ * Applies Unicode simple case folding to each codepoint starting at the given offset.
+ *
+ * @param buffer the char buffer to fold
+ * @param offset the offset to start at
+ * @param limit the max char in the buffer to fold
+ */
+ public static void simpleCaseFold(final char[] buffer, final int offset, final int limit) {
+ assert buffer.length >= limit;
+ assert 0 <= offset && offset <= buffer.length;
+ for (int i = offset; i < limit; i++) {
+ char c = buffer[i];
+ if (c > 127) { // non-ASCII: switch to full Unicode path from here onward
+ while (i < limit) {
+ i +=
+ Character.toChars(
+ UnicodeUtil.foldCase(Character.codePointAt(buffer, i, limit)), buffer, i);
+ }
+ return;
+ } else if (c >= 'A' && c <= 'Z') {
+ buffer[i] = (char) (c | 32); // set bit 5 to lowercase ASCII
+ }
+ }
+ }
+
/**
* Converts each unicode codepoint to UpperCase via {@link Character#toUpperCase(int)} starting at
* the given offset.
diff --git a/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java
new file mode 100644
index 00000000000..004fe585510
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/SimpleCaseFolding.java
@@ -0,0 +1,517 @@
+// DO NOT EDIT THIS FILE! Use "gradlew regenerateSimpleCaseFolding tidy" to recreate.
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util;
+
+/**
+ * Switch-based Unicode simple case folding for the 226 codepoints where folding differs from {@link
+ * Character#toLowerCase(int)}. Generated using ICU4J v78.3.0.0, unicode version: 17.0.0.0.
+ *
+ * @see <a href="https://www.unicode.org/Public/latest/ucd/CaseFolding.txt">Unicode Case Folding
+ * mappings</a>
+ */
+final class SimpleCaseFolding {
+ private SimpleCaseFolding() {}
+
+ /**
+ * Returns the simple case fold of {@code c}, falling back to {@link Character#toLowerCase(int)}
+ * for codepoints where fold equals lowercase.
+ */
+ static int fold(int c) {
+ switch (c) {
+ // Common
+ case 0x00B5:
+ return 0x03BC; // µ → μ
+
+ // Latin
+ case 0x0130:
+ return 0x0130; // İ → İ
+ case 0x017F:
+ return 0x0073; // ſ → s
+
+ // Inherited
+ case 0x0345:
+ return 0x03B9; // ͅ → ι
+
+ // Greek
+ case 0x03C2:
+ return 0x03C3; // ς → σ
+ case 0x03D0:
+ return 0x03B2; // ϐ → β
+ case 0x03D1:
+ return 0x03B8; // ϑ → θ
+ case 0x03D5:
+ return 0x03C6; // ϕ → φ
+ case 0x03D6:
+ return 0x03C0; // ϖ → π
+ case 0x03F0:
+ return 0x03BA; // ϰ → κ
+ case 0x03F1:
+ return 0x03C1; // ϱ → ρ
+ case 0x03F5:
+ return 0x03B5; // ϵ → ε
+
+ // Cherokee
+ case 0x13A0:
+ return 0x13A0; // Ꭰ → Ꭰ
+ case 0x13A1:
+ return 0x13A1; // Ꭱ → Ꭱ
+ case 0x13A2:
+ return 0x13A2; // Ꭲ → Ꭲ
+ case 0x13A3:
+ return 0x13A3; // Ꭳ → Ꭳ
+ case 0x13A4:
+ return 0x13A4; // Ꭴ → Ꭴ
+ case 0x13A5:
+ return 0x13A5; // Ꭵ → Ꭵ
+ case 0x13A6:
+ return 0x13A6; // Ꭶ → Ꭶ
+ case 0x13A7:
+ return 0x13A7; // Ꭷ → Ꭷ
+ case 0x13A8:
+ return 0x13A8; // Ꭸ → Ꭸ
+ case 0x13A9:
+ return 0x13A9; // Ꭹ → Ꭹ
+ case 0x13AA:
+ return 0x13AA; // Ꭺ → Ꭺ
+ case 0x13AB:
+ return 0x13AB; // Ꭻ → Ꭻ
+ case 0x13AC:
+ return 0x13AC; // Ꭼ → Ꭼ
+ case 0x13AD:
+ return 0x13AD; // Ꭽ → Ꭽ
+ case 0x13AE:
+ return 0x13AE; // Ꭾ → Ꭾ
+ case 0x13AF:
+ return 0x13AF; // Ꭿ → Ꭿ
+ case 0x13B0:
+ return 0x13B0; // Ꮀ → Ꮀ
+ case 0x13B1:
+ return 0x13B1; // Ꮁ → Ꮁ
+ case 0x13B2:
+ return 0x13B2; // Ꮂ → Ꮂ
+ case 0x13B3:
+ return 0x13B3; // Ꮃ → Ꮃ
+ case 0x13B4:
+ return 0x13B4; // Ꮄ → Ꮄ
+ case 0x13B5:
+ return 0x13B5; // Ꮅ → Ꮅ
+ case 0x13B6:
+ return 0x13B6; // Ꮆ → Ꮆ
+ case 0x13B7:
+ return 0x13B7; // Ꮇ → Ꮇ
+ case 0x13B8:
+ return 0x13B8; // Ꮈ → Ꮈ
+ case 0x13B9:
+ return 0x13B9; // Ꮉ → Ꮉ
+ case 0x13BA:
+ return 0x13BA; // Ꮊ → Ꮊ
+ case 0x13BB:
+ return 0x13BB; // Ꮋ → Ꮋ
+ case 0x13BC:
+ return 0x13BC; // Ꮌ → Ꮌ
+ case 0x13BD:
+ return 0x13BD; // Ꮍ → Ꮍ
+ case 0x13BE:
+ return 0x13BE; // Ꮎ → Ꮎ
+ case 0x13BF:
+ return 0x13BF; // Ꮏ → Ꮏ
+ case 0x13C0:
+ return 0x13C0; // Ꮐ → Ꮐ
+ case 0x13C1:
+ return 0x13C1; // Ꮑ → Ꮑ
+ case 0x13C2:
+ return 0x13C2; // Ꮒ → Ꮒ
+ case 0x13C3:
+ return 0x13C3; // Ꮓ → Ꮓ
+ case 0x13C4:
+ return 0x13C4; // Ꮔ → Ꮔ
+ case 0x13C5:
+ return 0x13C5; // Ꮕ → Ꮕ
+ case 0x13C6:
+ return 0x13C6; // Ꮖ → Ꮖ
+ case 0x13C7:
+ return 0x13C7; // Ꮗ → Ꮗ
+ case 0x13C8:
+ return 0x13C8; // Ꮘ → Ꮘ
+ case 0x13C9:
+ return 0x13C9; // Ꮙ → Ꮙ
+ case 0x13CA:
+ return 0x13CA; // Ꮚ → Ꮚ
+ case 0x13CB:
+ return 0x13CB; // Ꮛ → Ꮛ
+ case 0x13CC:
+ return 0x13CC; // Ꮜ → Ꮜ
+ case 0x13CD:
+ return 0x13CD; // Ꮝ → Ꮝ
+ case 0x13CE:
+ return 0x13CE; // Ꮞ → Ꮞ
+ case 0x13CF:
+ return 0x13CF; // Ꮟ → Ꮟ
+ case 0x13D0:
+ return 0x13D0; // Ꮠ → Ꮠ
+ case 0x13D1:
+ return 0x13D1; // Ꮡ → Ꮡ
+ case 0x13D2:
+ return 0x13D2; // Ꮢ → Ꮢ
+ case 0x13D3:
+ return 0x13D3; // Ꮣ → Ꮣ
+ case 0x13D4:
+ return 0x13D4; // Ꮤ → Ꮤ
+ case 0x13D5:
+ return 0x13D5; // Ꮥ → Ꮥ
+ case 0x13D6:
+ return 0x13D6; // Ꮦ → Ꮦ
+ case 0x13D7:
+ return 0x13D7; // Ꮧ → Ꮧ
+ case 0x13D8:
+ return 0x13D8; // Ꮨ → Ꮨ
+ case 0x13D9:
+ return 0x13D9; // Ꮩ → Ꮩ
+ case 0x13DA:
+ return 0x13DA; // Ꮪ → Ꮪ
+ case 0x13DB:
+ return 0x13DB; // Ꮫ → Ꮫ
+ case 0x13DC:
+ return 0x13DC; // Ꮬ → Ꮬ
+ case 0x13DD:
+ return 0x13DD; // Ꮭ → Ꮭ
+ case 0x13DE:
+ return 0x13DE; // Ꮮ → Ꮮ
+ case 0x13DF:
+ return 0x13DF; // Ꮯ → Ꮯ
+ case 0x13E0:
+ return 0x13E0; // Ꮰ → Ꮰ
+ case 0x13E1:
+ return 0x13E1; // Ꮱ → Ꮱ
+ case 0x13E2:
+ return 0x13E2; // Ꮲ → Ꮲ
+ case 0x13E3:
+ return 0x13E3; // Ꮳ → Ꮳ
+ case 0x13E4:
+ return 0x13E4; // Ꮴ → Ꮴ
+ case 0x13E5:
+ return 0x13E5; // Ꮵ → Ꮵ
+ case 0x13E6:
+ return 0x13E6; // Ꮶ → Ꮶ
+ case 0x13E7:
+ return 0x13E7; // Ꮷ → Ꮷ
+ case 0x13E8:
+ return 0x13E8; // Ꮸ → Ꮸ
+ case 0x13E9:
+ return 0x13E9; // Ꮹ → Ꮹ
+ case 0x13EA:
+ return 0x13EA; // Ꮺ → Ꮺ
+ case 0x13EB:
+ return 0x13EB; // Ꮻ → Ꮻ
+ case 0x13EC:
+ return 0x13EC; // Ꮼ → Ꮼ
+ case 0x13ED:
+ return 0x13ED; // Ꮽ → Ꮽ
+ case 0x13EE:
+ return 0x13EE; // Ꮾ → Ꮾ
+ case 0x13EF:
+ return 0x13EF; // Ꮿ → Ꮿ
+ case 0x13F0:
+ return 0x13F0; // Ᏸ → Ᏸ
+ case 0x13F1:
+ return 0x13F1; // Ᏹ → Ᏹ
+ case 0x13F2:
+ return 0x13F2; // Ᏺ → Ᏺ
+ case 0x13F3:
+ return 0x13F3; // Ᏻ → Ᏻ
+ case 0x13F4:
+ return 0x13F4; // Ᏼ → Ᏼ
+ case 0x13F5:
+ return 0x13F5; // Ᏽ → Ᏽ
+ case 0x13F8:
+ return 0x13F0; // ᏸ → Ᏸ
+ case 0x13F9:
+ return 0x13F1; // ᏹ → Ᏹ
+ case 0x13FA:
+ return 0x13F2; // ᏺ → Ᏺ
+ case 0x13FB:
+ return 0x13F3; // ᏻ → Ᏻ
+ case 0x13FC:
+ return 0x13F4; // ᏼ → Ᏼ
+ case 0x13FD:
+ return 0x13F5; // ᏽ → Ᏽ
+
+ // Cyrillic
+ case 0x1C80:
+ return 0x0432; // ᲀ → в
+ case 0x1C81:
+ return 0x0434; // ᲁ → д
+ case 0x1C82:
+ return 0x043E; // ᲂ → о
+ case 0x1C83:
+ return 0x0441; // ᲃ → с
+ case 0x1C84:
+ return 0x0442; // ᲄ → т
+ case 0x1C85:
+ return 0x0442; // ᲅ → т
+ case 0x1C86:
+ return 0x044A; // ᲆ → ъ
+ case 0x1C87:
+ return 0x0463; // ᲇ → ѣ
+ case 0x1C88:
+ return 0xA64B; // ᲈ → ꙋ
+
+ // Latin
+ case 0x1E9B:
+ return 0x1E61; // ẛ → ṡ
+
+ // Greek
+ case 0x1FBE:
+ return 0x03B9; // ι → ι
+ case 0x1FD3:
+ return 0x0390; // ΐ → ΐ
+ case 0x1FE3:
+ return 0x03B0; // ΰ → ΰ
+
+ // Latin
+ case 0xA7CE:
+ return 0xA7CF; // ꟎ → ꟏
+ case 0xA7D2:
+ return 0xA7D3; // ꟒ → ꟓ
+ case 0xA7D4:
+ return 0xA7D5; // ꟔ → ꟕ
+
+ // Cherokee
+ case 0xAB70:
+ return 0x13A0; // ꭰ → Ꭰ
+ case 0xAB71:
+ return 0x13A1; // ꭱ → Ꭱ
+ case 0xAB72:
+ return 0x13A2; // ꭲ → Ꭲ
+ case 0xAB73:
+ return 0x13A3; // ꭳ → Ꭳ
+ case 0xAB74:
+ return 0x13A4; // ꭴ → Ꭴ
+ case 0xAB75:
+ return 0x13A5; // ꭵ → Ꭵ
+ case 0xAB76:
+ return 0x13A6; // ꭶ → Ꭶ
+ case 0xAB77:
+ return 0x13A7; // ꭷ → Ꭷ
+ case 0xAB78:
+ return 0x13A8; // ꭸ → Ꭸ
+ case 0xAB79:
+ return 0x13A9; // ꭹ → Ꭹ
+ case 0xAB7A:
+ return 0x13AA; // ꭺ → Ꭺ
+ case 0xAB7B:
+ return 0x13AB; // ꭻ → Ꭻ
+ case 0xAB7C:
+ return 0x13AC; // ꭼ → Ꭼ
+ case 0xAB7D:
+ return 0x13AD; // ꭽ → Ꭽ
+ case 0xAB7E:
+ return 0x13AE; // ꭾ → Ꭾ
+ case 0xAB7F:
+ return 0x13AF; // ꭿ → Ꭿ
+ case 0xAB80:
+ return 0x13B0; // ꮀ → Ꮀ
+ case 0xAB81:
+ return 0x13B1; // ꮁ → Ꮁ
+ case 0xAB82:
+ return 0x13B2; // ꮂ → Ꮂ
+ case 0xAB83:
+ return 0x13B3; // ꮃ → Ꮃ
+ case 0xAB84:
+ return 0x13B4; // ꮄ → Ꮄ
+ case 0xAB85:
+ return 0x13B5; // ꮅ → Ꮅ
+ case 0xAB86:
+ return 0x13B6; // ꮆ → Ꮆ
+ case 0xAB87:
+ return 0x13B7; // ꮇ → Ꮇ
+ case 0xAB88:
+ return 0x13B8; // ꮈ → Ꮈ
+ case 0xAB89:
+ return 0x13B9; // ꮉ → Ꮉ
+ case 0xAB8A:
+ return 0x13BA; // ꮊ → Ꮊ
+ case 0xAB8B:
+ return 0x13BB; // ꮋ → Ꮋ
+ case 0xAB8C:
+ return 0x13BC; // ꮌ → Ꮌ
+ case 0xAB8D:
+ return 0x13BD; // ꮍ → Ꮍ
+ case 0xAB8E:
+ return 0x13BE; // ꮎ → Ꮎ
+ case 0xAB8F:
+ return 0x13BF; // ꮏ → Ꮏ
+ case 0xAB90:
+ return 0x13C0; // ꮐ → Ꮐ
+ case 0xAB91:
+ return 0x13C1; // ꮑ → Ꮑ
+ case 0xAB92:
+ return 0x13C2; // ꮒ → Ꮒ
+ case 0xAB93:
+ return 0x13C3; // ꮓ → Ꮓ
+ case 0xAB94:
+ return 0x13C4; // ꮔ → Ꮔ
+ case 0xAB95:
+ return 0x13C5; // ꮕ → Ꮕ
+ case 0xAB96:
+ return 0x13C6; // ꮖ → Ꮖ
+ case 0xAB97:
+ return 0x13C7; // ꮗ → Ꮗ
+ case 0xAB98:
+ return 0x13C8; // ꮘ → Ꮘ
+ case 0xAB99:
+ return 0x13C9; // ꮙ → Ꮙ
+ case 0xAB9A:
+ return 0x13CA; // ꮚ → Ꮚ
+ case 0xAB9B:
+ return 0x13CB; // ꮛ → Ꮛ
+ case 0xAB9C:
+ return 0x13CC; // ꮜ → Ꮜ
+ case 0xAB9D:
+ return 0x13CD; // ꮝ → Ꮝ
+ case 0xAB9E:
+ return 0x13CE; // ꮞ → Ꮞ
+ case 0xAB9F:
+ return 0x13CF; // ꮟ → Ꮟ
+ case 0xABA0:
+ return 0x13D0; // ꮠ → Ꮠ
+ case 0xABA1:
+ return 0x13D1; // ꮡ → Ꮡ
+ case 0xABA2:
+ return 0x13D2; // ꮢ → Ꮢ
+ case 0xABA3:
+ return 0x13D3; // ꮣ → Ꮣ
+ case 0xABA4:
+ return 0x13D4; // ꮤ → Ꮤ
+ case 0xABA5:
+ return 0x13D5; // ꮥ → Ꮥ
+ case 0xABA6:
+ return 0x13D6; // ꮦ → Ꮦ
+ case 0xABA7:
+ return 0x13D7; // ꮧ → Ꮧ
+ case 0xABA8:
+ return 0x13D8; // ꮨ → Ꮨ
+ case 0xABA9:
+ return 0x13D9; // ꮩ → Ꮩ
+ case 0xABAA:
+ return 0x13DA; // ꮪ → Ꮪ
+ case 0xABAB:
+ return 0x13DB; // ꮫ → Ꮫ
+ case 0xABAC:
+ return 0x13DC; // ꮬ → Ꮬ
+ case 0xABAD:
+ return 0x13DD; // ꮭ → Ꮭ
+ case 0xABAE:
+ return 0x13DE; // ꮮ → Ꮮ
+ case 0xABAF:
+ return 0x13DF; // ꮯ → Ꮯ
+ case 0xABB0:
+ return 0x13E0; // ꮰ → Ꮰ
+ case 0xABB1:
+ return 0x13E1; // ꮱ → Ꮱ
+ case 0xABB2:
+ return 0x13E2; // ꮲ → Ꮲ
+ case 0xABB3:
+ return 0x13E3; // ꮳ → Ꮳ
+ case 0xABB4:
+ return 0x13E4; // ꮴ → Ꮴ
+ case 0xABB5:
+ return 0x13E5; // ꮵ → Ꮵ
+ case 0xABB6:
+ return 0x13E6; // ꮶ → Ꮶ
+ case 0xABB7:
+ return 0x13E7; // ꮷ → Ꮷ
+ case 0xABB8:
+ return 0x13E8; // ꮸ → Ꮸ
+ case 0xABB9:
+ return 0x13E9; // ꮹ → Ꮹ
+ case 0xABBA:
+ return 0x13EA; // ꮺ → Ꮺ
+ case 0xABBB:
+ return 0x13EB; // ꮻ → Ꮻ
+ case 0xABBC:
+ return 0x13EC; // ꮼ → Ꮼ
+ case 0xABBD:
+ return 0x13ED; // ꮽ → Ꮽ
+ case 0xABBE:
+ return 0x13EE; // ꮾ → Ꮾ
+ case 0xABBF:
+ return 0x13EF; // ꮿ → Ꮿ
+
+ // Latin
+ case 0xFB05:
+ return 0xFB06; // ſt → st
+
+ // Beria_Erfe
+ case 0x16EA0:
+ return 0x16EBB; // 𖺠 → 𖺻
+ case 0x16EA1:
+ return 0x16EBC; // 𖺡 → 𖺼
+ case 0x16EA2:
+ return 0x16EBD; // 𖺢 → 𖺽
+ case 0x16EA3:
+ return 0x16EBE; // 𖺣 → 𖺾
+ case 0x16EA4:
+ return 0x16EBF; // 𖺤 → 𖺿
+ case 0x16EA5:
+ return 0x16EC0; // 𖺥 → 𖻀
+ case 0x16EA6:
+ return 0x16EC1; // 𖺦 → 𖻁
+ case 0x16EA7:
+ return 0x16EC2; // 𖺧 → 𖻂
+ case 0x16EA8:
+ return 0x16EC3; // 𖺨 → 𖻃
+ case 0x16EA9:
+ return 0x16EC4; // 𖺩 → 𖻄
+ case 0x16EAA:
+ return 0x16EC5; // 𖺪 → 𖻅
+ case 0x16EAB:
+ return 0x16EC6; // 𖺫 → 𖻆
+ case 0x16EAC:
+ return 0x16EC7; // 𖺬 → 𖻇
+ case 0x16EAD:
+ return 0x16EC8; // 𖺭 → 𖻈
+ case 0x16EAE:
+ return 0x16EC9; // 𖺮 → 𖻉
+ case 0x16EAF:
+ return 0x16ECA; // 𖺯 → 𖻊
+ case 0x16EB0:
+ return 0x16ECB; // 𖺰 → 𖻋
+ case 0x16EB1:
+ return 0x16ECC; // 𖺱 → 𖻌
+ case 0x16EB2:
+ return 0x16ECD; // 𖺲 → 𖻍
+ case 0x16EB3:
+ return 0x16ECE; // 𖺳 → 𖻎
+ case 0x16EB4:
+ return 0x16ECF; // 𖺴 → 𖻏
+ case 0x16EB5:
+ return 0x16ED0; // 𖺵 → 𖻐
+ case 0x16EB6:
+ return 0x16ED1; // 𖺶 → 𖻑
+ case 0x16EB7:
+ return 0x16ED2; // 𖺷 → 𖻒
+ case 0x16EB8:
+ return 0x16ED3; // 𖺸 → 𖻓
+ default:
+ return Character.toLowerCase(c);
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
index 6f493c0556d..758c9f0cb68 100644
--- a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
@@ -686,4 +686,9 @@ public final class UnicodeUtil {
public static int UTF8toUTF16(BytesRef bytesRef, char[] chars) {
return UTF8toUTF16(bytesRef.bytes, bytesRef.offset, bytesRef.length, chars);
}
+
+ /** Returns the Unicode simple case folding of {@code codepoint}. */
+ public static int foldCase(int codepoint) {
+ return SimpleCaseFolding.fold(codepoint);
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
index e359b6ddb63..3b36b9bcc39 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automata.java
@@ -699,7 +699,7 @@ public final class Automata {
*/
private static int[] toCaseInsensitiveChar(int codepoint) {
IntArrayList list = new IntArrayList();
- CaseFolding.expand(
+ CaseExpansion.expand(
codepoint,
(int variant) -> {
list.add(variant);
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java
rename to lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java
index f21e0ea3834..3477a610cf1 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseExpansion.java
@@ -1,4 +1,4 @@
-// DO NOT EDIT THIS FILE! Use "gradlew generateUnicodeProps tidy" to recreate.
+// DO NOT EDIT THIS FILE! Use "gradlew regenerateCaseExpansion tidy" to recreate.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -23,10 +23,10 @@ import java.util.function.IntConsumer;
/**
* This file contains unicode properties used by {@code RegExp}. The data was generated using ICU4J
- * v77.1.0.0, unicode version: 16.0.0.0.
+ * v78.3.0.0, unicode version: 17.0.0.0.
*/
-final class CaseFolding {
- private CaseFolding() {}
+final class CaseExpansion {
+ private CaseExpansion() {}
/** Calls {@code fn} consumer with {@code c} itself and its {@code scf} mappings. */
static void expand(int c, IntConsumer fn) {
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
index ce4d2d2d951..8367f81f0df 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
@@ -742,7 +742,7 @@ public class RegExp {
*/
private int[] toCaseInsensitiveChar(int codepoint) {
IntArrayList list = new IntArrayList();
- CaseFolding.expand(
+ CaseExpansion.expand(
codepoint,
(int variant) -> {
list.add(variant);
@@ -770,7 +770,7 @@ public class RegExp {
// iterate over range, adding codepoint and any alternatives as transitions
for (int i = start; i <= end; i++) {
- CaseFolding.expand(
+ CaseExpansion.expand(
i,
(int ch) -> {
scratch.addTransition(state, state, ch);
diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java b/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java
index 43352537d2a..e0de632d223 100644
--- a/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/analysis/TestCharacterUtils.java
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.CharacterUtils.CharacterBuffer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.UnicodeUtil;
/** TestCase for the {@link CharacterUtils} class. */
public class TestCharacterUtils extends LuceneTestCase {
@@ -109,4 +110,44 @@ public class TestCharacterUtils extends LuceneTestCase {
assertFalse(CharacterUtils.fill(buffer, reader));
assertEquals(0, buffer.getLength());
}
+
+ public void testAsciiFoldMatchesToLowerCase() {
+ for (int c = 0; c < 128; c++) {
+ assertEquals(Character.toLowerCase(c), UnicodeUtil.foldCase(c));
+ }
+ }
+
+ public void testFoldExceptions() {
+ assertEquals(0x03BC, UnicodeUtil.foldCase(0x00B5));
+ assertEquals(0x0073, UnicodeUtil.foldCase(0x017F));
+ assertEquals(0x03C3, UnicodeUtil.foldCase(0x03C2));
+ assertEquals('0', UnicodeUtil.foldCase('0'));
+ assertEquals('a', UnicodeUtil.foldCase('a'));
+ assertEquals('a', UnicodeUtil.foldCase('A'));
+ }
+
+ public void testFoldIdempotent() {
+ for (int c = 0; c <= 0xFFFF; c++) {
+ int folded = UnicodeUtil.foldCase(c);
+ assertEquals(
+ "Not idempotent at U+" + Integer.toHexString(c), folded, UnicodeUtil.foldCase(folded));
+ }
+ }
+
+ public void testSimpleCaseFold() {
+ char[] buf = "ABcΣσς".toCharArray();
+ CharacterUtils.simpleCaseFold(buf, 0, buf.length);
+ assertEquals("abcσσσ", new String(buf));
+ }
+
+ public void testSimpleCaseFoldRandom() {
+ for (int iter = 0; iter < 100; iter++) {
+ String s = TestUtil.randomUnicodeString(random(), 100);
+ char[] buf = s.toCharArray();
+ CharacterUtils.simpleCaseFold(buf, 0, buf.length);
+ char[] buf2 = new String(buf).toCharArray();
+ CharacterUtils.simpleCaseFold(buf2, 0, buf2.length);
+ assertArrayEquals(buf, buf2);
+ }
+ }
}