Skip to content

Apache Lucene open-source search software

Unicode Support for Case Insensitive Matching in RegExp (#14192)

John Wagster7c050f9c6e47
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 44c045a0b6e..199822b658a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -85,10 +85,12 @@ Improvements
the wrapped analyzer's strategy to decide if components can be reused or need
to be updated. (Mayya Sharipova)
+* GITHUB#14192: Added support for RegExp to handle case insensitive matching
+ across the full set of Unicode characters. (John Wagster)
+
* GITHUB#14239: Hunspell's option to tolerate affix rule count mismatches was
improved to tolerate more instances of this problem. (Robert Muir)
-
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java
new file mode 100644
index 00000000000..dba1e6438e2
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java
@@ -0,0 +1,746 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.automaton;
+
+class CaseFolding {
+
+ /**
+ * Generates the set of codepoints which represent the given codepoint that are case-insensitive
+ * matches within the Unicode table, which may not always be intuitive for instance Σ, σ, ς do all
+ * fold to one another and so would match one another
+ *
+ * <p>Known special cases derived from generating mappings using a combination of the Unicode
+ * 16.0.0 spec: https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt and
+ * https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt these are only the alternate
+ * mappings for each codepoint that are not supported by a transform using Character.toUpperCase
+ * or Character.toLowerCase
+ *
+ * @param codepoint the codepoint for the character to case fold
+ * @return an array of characters as codepoints that should match the given codepoint in a
+ * case-insensitive context or null if no alternates exist this does not include the given
+ * codepoint itself
+ */
+ static int[] lookupAlternates(int codepoint) {
+
+ int[] alts =
+ switch (codepoint) {
+ case 0x00049 -> // I [LATIN CAPITAL LETTER I]
+ new int[] {
+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]
+ 0x00069, // i [LATIN SMALL LETTER I]
+ };
+ case 0x0004B -> // K [LATIN CAPITAL LETTER K]
+ new int[] {
+ 0x0212A, // K [KELVIN SIGN]
+ 0x0006B, // k [LATIN SMALL LETTER K]
+ };
+ case 0x00053 -> // S [LATIN CAPITAL LETTER S]
+ new int[] {
+ 0x00073, // s [LATIN SMALL LETTER S]
+ 0x0017F, // ſ [LATIN SMALL LETTER LONG S]
+ };
+ case 0x00069 -> // i [LATIN SMALL LETTER I]
+ new int[] {
+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]
+ 0x00049, // I [LATIN CAPITAL LETTER I]
+ };
+ case 0x0006B -> // k [LATIN SMALL LETTER K]
+ new int[] {
+ 0x0212A, // K [KELVIN SIGN]
+ 0x0004B, // K [LATIN CAPITAL LETTER K]
+ };
+ case 0x00073 -> // s [LATIN SMALL LETTER S]
+ new int[] {
+ 0x00053, // S [LATIN CAPITAL LETTER S]
+ 0x0017F, // ſ [LATIN SMALL LETTER LONG S]
+ };
+ case 0x000B5 -> // µ [MICRO SIGN]
+ new int[] {
+ 0x0039C, // Μ [GREEK CAPITAL LETTER MU]
+ 0x003BC, // μ [GREEK SMALL LETTER MU]
+ };
+ case 0x000C5 -> // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]
+ new int[] {
+ 0x000E5, // å [LATIN SMALL LETTER A WITH RING ABOVE]
+ 0x0212B, // Å [ANGSTROM SIGN]
+ };
+ case 0x000DF -> // ß [LATIN SMALL LETTER SHARP S]
+ new int[] {
+ 0x01E9E, // ẞ [LATIN CAPITAL LETTER SHARP S]
+ };
+ case 0x000E5 -> // å [LATIN SMALL LETTER A WITH RING ABOVE]
+ new int[] {
+ 0x000C5, // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]
+ 0x0212B, // Å [ANGSTROM SIGN]
+ };
+ case 0x02126 -> // Ω [OHM SIGN]
+ new int[] {
+ 0x003A9, // Ω [GREEK CAPITAL LETTER OMEGA]
+ 0x003C9, // ω [GREEK SMALL LETTER OMEGA]
+ };
+ case 0x0212A -> // K [KELVIN SIGN]
+ new int[] {
+ 0x0004B, // K [LATIN CAPITAL LETTER K]
+ 0x0006B, // k [LATIN SMALL LETTER K]
+ };
+ case 0x0212B -> // Å [ANGSTROM SIGN]
+ new int[] {
+ 0x000C5, // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]
+ 0x000E5, // å [LATIN SMALL LETTER A WITH RING ABOVE]
+ };
+ case 0x00130 -> // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+ new int[] {
+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]
+ 0x00049, // I [LATIN CAPITAL LETTER I]
+ 0x00069, // i [LATIN SMALL LETTER I]
+ };
+ case 0x00131 -> // ı [LATIN SMALL LETTER DOTLESS I]
+ new int[] {
+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]
+ 0x00069, // i [LATIN SMALL LETTER I]
+ 0x00049, // I [LATIN CAPITAL LETTER I]
+ };
+ case 0x0017F -> // ſ [LATIN SMALL LETTER LONG S]
+ new int[] {
+ 0x00053, // S [LATIN CAPITAL LETTER S]
+ 0x00073, // s [LATIN SMALL LETTER S]
+ };
+ case 0x0019B -> // ƛ [LATIN SMALL LETTER LAMBDA WITH STROKE]
+ new int[] {
+ 0x0A7DC, // Ƛ [LATIN CAPITAL LETTER LAMBDA WITH STROKE]
+ };
+ case 0x001C4 -> // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]
+ new int[] {
+ 0x001C5, // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]
+ 0x001C6, // dž [LATIN SMALL LETTER DZ WITH CARON]
+ };
+ case 0x001C5 -> // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]
+ new int[] {
+ 0x001C4, // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]
+ 0x001C6, // dž [LATIN SMALL LETTER DZ WITH CARON]
+ };
+ case 0x001C6 -> // dž [LATIN SMALL LETTER DZ WITH CARON]
+ new int[] {
+ 0x001C4, // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]
+ 0x001C5, // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]
+ };
+ case 0x001C7 -> // LJ [LATIN CAPITAL LETTER LJ]
+ new int[] {
+ 0x001C8, // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]
+ 0x001C9, // lj [LATIN SMALL LETTER LJ]
+ };
+ case 0x001C8 -> // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]
+ new int[] {
+ 0x001C7, // LJ [LATIN CAPITAL LETTER LJ]
+ 0x001C9, // lj [LATIN SMALL LETTER LJ]
+ };
+ case 0x001C9 -> // lj [LATIN SMALL LETTER LJ]
+ new int[] {
+ 0x001C7, // LJ [LATIN CAPITAL LETTER LJ]
+ 0x001C8, // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]
+ };
+ case 0x001CA -> // NJ [LATIN CAPITAL LETTER NJ]
+ new int[] {
+ 0x001CB, // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]
+ 0x001CC, // nj [LATIN SMALL LETTER NJ]
+ };
+ case 0x001CB -> // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]
+ new int[] {
+ 0x001CA, // NJ [LATIN CAPITAL LETTER NJ]
+ 0x001CC, // nj [LATIN SMALL LETTER NJ]
+ };
+ case 0x001CC -> // nj [LATIN SMALL LETTER NJ]
+ new int[] {
+ 0x001CA, // NJ [LATIN CAPITAL LETTER NJ]
+ 0x001CB, // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]
+ };
+ case 0x001F1 -> // DZ [LATIN CAPITAL LETTER DZ]
+ new int[] {
+ 0x001F2, // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]
+ 0x001F3, // dz [LATIN SMALL LETTER DZ]
+ };
+ case 0x001F2 -> // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]
+ new int[] {
+ 0x001F1, // DZ [LATIN CAPITAL LETTER DZ]
+ 0x001F3, // dz [LATIN SMALL LETTER DZ]
+ };
+ case 0x001F3 -> // dz [LATIN SMALL LETTER DZ]
+ new int[] {
+ 0x001F1, // DZ [LATIN CAPITAL LETTER DZ]
+ 0x001F2, // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]
+ };
+ case 0x00264 -> // ɤ [LATIN SMALL LETTER RAMS HORN]
+ new int[] {
+ 0x0A7CB, // Ɤ [LATIN CAPITAL LETTER RAMS HORN]
+ };
+ case 0x00345 -> // ͅ [COMBINING GREEK YPOGEGRAMMENI]
+ new int[] {
+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]
+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]
+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]
+ };
+ case 0x00390 -> // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS]
+ new int[] {
+ 0x01FD3, // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA]
+ };
+ case 0x00392 -> // Β [GREEK CAPITAL LETTER BETA]
+ new int[] {
+ 0x003D0, // ϐ [GREEK BETA SYMBOL]
+ 0x003B2, // β [GREEK SMALL LETTER BETA]
+ };
+ case 0x00395 -> // Ε [GREEK CAPITAL LETTER EPSILON]
+ new int[] {
+ 0x003F5, // ϵ [GREEK LUNATE EPSILON SYMBOL]
+ 0x003B5, // ε [GREEK SMALL LETTER EPSILON]
+ };
+ case 0x00398 -> // Θ [GREEK CAPITAL LETTER THETA]
+ new int[] {
+ 0x003D1, // ϑ [GREEK THETA SYMBOL]
+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]
+ 0x003B8, // θ [GREEK SMALL LETTER THETA]
+ };
+ case 0x00399 -> // Ι [GREEK CAPITAL LETTER IOTA]
+ new int[] {
+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]
+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]
+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]
+ };
+ case 0x0039A -> // Κ [GREEK CAPITAL LETTER KAPPA]
+ new int[] {
+ 0x003F0, // ϰ [GREEK KAPPA SYMBOL]
+ 0x003BA, // κ [GREEK SMALL LETTER KAPPA]
+ };
+ case 0x0039C -> // Μ [GREEK CAPITAL LETTER MU]
+ new int[] {
+ 0x000B5, // µ [MICRO SIGN]
+ 0x003BC, // μ [GREEK SMALL LETTER MU]
+ };
+ case 0x003A0 -> // Π [GREEK CAPITAL LETTER PI]
+ new int[] {
+ 0x003C0, // π [GREEK SMALL LETTER PI]
+ 0x003D6, // ϖ [GREEK PI SYMBOL]
+ };
+ case 0x003A1 -> // Ρ [GREEK CAPITAL LETTER RHO]
+ new int[] {
+ 0x003F1, // ϱ [GREEK RHO SYMBOL]
+ 0x003C1, // ρ [GREEK SMALL LETTER RHO]
+ };
+ case 0x003A3 -> // Σ [GREEK CAPITAL LETTER SIGMA]
+ new int[] {
+ 0x003C2, // ς [GREEK SMALL LETTER FINAL SIGMA]
+ 0x003C3, // σ [GREEK SMALL LETTER SIGMA]
+ };
+ case 0x003A6 -> // Φ [GREEK CAPITAL LETTER PHI]
+ new int[] {
+ 0x003D5, // ϕ [GREEK PHI SYMBOL]
+ 0x003C6, // φ [GREEK SMALL LETTER PHI]
+ };
+ case 0x003A9 -> // Ω [GREEK CAPITAL LETTER OMEGA]
+ new int[] {
+ 0x02126, // Ω [OHM SIGN]
+ 0x003C9, // ω [GREEK SMALL LETTER OMEGA]
+ };
+ case 0x003B0 -> // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS]
+ new int[] {
+ 0x01FE3, // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA]
+ };
+ case 0x003B2 -> // β [GREEK SMALL LETTER BETA]
+ new int[] {
+ 0x003D0, // ϐ [GREEK BETA SYMBOL]
+ 0x00392, // Β [GREEK CAPITAL LETTER BETA]
+ };
+ case 0x003B5 -> // ε [GREEK SMALL LETTER EPSILON]
+ new int[] {
+ 0x00395, // Ε [GREEK CAPITAL LETTER EPSILON]
+ 0x003F5, // ϵ [GREEK LUNATE EPSILON SYMBOL]
+ };
+ case 0x003B8 -> // θ [GREEK SMALL LETTER THETA]
+ new int[] {
+ 0x003D1, // ϑ [GREEK THETA SYMBOL]
+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]
+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]
+ };
+ case 0x003B9 -> // ι [GREEK SMALL LETTER IOTA]
+ new int[] {
+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]
+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]
+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]
+ };
+ case 0x003BA -> // κ [GREEK SMALL LETTER KAPPA]
+ new int[] {
+ 0x003F0, // ϰ [GREEK KAPPA SYMBOL]
+ 0x0039A, // Κ [GREEK CAPITAL LETTER KAPPA]
+ };
+ case 0x003BC -> // μ [GREEK SMALL LETTER MU]
+ new int[] {
+ 0x000B5, // µ [MICRO SIGN]
+ 0x0039C, // Μ [GREEK CAPITAL LETTER MU]
+ };
+ case 0x003C0 -> // π [GREEK SMALL LETTER PI]
+ new int[] {
+ 0x003A0, // Π [GREEK CAPITAL LETTER PI]
+ 0x003D6, // ϖ [GREEK PI SYMBOL]
+ };
+ case 0x003C1 -> // ρ [GREEK SMALL LETTER RHO]
+ new int[] {
+ 0x003A1, // Ρ [GREEK CAPITAL LETTER RHO]
+ 0x003F1, // ϱ [GREEK RHO SYMBOL]
+ };
+ case 0x003C2 -> // ς [GREEK SMALL LETTER FINAL SIGMA]
+ new int[] {
+ 0x003A3, // Σ [GREEK CAPITAL LETTER SIGMA]
+ 0x003C3, // σ [GREEK SMALL LETTER SIGMA]
+ };
+ case 0x003C3 -> // σ [GREEK SMALL LETTER SIGMA]
+ new int[] {
+ 0x003C2, // ς [GREEK SMALL LETTER FINAL SIGMA]
+ 0x003A3, // Σ [GREEK CAPITAL LETTER SIGMA]
+ };
+ case 0x003C6 -> // φ [GREEK SMALL LETTER PHI]
+ new int[] {
+ 0x003D5, // ϕ [GREEK PHI SYMBOL]
+ 0x003A6, // Φ [GREEK CAPITAL LETTER PHI]
+ };
+ case 0x003C9 -> // ω [GREEK SMALL LETTER OMEGA]
+ new int[] {
+ 0x02126, // Ω [OHM SIGN]
+ 0x003A9, // Ω [GREEK CAPITAL LETTER OMEGA]
+ };
+ case 0x003D0 -> // ϐ [GREEK BETA SYMBOL]
+ new int[] {
+ 0x00392, // Β [GREEK CAPITAL LETTER BETA]
+ 0x003B2, // β [GREEK SMALL LETTER BETA]
+ };
+ case 0x003D1 -> // ϑ [GREEK THETA SYMBOL]
+ new int[] {
+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]
+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]
+ 0x003B8, // θ [GREEK SMALL LETTER THETA]
+ };
+ case 0x003D5 -> // ϕ [GREEK PHI SYMBOL]
+ new int[] {
+ 0x003A6, // Φ [GREEK CAPITAL LETTER PHI]
+ 0x003C6, // φ [GREEK SMALL LETTER PHI]
+ };
+ case 0x003D6 -> // ϖ [GREEK PI SYMBOL]
+ new int[] {
+ 0x003A0, // Π [GREEK CAPITAL LETTER PI]
+ 0x003C0, // π [GREEK SMALL LETTER PI]
+ };
+ case 0x003F0 -> // ϰ [GREEK KAPPA SYMBOL]
+ new int[] {
+ 0x0039A, // Κ [GREEK CAPITAL LETTER KAPPA]
+ 0x003BA, // κ [GREEK SMALL LETTER KAPPA]
+ };
+ case 0x003F1 -> // ϱ [GREEK RHO SYMBOL]
+ new int[] {
+ 0x003A1, // Ρ [GREEK CAPITAL LETTER RHO]
+ 0x003C1, // ρ [GREEK SMALL LETTER RHO]
+ };
+ case 0x003F4 -> // ϴ [GREEK CAPITAL THETA SYMBOL]
+ new int[] {
+ 0x003D1, // ϑ [GREEK THETA SYMBOL]
+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]
+ 0x003B8, // θ [GREEK SMALL LETTER THETA]
+ };
+ case 0x003F5 -> // ϵ [GREEK LUNATE EPSILON SYMBOL]
+ new int[] {
+ 0x00395, // Ε [GREEK CAPITAL LETTER EPSILON]
+ 0x003B5, // ε [GREEK SMALL LETTER EPSILON]
+ };
+ case 0x00412 -> // В [CYRILLIC CAPITAL LETTER VE]
+ new int[] {
+ 0x01C80, // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]
+ 0x00432, // в [CYRILLIC SMALL LETTER VE]
+ };
+ case 0x00414 -> // Д [CYRILLIC CAPITAL LETTER DE]
+ new int[] {
+ 0x01C81, // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]
+ 0x00434, // д [CYRILLIC SMALL LETTER DE]
+ };
+ case 0x0041E -> // О [CYRILLIC CAPITAL LETTER O]
+ new int[] {
+ 0x01C82, // ᲂ [CYRILLIC SMALL LETTER NARROW O]
+ 0x0043E, // о [CYRILLIC SMALL LETTER O]
+ };
+ case 0x00421 -> // С [CYRILLIC CAPITAL LETTER ES]
+ new int[] {
+ 0x00441, // с [CYRILLIC SMALL LETTER ES]
+ 0x01C83, // ᲃ [CYRILLIC SMALL LETTER WIDE ES]
+ };
+ case 0x00422 -> // Т [CYRILLIC CAPITAL LETTER TE]
+ new int[] {
+ 0x00442, // т [CYRILLIC SMALL LETTER TE]
+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]
+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]
+ };
+ case 0x0042A -> // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]
+ new int[] {
+ 0x01C86, // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]
+ 0x0044A, // ъ [CYRILLIC SMALL LETTER HARD SIGN]
+ };
+ case 0x00432 -> // в [CYRILLIC SMALL LETTER VE]
+ new int[] {
+ 0x01C80, // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]
+ 0x00412, // В [CYRILLIC CAPITAL LETTER VE]
+ };
+ case 0x00434 -> // д [CYRILLIC SMALL LETTER DE]
+ new int[] {
+ 0x01C81, // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]
+ 0x00414, // Д [CYRILLIC CAPITAL LETTER DE]
+ };
+ case 0x0043E -> // о [CYRILLIC SMALL LETTER O]
+ new int[] {
+ 0x01C82, // ᲂ [CYRILLIC SMALL LETTER NARROW O]
+ 0x0041E, // О [CYRILLIC CAPITAL LETTER O]
+ };
+ case 0x00441 -> // с [CYRILLIC SMALL LETTER ES]
+ new int[] {
+ 0x00421, // С [CYRILLIC CAPITAL LETTER ES]
+ 0x01C83, // ᲃ [CYRILLIC SMALL LETTER WIDE ES]
+ };
+ case 0x00442 -> // т [CYRILLIC SMALL LETTER TE]
+ new int[] {
+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]
+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]
+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]
+ };
+ case 0x0044A -> // ъ [CYRILLIC SMALL LETTER HARD SIGN]
+ new int[] {
+ 0x01C86, // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]
+ 0x0042A, // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]
+ };
+ case 0x00462 -> // Ѣ [CYRILLIC CAPITAL LETTER YAT]
+ new int[] {
+ 0x00463, // ѣ [CYRILLIC SMALL LETTER YAT]
+ 0x01C87, // ᲇ [CYRILLIC SMALL LETTER TALL YAT]
+ };
+ case 0x00463 -> // ѣ [CYRILLIC SMALL LETTER YAT]
+ new int[] {
+ 0x00462, // Ѣ [CYRILLIC CAPITAL LETTER YAT]
+ 0x01C87, // ᲇ [CYRILLIC SMALL LETTER TALL YAT]
+ };
+ case 0x0A64A -> // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]
+ new int[] {
+ 0x01C88, // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]
+ 0x0A64B, // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]
+ };
+ case 0x0A64B -> // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]
+ new int[] {
+ 0x01C88, // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]
+ 0x0A64A, // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]
+ };
+ case 0x0A7CB -> // Ɤ [LATIN CAPITAL LETTER RAMS HORN]
+ new int[] {
+ 0x00264, // ɤ [LATIN SMALL LETTER RAMS HORN]
+ };
+ case 0x0A7CC -> // Ꟍ [LATIN CAPITAL LETTER S WITH DIAGONAL STROKE]
+ new int[] {
+ 0x0A7CD, // ꟍ [LATIN SMALL LETTER S WITH DIAGONAL STROKE]
+ };
+ case 0x0A7CD -> // ꟍ [LATIN SMALL LETTER S WITH DIAGONAL STROKE]
+ new int[] {
+ 0x0A7CC, // Ꟍ [LATIN CAPITAL LETTER S WITH DIAGONAL STROKE]
+ };
+ case 0x0A7DA -> // Ꟛ [LATIN CAPITAL LETTER LAMBDA]
+ new int[] {
+ 0x0A7DB, // ꟛ [LATIN SMALL LETTER LAMBDA]
+ };
+ case 0x0A7DB -> // ꟛ [LATIN SMALL LETTER LAMBDA]
+ new int[] {
+ 0x0A7DA, // Ꟛ [LATIN CAPITAL LETTER LAMBDA]
+ };
+ case 0x0A7DC -> // Ƛ [LATIN CAPITAL LETTER LAMBDA WITH STROKE]
+ new int[] {
+ 0x0019B, // ƛ [LATIN SMALL LETTER LAMBDA WITH STROKE]
+ };
+ case 0x0FB05 -> // ſt [LATIN SMALL LIGATURE LONG S T]
+ new int[] {
+ 0x0FB06, // st [LATIN SMALL LIGATURE ST]
+ };
+ case 0x0FB06 -> // st [LATIN SMALL LIGATURE ST]
+ new int[] {
+ 0x0FB05, // ſt [LATIN SMALL LIGATURE LONG S T]
+ };
+ case 0x01C80 -> // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]
+ new int[] {
+ 0x00412, // В [CYRILLIC CAPITAL LETTER VE]
+ 0x00432, // в [CYRILLIC SMALL LETTER VE]
+ };
+ case 0x01C81 -> // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]
+ new int[] {
+ 0x00414, // Д [CYRILLIC CAPITAL LETTER DE]
+ 0x00434, // д [CYRILLIC SMALL LETTER DE]
+ };
+ case 0x01C82 -> // ᲂ [CYRILLIC SMALL LETTER NARROW O]
+ new int[] {
+ 0x0041E, // О [CYRILLIC CAPITAL LETTER O]
+ 0x0043E, // о [CYRILLIC SMALL LETTER O]
+ };
+ case 0x01C83 -> // ᲃ [CYRILLIC SMALL LETTER WIDE ES]
+ new int[] {
+ 0x00421, // С [CYRILLIC CAPITAL LETTER ES]
+ 0x00441, // с [CYRILLIC SMALL LETTER ES]
+ };
+ case 0x01C84 -> // ᲄ [CYRILLIC SMALL LETTER TALL TE]
+ new int[] {
+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]
+ 0x00442, // т [CYRILLIC SMALL LETTER TE]
+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]
+ };
+ case 0x01C85 -> // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]
+ new int[] {
+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]
+ 0x00442, // т [CYRILLIC SMALL LETTER TE]
+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]
+ };
+ case 0x01C86 -> // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]
+ new int[] {
+ 0x0042A, // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]
+ 0x0044A, // ъ [CYRILLIC SMALL LETTER HARD SIGN]
+ };
+ case 0x01C87 -> // ᲇ [CYRILLIC SMALL LETTER TALL YAT]
+ new int[] {
+ 0x00462, // Ѣ [CYRILLIC CAPITAL LETTER YAT]
+ 0x00463, // ѣ [CYRILLIC SMALL LETTER YAT]
+ };
+ case 0x01C88 -> // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]
+ new int[] {
+ 0x0A64A, // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]
+ 0x0A64B, // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]
+ };
+ case 0x01C89 -> // Ᲊ [CYRILLIC CAPITAL LETTER TJE]
+ new int[] {
+ 0x01C8A, // ᲊ [CYRILLIC SMALL LETTER TJE]
+ };
+ case 0x01C8A -> // ᲊ [CYRILLIC SMALL LETTER TJE]
+ new int[] {
+ 0x01C89, // Ᲊ [CYRILLIC CAPITAL LETTER TJE]
+ };
+ case 0x10D51 -> // 𐵑 [GARAY CAPITAL LETTER CA]
+ new int[] {
+ 0x10D71, // 𐵱 [GARAY SMALL LETTER CA]
+ };
+ case 0x10D50 -> // 𐵐 [GARAY CAPITAL LETTER A]
+ new int[] {
+ 0x10D70, // 𐵰 [GARAY SMALL LETTER A]
+ };
+ case 0x10D53 -> // 𐵓 [GARAY CAPITAL LETTER KA]
+ new int[] {
+ 0x10D73, // 𐵳 [GARAY SMALL LETTER KA]
+ };
+ case 0x10D52 -> // 𐵒 [GARAY CAPITAL LETTER MA]
+ new int[] {
+ 0x10D72, // 𐵲 [GARAY SMALL LETTER MA]
+ };
+ case 0x10D55 -> // 𐵕 [GARAY CAPITAL LETTER JA]
+ new int[] {
+ 0x10D75, // 𐵵 [GARAY SMALL LETTER JA]
+ };
+ case 0x10D54 -> // 𐵔 [GARAY CAPITAL LETTER BA]
+ new int[] {
+ 0x10D74, // 𐵴 [GARAY SMALL LETTER BA]
+ };
+ case 0x10D57 -> // 𐵗 [GARAY CAPITAL LETTER WA]
+ new int[] {
+ 0x10D77, // 𐵷 [GARAY SMALL LETTER WA]
+ };
+ case 0x10D56 -> // 𐵖 [GARAY CAPITAL LETTER SA]
+ new int[] {
+ 0x10D76, // 𐵶 [GARAY SMALL LETTER SA]
+ };
+ case 0x10D59 -> // 𐵙 [GARAY CAPITAL LETTER GA]
+ new int[] {
+ 0x10D79, // 𐵹 [GARAY SMALL LETTER GA]
+ };
+ case 0x10D58 -> // 𐵘 [GARAY CAPITAL LETTER LA]
+ new int[] {
+ 0x10D78, // 𐵸 [GARAY SMALL LETTER LA]
+ };
+ case 0x10D5B -> // 𐵛 [GARAY CAPITAL LETTER XA]
+ new int[] {
+ 0x10D7B, // 𐵻 [GARAY SMALL LETTER XA]
+ };
+ case 0x10D5A -> // 𐵚 [GARAY CAPITAL LETTER DA]
+ new int[] {
+ 0x10D7A, // 𐵺 [GARAY SMALL LETTER DA]
+ };
+ case 0x10D5D -> // 𐵝 [GARAY CAPITAL LETTER TA]
+ new int[] {
+ 0x10D7D, // 𐵽 [GARAY SMALL LETTER TA]
+ };
+ case 0x10D5C -> // 𐵜 [GARAY CAPITAL LETTER YA]
+ new int[] {
+ 0x10D7C, // 𐵼 [GARAY SMALL LETTER YA]
+ };
+ case 0x10D5F -> // 𐵟 [GARAY CAPITAL LETTER NYA]
+ new int[] {
+ 0x10D7F, // 𐵿 [GARAY SMALL LETTER NYA]
+ };
+ case 0x10D5E -> // 𐵞 [GARAY CAPITAL LETTER RA]
+ new int[] {
+ 0x10D7E, // 𐵾 [GARAY SMALL LETTER RA]
+ };
+ case 0x10D61 -> // 𐵡 [GARAY CAPITAL LETTER NA]
+ new int[] {
+ 0x10D81, // 𐶁 [GARAY SMALL LETTER NA]
+ };
+ case 0x10D60 -> // 𐵠 [GARAY CAPITAL LETTER FA]
+ new int[] {
+ 0x10D80, // 𐶀 [GARAY SMALL LETTER FA]
+ };
+ case 0x10D63 -> // 𐵣 [GARAY CAPITAL LETTER HA]
+ new int[] {
+ 0x10D83, // 𐶃 [GARAY SMALL LETTER HA]
+ };
+ case 0x10D62 -> // 𐵢 [GARAY CAPITAL LETTER PA]
+ new int[] {
+ 0x10D82, // 𐶂 [GARAY SMALL LETTER PA]
+ };
+ case 0x10D65 -> // 𐵥 [GARAY CAPITAL LETTER OLD NA]
+ new int[] {
+ 0x10D85, // 𐶅 [GARAY SMALL LETTER OLD NA]
+ };
+ case 0x10D64 -> // 𐵤 [GARAY CAPITAL LETTER OLD KA]
+ new int[] {
+ 0x10D84, // 𐶄 [GARAY SMALL LETTER OLD KA]
+ };
+ case 0x10D71 -> // 𐵱 [GARAY SMALL LETTER CA]
+ new int[] {
+ 0x10D51, // 𐵑 [GARAY CAPITAL LETTER CA]
+ };
+ case 0x10D70 -> // 𐵰 [GARAY SMALL LETTER A]
+ new int[] {
+ 0x10D50, // 𐵐 [GARAY CAPITAL LETTER A]
+ };
+ case 0x10D73 -> // 𐵳 [GARAY SMALL LETTER KA]
+ new int[] {
+ 0x10D53, // 𐵓 [GARAY CAPITAL LETTER KA]
+ };
+ case 0x10D72 -> // 𐵲 [GARAY SMALL LETTER MA]
+ new int[] {
+ 0x10D52, // 𐵒 [GARAY CAPITAL LETTER MA]
+ };
+ case 0x10D75 -> // 𐵵 [GARAY SMALL LETTER JA]
+ new int[] {
+ 0x10D55, // 𐵕 [GARAY CAPITAL LETTER JA]
+ };
+ case 0x10D74 -> // 𐵴 [GARAY SMALL LETTER BA]
+ new int[] {
+ 0x10D54, // 𐵔 [GARAY CAPITAL LETTER BA]
+ };
+ case 0x10D77 -> // 𐵷 [GARAY SMALL LETTER WA]
+ new int[] {
+ 0x10D57, // 𐵗 [GARAY CAPITAL LETTER WA]
+ };
+ case 0x10D76 -> // 𐵶 [GARAY SMALL LETTER SA]
+ new int[] {
+ 0x10D56, // 𐵖 [GARAY CAPITAL LETTER SA]
+ };
+ case 0x10D79 -> // 𐵹 [GARAY SMALL LETTER GA]
+ new int[] {
+ 0x10D59, // 𐵙 [GARAY CAPITAL LETTER GA]
+ };
+ case 0x10D78 -> // 𐵸 [GARAY SMALL LETTER LA]
+ new int[] {
+ 0x10D58, // 𐵘 [GARAY CAPITAL LETTER LA]
+ };
+ case 0x10D7B -> // 𐵻 [GARAY SMALL LETTER XA]
+ new int[] {
+ 0x10D5B, // 𐵛 [GARAY CAPITAL LETTER XA]
+ };
+ case 0x10D7A -> // 𐵺 [GARAY SMALL LETTER DA]
+ new int[] {
+ 0x10D5A, // 𐵚 [GARAY CAPITAL LETTER DA]
+ };
+ case 0x10D7D -> // 𐵽 [GARAY SMALL LETTER TA]
+ new int[] {
+ 0x10D5D, // 𐵝 [GARAY CAPITAL LETTER TA]
+ };
+ case 0x10D7C -> // 𐵼 [GARAY SMALL LETTER YA]
+ new int[] {
+ 0x10D5C, // 𐵜 [GARAY CAPITAL LETTER YA]
+ };
+ case 0x10D7F -> // 𐵿 [GARAY SMALL LETTER NYA]
+ new int[] {
+ 0x10D5F, // 𐵟 [GARAY CAPITAL LETTER NYA]
+ };
+ case 0x10D7E -> // 𐵾 [GARAY SMALL LETTER RA]
+ new int[] {
+ 0x10D5E, // 𐵞 [GARAY CAPITAL LETTER RA]
+ };
+ case 0x10D81 -> // 𐶁 [GARAY SMALL LETTER NA]
+ new int[] {
+ 0x10D61, // 𐵡 [GARAY CAPITAL LETTER NA]
+ };
+ case 0x10D80 -> // 𐶀 [GARAY SMALL LETTER FA]
+ new int[] {
+ 0x10D60, // 𐵠 [GARAY CAPITAL LETTER FA]
+ };
+ case 0x10D83 -> // 𐶃 [GARAY SMALL LETTER HA]
+ new int[] {
+ 0x10D63, // 𐵣 [GARAY CAPITAL LETTER HA]
+ };
+ case 0x10D82 -> // 𐶂 [GARAY SMALL LETTER PA]
+ new int[] {
+ 0x10D62, // 𐵢 [GARAY CAPITAL LETTER PA]
+ };
+ case 0x10D85 -> // 𐶅 [GARAY SMALL LETTER OLD NA]
+ new int[] {
+ 0x10D65, // 𐵥 [GARAY CAPITAL LETTER OLD NA]
+ };
+ case 0x10D84 -> // 𐶄 [GARAY SMALL LETTER OLD KA]
+ new int[] {
+ 0x10D64, // 𐵤 [GARAY CAPITAL LETTER OLD KA]
+ };
+ case 0x01E60 -> // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]
+ new int[] {
+ 0x01E61, // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]
+ 0x01E9B, // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]
+ };
+ case 0x01E61 -> // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]
+ new int[] {
+ 0x01E60, // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]
+ 0x01E9B, // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]
+ };
+ case 0x01E9B -> // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]
+ new int[] {
+ 0x01E60, // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]
+ 0x01E61, // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]
+ };
+ case 0x01FBE -> // ι [GREEK PROSGEGRAMMENI]
+ new int[] {
+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]
+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]
+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]
+ };
+ case 0x01FD3 -> // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA]
+ new int[] {
+ 0x00390, // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS]
+ };
+ case 0x01FE3 -> // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA]
+ new int[] {
+ 0x003B0, // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS]
+ };
+ default -> null;
+ };
+
+ return alts;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
index 8be1a0eb12c..342efe05978 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
@@ -422,8 +422,62 @@ public class RegExp {
// ----- Matching flags ( > 0xff <= 0xffff ) ------
- /** Allows case insensitive matching of ASCII characters. */
- public static final int ASCII_CASE_INSENSITIVE = 0x0100;
+ /**
+ * Allows case-insensitive matching of ASCII characters.
+ *
+ * <p>This flag has been deprecated in favor of {@link #CASE_INSENSITIVE} that supports the full
+ * range of Unicode characters. Usage of this flag now has the same behavior as {@link
+ * #CASE_INSENSITIVE}
+ */
+ @Deprecated public static final int ASCII_CASE_INSENSITIVE = 0x0100;
+
+ /**
+ * Allows case-insensitive matching of most Unicode characters.
+ *
+ * <p>In general the attempt is to reach parity with {@link java.util.regex.Pattern}
+ * Pattern.CASE_INSENSITIVE and Pattern.UNICODE_CASE flags when doing a case-insensitive match. We
+ * support common case folding in addition to simple case folding as defined by the common (C),
+ * simple (S) and special (T) mappings in
+ * https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt. This is in line with {@link
+ * java.util.regex.Pattern} and means characters like those representing the Greek symbol sigma
+ * (Σ, σ, ς) will all match one another despite σ and ς both being lowercase characters as
+ * detailed here: https://www.unicode.org/Public/UCD/latest/ucd/SpecialCasing.txt.
+ *
+ * <p>Some Unicode characters are difficult to correctly decode casing. In some cases Java's
+ * String class correctly handles decoding these but Java's {@link java.util.regex.Pattern} class
+ * does not. We make only a best effort to maintaining consistency with {@link
+ * java.util.regex.Pattern} and there may be differences.
+ *
+ * <p>There are three known special classes of these characters:
+ *
+ * <ul>
+ * <li>1. the set of characters whose casing matches across multiple characters such as the
+ * Greek sigma character mentioned above (Σ, σ, ς); we support these; notably some of these
+ * characters fall into the ASCII range and so will behave differently when this flag is
+ * enabled
+ * <li>2. the set of characters that are neither in an upper nor lower case stable state and can
+ * be both uppercased and lowercased from their current code point such as Dž which when
+ * uppercased produces DŽ and when lowercased produces dž; we support these
+ * <li>3. the set of characters that when uppercased produce more than 1 character. For
+ * performance reasons we ignore characters for now, which is consistent with {@link
+ * java.util.regex.Pattern}
+ * </ul>
+ *
+ * <p>Sometimes these classes of character will overlap; if a character is in both class 3 and any
+ * other case listed above it is ignored; this is consistent with {@link java.util.regex.Pattern}
+ * and C,S,T mappings in https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt. Support for
+ * class 3 is only available with full (F) mappings, which is not supported. For instance: this
+ * character ῼ will match it's lowercase form ῳ but not it's uppercase form: ΩΙ
+ *
+ * <p>Class 3 characters that when uppercased generate multiple characters such as ﬗ (0xFB17)
+ * which when uppercased produces ՄԽ (code points: 0x0544 0x053D) and are therefore ignored;
+ * however, lowercase matching on these values is supported: 0x00DF, 0x0130, 0x0149, 0x01F0,
+ * 0x0390, 0x03B0, 0x0587, 0x1E96-0x1E9A, 0x1F50, 0x1F52, 0x1F54, 0x1F56, 0x1F80-0x1FAF,
+ * 0x1FB2-0x1FB4, 0x1FB6, 0x1FB7, 0x1FBC, 0x1FC2-0x1FC4, 0x1FC6, 0x1FC7, 0x1FCC, 0x1FD2, 0x1FD3,
+ * 0x1FD6, 0x1FD7, 0x1FE2-0x1FE4, 0x1FE6, 0x1FE7, 0x1FF2-0x1FF4, 0x1FF6, 0x1FF7, 0x1FFC,
+ * 0xFB00-0xFB06, 0xFB13-0xFB17
+ */
+ public static final int CASE_INSENSITIVE = 0x0200;
// ----- Deprecated flags ( > 0xffff ) ------
@@ -644,7 +698,7 @@ public class RegExp {
a = Operations.complement(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
break;
case REGEXP_CHAR:
- if (check(ASCII_CASE_INSENSITIVE)) {
+ if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {
a = Automata.makeCharSet(toCaseInsensitiveChar(c));
} else {
a = Automata.makeChar(c);
@@ -663,7 +717,7 @@ public class RegExp {
a = Automata.makeEmpty();
break;
case REGEXP_STRING:
- if (check(ASCII_CASE_INSENSITIVE)) {
+ if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {
a = toCaseInsensitiveString();
} else {
a = Automata.makeString(s);
@@ -696,19 +750,31 @@ public class RegExp {
return a;
}
+ /**
+ * This function handles uses the Unicode spec for generating case-insensitive alternates.
+ *
+ * <p>See the {@link #CASE_INSENSITIVE} flag for details on case folding within the Unicode spec.
+ *
+ * @param codepoint the Character code point to encode as an Automaton
+ * @return the original codepoint and the set of alternates
+ */
private int[] toCaseInsensitiveChar(int codepoint) {
- // For now we only work with ASCII characters
- if (codepoint > 128) {
- return new int[] {codepoint};
- }
- int altCase =
- Character.isLowerCase(codepoint)
- ? Character.toUpperCase(codepoint)
- : Character.toLowerCase(codepoint);
- if (altCase != codepoint) {
- return new int[] {codepoint, altCase};
+ int[] altCodepoints = CaseFolding.lookupAlternates(codepoint);
+ if (altCodepoints != null) {
+ int[] concat = new int[altCodepoints.length + 1];
+ System.arraycopy(altCodepoints, 0, concat, 0, altCodepoints.length);
+ concat[altCodepoints.length] = codepoint;
+ return concat;
} else {
- return new int[] {codepoint};
+ int altCase =
+ Character.isLowerCase(codepoint)
+ ? Character.toUpperCase(codepoint)
+ : Character.toLowerCase(codepoint);
+ if (altCase != codepoint) {
+ return new int[] {altCase, codepoint};
+ } else {
+ return new int[] {codepoint};
+ }
}
}
@@ -717,7 +783,7 @@ public class RegExp {
Iterator<Integer> iter = s.codePoints().iterator();
while (iter.hasNext()) {
- int points[] = toCaseInsensitiveChar(iter.next());
+ int[] points = toCaseInsensitiveChar(iter.next());
list.add(Automata.makeCharSet(points));
}
return Operations.concatenate(list);
@@ -1260,7 +1326,7 @@ public class RegExp {
// range from c-d
starts.add(c);
ends.add(parseCharExp());
- } else if (check(ASCII_CASE_INSENSITIVE)) {
+ } else if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {
// single case-insensitive character
for (int form : toCaseInsensitiveChar(c)) {
starts.add(form);
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
index 5c7e527040c..623f7904366 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
@@ -50,7 +50,7 @@ public class TestRegexpQuery extends LuceneTestCase {
doc.add(
newTextField(
FN,
- "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\",
+ "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\ ς",
Field.Store.NO));
writer.addDocument(doc);
reader = writer.getReader();
@@ -79,7 +79,7 @@ public class TestRegexpQuery extends LuceneTestCase {
new RegexpQuery(
newTerm(regex),
RegExp.ALL,
- RegExp.ASCII_CASE_INSENSITIVE,
+ RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE,
RegexpQuery.DEFAULT_PROVIDER,
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
@@ -143,6 +143,8 @@ public class TestRegexpQuery extends LuceneTestCase {
public void testCaseInsensitive() throws IOException {
assertEquals(0, regexQueryNrHits("Quick"));
assertEquals(1, caseInsensitiveRegexQueryNrHits("Quick"));
+ assertEquals(1, caseInsensitiveRegexQueryNrHits("Σ"));
+ assertEquals(1, caseInsensitiveRegexQueryNrHits("σ"));
}
public void testRegexNegatedCharacterClass() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
index 1747926295e..234aa1525c7 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
@@ -16,6 +16,11 @@
*/
package org.apache.lucene.util.automaton;
+import java.io.IOException;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -35,6 +40,133 @@ public class TestRegExp extends LuceneTestCase {
assertFalse(run.run("ad"));
}
+ public void testUnicodeAsciiInsensitiveFlags() {
+ RegExp r;
+ // ASCII behaves appropriately with different flags
+ r = new RegExp("A");
+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("a"));
+
+ r = new RegExp("A", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));
+
+ r = new RegExp("A", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));
+
+ r = new RegExp("A", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));
+
+ // class 1 Unicode characters behaves appropriately with different flags
+ r = new RegExp("Σ");
+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));
+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("σ");
+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("Σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("Σ", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("σ", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("Σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ r = new RegExp("σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));
+
+ // class 2 Unicode characters behaves appropriately with different flags
+ r = new RegExp("ῼ", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ῳ"));
+
+ r = new RegExp("ῼ", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertFalse(
+ new CharacterRunAutomaton(r.toAutomaton()).run("ῼ".toUpperCase(Locale.ROOT))); // "ΩΙ"
+
+ // class 3 Unicode characters behaves appropriately with different flags
+ r = new RegExp("ﬗ", RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ﬗ".toUpperCase(Locale.ROOT)));
+
+ r = new RegExp(new String(Character.toChars(0x1C8A)), RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ assertTrue(
+ new CharacterRunAutomaton(r.toAutomaton()).run(new String(Character.toChars(0x1C89))));
+ }
+
+ public void testRandomUnicodeInsensitiveMatchPatternParity() {
+ HexFormat hexFormat = HexFormat.of().withUpperCase();
+ int maxIters = 1000;
+ List<Integer> reservedCharacters =
+ Set.of(
+ '.', '^', '$', '*', '+', '?', '(', ')', '[', '{', '\\', '|', '-', '"', '<', '>',
+ '#', '@', '&', '~')
+ .stream()
+ .map(c -> (int) c)
+ .toList();
+ for (int i = 0; i < maxIters; i++) {
+ int nextCode1 = random().nextInt(0, Character.MAX_CODE_POINT + 1);
+ int nextCode2 = random().nextInt(0, Character.MAX_CODE_POINT + 1);
+
+ // skip if we select a reserved character that blows up .^$*+?()[{\|-]"<
+ if (reservedCharacters.contains(nextCode1)) {
+ continue;
+ }
+
+ String pattern = new String(Character.toChars(nextCode1));
+ String altString = new String(Character.toChars(nextCode2));
+
+ Pattern javaRegex = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
+ RegExp r = new RegExp(pattern, RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ CharacterRunAutomaton cra = new CharacterRunAutomaton(r.toAutomaton());
+
+ // Pattern doesn't respect the Unicode spec so some things will not match
+ if (javaRegex.matcher(altString).matches()) {
+ // ... but if they do match then we must agree
+ assertTrue(
+ "Pattern and RegExp disagree on pattern: "
+ + hexFormat.toHexDigits(nextCode1, 4)
+ + " :text: "
+ + hexFormat.toHexDigits(nextCode2, 4),
+ cra.run(altString));
+ }
+ }
+ }
+
+ public void testUnicodeInsensitiveMatchPatternParity() throws IOException {
+ // this ensures that if the Pattern class behavior were to change with a change to the Unicode
+ // spec then we would pick it up. It may help indicate in the future if we don't notice
+ // that the spec has changed and Pattern picks up the change first
+ for (int codepoint = 0; codepoint < Character.MAX_CODE_POINT + 1; codepoint++) {
+ int[] caseInsensitiveAlternatives = CaseFolding.lookupAlternates(codepoint);
+ if (caseInsensitiveAlternatives != null) {
+ String pattern = new String(Character.toChars(codepoint));
+ Pattern javaRegex =
+ Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
+ RegExp r = new RegExp(pattern, RegExp.ALL, RegExp.CASE_INSENSITIVE);
+ CharacterRunAutomaton cra = new CharacterRunAutomaton(r.toAutomaton());
+ for (int i = 0; i < caseInsensitiveAlternatives.length; i++) {
+
+ int alt = caseInsensitiveAlternatives[i];
+ String altString = new String(Character.toChars(alt));
+
+ // Pattern doesn't respect the Unicode spec so some things will not match
+ if (javaRegex.matcher(altString).matches()) {
+ // ... but if they do match then we must agree
+ assertTrue(cra.run(altString));
+ }
+ }
+ }
+ }
+ }
+
// LUCENE-6046
public void testRepeatWithEmptyString() throws Exception {
Automaton a = new RegExp("[^y]*{1,2}").toAutomaton();
@@ -55,13 +187,20 @@ public class TestRegExp extends LuceneTestCase {
}
boolean caseSensitiveQuery = true;
+ boolean unicodeCaseQuery = true;
public void testCoreJavaParity() {
// Generate random doc values and random regular expressions
// and check for same matching behaviour as Java's Pattern class.
for (int i = 0; i < 1000; i++) {
caseSensitiveQuery = true;
- checkRandomExpression(randomDocValue(1 + random().nextInt(30)));
+ checkRandomExpression(randomDocValue(1 + random().nextInt(30), false));
+ }
+
+ for (int i = 0; i < 1000; i++) {
+ caseSensitiveQuery = true;
+ unicodeCaseQuery = true;
+ checkRandomExpression(randomDocValue(1 + random().nextInt(30), true));
}
}
@@ -98,8 +237,11 @@ public class TestRegExp extends LuceneTestCase {
assertTrue(expected.getMessage().contains("out of order"));
}
- static String randomDocValue(int minLength) {
+ static String randomDocValue(int minLength, boolean includeUnicode) {
String charPalette = "AAAaaaBbbCccc123456 \t";
+ if (includeUnicode) {
+ charPalette += "Σσςῼῳ";
+ }
StringBuilder sb = new StringBuilder();
for (int i = 0; i < minLength; i++) {
sb.append(charPalette.charAt(randomInt(charPalette.length() - 1)));
@@ -131,7 +273,7 @@ public class TestRegExp extends LuceneTestCase {
case 0:
// OR with random alpha of same length
result.append(
- "(" + replacementPart + "|d" + randomDocValue(replacementPart.length()) + ")");
+ "(" + replacementPart + "|d" + randomDocValue(replacementPart.length(), false) + ")");
break;
case 1:
// OR with non-existant value
@@ -222,12 +364,13 @@ public class TestRegExp extends LuceneTestCase {
Pattern pattern =
caseSensitiveQuery
? Pattern.compile(regexPattern)
- : Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
+ : Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
Matcher matcher = pattern.matcher(docValue);
assertTrue(
"Java regex " + regexPattern + " did not match doc value " + docValue, matcher.matches());
- int matchFlags = caseSensitiveQuery ? 0 : RegExp.ASCII_CASE_INSENSITIVE;
+ int matchFlags =
+ caseSensitiveQuery ? 0 : RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE;
RegExp regex = new RegExp(regexPattern, RegExp.ALL, matchFlags);
Automaton automaton =
Operations.determinize(regex.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
index c5e269b51f9..77c315c45e9 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java
@@ -75,7 +75,7 @@ public class TestRegExpParsing extends LuceneTestCase {
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
- Automaton expected = Operations.union(List.of(Automata.makeChar('c'), Automata.makeChar('C')));
+ Automaton expected = Automata.makeCharSet(new int[] {'c', 'C'});
assertSameLanguage(expected, actual);
}
@@ -83,7 +83,7 @@ public class TestRegExpParsing extends LuceneTestCase {
public void testCaseInsensitiveClassChar() {
RegExp re = new RegExp("[c]", RegExp.NONE, RegExp.ASCII_CASE_INSENSITIVE);
assertEquals(
- "REGEXP_CHAR_CLASS starts=[U+0063 U+0043] ends=[U+0063 U+0043]\n", re.toStringTree());
+ "REGEXP_CHAR_CLASS starts=[U+0043 U+0063] ends=[U+0043 U+0063]\n", re.toStringTree());
}
// ranges aren't treated as case-insensitive, but maybe ok with charclass
@@ -101,7 +101,7 @@ public class TestRegExpParsing extends LuceneTestCase {
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
- Automaton expected = Operations.union(List.of(Automata.makeChar('c'), Automata.makeChar('C')));
+ Automaton expected = Automata.makeCharSet(new int[] {'c', 'C'});
assertSameLanguage(expected, actual);
}
@@ -125,7 +125,31 @@ public class TestRegExpParsing extends LuceneTestCase {
Automaton actual = re.toAutomaton();
assertTrue(actual.isDeterministic());
- Automaton expected = Automata.makeChar('Ж');
+ Automaton expected = Automata.makeCharSet(new int[] {'Ж', 'ж'});
+ assertSameLanguage(expected, actual);
+ }
+
+ public void testCaseInsensitiveCharUnicode() {
+ RegExp re = new RegExp("Ж", RegExp.NONE, RegExp.CASE_INSENSITIVE);
+ assertEquals("\\Ж", re.toString());
+ assertEquals("REGEXP_CHAR char=Ж\n", re.toStringTree());
+
+ Automaton actual = re.toAutomaton();
+ assertTrue(actual.isDeterministic());
+
+ Automaton expected = Automata.makeCharSet(new int[] {'Ж', 'ж'});
+ assertSameLanguage(expected, actual);
+ }
+
+ public void testCaseInsensitiveCharUnicodeSigma() {
+ RegExp re = new RegExp("σ", RegExp.NONE, RegExp.CASE_INSENSITIVE);
+ assertEquals("\\σ", re.toString());
+ assertEquals("REGEXP_CHAR char=σ\n", re.toStringTree());
+
+ Automaton actual = re.toAutomaton();
+ assertTrue(actual.isDeterministic());
+
+ Automaton expected = Automata.makeCharSet(new int[] {'Σ', 'σ', 'ς'});
assertSameLanguage(expected, actual);
}