Apache Lucene open-source search software
Unicode Support for Case Insensitive Matching in RegExp (#14192)
John Wagster7c050f9c6e47
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 44c045a0b6e..199822b658a 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -85,10 +85,12 @@ Improvementsthe wrapped analyzer's strategy to decide if components can be reused or needto be updated. (Mayya Sharipova)+* GITHUB#14192: Added support for RegExp to handle case insensitive matching+ across the full set of Unicode characters. (John Wagster)+* GITHUB#14239: Hunspell's option to tolerate affix rule count mismatches wasimproved to tolerate more instances of this problem. (Robert Muir)-Optimizations---------------------diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.javanew file mode 100644index 00000000000..dba1e6438e2--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CaseFolding.java@@ -0,0 +1,746 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.util.automaton;++class CaseFolding {++ /**+ * Generates the set of codepoints which represent the given codepoint that are case-insensitive+ * matches within the Unicode table, which may not always be intuitive for instance Σ, σ, ς do all+ * fold to one another and so would match one another+ *+ * <p>Known special cases derived from generating mappings using a combination of the Unicode+ * 16.0.0 spec: https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt and+ * https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt these are only the alternate+ * mappings for each codepoint that are not supported by a transform using Character.toUpperCase+ * or Character.toLowerCase+ *+ * @param codepoint the codepoint for the character to case fold+ * @return an array of characters as codepoints that should match the given codepoint in a+ * case-insensitive context or null if no alternates exist this does not include the given+ * codepoint itself+ */+ static int[] lookupAlternates(int codepoint) {++ int[] alts =+ switch (codepoint) {+ case 0x00049 -> // I [LATIN CAPITAL LETTER I]+ new int[] {+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]+ 0x00069, // i [LATIN SMALL LETTER I]+ };+ case 0x0004B -> // K [LATIN CAPITAL LETTER K]+ new int[] {+ 0x0212A, // K [KELVIN SIGN]+ 0x0006B, // k [LATIN SMALL LETTER K]+ };+ case 0x00053 -> // S [LATIN CAPITAL LETTER S]+ new int[] {+ 0x00073, // s [LATIN SMALL LETTER S]+ 0x0017F, // ſ [LATIN SMALL LETTER LONG S]+ };+ case 0x00069 -> // i [LATIN SMALL LETTER I]+ new int[] {+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]+ 0x00049, // I [LATIN CAPITAL LETTER I]+ };+ case 0x0006B -> // k [LATIN SMALL LETTER K]+ new int[] {+ 0x0212A, // K [KELVIN SIGN]+ 0x0004B, // K [LATIN CAPITAL LETTER K]+ };+ case 0x00073 -> // s [LATIN SMALL LETTER S]+ new int[] {+ 0x00053, // S [LATIN CAPITAL LETTER S]+ 0x0017F, // ſ [LATIN SMALL LETTER LONG S]+ };+ case 0x000B5 -> // µ [MICRO SIGN]+ new int[] {+ 0x0039C, // Μ [GREEK CAPITAL LETTER MU]+ 0x003BC, // μ [GREEK SMALL LETTER MU]+ };+ case 0x000C5 -> // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]+ new int[] {+ 0x000E5, // å [LATIN SMALL LETTER A WITH RING ABOVE]+ 0x0212B, // Å [ANGSTROM SIGN]+ };+ case 0x000DF -> // ß [LATIN SMALL LETTER SHARP S]+ new int[] {+ 0x01E9E, // ẞ [LATIN CAPITAL LETTER SHARP S]+ };+ case 0x000E5 -> // å [LATIN SMALL LETTER A WITH RING ABOVE]+ new int[] {+ 0x000C5, // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]+ 0x0212B, // Å [ANGSTROM SIGN]+ };+ case 0x02126 -> // Ω [OHM SIGN]+ new int[] {+ 0x003A9, // Ω [GREEK CAPITAL LETTER OMEGA]+ 0x003C9, // ω [GREEK SMALL LETTER OMEGA]+ };+ case 0x0212A -> // K [KELVIN SIGN]+ new int[] {+ 0x0004B, // K [LATIN CAPITAL LETTER K]+ 0x0006B, // k [LATIN SMALL LETTER K]+ };+ case 0x0212B -> // Å [ANGSTROM SIGN]+ new int[] {+ 0x000C5, // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]+ 0x000E5, // å [LATIN SMALL LETTER A WITH RING ABOVE]+ };+ case 0x00130 -> // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]+ new int[] {+ 0x00131, // ı [LATIN SMALL LETTER DOTLESS I]+ 0x00049, // I [LATIN CAPITAL LETTER I]+ 0x00069, // i [LATIN SMALL LETTER I]+ };+ case 0x00131 -> // ı [LATIN SMALL LETTER DOTLESS I]+ new int[] {+ 0x00130, // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]+ 0x00069, // i [LATIN SMALL LETTER I]+ 0x00049, // I [LATIN CAPITAL LETTER I]+ };+ case 0x0017F -> // ſ [LATIN SMALL LETTER LONG S]+ new int[] {+ 0x00053, // S [LATIN CAPITAL LETTER S]+ 0x00073, // s [LATIN SMALL LETTER S]+ };+ case 0x0019B -> // ƛ [LATIN SMALL LETTER LAMBDA WITH STROKE]+ new int[] {+ 0x0A7DC, // [LATIN CAPITAL LETTER LAMBDA WITH STROKE]+ };+ case 0x001C4 -> // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]+ new int[] {+ 0x001C5, // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]+ 0x001C6, // dž [LATIN SMALL LETTER DZ WITH CARON]+ };+ case 0x001C5 -> // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]+ new int[] {+ 0x001C4, // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]+ 0x001C6, // dž [LATIN SMALL LETTER DZ WITH CARON]+ };+ case 0x001C6 -> // dž [LATIN SMALL LETTER DZ WITH CARON]+ new int[] {+ 0x001C4, // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]+ 0x001C5, // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]+ };+ case 0x001C7 -> // LJ [LATIN CAPITAL LETTER LJ]+ new int[] {+ 0x001C8, // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]+ 0x001C9, // lj [LATIN SMALL LETTER LJ]+ };+ case 0x001C8 -> // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]+ new int[] {+ 0x001C7, // LJ [LATIN CAPITAL LETTER LJ]+ 0x001C9, // lj [LATIN SMALL LETTER LJ]+ };+ case 0x001C9 -> // lj [LATIN SMALL LETTER LJ]+ new int[] {+ 0x001C7, // LJ [LATIN CAPITAL LETTER LJ]+ 0x001C8, // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]+ };+ case 0x001CA -> // NJ [LATIN CAPITAL LETTER NJ]+ new int[] {+ 0x001CB, // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]+ 0x001CC, // nj [LATIN SMALL LETTER NJ]+ };+ case 0x001CB -> // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]+ new int[] {+ 0x001CA, // NJ [LATIN CAPITAL LETTER NJ]+ 0x001CC, // nj [LATIN SMALL LETTER NJ]+ };+ case 0x001CC -> // nj [LATIN SMALL LETTER NJ]+ new int[] {+ 0x001CA, // NJ [LATIN CAPITAL LETTER NJ]+ 0x001CB, // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]+ };+ case 0x001F1 -> // DZ [LATIN CAPITAL LETTER DZ]+ new int[] {+ 0x001F2, // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]+ 0x001F3, // dz [LATIN SMALL LETTER DZ]+ };+ case 0x001F2 -> // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]+ new int[] {+ 0x001F1, // DZ [LATIN CAPITAL LETTER DZ]+ 0x001F3, // dz [LATIN SMALL LETTER DZ]+ };+ case 0x001F3 -> // dz [LATIN SMALL LETTER DZ]+ new int[] {+ 0x001F1, // DZ [LATIN CAPITAL LETTER DZ]+ 0x001F2, // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]+ };+ case 0x00264 -> // ɤ [LATIN SMALL LETTER RAMS HORN]+ new int[] {+ 0x0A7CB, // [LATIN CAPITAL LETTER RAMS HORN]+ };+ case 0x00345 -> // ͅ [COMBINING GREEK YPOGEGRAMMENI]+ new int[] {+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]+ };+ case 0x00390 -> // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS]+ new int[] {+ 0x01FD3, // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA]+ };+ case 0x00392 -> // Β [GREEK CAPITAL LETTER BETA]+ new int[] {+ 0x003D0, // ϐ [GREEK BETA SYMBOL]+ 0x003B2, // β [GREEK SMALL LETTER BETA]+ };+ case 0x00395 -> // Ε [GREEK CAPITAL LETTER EPSILON]+ new int[] {+ 0x003F5, // ϵ [GREEK LUNATE EPSILON SYMBOL]+ 0x003B5, // ε [GREEK SMALL LETTER EPSILON]+ };+ case 0x00398 -> // Θ [GREEK CAPITAL LETTER THETA]+ new int[] {+ 0x003D1, // ϑ [GREEK THETA SYMBOL]+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]+ 0x003B8, // θ [GREEK SMALL LETTER THETA]+ };+ case 0x00399 -> // Ι [GREEK CAPITAL LETTER IOTA]+ new int[] {+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]+ };+ case 0x0039A -> // Κ [GREEK CAPITAL LETTER KAPPA]+ new int[] {+ 0x003F0, // ϰ [GREEK KAPPA SYMBOL]+ 0x003BA, // κ [GREEK SMALL LETTER KAPPA]+ };+ case 0x0039C -> // Μ [GREEK CAPITAL LETTER MU]+ new int[] {+ 0x000B5, // µ [MICRO SIGN]+ 0x003BC, // μ [GREEK SMALL LETTER MU]+ };+ case 0x003A0 -> // Π [GREEK CAPITAL LETTER PI]+ new int[] {+ 0x003C0, // π [GREEK SMALL LETTER PI]+ 0x003D6, // ϖ [GREEK PI SYMBOL]+ };+ case 0x003A1 -> // Ρ [GREEK CAPITAL LETTER RHO]+ new int[] {+ 0x003F1, // ϱ [GREEK RHO SYMBOL]+ 0x003C1, // ρ [GREEK SMALL LETTER RHO]+ };+ case 0x003A3 -> // Σ [GREEK CAPITAL LETTER SIGMA]+ new int[] {+ 0x003C2, // ς [GREEK SMALL LETTER FINAL SIGMA]+ 0x003C3, // σ [GREEK SMALL LETTER SIGMA]+ };+ case 0x003A6 -> // Φ [GREEK CAPITAL LETTER PHI]+ new int[] {+ 0x003D5, // ϕ [GREEK PHI SYMBOL]+ 0x003C6, // φ [GREEK SMALL LETTER PHI]+ };+ case 0x003A9 -> // Ω [GREEK CAPITAL LETTER OMEGA]+ new int[] {+ 0x02126, // Ω [OHM SIGN]+ 0x003C9, // ω [GREEK SMALL LETTER OMEGA]+ };+ case 0x003B0 -> // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS]+ new int[] {+ 0x01FE3, // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA]+ };+ case 0x003B2 -> // β [GREEK SMALL LETTER BETA]+ new int[] {+ 0x003D0, // ϐ [GREEK BETA SYMBOL]+ 0x00392, // Β [GREEK CAPITAL LETTER BETA]+ };+ case 0x003B5 -> // ε [GREEK SMALL LETTER EPSILON]+ new int[] {+ 0x00395, // Ε [GREEK CAPITAL LETTER EPSILON]+ 0x003F5, // ϵ [GREEK LUNATE EPSILON SYMBOL]+ };+ case 0x003B8 -> // θ [GREEK SMALL LETTER THETA]+ new int[] {+ 0x003D1, // ϑ [GREEK THETA SYMBOL]+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]+ };+ case 0x003B9 -> // ι [GREEK SMALL LETTER IOTA]+ new int[] {+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]+ 0x01FBE, // ι [GREEK PROSGEGRAMMENI]+ };+ case 0x003BA -> // κ [GREEK SMALL LETTER KAPPA]+ new int[] {+ 0x003F0, // ϰ [GREEK KAPPA SYMBOL]+ 0x0039A, // Κ [GREEK CAPITAL LETTER KAPPA]+ };+ case 0x003BC -> // μ [GREEK SMALL LETTER MU]+ new int[] {+ 0x000B5, // µ [MICRO SIGN]+ 0x0039C, // Μ [GREEK CAPITAL LETTER MU]+ };+ case 0x003C0 -> // π [GREEK SMALL LETTER PI]+ new int[] {+ 0x003A0, // Π [GREEK CAPITAL LETTER PI]+ 0x003D6, // ϖ [GREEK PI SYMBOL]+ };+ case 0x003C1 -> // ρ [GREEK SMALL LETTER RHO]+ new int[] {+ 0x003A1, // Ρ [GREEK CAPITAL LETTER RHO]+ 0x003F1, // ϱ [GREEK RHO SYMBOL]+ };+ case 0x003C2 -> // ς [GREEK SMALL LETTER FINAL SIGMA]+ new int[] {+ 0x003A3, // Σ [GREEK CAPITAL LETTER SIGMA]+ 0x003C3, // σ [GREEK SMALL LETTER SIGMA]+ };+ case 0x003C3 -> // σ [GREEK SMALL LETTER SIGMA]+ new int[] {+ 0x003C2, // ς [GREEK SMALL LETTER FINAL SIGMA]+ 0x003A3, // Σ [GREEK CAPITAL LETTER SIGMA]+ };+ case 0x003C6 -> // φ [GREEK SMALL LETTER PHI]+ new int[] {+ 0x003D5, // ϕ [GREEK PHI SYMBOL]+ 0x003A6, // Φ [GREEK CAPITAL LETTER PHI]+ };+ case 0x003C9 -> // ω [GREEK SMALL LETTER OMEGA]+ new int[] {+ 0x02126, // Ω [OHM SIGN]+ 0x003A9, // Ω [GREEK CAPITAL LETTER OMEGA]+ };+ case 0x003D0 -> // ϐ [GREEK BETA SYMBOL]+ new int[] {+ 0x00392, // Β [GREEK CAPITAL LETTER BETA]+ 0x003B2, // β [GREEK SMALL LETTER BETA]+ };+ case 0x003D1 -> // ϑ [GREEK THETA SYMBOL]+ new int[] {+ 0x003F4, // ϴ [GREEK CAPITAL THETA SYMBOL]+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]+ 0x003B8, // θ [GREEK SMALL LETTER THETA]+ };+ case 0x003D5 -> // ϕ [GREEK PHI SYMBOL]+ new int[] {+ 0x003A6, // Φ [GREEK CAPITAL LETTER PHI]+ 0x003C6, // φ [GREEK SMALL LETTER PHI]+ };+ case 0x003D6 -> // ϖ [GREEK PI SYMBOL]+ new int[] {+ 0x003A0, // Π [GREEK CAPITAL LETTER PI]+ 0x003C0, // π [GREEK SMALL LETTER PI]+ };+ case 0x003F0 -> // ϰ [GREEK KAPPA SYMBOL]+ new int[] {+ 0x0039A, // Κ [GREEK CAPITAL LETTER KAPPA]+ 0x003BA, // κ [GREEK SMALL LETTER KAPPA]+ };+ case 0x003F1 -> // ϱ [GREEK RHO SYMBOL]+ new int[] {+ 0x003A1, // Ρ [GREEK CAPITAL LETTER RHO]+ 0x003C1, // ρ [GREEK SMALL LETTER RHO]+ };+ case 0x003F4 -> // ϴ [GREEK CAPITAL THETA SYMBOL]+ new int[] {+ 0x003D1, // ϑ [GREEK THETA SYMBOL]+ 0x00398, // Θ [GREEK CAPITAL LETTER THETA]+ 0x003B8, // θ [GREEK SMALL LETTER THETA]+ };+ case 0x003F5 -> // ϵ [GREEK LUNATE EPSILON SYMBOL]+ new int[] {+ 0x00395, // Ε [GREEK CAPITAL LETTER EPSILON]+ 0x003B5, // ε [GREEK SMALL LETTER EPSILON]+ };+ case 0x00412 -> // В [CYRILLIC CAPITAL LETTER VE]+ new int[] {+ 0x01C80, // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]+ 0x00432, // в [CYRILLIC SMALL LETTER VE]+ };+ case 0x00414 -> // Д [CYRILLIC CAPITAL LETTER DE]+ new int[] {+ 0x01C81, // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]+ 0x00434, // д [CYRILLIC SMALL LETTER DE]+ };+ case 0x0041E -> // О [CYRILLIC CAPITAL LETTER O]+ new int[] {+ 0x01C82, // ᲂ [CYRILLIC SMALL LETTER NARROW O]+ 0x0043E, // о [CYRILLIC SMALL LETTER O]+ };+ case 0x00421 -> // С [CYRILLIC CAPITAL LETTER ES]+ new int[] {+ 0x00441, // с [CYRILLIC SMALL LETTER ES]+ 0x01C83, // ᲃ [CYRILLIC SMALL LETTER WIDE ES]+ };+ case 0x00422 -> // Т [CYRILLIC CAPITAL LETTER TE]+ new int[] {+ 0x00442, // т [CYRILLIC SMALL LETTER TE]+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]+ };+ case 0x0042A -> // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]+ new int[] {+ 0x01C86, // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]+ 0x0044A, // ъ [CYRILLIC SMALL LETTER HARD SIGN]+ };+ case 0x00432 -> // в [CYRILLIC SMALL LETTER VE]+ new int[] {+ 0x01C80, // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]+ 0x00412, // В [CYRILLIC CAPITAL LETTER VE]+ };+ case 0x00434 -> // д [CYRILLIC SMALL LETTER DE]+ new int[] {+ 0x01C81, // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]+ 0x00414, // Д [CYRILLIC CAPITAL LETTER DE]+ };+ case 0x0043E -> // о [CYRILLIC SMALL LETTER O]+ new int[] {+ 0x01C82, // ᲂ [CYRILLIC SMALL LETTER NARROW O]+ 0x0041E, // О [CYRILLIC CAPITAL LETTER O]+ };+ case 0x00441 -> // с [CYRILLIC SMALL LETTER ES]+ new int[] {+ 0x00421, // С [CYRILLIC CAPITAL LETTER ES]+ 0x01C83, // ᲃ [CYRILLIC SMALL LETTER WIDE ES]+ };+ case 0x00442 -> // т [CYRILLIC SMALL LETTER TE]+ new int[] {+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]+ };+ case 0x0044A -> // ъ [CYRILLIC SMALL LETTER HARD SIGN]+ new int[] {+ 0x01C86, // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]+ 0x0042A, // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]+ };+ case 0x00462 -> // Ѣ [CYRILLIC CAPITAL LETTER YAT]+ new int[] {+ 0x00463, // ѣ [CYRILLIC SMALL LETTER YAT]+ 0x01C87, // ᲇ [CYRILLIC SMALL LETTER TALL YAT]+ };+ case 0x00463 -> // ѣ [CYRILLIC SMALL LETTER YAT]+ new int[] {+ 0x00462, // Ѣ [CYRILLIC CAPITAL LETTER YAT]+ 0x01C87, // ᲇ [CYRILLIC SMALL LETTER TALL YAT]+ };+ case 0x0A64A -> // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]+ new int[] {+ 0x01C88, // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]+ 0x0A64B, // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]+ };+ case 0x0A64B -> // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]+ new int[] {+ 0x01C88, // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]+ 0x0A64A, // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]+ };+ case 0x0A7CB -> // [LATIN CAPITAL LETTER RAMS HORN]+ new int[] {+ 0x00264, // ɤ [LATIN SMALL LETTER RAMS HORN]+ };+ case 0x0A7CC -> // [LATIN CAPITAL LETTER S WITH DIAGONAL STROKE]+ new int[] {+ 0x0A7CD, // [LATIN SMALL LETTER S WITH DIAGONAL STROKE]+ };+ case 0x0A7CD -> // [LATIN SMALL LETTER S WITH DIAGONAL STROKE]+ new int[] {+ 0x0A7CC, // [LATIN CAPITAL LETTER S WITH DIAGONAL STROKE]+ };+ case 0x0A7DA -> // [LATIN CAPITAL LETTER LAMBDA]+ new int[] {+ 0x0A7DB, // [LATIN SMALL LETTER LAMBDA]+ };+ case 0x0A7DB -> // [LATIN SMALL LETTER LAMBDA]+ new int[] {+ 0x0A7DA, // [LATIN CAPITAL LETTER LAMBDA]+ };+ case 0x0A7DC -> // [LATIN CAPITAL LETTER LAMBDA WITH STROKE]+ new int[] {+ 0x0019B, // ƛ [LATIN SMALL LETTER LAMBDA WITH STROKE]+ };+ case 0x0FB05 -> // ſt [LATIN SMALL LIGATURE LONG S T]+ new int[] {+ 0x0FB06, // st [LATIN SMALL LIGATURE ST]+ };+ case 0x0FB06 -> // st [LATIN SMALL LIGATURE ST]+ new int[] {+ 0x0FB05, // ſt [LATIN SMALL LIGATURE LONG S T]+ };+ case 0x01C80 -> // ᲀ [CYRILLIC SMALL LETTER ROUNDED VE]+ new int[] {+ 0x00412, // В [CYRILLIC CAPITAL LETTER VE]+ 0x00432, // в [CYRILLIC SMALL LETTER VE]+ };+ case 0x01C81 -> // ᲁ [CYRILLIC SMALL LETTER LONG-LEGGED DE]+ new int[] {+ 0x00414, // Д [CYRILLIC CAPITAL LETTER DE]+ 0x00434, // д [CYRILLIC SMALL LETTER DE]+ };+ case 0x01C82 -> // ᲂ [CYRILLIC SMALL LETTER NARROW O]+ new int[] {+ 0x0041E, // О [CYRILLIC CAPITAL LETTER O]+ 0x0043E, // о [CYRILLIC SMALL LETTER O]+ };+ case 0x01C83 -> // ᲃ [CYRILLIC SMALL LETTER WIDE ES]+ new int[] {+ 0x00421, // С [CYRILLIC CAPITAL LETTER ES]+ 0x00441, // с [CYRILLIC SMALL LETTER ES]+ };+ case 0x01C84 -> // ᲄ [CYRILLIC SMALL LETTER TALL TE]+ new int[] {+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]+ 0x00442, // т [CYRILLIC SMALL LETTER TE]+ 0x01C85, // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]+ };+ case 0x01C85 -> // ᲅ [CYRILLIC SMALL LETTER THREE-LEGGED TE]+ new int[] {+ 0x00422, // Т [CYRILLIC CAPITAL LETTER TE]+ 0x00442, // т [CYRILLIC SMALL LETTER TE]+ 0x01C84, // ᲄ [CYRILLIC SMALL LETTER TALL TE]+ };+ case 0x01C86 -> // ᲆ [CYRILLIC SMALL LETTER TALL HARD SIGN]+ new int[] {+ 0x0042A, // Ъ [CYRILLIC CAPITAL LETTER HARD SIGN]+ 0x0044A, // ъ [CYRILLIC SMALL LETTER HARD SIGN]+ };+ case 0x01C87 -> // ᲇ [CYRILLIC SMALL LETTER TALL YAT]+ new int[] {+ 0x00462, // Ѣ [CYRILLIC CAPITAL LETTER YAT]+ 0x00463, // ѣ [CYRILLIC SMALL LETTER YAT]+ };+ case 0x01C88 -> // ᲈ [CYRILLIC SMALL LETTER UNBLENDED UK]+ new int[] {+ 0x0A64A, // Ꙋ [CYRILLIC CAPITAL LETTER MONOGRAPH UK]+ 0x0A64B, // ꙋ [CYRILLIC SMALL LETTER MONOGRAPH UK]+ };+ case 0x01C89 -> // [CYRILLIC CAPITAL LETTER TJE]+ new int[] {+ 0x01C8A, // [CYRILLIC SMALL LETTER TJE]+ };+ case 0x01C8A -> // [CYRILLIC SMALL LETTER TJE]+ new int[] {+ 0x01C89, // [CYRILLIC CAPITAL LETTER TJE]+ };+ case 0x10D51 -> // [GARAY CAPITAL LETTER CA]+ new int[] {+ 0x10D71, // [GARAY SMALL LETTER CA]+ };+ case 0x10D50 -> // [GARAY CAPITAL LETTER A]+ new int[] {+ 0x10D70, // [GARAY SMALL LETTER A]+ };+ case 0x10D53 -> // [GARAY CAPITAL LETTER KA]+ new int[] {+ 0x10D73, // [GARAY SMALL LETTER KA]+ };+ case 0x10D52 -> // [GARAY CAPITAL LETTER MA]+ new int[] {+ 0x10D72, // [GARAY SMALL LETTER MA]+ };+ case 0x10D55 -> // [GARAY CAPITAL LETTER JA]+ new int[] {+ 0x10D75, // [GARAY SMALL LETTER JA]+ };+ case 0x10D54 -> // [GARAY CAPITAL LETTER BA]+ new int[] {+ 0x10D74, // [GARAY SMALL LETTER BA]+ };+ case 0x10D57 -> // [GARAY CAPITAL LETTER WA]+ new int[] {+ 0x10D77, // [GARAY SMALL LETTER WA]+ };+ case 0x10D56 -> // [GARAY CAPITAL LETTER SA]+ new int[] {+ 0x10D76, // [GARAY SMALL LETTER SA]+ };+ case 0x10D59 -> // [GARAY CAPITAL LETTER GA]+ new int[] {+ 0x10D79, // [GARAY SMALL LETTER GA]+ };+ case 0x10D58 -> // [GARAY CAPITAL LETTER LA]+ new int[] {+ 0x10D78, // [GARAY SMALL LETTER LA]+ };+ case 0x10D5B -> // [GARAY CAPITAL LETTER XA]+ new int[] {+ 0x10D7B, // [GARAY SMALL LETTER XA]+ };+ case 0x10D5A -> // [GARAY CAPITAL LETTER DA]+ new int[] {+ 0x10D7A, // [GARAY SMALL LETTER DA]+ };+ case 0x10D5D -> // [GARAY CAPITAL LETTER TA]+ new int[] {+ 0x10D7D, // [GARAY SMALL LETTER TA]+ };+ case 0x10D5C -> // [GARAY CAPITAL LETTER YA]+ new int[] {+ 0x10D7C, // [GARAY SMALL LETTER YA]+ };+ case 0x10D5F -> // [GARAY CAPITAL LETTER NYA]+ new int[] {+ 0x10D7F, // [GARAY SMALL LETTER NYA]+ };+ case 0x10D5E -> // [GARAY CAPITAL LETTER RA]+ new int[] {+ 0x10D7E, // [GARAY SMALL LETTER RA]+ };+ case 0x10D61 -> // [GARAY CAPITAL LETTER NA]+ new int[] {+ 0x10D81, // [GARAY SMALL LETTER NA]+ };+ case 0x10D60 -> // [GARAY CAPITAL LETTER FA]+ new int[] {+ 0x10D80, // [GARAY SMALL LETTER FA]+ };+ case 0x10D63 -> // [GARAY CAPITAL LETTER HA]+ new int[] {+ 0x10D83, // [GARAY SMALL LETTER HA]+ };+ case 0x10D62 -> // [GARAY CAPITAL LETTER PA]+ new int[] {+ 0x10D82, // [GARAY SMALL LETTER PA]+ };+ case 0x10D65 -> // [GARAY CAPITAL LETTER OLD NA]+ new int[] {+ 0x10D85, // [GARAY SMALL LETTER OLD NA]+ };+ case 0x10D64 -> // [GARAY CAPITAL LETTER OLD KA]+ new int[] {+ 0x10D84, // [GARAY SMALL LETTER OLD KA]+ };+ case 0x10D71 -> // [GARAY SMALL LETTER CA]+ new int[] {+ 0x10D51, // [GARAY CAPITAL LETTER CA]+ };+ case 0x10D70 -> // [GARAY SMALL LETTER A]+ new int[] {+ 0x10D50, // [GARAY CAPITAL LETTER A]+ };+ case 0x10D73 -> // [GARAY SMALL LETTER KA]+ new int[] {+ 0x10D53, // [GARAY CAPITAL LETTER KA]+ };+ case 0x10D72 -> // [GARAY SMALL LETTER MA]+ new int[] {+ 0x10D52, // [GARAY CAPITAL LETTER MA]+ };+ case 0x10D75 -> // [GARAY SMALL LETTER JA]+ new int[] {+ 0x10D55, // [GARAY CAPITAL LETTER JA]+ };+ case 0x10D74 -> // [GARAY SMALL LETTER BA]+ new int[] {+ 0x10D54, // [GARAY CAPITAL LETTER BA]+ };+ case 0x10D77 -> // [GARAY SMALL LETTER WA]+ new int[] {+ 0x10D57, // [GARAY CAPITAL LETTER WA]+ };+ case 0x10D76 -> // [GARAY SMALL LETTER SA]+ new int[] {+ 0x10D56, // [GARAY CAPITAL LETTER SA]+ };+ case 0x10D79 -> // [GARAY SMALL LETTER GA]+ new int[] {+ 0x10D59, // [GARAY CAPITAL LETTER GA]+ };+ case 0x10D78 -> // [GARAY SMALL LETTER LA]+ new int[] {+ 0x10D58, // [GARAY CAPITAL LETTER LA]+ };+ case 0x10D7B -> // [GARAY SMALL LETTER XA]+ new int[] {+ 0x10D5B, // [GARAY CAPITAL LETTER XA]+ };+ case 0x10D7A -> // [GARAY SMALL LETTER DA]+ new int[] {+ 0x10D5A, // [GARAY CAPITAL LETTER DA]+ };+ case 0x10D7D -> // [GARAY SMALL LETTER TA]+ new int[] {+ 0x10D5D, // [GARAY CAPITAL LETTER TA]+ };+ case 0x10D7C -> // [GARAY SMALL LETTER YA]+ new int[] {+ 0x10D5C, // [GARAY CAPITAL LETTER YA]+ };+ case 0x10D7F -> // [GARAY SMALL LETTER NYA]+ new int[] {+ 0x10D5F, // [GARAY CAPITAL LETTER NYA]+ };+ case 0x10D7E -> // [GARAY SMALL LETTER RA]+ new int[] {+ 0x10D5E, // [GARAY CAPITAL LETTER RA]+ };+ case 0x10D81 -> // [GARAY SMALL LETTER NA]+ new int[] {+ 0x10D61, // [GARAY CAPITAL LETTER NA]+ };+ case 0x10D80 -> // [GARAY SMALL LETTER FA]+ new int[] {+ 0x10D60, // [GARAY CAPITAL LETTER FA]+ };+ case 0x10D83 -> // [GARAY SMALL LETTER HA]+ new int[] {+ 0x10D63, // [GARAY CAPITAL LETTER HA]+ };+ case 0x10D82 -> // [GARAY SMALL LETTER PA]+ new int[] {+ 0x10D62, // [GARAY CAPITAL LETTER PA]+ };+ case 0x10D85 -> // [GARAY SMALL LETTER OLD NA]+ new int[] {+ 0x10D65, // [GARAY CAPITAL LETTER OLD NA]+ };+ case 0x10D84 -> // [GARAY SMALL LETTER OLD KA]+ new int[] {+ 0x10D64, // [GARAY CAPITAL LETTER OLD KA]+ };+ case 0x01E60 -> // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]+ new int[] {+ 0x01E61, // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]+ 0x01E9B, // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]+ };+ case 0x01E61 -> // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]+ new int[] {+ 0x01E60, // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]+ 0x01E9B, // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]+ };+ case 0x01E9B -> // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]+ new int[] {+ 0x01E60, // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]+ 0x01E61, // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]+ };+ case 0x01FBE -> // ι [GREEK PROSGEGRAMMENI]+ new int[] {+ 0x00345, // ͅ [COMBINING GREEK YPOGEGRAMMENI]+ 0x00399, // Ι [GREEK CAPITAL LETTER IOTA]+ 0x003B9, // ι [GREEK SMALL LETTER IOTA]+ };+ case 0x01FD3 -> // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA]+ new int[] {+ 0x00390, // ΐ [GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS]+ };+ case 0x01FE3 -> // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA]+ new int[] {+ 0x003B0, // ΰ [GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS]+ };+ default -> null;+ };++ return alts;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.javaindex 8be1a0eb12c..342efe05978 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java@@ -422,8 +422,62 @@ public class RegExp {// ----- Matching flags ( > 0xff <= 0xffff ) ------- /** Allows case insensitive matching of ASCII characters. */- public static final int ASCII_CASE_INSENSITIVE = 0x0100;+ /**+ * Allows case-insensitive matching of ASCII characters.+ *+ * <p>This flag has been deprecated in favor of {@link #CASE_INSENSITIVE} that supports the full+ * range of Unicode characters. Usage of this flag now has the same behavior as {@link+ * #CASE_INSENSITIVE}+ */+ @Deprecated public static final int ASCII_CASE_INSENSITIVE = 0x0100;++ /**+ * Allows case-insensitive matching of most Unicode characters.+ *+ * <p>In general the attempt is to reach parity with {@link java.util.regex.Pattern}+ * Pattern.CASE_INSENSITIVE and Pattern.UNICODE_CASE flags when doing a case-insensitive match. We+ * support common case folding in addition to simple case folding as defined by the common (C),+ * simple (S) and special (T) mappings in+ * https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt. This is in line with {@link+ * java.util.regex.Pattern} and means characters like those representing the Greek symbol sigma+ * (Σ, σ, ς) will all match one another despite σ and ς both being lowercase characters as+ * detailed here: https://www.unicode.org/Public/UCD/latest/ucd/SpecialCasing.txt.+ *+ * <p>Some Unicode characters are difficult to correctly decode casing. In some cases Java's+ * String class correctly handles decoding these but Java's {@link java.util.regex.Pattern} class+ * does not. We make only a best effort to maintaining consistency with {@link+ * java.util.regex.Pattern} and there may be differences.+ *+ * <p>There are three known special classes of these characters:+ *+ * <ul>+ * <li>1. the set of characters whose casing matches across multiple characters such as the+ * Greek sigma character mentioned above (Σ, σ, ς); we support these; notably some of these+ * characters fall into the ASCII range and so will behave differently when this flag is+ * enabled+ * <li>2. the set of characters that are neither in an upper nor lower case stable state and can+ * be both uppercased and lowercased from their current code point such as Dž which when+ * uppercased produces DŽ and when lowercased produces dž; we support these+ * <li>3. the set of characters that when uppercased produce more than 1 character. For+ * performance reasons we ignore characters for now, which is consistent with {@link+ * java.util.regex.Pattern}+ * </ul>+ *+ * <p>Sometimes these classes of character will overlap; if a character is in both class 3 and any+ * other case listed above it is ignored; this is consistent with {@link java.util.regex.Pattern}+ * and C,S,T mappings in https://www.unicode.org/Public/16.0.0/ucd/CaseFolding.txt. Support for+ * class 3 is only available with full (F) mappings, which is not supported. For instance: this+ * character ῼ will match it's lowercase form ῳ but not it's uppercase form: ΩΙ+ *+ * <p>Class 3 characters that when uppercased generate multiple characters such as ﬗ (0xFB17)+ * which when uppercased produces ՄԽ (code points: 0x0544 0x053D) and are therefore ignored;+ * however, lowercase matching on these values is supported: 0x00DF, 0x0130, 0x0149, 0x01F0,+ * 0x0390, 0x03B0, 0x0587, 0x1E96-0x1E9A, 0x1F50, 0x1F52, 0x1F54, 0x1F56, 0x1F80-0x1FAF,+ * 0x1FB2-0x1FB4, 0x1FB6, 0x1FB7, 0x1FBC, 0x1FC2-0x1FC4, 0x1FC6, 0x1FC7, 0x1FCC, 0x1FD2, 0x1FD3,+ * 0x1FD6, 0x1FD7, 0x1FE2-0x1FE4, 0x1FE6, 0x1FE7, 0x1FF2-0x1FF4, 0x1FF6, 0x1FF7, 0x1FFC,+ * 0xFB00-0xFB06, 0xFB13-0xFB17+ */+ public static final int CASE_INSENSITIVE = 0x0200;// ----- Deprecated flags ( > 0xffff ) ------@@ -644,7 +698,7 @@ public class RegExp {a = Operations.complement(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);break;case REGEXP_CHAR:- if (check(ASCII_CASE_INSENSITIVE)) {+ if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {a = Automata.makeCharSet(toCaseInsensitiveChar(c));} else {a = Automata.makeChar(c);@@ -663,7 +717,7 @@ public class RegExp {a = Automata.makeEmpty();break;case REGEXP_STRING:- if (check(ASCII_CASE_INSENSITIVE)) {+ if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {a = toCaseInsensitiveString();} else {a = Automata.makeString(s);@@ -696,19 +750,31 @@ public class RegExp {return a;}+ /**+ * This function handles uses the Unicode spec for generating case-insensitive alternates.+ *+ * <p>See the {@link #CASE_INSENSITIVE} flag for details on case folding within the Unicode spec.+ *+ * @param codepoint the Character code point to encode as an Automaton+ * @return the original codepoint and the set of alternates+ */private int[] toCaseInsensitiveChar(int codepoint) {- // For now we only work with ASCII characters- if (codepoint > 128) {- return new int[] {codepoint};- }- int altCase =- Character.isLowerCase(codepoint)- ? Character.toUpperCase(codepoint)- : Character.toLowerCase(codepoint);- if (altCase != codepoint) {- return new int[] {codepoint, altCase};+ int[] altCodepoints = CaseFolding.lookupAlternates(codepoint);+ if (altCodepoints != null) {+ int[] concat = new int[altCodepoints.length + 1];+ System.arraycopy(altCodepoints, 0, concat, 0, altCodepoints.length);+ concat[altCodepoints.length] = codepoint;+ return concat;} else {- return new int[] {codepoint};+ int altCase =+ Character.isLowerCase(codepoint)+ ? Character.toUpperCase(codepoint)+ : Character.toLowerCase(codepoint);+ if (altCase != codepoint) {+ return new int[] {altCase, codepoint};+ } else {+ return new int[] {codepoint};+ }}}@@ -717,7 +783,7 @@ public class RegExp {Iterator<Integer> iter = s.codePoints().iterator();while (iter.hasNext()) {- int points[] = toCaseInsensitiveChar(iter.next());+ int[] points = toCaseInsensitiveChar(iter.next());list.add(Automata.makeCharSet(points));}return Operations.concatenate(list);@@ -1260,7 +1326,7 @@ public class RegExp {// range from c-dstarts.add(c);ends.add(parseCharExp());- } else if (check(ASCII_CASE_INSENSITIVE)) {+ } else if (check(ASCII_CASE_INSENSITIVE | CASE_INSENSITIVE)) {// single case-insensitive characterfor (int form : toCaseInsensitiveChar(c)) {starts.add(form);diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.javaindex 5c7e527040c..623f7904366 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java@@ -50,7 +50,7 @@ public class TestRegexpQuery extends LuceneTestCase {doc.add(newTextField(FN,- "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\",+ "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\ ς",Field.Store.NO));writer.addDocument(doc);reader = writer.getReader();@@ -79,7 +79,7 @@ public class TestRegexpQuery extends LuceneTestCase {new RegexpQuery(newTerm(regex),RegExp.ALL,- RegExp.ASCII_CASE_INSENSITIVE,+ RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE,RegexpQuery.DEFAULT_PROVIDER,Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,@@ -143,6 +143,8 @@ public class TestRegexpQuery extends LuceneTestCase {public void testCaseInsensitive() throws IOException {assertEquals(0, regexQueryNrHits("Quick"));assertEquals(1, caseInsensitiveRegexQueryNrHits("Quick"));+ assertEquals(1, caseInsensitiveRegexQueryNrHits("Σ"));+ assertEquals(1, caseInsensitiveRegexQueryNrHits("σ"));}public void testRegexNegatedCharacterClass() throws IOException {diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.javaindex 1747926295e..234aa1525c7 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java@@ -16,6 +16,11 @@*/package org.apache.lucene.util.automaton;+import java.io.IOException;+import java.util.HexFormat;+import java.util.List;+import java.util.Locale;+import java.util.Set;import java.util.regex.Matcher;import java.util.regex.Pattern;import org.apache.lucene.tests.util.LuceneTestCase;@@ -35,6 +40,133 @@ public class TestRegExp extends LuceneTestCase {assertFalse(run.run("ad"));}+ public void testUnicodeAsciiInsensitiveFlags() {+ RegExp r;+ // ASCII behaves appropriately with different flags+ r = new RegExp("A");+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("a"));++ r = new RegExp("A", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));++ r = new RegExp("A", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));++ r = new RegExp("A", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("a"));++ // class 1 Unicode characters behaves appropriately with different flags+ r = new RegExp("Σ");+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("σ");+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("Σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("Σ", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("σ", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("Σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("σ"));+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ r = new RegExp("σ", RegExp.ALL, RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ς"));++ // class 2 Unicode characters behaves appropriately with different flags+ r = new RegExp("ῼ", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertTrue(new CharacterRunAutomaton(r.toAutomaton()).run("ῳ"));++ r = new RegExp("ῼ", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertFalse(+ new CharacterRunAutomaton(r.toAutomaton()).run("ῼ".toUpperCase(Locale.ROOT))); // "ΩΙ"++ // class 3 Unicode characters behaves appropriately with different flags+ r = new RegExp("ﬗ", RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertFalse(new CharacterRunAutomaton(r.toAutomaton()).run("ﬗ".toUpperCase(Locale.ROOT)));++ r = new RegExp(new String(Character.toChars(0x1C8A)), RegExp.ALL, RegExp.CASE_INSENSITIVE);+ assertTrue(+ new CharacterRunAutomaton(r.toAutomaton()).run(new String(Character.toChars(0x1C89))));+ }++ public void testRandomUnicodeInsensitiveMatchPatternParity() {+ HexFormat hexFormat = HexFormat.of().withUpperCase();+ int maxIters = 1000;+ List<Integer> reservedCharacters =+ Set.of(+ '.', '^', '$', '*', '+', '?', '(', ')', '[', '{', '\\', '|', '-', '"', '<', '>',+ '#', '@', '&', '~')+ .stream()+ .map(c -> (int) c)+ .toList();+ for (int i = 0; i < maxIters; i++) {+ int nextCode1 = random().nextInt(0, Character.MAX_CODE_POINT + 1);+ int nextCode2 = random().nextInt(0, Character.MAX_CODE_POINT + 1);++ // skip if we select a reserved character that blows up .^$*+?()[{\|-]"<+ if (reservedCharacters.contains(nextCode1)) {+ continue;+ }++ String pattern = new String(Character.toChars(nextCode1));+ String altString = new String(Character.toChars(nextCode2));++ Pattern javaRegex = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);+ RegExp r = new RegExp(pattern, RegExp.ALL, RegExp.CASE_INSENSITIVE);+ CharacterRunAutomaton cra = new CharacterRunAutomaton(r.toAutomaton());++ // Pattern doesn't respect the Unicode spec so some things will not match+ if (javaRegex.matcher(altString).matches()) {+ // ... but if they do match then we must agree+ assertTrue(+ "Pattern and RegExp disagree on pattern: "+ + hexFormat.toHexDigits(nextCode1, 4)+ + " :text: "+ + hexFormat.toHexDigits(nextCode2, 4),+ cra.run(altString));+ }+ }+ }++ public void testUnicodeInsensitiveMatchPatternParity() throws IOException {+ // this ensures that if the Pattern class behavior were to change with a change to the Unicode+ // spec then we would pick it up. It may help indicate in the future if we don't notice+ // that the spec has changed and Pattern picks up the change first+ for (int codepoint = 0; codepoint < Character.MAX_CODE_POINT + 1; codepoint++) {+ int[] caseInsensitiveAlternatives = CaseFolding.lookupAlternates(codepoint);+ if (caseInsensitiveAlternatives != null) {+ String pattern = new String(Character.toChars(codepoint));+ Pattern javaRegex =+ Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);+ RegExp r = new RegExp(pattern, RegExp.ALL, RegExp.CASE_INSENSITIVE);+ CharacterRunAutomaton cra = new CharacterRunAutomaton(r.toAutomaton());+ for (int i = 0; i < caseInsensitiveAlternatives.length; i++) {++ int alt = caseInsensitiveAlternatives[i];+ String altString = new String(Character.toChars(alt));++ // Pattern doesn't respect the Unicode spec so some things will not match+ if (javaRegex.matcher(altString).matches()) {+ // ... but if they do match then we must agree+ assertTrue(cra.run(altString));+ }+ }+ }+ }+ }+// LUCENE-6046public void testRepeatWithEmptyString() throws Exception {Automaton a = new RegExp("[^y]*{1,2}").toAutomaton();@@ -55,13 +187,20 @@ public class TestRegExp extends LuceneTestCase {}boolean caseSensitiveQuery = true;+ boolean unicodeCaseQuery = true;public void testCoreJavaParity() {// Generate random doc values and random regular expressions// and check for same matching behaviour as Java's Pattern class.for (int i = 0; i < 1000; i++) {caseSensitiveQuery = true;- checkRandomExpression(randomDocValue(1 + random().nextInt(30)));+ checkRandomExpression(randomDocValue(1 + random().nextInt(30), false));+ }++ for (int i = 0; i < 1000; i++) {+ caseSensitiveQuery = true;+ unicodeCaseQuery = true;+ checkRandomExpression(randomDocValue(1 + random().nextInt(30), true));}}@@ -98,8 +237,11 @@ public class TestRegExp extends LuceneTestCase {assertTrue(expected.getMessage().contains("out of order"));}- static String randomDocValue(int minLength) {+ static String randomDocValue(int minLength, boolean includeUnicode) {String charPalette = "AAAaaaBbbCccc123456 \t";+ if (includeUnicode) {+ charPalette += "Σσςῼῳ";+ }StringBuilder sb = new StringBuilder();for (int i = 0; i < minLength; i++) {sb.append(charPalette.charAt(randomInt(charPalette.length() - 1)));@@ -131,7 +273,7 @@ public class TestRegExp extends LuceneTestCase {case 0:// OR with random alpha of same lengthresult.append(- "(" + replacementPart + "|d" + randomDocValue(replacementPart.length()) + ")");+ "(" + replacementPart + "|d" + randomDocValue(replacementPart.length(), false) + ")");break;case 1:// OR with non-existant value@@ -222,12 +364,13 @@ public class TestRegExp extends LuceneTestCase {Pattern pattern =caseSensitiveQuery? Pattern.compile(regexPattern)- : Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);+ : Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);Matcher matcher = pattern.matcher(docValue);assertTrue("Java regex " + regexPattern + " did not match doc value " + docValue, matcher.matches());- int matchFlags = caseSensitiveQuery ? 0 : RegExp.ASCII_CASE_INSENSITIVE;+ int matchFlags =+ caseSensitiveQuery ? 0 : RegExp.ASCII_CASE_INSENSITIVE | RegExp.CASE_INSENSITIVE;RegExp regex = new RegExp(regexPattern, RegExp.ALL, matchFlags);Automaton automaton =Operations.determinize(regex.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.javaindex c5e269b51f9..77c315c45e9 100644--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExpParsing.java@@ -75,7 +75,7 @@ public class TestRegExpParsing extends LuceneTestCase {Automaton actual = re.toAutomaton();assertTrue(actual.isDeterministic());- Automaton expected = Operations.union(List.of(Automata.makeChar('c'), Automata.makeChar('C')));+ Automaton expected = Automata.makeCharSet(new int[] {'c', 'C'});assertSameLanguage(expected, actual);}@@ -83,7 +83,7 @@ public class TestRegExpParsing extends LuceneTestCase {public void testCaseInsensitiveClassChar() {RegExp re = new RegExp("[c]", RegExp.NONE, RegExp.ASCII_CASE_INSENSITIVE);assertEquals(- "REGEXP_CHAR_CLASS starts=[U+0063 U+0043] ends=[U+0063 U+0043]\n", re.toStringTree());+ "REGEXP_CHAR_CLASS starts=[U+0043 U+0063] ends=[U+0043 U+0063]\n", re.toStringTree());}// ranges aren't treated as case-insensitive, but maybe ok with charclass@@ -101,7 +101,7 @@ public class TestRegExpParsing extends LuceneTestCase {Automaton actual = re.toAutomaton();assertTrue(actual.isDeterministic());- Automaton expected = Operations.union(List.of(Automata.makeChar('c'), Automata.makeChar('C')));+ Automaton expected = Automata.makeCharSet(new int[] {'c', 'C'});assertSameLanguage(expected, actual);}@@ -125,7 +125,31 @@ public class TestRegExpParsing extends LuceneTestCase {Automaton actual = re.toAutomaton();assertTrue(actual.isDeterministic());- Automaton expected = Automata.makeChar('Ж');+ Automaton expected = Automata.makeCharSet(new int[] {'Ж', 'ж'});+ assertSameLanguage(expected, actual);+ }++ public void testCaseInsensitiveCharUnicode() {+ RegExp re = new RegExp("Ж", RegExp.NONE, RegExp.CASE_INSENSITIVE);+ assertEquals("\\Ж", re.toString());+ assertEquals("REGEXP_CHAR char=Ж\n", re.toStringTree());++ Automaton actual = re.toAutomaton();+ assertTrue(actual.isDeterministic());++ Automaton expected = Automata.makeCharSet(new int[] {'Ж', 'ж'});+ assertSameLanguage(expected, actual);+ }++ public void testCaseInsensitiveCharUnicodeSigma() {+ RegExp re = new RegExp("σ", RegExp.NONE, RegExp.CASE_INSENSITIVE);+ assertEquals("\\σ", re.toString());+ assertEquals("REGEXP_CHAR char=σ\n", re.toStringTree());++ Automaton actual = re.toAutomaton();+ assertTrue(actual.isDeterministic());++ Automaton expected = Automata.makeCharSet(new int[] {'Σ', 'σ', 'ς'});assertSameLanguage(expected, actual);}