Apache Lucene open-source search software
Remove deprecated CharsRef UTF16-UTF8 Comparator (#16834)
Alan Woodward2eb462541b1c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 9c22924b8c1..d107f50a94d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -135,6 +135,9 @@ API Changes* GITHUB#16814: Remove unused VectorValueSource and MultiValueSource. They weredeprecated in 10.x (GITHUB#16813). (Luca Cavanna)+* GITHUB#16834: Remove deprecated CharsRef.getUTF16SortedAsUTF8Comparator() method.+ (Alan Woodward)+New Features---------------------* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition ofdiff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.javaindex 46d9a62b141..e318dbad1a5 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java@@ -193,6 +193,24 @@ public class SynonymMap {maxHorizontalContext = Math.max(maxHorizontalContext, numOutputWords);}+ /** Compares by Unicode code point, which matches the UTF-8 byte order the FST requires. */+ static int compareCodePoints(CharsRef a, CharsRef b) {+ int aEnd = a.offset + a.length;+ int bEnd = b.offset + b.length;+ int i = a.offset;+ int j = b.offset;+ while (i < aEnd && j < bEnd) {+ int aCp = Character.codePointAt(a.chars, i, aEnd);+ int bCp = Character.codePointAt(b.chars, j, bEnd);+ if (aCp != bCp) {+ return Integer.compare(aCp, bCp);+ }+ i += Character.charCount(aCp);+ j += Character.charCount(bCp);+ }+ return Integer.compare(aEnd - i, bEnd - j);+ }+private int countWords(CharsRef chars) {int wordCount = 1;int upto = chars.offset;@@ -240,7 +258,7 @@ public class SynonymMap {Set<CharsRef> keys = workingSet.keySet();CharsRef[] sortedKeys = keys.toArray(CharsRef[]::new);- Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());+ Arrays.sort(sortedKeys, SynonymMap.Builder::compareCodePoints);final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.javanew file mode 100644index 00000000000..e8751ba0ffd--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.java@@ -0,0 +1,66 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.synonym;++import java.util.Arrays;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.CharsRef;++public class TestSynonymMap extends LuceneTestCase {++ public void testCompareCodePointsMatchesUTF8Order() {+ final int numStrings = atLeast(1000);+ BytesRef[] utf8 = new BytesRef[numStrings];+ CharsRef[] utf16 = new CharsRef[numStrings];++ for (int i = 0; i < numStrings; i++) {+ String s = TestUtil.randomUnicodeString(random());+ utf8[i] = new BytesRef(s);+ utf16[i] = new CharsRef(s);+ }++ Arrays.sort(utf8);+ Arrays.sort(utf16, SynonymMap.Builder::compareCodePoints);++ for (int i = 0; i < numStrings; i++) {+ assertEquals(utf8[i].utf8ToString(), utf16[i].toString());+ }+ }++ public void testSupplementaryCharsSortAfterHighBMP() {+ // U+FFFD is a single char above the surrogate range; U+10000 is a surrogate pair.+ CharsRef bmp = new CharsRef("\uFFFD");+ CharsRef supplementary = new CharsRef("\uD800\uDC00");+ assertTrue(SynonymMap.Builder.compareCodePoints(bmp, supplementary) < 0);+ assertTrue(SynonymMap.Builder.compareCodePoints(supplementary, bmp) > 0);+ }++ public void testPrefixAndEquality() {+ assertEquals(0, SynonymMap.Builder.compareCodePoints(new CharsRef("abc"), new CharsRef("abc")));+ assertTrue(SynonymMap.Builder.compareCodePoints(new CharsRef("ab"), new CharsRef("abc")) < 0);+ assertTrue(SynonymMap.Builder.compareCodePoints(new CharsRef("abc"), new CharsRef("ab")) > 0);+ assertEquals(0, SynonymMap.Builder.compareCodePoints(new CharsRef(""), new CharsRef("")));+ }++ public void testRespectsOffsetAndLength() {+ CharsRef a = new CharsRef("xx\uD800\uDC00yy".toCharArray(), 2, 2);+ CharsRef b = new CharsRef("zzz\uFFFDz".toCharArray(), 3, 1);+ assertTrue(SynonymMap.Builder.compareCodePoints(a, b) > 0);+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java b/lucene/core/src/java/org/apache/lucene/util/CharsRef.javaindex f8064d8bc6f..858b0f78531 100644--- a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java+++ b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java@@ -17,7 +17,6 @@package org.apache.lucene.util;import java.util.Arrays;-import java.util.Comparator;import java.util.Objects;/**@@ -150,64 +149,6 @@ public final class CharsRef implements Comparable<CharsRef>, CharSequence, Clonereturn new CharsRef(chars, offset + start, end - start);}- /**- * @deprecated This comparator is only a transition mechanism- */- @Deprecated- private static final Comparator<CharsRef> utf16SortedAsUTF8SortOrder =- new UTF16SortedAsUTF8Comparator();-- /**- * @deprecated This comparator is only a transition mechanism- */- @Deprecated- public static Comparator<CharsRef> getUTF16SortedAsUTF8Comparator() {- return utf16SortedAsUTF8SortOrder;- }-- /**- * @deprecated This comparator is only a transition mechanism- */- @Deprecated- private static class UTF16SortedAsUTF8Comparator implements Comparator<CharsRef> {- // Only singleton- private UTF16SortedAsUTF8Comparator() {}-- @Override- public int compare(CharsRef a, CharsRef b) {- int aEnd = a.offset + a.length;- int bEnd = b.offset + b.length;- int i = Arrays.mismatch(a.chars, a.offset, aEnd, b.chars, b.offset, bEnd);-- if (i >= 0 && i < Math.min(a.length, b.length)) {- // http://icu-project.org/docs/papers/utf16_code_point_order.html-- char aChar = a.chars[a.offset + i];- char bChar = b.chars[b.offset + i];- /* aChar != bChar, fix up each one if they're both in or above the surrogate range, then compare them */- if (aChar >= 0xd800 && bChar >= 0xd800) {- if (aChar >= 0xe000) {- aChar -= 0x800;- } else {- aChar += 0x2000;- }-- if (bChar >= 0xe000) {- bChar -= 0x800;- } else {- bChar += 0x2000;- }- }-- /* now aChar and bChar are in code point order */- return (int) aChar - (int) bChar; /* int must be 32 bits wide */- }-- // One is a prefix of the other, or, they are equal:- return a.length - b.length;- }- }-/*** Creates a new CharsRef that points to a copy of the chars from <code>other</code>*diff --git a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.javaindex 404fe73f61d..5a319904293 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java@@ -16,32 +16,11 @@*/package org.apache.lucene.util;-import java.util.Arrays;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;public class TestCharsRef extends LuceneTestCase {- @SuppressWarnings("deprecation")- public void testUTF16InUTF8Order() {- final int numStrings = atLeast(1000);- BytesRef[] utf8 = new BytesRef[numStrings];- CharsRef[] utf16 = new CharsRef[numStrings];-- for (int i = 0; i < numStrings; i++) {- String s = TestUtil.randomUnicodeString(random());- utf8[i] = new BytesRef(s);- utf16[i] = new CharsRef(s);- }-- Arrays.sort(utf8);- Arrays.sort(utf16, CharsRef.getUTF16SortedAsUTF8Comparator());-- for (int i = 0; i < numStrings; i++) {- assertEquals(utf8[i].utf8ToString(), utf16[i].toString());- }- }-public void testAppend() {CharsRefBuilder ref = new CharsRefBuilder();StringBuilder builder = new StringBuilder();