Skip to content

Apache Lucene open-source search software

Remove deprecated CharsRef UTF16-UTF8 Comparator (#16834)

Alan Woodward2eb462541b1c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9c22924b8c1..d107f50a94d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -135,6 +135,9 @@ API Changes
* GITHUB#16814: Remove unused VectorValueSource and MultiValueSource. They were
deprecated in 10.x (GITHUB#16813). (Luca Cavanna)
+* GITHUB#16834: Remove deprecated CharsRef.getUTF16SortedAsUTF8Comparator() method.
+ (Alan Woodward)
+
New Features
---------------------
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
index 46d9a62b141..e318dbad1a5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
@@ -193,6 +193,24 @@ public class SynonymMap {
maxHorizontalContext = Math.max(maxHorizontalContext, numOutputWords);
}
+ /** Compares by Unicode code point, which matches the UTF-8 byte order the FST requires. */
+ static int compareCodePoints(CharsRef a, CharsRef b) {
+ int aEnd = a.offset + a.length;
+ int bEnd = b.offset + b.length;
+ int i = a.offset;
+ int j = b.offset;
+ while (i < aEnd && j < bEnd) {
+ int aCp = Character.codePointAt(a.chars, i, aEnd);
+ int bCp = Character.codePointAt(b.chars, j, bEnd);
+ if (aCp != bCp) {
+ return Integer.compare(aCp, bCp);
+ }
+ i += Character.charCount(aCp);
+ j += Character.charCount(bCp);
+ }
+ return Integer.compare(aEnd - i, bEnd - j);
+ }
+
private int countWords(CharsRef chars) {
int wordCount = 1;
int upto = chars.offset;
@@ -240,7 +258,7 @@ public class SynonymMap {
Set<CharsRef> keys = workingSet.keySet();
CharsRef[] sortedKeys = keys.toArray(CharsRef[]::new);
- Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());
+ Arrays.sort(sortedKeys, SynonymMap.Builder::compareCodePoints);
final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.java
new file mode 100644
index 00000000000..e8751ba0ffd
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMap.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.synonym;
+
+import java.util.Arrays;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+
+public class TestSynonymMap extends LuceneTestCase {
+
+ public void testCompareCodePointsMatchesUTF8Order() {
+ final int numStrings = atLeast(1000);
+ BytesRef[] utf8 = new BytesRef[numStrings];
+ CharsRef[] utf16 = new CharsRef[numStrings];
+
+ for (int i = 0; i < numStrings; i++) {
+ String s = TestUtil.randomUnicodeString(random());
+ utf8[i] = new BytesRef(s);
+ utf16[i] = new CharsRef(s);
+ }
+
+ Arrays.sort(utf8);
+ Arrays.sort(utf16, SynonymMap.Builder::compareCodePoints);
+
+ for (int i = 0; i < numStrings; i++) {
+ assertEquals(utf8[i].utf8ToString(), utf16[i].toString());
+ }
+ }
+
+ public void testSupplementaryCharsSortAfterHighBMP() {
+ // U+FFFD is a single char above the surrogate range; U+10000 is a surrogate pair.
+ CharsRef bmp = new CharsRef("\uFFFD");
+ CharsRef supplementary = new CharsRef("\uD800\uDC00");
+ assertTrue(SynonymMap.Builder.compareCodePoints(bmp, supplementary) < 0);
+ assertTrue(SynonymMap.Builder.compareCodePoints(supplementary, bmp) > 0);
+ }
+
+ public void testPrefixAndEquality() {
+ assertEquals(0, SynonymMap.Builder.compareCodePoints(new CharsRef("abc"), new CharsRef("abc")));
+ assertTrue(SynonymMap.Builder.compareCodePoints(new CharsRef("ab"), new CharsRef("abc")) < 0);
+ assertTrue(SynonymMap.Builder.compareCodePoints(new CharsRef("abc"), new CharsRef("ab")) > 0);
+ assertEquals(0, SynonymMap.Builder.compareCodePoints(new CharsRef(""), new CharsRef("")));
+ }
+
+ public void testRespectsOffsetAndLength() {
+ CharsRef a = new CharsRef("xx\uD800\uDC00yy".toCharArray(), 2, 2);
+ CharsRef b = new CharsRef("zzz\uFFFDz".toCharArray(), 3, 1);
+ assertTrue(SynonymMap.Builder.compareCodePoints(a, b) > 0);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
index f8064d8bc6f..858b0f78531 100644
--- a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
+++ b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
@@ -17,7 +17,6 @@
package org.apache.lucene.util;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.Objects;
/**
@@ -150,64 +149,6 @@ public final class CharsRef implements Comparable<CharsRef>, CharSequence, Clone
return new CharsRef(chars, offset + start, end - start);
}
- /**
- * @deprecated This comparator is only a transition mechanism
- */
- @Deprecated
- private static final Comparator<CharsRef> utf16SortedAsUTF8SortOrder =
- new UTF16SortedAsUTF8Comparator();
-
- /**
- * @deprecated This comparator is only a transition mechanism
- */
- @Deprecated
- public static Comparator<CharsRef> getUTF16SortedAsUTF8Comparator() {
- return utf16SortedAsUTF8SortOrder;
- }
-
- /**
- * @deprecated This comparator is only a transition mechanism
- */
- @Deprecated
- private static class UTF16SortedAsUTF8Comparator implements Comparator<CharsRef> {
- // Only singleton
- private UTF16SortedAsUTF8Comparator() {}
-
- @Override
- public int compare(CharsRef a, CharsRef b) {
- int aEnd = a.offset + a.length;
- int bEnd = b.offset + b.length;
- int i = Arrays.mismatch(a.chars, a.offset, aEnd, b.chars, b.offset, bEnd);
-
- if (i >= 0 && i < Math.min(a.length, b.length)) {
- // http://icu-project.org/docs/papers/utf16_code_point_order.html
-
- char aChar = a.chars[a.offset + i];
- char bChar = b.chars[b.offset + i];
- /* aChar != bChar, fix up each one if they're both in or above the surrogate range, then compare them */
- if (aChar >= 0xd800 && bChar >= 0xd800) {
- if (aChar >= 0xe000) {
- aChar -= 0x800;
- } else {
- aChar += 0x2000;
- }
-
- if (bChar >= 0xe000) {
- bChar -= 0x800;
- } else {
- bChar += 0x2000;
- }
- }
-
- /* now aChar and bChar are in code point order */
- return (int) aChar - (int) bChar; /* int must be 32 bits wide */
- }
-
- // One is a prefix of the other, or, they are equal:
- return a.length - b.length;
- }
- }
-
/**
* Creates a new CharsRef that points to a copy of the chars from <code>other</code>
*
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
index 404fe73f61d..5a319904293 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
@@ -16,32 +16,11 @@
*/
package org.apache.lucene.util;
-import java.util.Arrays;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
public class TestCharsRef extends LuceneTestCase {
- @SuppressWarnings("deprecation")
- public void testUTF16InUTF8Order() {
- final int numStrings = atLeast(1000);
- BytesRef[] utf8 = new BytesRef[numStrings];
- CharsRef[] utf16 = new CharsRef[numStrings];
-
- for (int i = 0; i < numStrings; i++) {
- String s = TestUtil.randomUnicodeString(random());
- utf8[i] = new BytesRef(s);
- utf16[i] = new CharsRef(s);
- }
-
- Arrays.sort(utf8);
- Arrays.sort(utf16, CharsRef.getUTF16SortedAsUTF8Comparator());
-
- for (int i = 0; i < numStrings; i++) {
- assertEquals(utf8[i].utf8ToString(), utf16[i].toString());
- }
- }
-
public void testAppend() {
CharsRefBuilder ref = new CharsRefBuilder();
StringBuilder builder = new StringBuilder();