Apache Lucene open-source search software
Improve TruncateTokenFilter to truncate on codepoints (and no longer produce half surrogates) or legacy utf-16 chars (#15900)
Michael Sokolov86ba05a59825
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 172a8ece3e4..8cdfb9e4f48 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -329,6 +329,10 @@ Bug FixesRefCountedSharedArena.DEFAULT_MAX_PERMITS instead of a hardcoded value, so thatthe default change from GITHUB#15078 takes effect. (Huaixinww)+* GITHUB#15899, GITHUB#15900: Improve TruncateTokenFilter to truncate on codepoints not+ chars and no longer produce half surrogates. There are new factory parameters available+ to configure legacy prefix chars and new codepoint behaviour. (Uwe Schindler, Michael Sokolov)+Other---------------------* GITHUB#15586: Document that scoring and ranking may change across major Lucene versions, and that applications requiring stable ranking should explicitly configure Similarity. (Parveen Saini)diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilter.javaindex bc16173c648..9439540eede 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilter.java@@ -16,6 +16,9 @@*/package org.apache.lucene.analysis.miscellaneous;+import static java.lang.Character.isHighSurrogate;+import static java.lang.Character.isLowSurrogate;+import java.io.IOException;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;@@ -23,34 +26,91 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;/**- * A token filter for truncating the terms into a specific length. Fixed prefix truncation, as a- * stemming method, produces good results on Turkish language. It is reported that F5, using first 5- * characters, produced best results in <a- * href="http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf">Information Retrieval on- * Turkish Texts</a>+ * A token filter for truncating the terms into a specific length (number of codepoints). Fixed+ * prefix truncation, as a stemming method, produces good results on Turkish language. It is+ * reported that F5, using first 5 characters, produced best results in <a+ * href="https://doi.org/10.1002/asi.20750">Information Retrieval on Turkish Texts</a>+ *+ * <p>Since Lucene 10.5, the filter is able to correctly handle codepoints and truncates after the+ * given number of codepoints, no longer producing incomplete surrogate pairs. Use the modern+ * factory method {@link #truncateAfterCodePoints(TokenStream, int)} to enable this mode. Legacy+ * behaviour is still available with {@link #truncateAfterChars(TokenStream, int)}*/public final class TruncateTokenFilter extends TokenFilter {private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);- private final int length;+ private final int truncateAfter;+ private final boolean useCodePoints;++ /** Returns a filter with a prefix of {@code nCodePoints}. */+ public static TruncateTokenFilter truncateAfterCodePoints(TokenStream input, int nCodePoints) {+ return new TruncateTokenFilter(input, nCodePoints, true);+ }++ /**+ * Returns a filter with a prefix of {@code nChars} Java Characters. This may split surrogate+ * pairs.+ */+ public static TruncateTokenFilter truncateAfterChars(TokenStream input, int nChars) {+ return new TruncateTokenFilter(input, nChars, false);+ }++ /**+ * Instantiates filter with a prefix of {@code nChars} Java Characters. This may split surrogate+ * pairs.+ *+ * @deprecated This constructor is deprecated, use {@link #truncateAfterChars(TokenStream, int)}+ * for backwards compatibility, or {@link #truncateAfterCodePoints(TokenStream, int)} to be+ * unicode conformant.+ */+ @Deprecated+ public TruncateTokenFilter(TokenStream input, int nChars) {+ this(input, nChars, false);+ }- public TruncateTokenFilter(TokenStream input, int length) {+ private TruncateTokenFilter(TokenStream input, int truncateAfter, boolean useCodePoints) {super(input);- if (length < 1)- throw new IllegalArgumentException("length parameter must be a positive number: " + length);- this.length = length;+ if (truncateAfter < 1) {+ throw new IllegalArgumentException(+ "truncateAfter parameter must be a positive number: " + truncateAfter);+ }+ this.truncateAfter = truncateAfter;+ this.useCodePoints = useCodePoints;}@Overridepublic final boolean incrementToken() throws IOException {- if (input.incrementToken()) {- if (!keywordAttr.isKeyword() && termAttribute.length() > length)- termAttribute.setLength(length);+ if (!input.incrementToken()) {+ return false;+ }+ if (keywordAttr.isKeyword()) {return true;+ }+ final int len = termAttribute.length();+ if (len <= truncateAfter) {+ // the term is short enough, so we do not need to modify it+ // (works for both chars and codepoints)+ return true;+ }+ if (useCodePoints) {+ // code based on ICU4J's com.ibm.icu.text.UTF16#findOffsetFromCodePoint(...) implementation:+ final char[] arr = termAttribute.buffer();+ int ofs = 0, remaining = truncateAfter;+ while (ofs < len && remaining > 0) {+ if (isHighSurrogate(arr[ofs++]) && ofs < len && isLowSurrogate(arr[ofs])) {+ ofs++;+ }+ remaining--;+ }+ // check if we actually reached the limit and set new length based on calculated offset:+ if (remaining == 0) {+ termAttribute.setLength(ofs);+ }} else {- return false;+ termAttribute.setLength(truncateAfter);}+ return true;}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.javaindex 11ae978532f..2a3743a2460 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterFactory.java@@ -17,12 +17,26 @@package org.apache.lucene.analysis.miscellaneous;import java.util.Map;+import java.util.function.BiFunction;import org.apache.lucene.analysis.TokenFilterFactory;import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.util.Version;/**- * Factory for {@link org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter}. The following- * type is recommended for "<i>diacritics-insensitive search</i>" for Turkish.+ * Factory for {@link org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter}.+ *+ * <p>Fixed prefix truncation, as a stemming method, produces good results on Turkish language. It+ * is reported that F5, using first 5 characters, produced best results in <a+ * href="https://doi.org/10.1002/asi.20750">Information Retrieval on Turkish Texts</a>+ *+ * <p>Since Lucene 10.5, the filter correctly handles codepoints and truncates after {@code+ * truncateAfterCodePoints} codepoints, no longer producing incomplete surrogate pairs. For+ * backwards compatibility the old {@code prefixLength} is still supported and its behaviour depends+ * on the {@code luceneMatchVersion} parameter. If no parameter is given, it uses a prefix length of+ * 5. In case you change to the more modern codepoint behaviour, reindexing may be required if your+ * documents contain surrogate pairs (like emojis).+ *+ * <p>The following type is recommended for "<i>diacritics-insensitive search</i>" for Turkish:** <pre><code class="language-xml">* <fieldType name="text_tr_ascii_f5" class="solr.TextField" positionIncrementGap="100">@@ -32,7 +46,7 @@ import org.apache.lucene.analysis.TokenStream;* <filter class="solr.TurkishLowerCaseFilterFactory"/>* <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>* <filter class="solr.KeywordRepeatFilterFactory"/>- * <filter class="solr.TruncateTokenFilterFactory" prefixLength="5"/>+ * <filter class="solr.TruncateTokenFilterFactory" truncateAfterCodePoints="5"/>* <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>* </analyzer>* </fieldType></code></pre>@@ -45,20 +59,43 @@ public class TruncateTokenFilterFactory extends TokenFilterFactory {/** SPI name */public static final String NAME = "truncate";- public static final String PREFIX_LENGTH_KEY = "prefixLength";- private final int prefixLength;+ @Deprecated public static final String PREFIX_LENGTH_KEY = "prefixLength";+ public static final String TRUNCATE_AFTER_CODEPOINTS_KEY = "truncateAfterCodePoints";+ public static final String TRUNCATE_AFTER_CHARS_KEY = "truncateAfterChars";++ private final int truncateAfter;+ private final BiFunction<TokenStream, Integer, TruncateTokenFilter> factory;public TruncateTokenFilterFactory(Map<String, String> args) {super(args);- prefixLength = Integer.parseInt(get(args, PREFIX_LENGTH_KEY, "5"));- if (prefixLength < 1)+ Map<String, BiFunction<TokenStream, Integer, TruncateTokenFilter>> paramMapping =+ Map.of(+ TRUNCATE_AFTER_CODEPOINTS_KEY, TruncateTokenFilter::truncateAfterCodePoints,+ TRUNCATE_AFTER_CHARS_KEY, TruncateTokenFilter::truncateAfterChars,+ PREFIX_LENGTH_KEY, this::legacyPrefixLengthFactory);+ var avail = paramMapping.keySet().stream().filter(args::containsKey).toList();+ if (avail.size() > 1) {throw new IllegalArgumentException(- PREFIX_LENGTH_KEY + " parameter must be a positive number: " + prefixLength);+ "Can only give one of the following parameters: " + paramMapping.keySet());+ }+ String param = avail.stream().findFirst().orElse(PREFIX_LENGTH_KEY);+ this.truncateAfter = getInt(args, param, 5);+ this.factory = paramMapping.get(param);+ if (truncateAfter < 1) {+ throw new IllegalArgumentException(+ param + " parameter must be a positive number: " + truncateAfter);+ }if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameter(s): " + args);}}+ private TruncateTokenFilter legacyPrefixLengthFactory(TokenStream input, int prefixChars) {+ return (luceneMatchVersion.onOrAfter(Version.LUCENE_10_5_0))+ ? TruncateTokenFilter.truncateAfterCodePoints(input, prefixChars)+ : TruncateTokenFilter.truncateAfterChars(input, prefixChars);+ }+/** Default ctor for compatibility with SPI */public TruncateTokenFilterFactory() {throw defaultCtorException();@@ -66,6 +103,6 @@ public class TruncateTokenFilterFactory extends TokenFilterFactory {@Overridepublic TokenStream create(TokenStream input) {- return new TruncateTokenFilter(input, prefixLength);+ return factory.apply(input, truncateAfter);}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestConditionalTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestConditionalTokenFilter.javaindex 272dfd6a546..4e30fe3647f 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestConditionalTokenFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestConditionalTokenFilter.java@@ -323,7 +323,8 @@ public class TestConditionalTokenFilter extends BaseTokenStreamTestCase {new SkipMatchingFilter(stream,in -> {- TruncateTokenFilter truncateFilter = new TruncateTokenFilter(in, 2);+ TruncateTokenFilter truncateFilter =+ TruncateTokenFilter.truncateAfterCodePoints(in, 2);return new AssertingLowerCaseFilter(truncateFilter);},".*o.*");diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.javaindex 0ce3932bd91..631e4d3f8b6 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilter.java@@ -16,22 +16,123 @@*/package org.apache.lucene.analysis.miscellaneous;+import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;+import org.apache.lucene.tests.analysis.MockTokenizer;+import org.apache.lucene.tests.util.TestUtil;import org.junit.Test;/** Test the truncate token filter. */public class TestTruncateTokenFilter extends BaseTokenStreamTestCase {- public void testTruncating() throws Exception {- TokenStream stream = whitespaceMockTokenizer("abcdefg 1234567 ABCDEFG abcde abc 12345 123");- stream = new TruncateTokenFilter(stream, 5);+ public void testLegacyTruncating() throws Exception {+ TokenStream stream =+ whitespaceMockTokenizer("abcdefg 1234567 ABCDEFG abcde abc 12345 123 1234567 1234π5");+ stream = TruncateTokenFilter.truncateAfterChars(stream, 5);assertTokenStreamContents(- stream, new String[] {"abcde", "12345", "ABCDE", "abcde", "abc", "12345", "123"});+ stream,+ new String[] {+ "abcde",+ "12345",+ "ABCDE",+ "abcde",+ "abc",+ "12345",+ "123",+ "12345",+ "1234" + "π".charAt(0)+ });+ }++ public void testCodePointTruncating() throws Exception {+ TokenStream stream =+ whitespaceMockTokenizer(+ "abcdefg 1234567 ABCDEFG abcde abc 12345 123 1234π5 1 π π12345 ππ πππ ππππ πππππ ππππππ");+ stream = TruncateTokenFilter.truncateAfterCodePoints(stream, 5);+ assertTokenStreamContents(+ stream,+ new String[] {+ "abcde",+ "12345",+ "ABCDE",+ "abcde",+ "abc",+ "12345",+ "123",+ "1234π",+ "1",+ "π",+ "π1234",+ "ππ",+ "πππ",+ "ππππ",+ "πππππ",+ "πππππ"+ });+ }++ public void testRandom() throws Exception {+ var rnd = random();+ for (int i = 0; i < 50 * RANDOM_MULTIPLIER; i++) {+ var truncateLength = rnd.nextInt(5) + 1;+ String text = TestUtil.randomAnalysisString(rnd, 200, false);++ TokenStream ts1 = whitespaceMockTokenizer(text);+ CharTermAttribute termAtt1 = ts1.addAttribute(CharTermAttribute.class);+ TokenStream ts2 =+ TruncateTokenFilter.truncateAfterCodePoints(+ whitespaceMockTokenizer(text), truncateLength);+ CharTermAttribute termAtt2 = ts2.addAttribute(CharTermAttribute.class);++ ts1.reset();+ ts2.reset();+ while (ts2.incrementToken()) {+ assertTrue(ts1.incrementToken());+ int len1 = Character.codePointCount(termAtt1, 0, termAtt1.length());+ int len2 = Character.codePointCount(termAtt2, 0, termAtt2.length());+ if (len1 <= truncateLength) {+ assertEquals(len1, len2);+ assertEquals(termAtt1.toString(), termAtt2.toString());+ } else {+ assertEquals(truncateLength, len2);+ assertTrue(termAtt1.toString().startsWith(termAtt2.toString()));+ }+ }+ assertFalse(ts1.incrementToken());+ ts1.end();+ ts2.end();+ ts1.close();+ ts2.close();+ }+ }++ public void testStressRandom() throws Exception {+ var rnd = random();+ var truncateLength = rnd.nextInt(5) + 1;+ Analyzer a =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ return new TokenStreamComponents(+ tokenizer, TruncateTokenFilter.truncateAfterCodePoints(tokenizer, truncateLength));+ }+ };+ checkRandomData(rnd, a, 20 * RANDOM_MULTIPLIER, truncateLength * 2);+ }++ @Test(expected = IllegalArgumentException.class)+ public void testLegacyNonPositiveLength() throws Exception {+ TruncateTokenFilter.truncateAfterChars(+ whitespaceMockTokenizer("param must be a positive number"), -48);}@Test(expected = IllegalArgumentException.class)public void testNonPositiveLength() throws Exception {- new TruncateTokenFilter(whitespaceMockTokenizer("length must be a positive number"), -48);+ TruncateTokenFilter.truncateAfterCodePoints(+ whitespaceMockTokenizer("param must be a positive number"), -48);}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.javaindex f537b93a156..adc172666aa 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTruncateTokenFilterFactory.java@@ -19,22 +19,80 @@ package org.apache.lucene.analysis.miscellaneous;import java.io.Reader;import java.io.StringReader;import org.apache.lucene.analysis.TokenStream;-import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;import org.apache.lucene.tests.analysis.MockTokenizer;+import org.apache.lucene.util.Version;/** Simple tests to ensure the simple truncation filter factory is working. */public class TestTruncateTokenFilterFactory extends BaseTokenStreamFactoryTestCase {- /** Ensure the filter actually truncates text. */- public void testTruncating() throws Exception {- Reader reader = new StringReader("abcdefg 1234567 ABCDEFG abcde abc 12345 123");- TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);- ((Tokenizer) stream).setReader(reader);- stream =- tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "5")- .create(stream);++ public void testCharTruncating() throws Exception {+ testCharTruncating(TruncateTokenFilterFactory.TRUNCATE_AFTER_CHARS_KEY);+ testCharTruncating(TruncateTokenFilterFactory.PREFIX_LENGTH_KEY);+ testCharTruncating(null);+ }++ private void testCharTruncating(String param) throws Exception {+ Reader reader = new StringReader("abcdefg 1234567 ABCDEFG abcde abc 12345 123 1234567 1234π5");+ var tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ tokenizer.setReader(reader);+ TokenStream stream =+ (param == null+ ? tokenFilterFactory("truncate", Version.LUCENE_10_0_0)+ : tokenFilterFactory("Truncate", Version.LUCENE_10_0_0, param, "5"))+ .create(tokenizer);assertTokenStreamContents(- stream, new String[] {"abcde", "12345", "ABCDE", "abcde", "abc", "12345", "123"});+ stream,+ new String[] {+ "abcde",+ "12345",+ "ABCDE",+ "abcde",+ "abc",+ "12345",+ "123",+ "12345",+ "1234" + "π".charAt(0)+ });+ }++ public void testCodePointTruncating() throws Exception {+ testCodePointTruncating(TruncateTokenFilterFactory.TRUNCATE_AFTER_CODEPOINTS_KEY);+ testCodePointTruncating(TruncateTokenFilterFactory.PREFIX_LENGTH_KEY);+ testCodePointTruncating(null);+ }++ public void testCodePointTruncating(String param) throws Exception {+ Reader reader =+ new StringReader(+ "abcdefg 1234567 ABCDEFG abcde abc 12345 123 1234π5 1 π π12345 ππ πππ ππππ πππππ ππππππ");+ var tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ tokenizer.setReader(reader);+ TokenStream stream =+ (param == null+ ? tokenFilterFactory("truncate", Version.LATEST)+ : tokenFilterFactory("Truncate", Version.LATEST, param, "5"))+ .create(tokenizer);+ assertTokenStreamContents(+ stream,+ new String[] {+ "abcde",+ "12345",+ "ABCDE",+ "abcde",+ "abc",+ "12345",+ "123",+ "1234π",+ "1",+ "π",+ "π1234",+ "ππ",+ "πππ",+ "ππππ",+ "πππππ",+ "πππππ"+ });}/** Test that bogus arguments result in exception */@@ -45,12 +103,25 @@ public class TestTruncateTokenFilterFactory extends BaseTokenStreamFactoryTestCa() -> {tokenFilterFactory("Truncate",- TruncateTokenFilterFactory.PREFIX_LENGTH_KEY,+ TruncateTokenFilterFactory.TRUNCATE_AFTER_CODEPOINTS_KEY,"5","bogusArg","bogusValue");});assertTrue(expected.getMessage().contains("Unknown parameter(s):"));++ expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ tokenFilterFactory(+ "Truncate",+ TruncateTokenFilterFactory.TRUNCATE_AFTER_CODEPOINTS_KEY,+ "5",+ TruncateTokenFilterFactory.TRUNCATE_AFTER_CHARS_KEY,+ "6");+ });+ assertTrue(expected.getMessage().contains("Can only give one of the following parameters:"));}/** Test that negative prefix length result in exception */@@ -59,32 +130,14 @@ public class TestTruncateTokenFilterFactory extends BaseTokenStreamFactoryTestCaexpectThrows(IllegalArgumentException.class,() -> {- tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "-5");+ tokenFilterFactory(+ "Truncate", TruncateTokenFilterFactory.TRUNCATE_AFTER_CODEPOINTS_KEY, "-5");});assertTrue(expected.getMessage().contains(- TruncateTokenFilterFactory.PREFIX_LENGTH_KEY+ TruncateTokenFilterFactory.TRUNCATE_AFTER_CODEPOINTS_KEY+ " parameter must be a positive number: -5"));}-- /** Test that takes length greater than byte limit accepts it */- public void testLengthGreaterThanByteLimitArgument() throws Exception {- Reader reader =- new StringReader(- "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw128characters From here");- TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);- ((Tokenizer) stream).setReader(reader);- stream =- tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "128")- .create(stream);- assertTokenStreamContents(- stream,- new String[] {- "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw1",- "From",- "here"- });- }}