Apache Lucene open-source search software
Remove obsolete 'mappingRules' in Tokenizer tests (#12972)
sabi0ec9e593dc493
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.javaindex 620cdd860c1..5e07df48259 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java@@ -46,10 +46,10 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {};for (String[] test : tests) {- TokenStream stream =+ Tokenizer stream =new PatternTokenizer(newAttributeFactory(), Pattern.compile(test[1]), Integer.parseInt(test[0]));- ((Tokenizer) stream).setReader(new StringReader(test[2]));+ stream.setReader(new StringReader(test[2]));String out = tsToString(stream);// System.out.println( test[2] + " ==> " + out );@@ -73,8 +73,6 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {final String INPUT = "Günther Günther is here";// create MappingCharFilter- List<String> mappingRules = new ArrayList<>();- mappingRules.add("\"ü\" => \"ü\"");NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();builder.add("ü", "ü");NormalizeCharMap normMap = builder.build();@@ -113,7 +111,7 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {in.reset();while (in.incrementToken()) {if (out.length() > 0) out.append(' ');- out.append(termAtt.toString());+ out.append(termAtt);in.clearAttributes();termAtt.setEmpty().append("bogusTerm");}@@ -155,7 +153,7 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {// TODO: can we move this to BaseTSTC to catch other "hangs onto heap"ers?// Build a 1MB string:- StringBuilder b = new StringBuilder();+ StringBuilder b = new StringBuilder(1024 * 1024);for (int i = 0; i < 1024; i++) {// 1023 spaces, then an xfor (int j = 0; j < 1023; j++) {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.javaindex 3de03f4853a..f5fc6a0881e 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.pattern;import java.io.IOException;import java.io.StringReader;-import java.util.ArrayList;-import java.util.List;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CharFilter;import org.apache.lucene.analysis.TokenStream;@@ -72,12 +70,9 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {Tokenizer t = new SimplePatternSplitTokenizer(".*");t.getAttribute(CharTermAttribute.class);String s;- while (true) {+ do {s = TestUtil.randomUnicodeString(random());- if (s.length() > 0) {- break;- }- }+ } while (s.isEmpty());t.setReader(new StringReader(s));t.reset();assertFalse(t.incrementToken());@@ -160,8 +155,8 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {};for (String[] test : tests) {- TokenStream stream = new SimplePatternSplitTokenizer(test[0]);- ((Tokenizer) stream).setReader(new StringReader(test[1]));+ Tokenizer stream = new SimplePatternSplitTokenizer(test[0]);+ stream.setReader(new StringReader(test[1]));String out = tsToString(stream);assertEquals("pattern: " + test[0] + " with input: " + test[1], test[2], out);}@@ -178,19 +173,13 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {a.addTransition(start, mid2, 'a', 'z');a.addTransition(mid1, end, 'b');a.addTransition(mid2, end, 'b');- expectThrows(- IllegalArgumentException.class,- () -> {- new SimplePatternSplitTokenizer(a);- });+ expectThrows(IllegalArgumentException.class, () -> new SimplePatternSplitTokenizer(a));}public void testOffsetCorrection() throws Exception {final String INPUT = "Günther Günther is here";// create MappingCharFilter- List<String> mappingRules = new ArrayList<>();- mappingRules.add("\"ü\" => \"ü\"");NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();builder.add("ü", "ü");NormalizeCharMap normMap = builder.build();@@ -220,7 +209,7 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {if (out.length() > 0) {out.append(' ');}- out.append(termAtt.toString());+ out.append(termAtt);in.clearAttributes();termAtt.setEmpty().append("bogusTerm");}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.javaindex e66ae7a049d..6a6584be56d 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.pattern;import java.io.IOException;import java.io.StringReader;-import java.util.ArrayList;-import java.util.List;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CharFilter;import org.apache.lucene.analysis.TokenStream;@@ -61,12 +59,9 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {Tokenizer t = new SimplePatternTokenizer(".*");CharTermAttribute termAtt = t.getAttribute(CharTermAttribute.class);String s;- while (true) {+ do {s = TestUtil.randomUnicodeString(random());- if (s.length() > 0) {- break;- }- }+ } while (s.isEmpty());t.setReader(new StringReader(s));t.reset();assertTrue(t.incrementToken());@@ -120,8 +115,8 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {};for (String[] test : tests) {- TokenStream stream = new SimplePatternTokenizer(test[0]);- ((Tokenizer) stream).setReader(new StringReader(test[1]));+ Tokenizer stream = new SimplePatternTokenizer(test[0]);+ stream.setReader(new StringReader(test[1]));String out = tsToString(stream);assertEquals("pattern: " + test[0] + " with input: " + test[1], test[2], out);@@ -139,19 +134,13 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {a.addTransition(start, mid2, 'a', 'z');a.addTransition(mid1, end, 'b');a.addTransition(mid2, end, 'b');- expectThrows(- IllegalArgumentException.class,- () -> {- new SimplePatternTokenizer(a);- });+ expectThrows(IllegalArgumentException.class, () -> new SimplePatternTokenizer(a));}public void testOffsetCorrection() throws Exception {final String INPUT = "Günther Günther is here";// create MappingCharFilter- List<String> mappingRules = new ArrayList<>();- mappingRules.add("\"ü\" => \"ü\"");NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();builder.add("ü", "ü");NormalizeCharMap normMap = builder.build();@@ -181,7 +170,7 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {if (out.length() > 0) {out.append(' ');}- out.append(termAtt.toString());+ out.append(termAtt);in.clearAttributes();termAtt.setEmpty().append("bogusTerm");}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.javaindex d6c2a5330c9..321c31cb9fb 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java@@ -17,8 +17,6 @@package org.apache.lucene.analysis.wikipedia;import java.io.StringReader;-import java.util.HashSet;-import java.util.Set;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;@@ -66,9 +64,6 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCaspublic void testTokenizerUntokenizedOnly() throws Exception {String test ="[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";- Set<String> untoks = new HashSet<>();- untoks.add(WikipediaTokenizer.CATEGORY);- untoks.add(WikipediaTokenizer.ITALICS);Tokenizer tf =tokenizerFactory(WIKIPEDIA,@@ -145,9 +140,9 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCasIllegalArgumentException expected =expectThrows(IllegalArgumentException.class,- () -> {- tokenizerFactory(WIKIPEDIA, "bogusArg", "bogusValue").create(newAttributeFactory());- });+ () ->+ tokenizerFactory(WIKIPEDIA, "bogusArg", "bogusValue")+ .create(newAttributeFactory()));assertTrue(expected.getMessage().contains("Unknown parameters"));}@@ -155,9 +150,7 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCasIllegalArgumentException expected =expectThrows(IllegalArgumentException.class,- () -> {- tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, "-1").create(newAttributeFactory());- });+ () -> tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, "-1").create(newAttributeFactory()));assertTrue(expected.getMessage()