Skip to content

Apache Lucene open-source search software

Remove obsolete 'mappingRules' in Tokenizer tests (#12972)

sabi0ec9e593dc493
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
index 620cdd860c1..5e07df48259 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
@@ -46,10 +46,10 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {
};
for (String[] test : tests) {
- TokenStream stream =
+ Tokenizer stream =
new PatternTokenizer(
newAttributeFactory(), Pattern.compile(test[1]), Integer.parseInt(test[0]));
- ((Tokenizer) stream).setReader(new StringReader(test[2]));
+ stream.setReader(new StringReader(test[2]));
String out = tsToString(stream);
// System.out.println( test[2] + " ==> " + out );
@@ -73,8 +73,6 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {
final String INPUT = "Günther Günther is here";
// create MappingCharFilter
- List<String> mappingRules = new ArrayList<>();
- mappingRules.add("\"&uuml;\" => \"ü\"");
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
builder.add("&uuml;", "ü");
NormalizeCharMap normMap = builder.build();
@@ -113,7 +111,7 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {
in.reset();
while (in.incrementToken()) {
if (out.length() > 0) out.append(' ');
- out.append(termAtt.toString());
+ out.append(termAtt);
in.clearAttributes();
termAtt.setEmpty().append("bogusTerm");
}
@@ -155,7 +153,7 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase {
// TODO: can we move this to BaseTSTC to catch other "hangs onto heap"ers?
// Build a 1MB string:
- StringBuilder b = new StringBuilder();
+ StringBuilder b = new StringBuilder(1024 * 1024);
for (int i = 0; i < 1024; i++) {
// 1023 spaces, then an x
for (int j = 0; j < 1023; j++) {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java
index 3de03f4853a..f5fc6a0881e 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternSplitTokenizer.java
@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.pattern;
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -72,12 +70,9 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {
Tokenizer t = new SimplePatternSplitTokenizer(".*");
t.getAttribute(CharTermAttribute.class);
String s;
- while (true) {
+ do {
s = TestUtil.randomUnicodeString(random());
- if (s.length() > 0) {
- break;
- }
- }
+ } while (s.isEmpty());
t.setReader(new StringReader(s));
t.reset();
assertFalse(t.incrementToken());
@@ -160,8 +155,8 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {
};
for (String[] test : tests) {
- TokenStream stream = new SimplePatternSplitTokenizer(test[0]);
- ((Tokenizer) stream).setReader(new StringReader(test[1]));
+ Tokenizer stream = new SimplePatternSplitTokenizer(test[0]);
+ stream.setReader(new StringReader(test[1]));
String out = tsToString(stream);
assertEquals("pattern: " + test[0] + " with input: " + test[1], test[2], out);
}
@@ -178,19 +173,13 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {
a.addTransition(start, mid2, 'a', 'z');
a.addTransition(mid1, end, 'b');
a.addTransition(mid2, end, 'b');
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- new SimplePatternSplitTokenizer(a);
- });
+ expectThrows(IllegalArgumentException.class, () -> new SimplePatternSplitTokenizer(a));
}
public void testOffsetCorrection() throws Exception {
final String INPUT = "G&uuml;nther G&uuml;nther is here";
// create MappingCharFilter
- List<String> mappingRules = new ArrayList<>();
- mappingRules.add("\"&uuml;\" => \"ü\"");
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
builder.add("&uuml;", "ü");
NormalizeCharMap normMap = builder.build();
@@ -220,7 +209,7 @@ public class TestSimplePatternSplitTokenizer extends BaseTokenStreamTestCase {
if (out.length() > 0) {
out.append(' ');
}
- out.append(termAtt.toString());
+ out.append(termAtt);
in.clearAttributes();
termAtt.setEmpty().append("bogusTerm");
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java
index e66ae7a049d..6a6584be56d 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestSimplePatternTokenizer.java
@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.pattern;
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -61,12 +59,9 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {
Tokenizer t = new SimplePatternTokenizer(".*");
CharTermAttribute termAtt = t.getAttribute(CharTermAttribute.class);
String s;
- while (true) {
+ do {
s = TestUtil.randomUnicodeString(random());
- if (s.length() > 0) {
- break;
- }
- }
+ } while (s.isEmpty());
t.setReader(new StringReader(s));
t.reset();
assertTrue(t.incrementToken());
@@ -120,8 +115,8 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {
};
for (String[] test : tests) {
- TokenStream stream = new SimplePatternTokenizer(test[0]);
- ((Tokenizer) stream).setReader(new StringReader(test[1]));
+ Tokenizer stream = new SimplePatternTokenizer(test[0]);
+ stream.setReader(new StringReader(test[1]));
String out = tsToString(stream);
assertEquals("pattern: " + test[0] + " with input: " + test[1], test[2], out);
@@ -139,19 +134,13 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {
a.addTransition(start, mid2, 'a', 'z');
a.addTransition(mid1, end, 'b');
a.addTransition(mid2, end, 'b');
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- new SimplePatternTokenizer(a);
- });
+ expectThrows(IllegalArgumentException.class, () -> new SimplePatternTokenizer(a));
}
public void testOffsetCorrection() throws Exception {
final String INPUT = "G&uuml;nther G&uuml;nther is here";
// create MappingCharFilter
- List<String> mappingRules = new ArrayList<>();
- mappingRules.add("\"&uuml;\" => \"ü\"");
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
builder.add("&uuml;", "ü");
NormalizeCharMap normMap = builder.build();
@@ -181,7 +170,7 @@ public class TestSimplePatternTokenizer extends BaseTokenStreamTestCase {
if (out.length() > 0) {
out.append(' ');
}
- out.append(termAtt.toString());
+ out.append(termAtt);
in.clearAttributes();
termAtt.setEmpty().append("bogusTerm");
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
index d6c2a5330c9..321c31cb9fb 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
@@ -17,8 +17,6 @@
package org.apache.lucene.analysis.wikipedia;
import java.io.StringReader;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;
@@ -66,9 +64,6 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCas
public void testTokenizerUntokenizedOnly() throws Exception {
String test =
"[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
- Set<String> untoks = new HashSet<>();
- untoks.add(WikipediaTokenizer.CATEGORY);
- untoks.add(WikipediaTokenizer.ITALICS);
Tokenizer tf =
tokenizerFactory(
WIKIPEDIA,
@@ -145,9 +140,9 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCas
IllegalArgumentException expected =
expectThrows(
IllegalArgumentException.class,
- () -> {
- tokenizerFactory(WIKIPEDIA, "bogusArg", "bogusValue").create(newAttributeFactory());
- });
+ () ->
+ tokenizerFactory(WIKIPEDIA, "bogusArg", "bogusValue")
+ .create(newAttributeFactory()));
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
@@ -155,9 +150,7 @@ public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCas
IllegalArgumentException expected =
expectThrows(
IllegalArgumentException.class,
- () -> {
- tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, "-1").create(newAttributeFactory());
- });
+ () -> tokenizerFactory(WIKIPEDIA, TOKEN_OUTPUT, "-1").create(newAttributeFactory()));
assertTrue(
expected
.getMessage()